xref: /petsc/src/dm/impls/plex/plex.c (revision ccd65f6325a2212a5fe1f4fe4d73f30fbecd7d64) 
1 #include <petsc/private/dmpleximpl.h>   /*I      "petscdmplex.h"   I*/
2 #include <petsc/private/isimpl.h>
3 #include <petsc/private/vecimpl.h>
4 #include <petsc/private/glvisvecimpl.h>
5 #include <petscsf.h>
6 #include <petscds.h>
7 #include <petscdraw.h>
8 #include <petscdmfield.h>
9 
10 /* Logging support */
11 PetscLogEvent DMPLEX_Interpolate, DMPLEX_Partition, DMPLEX_Distribute, DMPLEX_DistributeCones, DMPLEX_DistributeLabels, DMPLEX_DistributeSF, DMPLEX_DistributeOverlap, DMPLEX_DistributeField, DMPLEX_DistributeData, DMPLEX_Migrate, DMPLEX_InterpolateSF, DMPLEX_GlobalToNaturalBegin, DMPLEX_GlobalToNaturalEnd, DMPLEX_NaturalToGlobalBegin, DMPLEX_NaturalToGlobalEnd, DMPLEX_Stratify, DMPLEX_Symmetrize, DMPLEX_Preallocate, DMPLEX_ResidualFEM, DMPLEX_JacobianFEM, DMPLEX_InterpolatorFEM, DMPLEX_InjectorFEM, DMPLEX_IntegralFEM, DMPLEX_CreateGmsh, DMPLEX_RebalanceSharedPoints, DMPLEX_PartSelf, DMPLEX_PartLabelInvert, DMPLEX_PartLabelCreateSF, DMPLEX_PartStratSF, DMPLEX_CreatePointSF,DMPLEX_LocatePoints;
12 
13 PETSC_EXTERN PetscErrorCode VecView_MPI(Vec, PetscViewer);
14 
15 /*@
16   DMPlexIsSimplex - Is the first cell in this mesh a simplex?
17 
18   Input Parameter:
19 . dm      - The DMPlex object
20 
21   Output Parameter:
22 . simplex - Flag checking for a simplex
23 
24   Note: This just gives the first range of cells found. If the mesh has several cell types, it will only give the first.
25   If the mesh has no cells, this returns PETSC_FALSE.
26 
27   Level: intermediate
28 
29 .seealso DMPlexGetSimplexOrBoxCells(), DMPlexGetCellType(), DMPlexGetHeightStratum(), DMPolytopeTypeGetNumVertices()
30 @*/
31 PetscErrorCode DMPlexIsSimplex(DM dm, PetscBool *simplex)
32 {
33   DMPolytopeType ct;
34   PetscInt       cStart, cEnd;
35   PetscErrorCode ierr;
36 
37   PetscFunctionBegin;
38   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
39   if (cEnd <= cStart) {*simplex = PETSC_FALSE; PetscFunctionReturn(0);}
40   ierr = DMPlexGetCellType(dm, cStart, &ct);CHKERRQ(ierr);
41   *simplex = DMPolytopeTypeGetNumVertices(ct) == DMPolytopeTypeGetDim(ct)+1 ? PETSC_TRUE : PETSC_FALSE;
42   PetscFunctionReturn(0);
43 }
44 
45 /*@
46   DMPlexGetSimplexOrBoxCells - Get the range of cells which are neither prisms nor ghost FV cells
47 
48   Input Parameter:
49 + dm     - The DMPlex object
50 - height - The cell height in the Plex, 0 is the default
51 
52   Output Parameters:
53 + cStart - The first "normal" cell
54 - cEnd   - The upper bound on "normal"" cells
55 
56   Note: This just gives the first range of cells found. If the mesh has several cell types, it will only give the first.
57 
58   Level: developer
59 
60 .seealso DMPlexConstructGhostCells(), DMPlexGetGhostCellStratum()
61 @*/
62 PetscErrorCode DMPlexGetSimplexOrBoxCells(DM dm, PetscInt height, PetscInt *cStart, PetscInt *cEnd)
63 {
64   DMPolytopeType ct = DM_POLYTOPE_UNKNOWN;
65   PetscInt       cS, cE, c;
66   PetscErrorCode ierr;
67 
68   PetscFunctionBegin;
69   ierr = DMPlexGetHeightStratum(dm, PetscMax(height, 0), &cS, &cE);CHKERRQ(ierr);
70   for (c = cS; c < cE; ++c) {
71     DMPolytopeType cct;
72 
73     ierr = DMPlexGetCellType(dm, c, &cct);CHKERRQ(ierr);
74     if ((PetscInt) cct < 0) break;
75     switch (cct) {
76       case DM_POLYTOPE_POINT:
77       case DM_POLYTOPE_SEGMENT:
78       case DM_POLYTOPE_TRIANGLE:
79       case DM_POLYTOPE_QUADRILATERAL:
80       case DM_POLYTOPE_TETRAHEDRON:
81       case DM_POLYTOPE_HEXAHEDRON:
82         ct = cct;
83         break;
84       default: break;
85     }
86     if (ct != DM_POLYTOPE_UNKNOWN) break;
87   }
88   if (ct != DM_POLYTOPE_UNKNOWN) {
89     DMLabel ctLabel;
90 
91     ierr = DMPlexGetCellTypeLabel(dm, &ctLabel);CHKERRQ(ierr);
92     ierr = DMLabelGetStratumBounds(ctLabel, ct, &cS, &cE);CHKERRQ(ierr);
93   }
94   if (cStart) *cStart = cS;
95   if (cEnd)   *cEnd   = cE;
96   PetscFunctionReturn(0);
97 }
98 
99 PetscErrorCode DMPlexGetFieldType_Internal(DM dm, PetscSection section, PetscInt field, PetscInt *sStart, PetscInt *sEnd, PetscViewerVTKFieldType *ft)
100 {
101   PetscInt       cdim, pStart, pEnd, vStart, vEnd, cStart, cEnd;
102   PetscInt       vcdof[2] = {0,0}, globalvcdof[2];
103   PetscErrorCode ierr;
104 
105   PetscFunctionBegin;
106   *ft  = PETSC_VTK_INVALID;
107   ierr = DMGetCoordinateDim(dm, &cdim);CHKERRQ(ierr);
108   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
109   ierr = DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
110   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
111   if (field >= 0) {
112     if ((vStart >= pStart) && (vStart < pEnd)) {ierr = PetscSectionGetFieldDof(section, vStart, field, &vcdof[0]);CHKERRQ(ierr);}
113     if ((cStart >= pStart) && (cStart < pEnd)) {ierr = PetscSectionGetFieldDof(section, cStart, field, &vcdof[1]);CHKERRQ(ierr);}
114   } else {
115     if ((vStart >= pStart) && (vStart < pEnd)) {ierr = PetscSectionGetDof(section, vStart, &vcdof[0]);CHKERRQ(ierr);}
116     if ((cStart >= pStart) && (cStart < pEnd)) {ierr = PetscSectionGetDof(section, cStart, &vcdof[1]);CHKERRQ(ierr);}
117   }
118   ierr = MPI_Allreduce(vcdof, globalvcdof, 2, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject)dm));CHKERRMPI(ierr);
119   if (globalvcdof[0]) {
120     *sStart = vStart;
121     *sEnd   = vEnd;
122     if (globalvcdof[0] == cdim) *ft = PETSC_VTK_POINT_VECTOR_FIELD;
123     else                        *ft = PETSC_VTK_POINT_FIELD;
124   } else if (globalvcdof[1]) {
125     *sStart = cStart;
126     *sEnd   = cEnd;
127     if (globalvcdof[1] == cdim) *ft = PETSC_VTK_CELL_VECTOR_FIELD;
128     else                        *ft = PETSC_VTK_CELL_FIELD;
129   } else {
130     if (field >= 0) {
131       const char *fieldname;
132 
133       ierr = PetscSectionGetFieldName(section, field, &fieldname);CHKERRQ(ierr);
134       ierr = PetscInfo2((PetscObject) dm, "Could not classify VTK output type of section field %D \"%s\"\n", field, fieldname);CHKERRQ(ierr);
135     } else {
136       ierr = PetscInfo((PetscObject) dm, "Could not classify VTK output type of section\"%s\"\n");CHKERRQ(ierr);
137     }
138   }
139   PetscFunctionReturn(0);
140 }
141 
142 static PetscErrorCode VecView_Plex_Local_Draw(Vec v, PetscViewer viewer)
143 {
144   DM                 dm;
145   PetscSection       s;
146   PetscDraw          draw, popup;
147   DM                 cdm;
148   PetscSection       coordSection;
149   Vec                coordinates;
150   const PetscScalar *coords, *array;
151   PetscReal          bound[4] = {PETSC_MAX_REAL, PETSC_MAX_REAL, PETSC_MIN_REAL, PETSC_MIN_REAL};
152   PetscReal          vbound[2], time;
153   PetscBool          isnull, flg;
154   PetscInt           dim, Nf, f, Nc, comp, vStart, vEnd, cStart, cEnd, c, N, level, step, w = 0;
155   const char        *name;
156   char               title[PETSC_MAX_PATH_LEN];
157   PetscErrorCode     ierr;
158 
159   PetscFunctionBegin;
160   ierr = PetscViewerDrawGetDraw(viewer, 0, &draw);CHKERRQ(ierr);
161   ierr = PetscDrawIsNull(draw, &isnull);CHKERRQ(ierr);
162   if (isnull) PetscFunctionReturn(0);
163 
164   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
165   ierr = DMGetCoordinateDim(dm, &dim);CHKERRQ(ierr);
166   if (dim != 2) SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Cannot draw meshes of dimension %D. Use PETSCVIEWERGLVIS", dim);
167   ierr = DMGetLocalSection(dm, &s);CHKERRQ(ierr);
168   ierr = PetscSectionGetNumFields(s, &Nf);CHKERRQ(ierr);
169   ierr = DMGetCoarsenLevel(dm, &level);CHKERRQ(ierr);
170   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
171   ierr = DMGetLocalSection(cdm, &coordSection);CHKERRQ(ierr);
172   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
173   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
174   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
175 
176   ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
177   ierr = DMGetOutputSequenceNumber(dm, &step, &time);CHKERRQ(ierr);
178 
179   ierr = VecGetLocalSize(coordinates, &N);CHKERRQ(ierr);
180   ierr = VecGetArrayRead(coordinates, &coords);CHKERRQ(ierr);
181   for (c = 0; c < N; c += dim) {
182     bound[0] = PetscMin(bound[0], PetscRealPart(coords[c]));   bound[2] = PetscMax(bound[2], PetscRealPart(coords[c]));
183     bound[1] = PetscMin(bound[1], PetscRealPart(coords[c+1])); bound[3] = PetscMax(bound[3], PetscRealPart(coords[c+1]));
184   }
185   ierr = VecRestoreArrayRead(coordinates, &coords);CHKERRQ(ierr);
186   ierr = PetscDrawClear(draw);CHKERRQ(ierr);
187 
188   /* Could implement something like DMDASelectFields() */
189   for (f = 0; f < Nf; ++f) {
190     DM   fdm = dm;
191     Vec  fv  = v;
192     IS   fis;
193     char prefix[PETSC_MAX_PATH_LEN];
194     const char *fname;
195 
196     ierr = PetscSectionGetFieldComponents(s, f, &Nc);CHKERRQ(ierr);
197     ierr = PetscSectionGetFieldName(s, f, &fname);CHKERRQ(ierr);
198 
199     if (v->hdr.prefix) {ierr = PetscStrncpy(prefix, v->hdr.prefix,sizeof(prefix));CHKERRQ(ierr);}
200     else               {prefix[0] = '\0';}
201     if (Nf > 1) {
202       ierr = DMCreateSubDM(dm, 1, &f, &fis, &fdm);CHKERRQ(ierr);
203       ierr = VecGetSubVector(v, fis, &fv);CHKERRQ(ierr);
204       ierr = PetscStrlcat(prefix, fname,sizeof(prefix));CHKERRQ(ierr);
205       ierr = PetscStrlcat(prefix, "_",sizeof(prefix));CHKERRQ(ierr);
206     }
207     for (comp = 0; comp < Nc; ++comp, ++w) {
208       PetscInt nmax = 2;
209 
210       ierr = PetscViewerDrawGetDraw(viewer, w, &draw);CHKERRQ(ierr);
211       if (Nc > 1) {ierr = PetscSNPrintf(title, sizeof(title), "%s:%s_%D Step: %D Time: %.4g", name, fname, comp, step, time);CHKERRQ(ierr);}
212       else        {ierr = PetscSNPrintf(title, sizeof(title), "%s:%s Step: %D Time: %.4g", name, fname, step, time);CHKERRQ(ierr);}
213       ierr = PetscDrawSetTitle(draw, title);CHKERRQ(ierr);
214 
215       /* TODO Get max and min only for this component */
216       ierr = PetscOptionsGetRealArray(NULL, prefix, "-vec_view_bounds", vbound, &nmax, &flg);CHKERRQ(ierr);
217       if (!flg) {
218         ierr = VecMin(fv, NULL, &vbound[0]);CHKERRQ(ierr);
219         ierr = VecMax(fv, NULL, &vbound[1]);CHKERRQ(ierr);
220         if (vbound[1] <= vbound[0]) vbound[1] = vbound[0] + 1.0;
221       }
222       ierr = PetscDrawGetPopup(draw, &popup);CHKERRQ(ierr);
223       ierr = PetscDrawScalePopup(popup, vbound[0], vbound[1]);CHKERRQ(ierr);
224       ierr = PetscDrawSetCoordinates(draw, bound[0], bound[1], bound[2], bound[3]);CHKERRQ(ierr);
225 
226       ierr = VecGetArrayRead(fv, &array);CHKERRQ(ierr);
227       for (c = cStart; c < cEnd; ++c) {
228         PetscScalar *coords = NULL, *a = NULL;
229         PetscInt     numCoords, color[4] = {-1,-1,-1,-1};
230 
231         ierr = DMPlexPointLocalRead(fdm, c, array, &a);CHKERRQ(ierr);
232         if (a) {
233           color[0] = PetscDrawRealToColor(PetscRealPart(a[comp]), vbound[0], vbound[1]);
234           color[1] = color[2] = color[3] = color[0];
235         } else {
236           PetscScalar *vals = NULL;
237           PetscInt     numVals, va;
238 
239           ierr = DMPlexVecGetClosure(fdm, NULL, fv, c, &numVals, &vals);CHKERRQ(ierr);
240           if (numVals % Nc) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "The number of components %D does not divide the number of values in the closure %D", Nc, numVals);
241           switch (numVals/Nc) {
242           case 3: /* P1 Triangle */
243           case 4: /* P1 Quadrangle */
244             for (va = 0; va < numVals/Nc; ++va) color[va] = PetscDrawRealToColor(PetscRealPart(vals[va*Nc+comp]), vbound[0], vbound[1]);
245             break;
246           case 6: /* P2 Triangle */
247           case 8: /* P2 Quadrangle */
248             for (va = 0; va < numVals/(Nc*2); ++va) color[va] = PetscDrawRealToColor(PetscRealPart(vals[va*Nc+comp + numVals/(Nc*2)]), vbound[0], vbound[1]);
249             break;
250           default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of values for cell closure %D cannot be handled", numVals/Nc);
251           }
252           ierr = DMPlexVecRestoreClosure(fdm, NULL, fv, c, &numVals, &vals);CHKERRQ(ierr);
253         }
254         ierr = DMPlexVecGetClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
255         switch (numCoords) {
256         case 6:
257         case 12: /* Localized triangle */
258           ierr = PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), color[0], color[1], color[2]);CHKERRQ(ierr);
259           break;
260         case 8:
261         case 16: /* Localized quadrilateral */
262           ierr = PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), color[0], color[1], color[2]);CHKERRQ(ierr);
263           ierr = PetscDrawTriangle(draw, PetscRealPart(coords[4]), PetscRealPart(coords[5]), PetscRealPart(coords[6]), PetscRealPart(coords[7]), PetscRealPart(coords[0]), PetscRealPart(coords[1]), color[2], color[3], color[0]);CHKERRQ(ierr);
264           break;
265         default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_SUP, "Cannot draw cells with %D coordinates", numCoords);
266         }
267         ierr = DMPlexVecRestoreClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
268       }
269       ierr = VecRestoreArrayRead(fv, &array);CHKERRQ(ierr);
270       ierr = PetscDrawFlush(draw);CHKERRQ(ierr);
271       ierr = PetscDrawPause(draw);CHKERRQ(ierr);
272       ierr = PetscDrawSave(draw);CHKERRQ(ierr);
273     }
274     if (Nf > 1) {
275       ierr = VecRestoreSubVector(v, fis, &fv);CHKERRQ(ierr);
276       ierr = ISDestroy(&fis);CHKERRQ(ierr);
277       ierr = DMDestroy(&fdm);CHKERRQ(ierr);
278     }
279   }
280   PetscFunctionReturn(0);
281 }
282 
283 static PetscErrorCode VecView_Plex_Local_VTK(Vec v, PetscViewer viewer)
284 {
285   DM                      dm;
286   Vec                     locv;
287   const char              *name;
288   PetscSection            section;
289   PetscInt                pStart, pEnd;
290   PetscInt                numFields;
291   PetscViewerVTKFieldType ft;
292   PetscErrorCode          ierr;
293 
294   PetscFunctionBegin;
295   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
296   ierr = DMCreateLocalVector(dm, &locv);CHKERRQ(ierr); /* VTK viewer requires exclusive ownership of the vector */
297   ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
298   ierr = PetscObjectSetName((PetscObject) locv, name);CHKERRQ(ierr);
299   ierr = VecCopy(v, locv);CHKERRQ(ierr);
300   ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);
301   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
302   if (!numFields) {
303     ierr = DMPlexGetFieldType_Internal(dm, section, PETSC_DETERMINE, &pStart, &pEnd, &ft);CHKERRQ(ierr);
304     ierr = PetscViewerVTKAddField(viewer, (PetscObject) dm, DMPlexVTKWriteAll, PETSC_DEFAULT, ft, PETSC_TRUE,(PetscObject) locv);CHKERRQ(ierr);
305   } else {
306     PetscInt f;
307 
308     for (f = 0; f < numFields; f++) {
309       ierr = DMPlexGetFieldType_Internal(dm, section, f, &pStart, &pEnd, &ft);CHKERRQ(ierr);
310       if (ft == PETSC_VTK_INVALID) continue;
311       ierr = PetscObjectReference((PetscObject)locv);CHKERRQ(ierr);
312       ierr = PetscViewerVTKAddField(viewer, (PetscObject) dm, DMPlexVTKWriteAll, f, ft, PETSC_TRUE,(PetscObject) locv);CHKERRQ(ierr);
313     }
314     ierr = VecDestroy(&locv);CHKERRQ(ierr);
315   }
316   PetscFunctionReturn(0);
317 }
318 
319 PetscErrorCode VecView_Plex_Local(Vec v, PetscViewer viewer)
320 {
321   DM             dm;
322   PetscBool      isvtk, ishdf5, isdraw, isglvis;
323   PetscErrorCode ierr;
324 
325   PetscFunctionBegin;
326   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
327   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
328   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,   &isvtk);CHKERRQ(ierr);
329   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,  &ishdf5);CHKERRQ(ierr);
330   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERDRAW,  &isdraw);CHKERRQ(ierr);
331   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERGLVIS, &isglvis);CHKERRQ(ierr);
332   if (isvtk || ishdf5 || isdraw || isglvis) {
333     PetscInt    i,numFields;
334     PetscObject fe;
335     PetscBool   fem = PETSC_FALSE;
336     Vec         locv = v;
337     const char  *name;
338     PetscInt    step;
339     PetscReal   time;
340 
341     ierr = DMGetNumFields(dm, &numFields);CHKERRQ(ierr);
342     for (i=0; i<numFields; i++) {
343       ierr = DMGetField(dm, i, NULL, &fe);CHKERRQ(ierr);
344       if (fe->classid == PETSCFE_CLASSID) { fem = PETSC_TRUE; break; }
345     }
346     if (fem) {
347       PetscObject isZero;
348 
349       ierr = DMGetLocalVector(dm, &locv);CHKERRQ(ierr);
350       ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
351       ierr = PetscObjectSetName((PetscObject) locv, name);CHKERRQ(ierr);
352       ierr = PetscObjectQuery((PetscObject) v, "__Vec_bc_zero__", &isZero);CHKERRQ(ierr);
353       ierr = PetscObjectCompose((PetscObject) locv, "__Vec_bc_zero__", isZero);CHKERRQ(ierr);
354       ierr = VecCopy(v, locv);CHKERRQ(ierr);
355       ierr = DMGetOutputSequenceNumber(dm, NULL, &time);CHKERRQ(ierr);
356       ierr = DMPlexInsertBoundaryValues(dm, PETSC_TRUE, locv, time, NULL, NULL, NULL);CHKERRQ(ierr);
357     }
358     if (isvtk) {
359       ierr = VecView_Plex_Local_VTK(locv, viewer);CHKERRQ(ierr);
360     } else if (ishdf5) {
361 #if defined(PETSC_HAVE_HDF5)
362       ierr = VecView_Plex_Local_HDF5_Internal(locv, viewer);CHKERRQ(ierr);
363 #else
364       SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
365 #endif
366     } else if (isdraw) {
367       ierr = VecView_Plex_Local_Draw(locv, viewer);CHKERRQ(ierr);
368     } else if (isglvis) {
369       ierr = DMGetOutputSequenceNumber(dm, &step, NULL);CHKERRQ(ierr);
370       ierr = PetscViewerGLVisSetSnapId(viewer, step);CHKERRQ(ierr);
371       ierr = VecView_GLVis(locv, viewer);CHKERRQ(ierr);
372     }
373     if (fem) {
374       ierr = PetscObjectCompose((PetscObject) locv, "__Vec_bc_zero__", NULL);CHKERRQ(ierr);
375       ierr = DMRestoreLocalVector(dm, &locv);CHKERRQ(ierr);
376     }
377   } else {
378     PetscBool isseq;
379 
380     ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
381     if (isseq) {ierr = VecView_Seq(v, viewer);CHKERRQ(ierr);}
382     else       {ierr = VecView_MPI(v, viewer);CHKERRQ(ierr);}
383   }
384   PetscFunctionReturn(0);
385 }
386 
387 PetscErrorCode VecView_Plex(Vec v, PetscViewer viewer)
388 {
389   DM             dm;
390   PetscBool      isvtk, ishdf5, isdraw, isglvis, isexodusii;
391   PetscErrorCode ierr;
392 
393   PetscFunctionBegin;
394   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
395   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
396   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,      &isvtk);CHKERRQ(ierr);
397   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,     &ishdf5);CHKERRQ(ierr);
398   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERDRAW,     &isdraw);CHKERRQ(ierr);
399   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERGLVIS,    &isglvis);CHKERRQ(ierr);
400   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWEREXODUSII, &isexodusii);CHKERRQ(ierr);
401   if (isvtk || isdraw || isglvis) {
402     Vec         locv;
403     PetscObject isZero;
404     const char *name;
405 
406     ierr = DMGetLocalVector(dm, &locv);CHKERRQ(ierr);
407     ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
408     ierr = PetscObjectSetName((PetscObject) locv, name);CHKERRQ(ierr);
409     ierr = DMGlobalToLocalBegin(dm, v, INSERT_VALUES, locv);CHKERRQ(ierr);
410     ierr = DMGlobalToLocalEnd(dm, v, INSERT_VALUES, locv);CHKERRQ(ierr);
411     ierr = PetscObjectQuery((PetscObject) v, "__Vec_bc_zero__", &isZero);CHKERRQ(ierr);
412     ierr = PetscObjectCompose((PetscObject) locv, "__Vec_bc_zero__", isZero);CHKERRQ(ierr);
413     ierr = VecView_Plex_Local(locv, viewer);CHKERRQ(ierr);
414     ierr = PetscObjectCompose((PetscObject) locv, "__Vec_bc_zero__", NULL);CHKERRQ(ierr);
415     ierr = DMRestoreLocalVector(dm, &locv);CHKERRQ(ierr);
416   } else if (ishdf5) {
417 #if defined(PETSC_HAVE_HDF5)
418     ierr = VecView_Plex_HDF5_Internal(v, viewer);CHKERRQ(ierr);
419 #else
420     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
421 #endif
422   } else if (isexodusii) {
423 #if defined(PETSC_HAVE_EXODUSII)
424     ierr = VecView_PlexExodusII_Internal(v, viewer);CHKERRQ(ierr);
425 #else
426     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "ExodusII not supported in this build.\nPlease reconfigure using --download-exodusii");
427 #endif
428   } else {
429     PetscBool isseq;
430 
431     ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
432     if (isseq) {ierr = VecView_Seq(v, viewer);CHKERRQ(ierr);}
433     else       {ierr = VecView_MPI(v, viewer);CHKERRQ(ierr);}
434   }
435   PetscFunctionReturn(0);
436 }
437 
438 PetscErrorCode VecView_Plex_Native(Vec originalv, PetscViewer viewer)
439 {
440   DM                dm;
441   MPI_Comm          comm;
442   PetscViewerFormat format;
443   Vec               v;
444   PetscBool         isvtk, ishdf5;
445   PetscErrorCode    ierr;
446 
447   PetscFunctionBegin;
448   ierr = VecGetDM(originalv, &dm);CHKERRQ(ierr);
449   ierr = PetscObjectGetComm((PetscObject) originalv, &comm);CHKERRQ(ierr);
450   if (!dm) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
451   ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
452   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
453   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,  &isvtk);CHKERRQ(ierr);
454   if (format == PETSC_VIEWER_NATIVE) {
455     /* Natural ordering is the common case for DMDA, NATIVE means plain vector, for PLEX is the opposite */
456     /* this need a better fix */
457     if (dm->useNatural) {
458       if (dm->sfNatural) {
459         const char *vecname;
460         PetscInt    n, nroots;
461 
462         ierr = VecGetLocalSize(originalv, &n);CHKERRQ(ierr);
463         ierr = PetscSFGetGraph(dm->sfNatural, &nroots, NULL, NULL, NULL);CHKERRQ(ierr);
464         if (n == nroots) {
465           ierr = DMGetGlobalVector(dm, &v);CHKERRQ(ierr);
466           ierr = DMPlexGlobalToNaturalBegin(dm, originalv, v);CHKERRQ(ierr);
467           ierr = DMPlexGlobalToNaturalEnd(dm, originalv, v);CHKERRQ(ierr);
468           ierr = PetscObjectGetName((PetscObject) originalv, &vecname);CHKERRQ(ierr);
469           ierr = PetscObjectSetName((PetscObject) v, vecname);CHKERRQ(ierr);
470         } else SETERRQ(comm, PETSC_ERR_ARG_WRONG, "DM global to natural SF only handles global vectors");
471       } else SETERRQ(comm, PETSC_ERR_ARG_WRONGSTATE, "DM global to natural SF was not created");
472     } else v = originalv;
473   } else v = originalv;
474 
475   if (ishdf5) {
476 #if defined(PETSC_HAVE_HDF5)
477     ierr = VecView_Plex_HDF5_Native_Internal(v, viewer);CHKERRQ(ierr);
478 #else
479     SETERRQ(comm, PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
480 #endif
481   } else if (isvtk) {
482     SETERRQ(comm, PETSC_ERR_SUP, "VTK format does not support viewing in natural order. Please switch to HDF5.");
483   } else {
484     PetscBool isseq;
485 
486     ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
487     if (isseq) {ierr = VecView_Seq(v, viewer);CHKERRQ(ierr);}
488     else       {ierr = VecView_MPI(v, viewer);CHKERRQ(ierr);}
489   }
490   if (v != originalv) {ierr = DMRestoreGlobalVector(dm, &v);CHKERRQ(ierr);}
491   PetscFunctionReturn(0);
492 }
493 
494 PetscErrorCode VecLoad_Plex_Local(Vec v, PetscViewer viewer)
495 {
496   DM             dm;
497   PetscBool      ishdf5;
498   PetscErrorCode ierr;
499 
500   PetscFunctionBegin;
501   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
502   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
503   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
504   if (ishdf5) {
505     DM          dmBC;
506     Vec         gv;
507     const char *name;
508 
509     ierr = DMGetOutputDM(dm, &dmBC);CHKERRQ(ierr);
510     ierr = DMGetGlobalVector(dmBC, &gv);CHKERRQ(ierr);
511     ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
512     ierr = PetscObjectSetName((PetscObject) gv, name);CHKERRQ(ierr);
513     ierr = VecLoad_Default(gv, viewer);CHKERRQ(ierr);
514     ierr = DMGlobalToLocalBegin(dmBC, gv, INSERT_VALUES, v);CHKERRQ(ierr);
515     ierr = DMGlobalToLocalEnd(dmBC, gv, INSERT_VALUES, v);CHKERRQ(ierr);
516     ierr = DMRestoreGlobalVector(dmBC, &gv);CHKERRQ(ierr);
517   } else {
518     ierr = VecLoad_Default(v, viewer);CHKERRQ(ierr);
519   }
520   PetscFunctionReturn(0);
521 }
522 
523 PetscErrorCode VecLoad_Plex(Vec v, PetscViewer viewer)
524 {
525   DM             dm;
526   PetscBool      ishdf5,isexodusii;
527   PetscErrorCode ierr;
528 
529   PetscFunctionBegin;
530   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
531   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
532   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,     &ishdf5);CHKERRQ(ierr);
533   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWEREXODUSII, &isexodusii);CHKERRQ(ierr);
534   if (ishdf5) {
535 #if defined(PETSC_HAVE_HDF5)
536     ierr = VecLoad_Plex_HDF5_Internal(v, viewer);CHKERRQ(ierr);
537 #else
538     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
539 #endif
540   } else if (isexodusii) {
541 #if defined(PETSC_HAVE_EXODUSII)
542     ierr = VecLoad_PlexExodusII_Internal(v, viewer);CHKERRQ(ierr);
543 #else
544     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "ExodusII not supported in this build.\nPlease reconfigure using --download-exodusii");
545 #endif
546   } else {
547     ierr = VecLoad_Default(v, viewer);CHKERRQ(ierr);
548   }
549   PetscFunctionReturn(0);
550 }
551 
552 PetscErrorCode VecLoad_Plex_Native(Vec originalv, PetscViewer viewer)
553 {
554   DM                dm;
555   PetscViewerFormat format;
556   PetscBool         ishdf5;
557   PetscErrorCode    ierr;
558 
559   PetscFunctionBegin;
560   ierr = VecGetDM(originalv, &dm);CHKERRQ(ierr);
561   if (!dm) SETERRQ(PetscObjectComm((PetscObject) originalv), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
562   ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
563   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
564   if (format == PETSC_VIEWER_NATIVE) {
565     if (dm->useNatural) {
566       if (dm->sfNatural) {
567         if (ishdf5) {
568 #if defined(PETSC_HAVE_HDF5)
569           Vec         v;
570           const char *vecname;
571 
572           ierr = DMGetGlobalVector(dm, &v);CHKERRQ(ierr);
573           ierr = PetscObjectGetName((PetscObject) originalv, &vecname);CHKERRQ(ierr);
574           ierr = PetscObjectSetName((PetscObject) v, vecname);CHKERRQ(ierr);
575           ierr = VecLoad_Plex_HDF5_Native_Internal(v, viewer);CHKERRQ(ierr);
576           ierr = DMPlexNaturalToGlobalBegin(dm, v, originalv);CHKERRQ(ierr);
577           ierr = DMPlexNaturalToGlobalEnd(dm, v, originalv);CHKERRQ(ierr);
578           ierr = DMRestoreGlobalVector(dm, &v);CHKERRQ(ierr);
579 #else
580           SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
581 #endif
582         } else SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Reading in natural order is not supported for anything but HDF5.");
583       }
584     } else {
585       ierr = VecLoad_Default(originalv, viewer);CHKERRQ(ierr);
586     }
587   }
588   PetscFunctionReturn(0);
589 }
590 
591 PETSC_UNUSED static PetscErrorCode DMPlexView_Ascii_Geometry(DM dm, PetscViewer viewer)
592 {
593   PetscSection       coordSection;
594   Vec                coordinates;
595   DMLabel            depthLabel, celltypeLabel;
596   const char        *name[4];
597   const PetscScalar *a;
598   PetscInt           dim, pStart, pEnd, cStart, cEnd, c;
599   PetscErrorCode     ierr;
600 
601   PetscFunctionBegin;
602   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
603   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
604   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
605   ierr = DMPlexGetDepthLabel(dm, &depthLabel);CHKERRQ(ierr);
606   ierr = DMPlexGetCellTypeLabel(dm, &celltypeLabel);CHKERRQ(ierr);
607   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
608   ierr = PetscSectionGetChart(coordSection, &pStart, &pEnd);CHKERRQ(ierr);
609   ierr = VecGetArrayRead(coordinates, &a);CHKERRQ(ierr);
610   name[0]     = "vertex";
611   name[1]     = "edge";
612   name[dim-1] = "face";
613   name[dim]   = "cell";
614   for (c = cStart; c < cEnd; ++c) {
615     PetscInt *closure = NULL;
616     PetscInt  closureSize, cl, ct;
617 
618     ierr = DMLabelGetValue(celltypeLabel, c, &ct);CHKERRQ(ierr);
619     ierr = PetscViewerASCIIPrintf(viewer, "Geometry for cell %D polytope type %s:\n", c, DMPolytopeTypes[ct]);CHKERRQ(ierr);
620     ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
621     ierr = PetscViewerASCIIPushTab(viewer);CHKERRQ(ierr);
622     for (cl = 0; cl < closureSize*2; cl += 2) {
623       PetscInt point = closure[cl], depth, dof, off, d, p;
624 
625       if ((point < pStart) || (point >= pEnd)) continue;
626       ierr = PetscSectionGetDof(coordSection, point, &dof);CHKERRQ(ierr);
627       if (!dof) continue;
628       ierr = DMLabelGetValue(depthLabel, point, &depth);CHKERRQ(ierr);
629       ierr = PetscSectionGetOffset(coordSection, point, &off);CHKERRQ(ierr);
630       ierr = PetscViewerASCIIPrintf(viewer, "%s %D coords:", name[depth], point);CHKERRQ(ierr);
631       for (p = 0; p < dof/dim; ++p) {
632         ierr = PetscViewerASCIIPrintf(viewer, " (");CHKERRQ(ierr);
633         for (d = 0; d < dim; ++d) {
634           if (d > 0) {ierr = PetscViewerASCIIPrintf(viewer, ", ");CHKERRQ(ierr);}
635           ierr = PetscViewerASCIIPrintf(viewer, "%g", (double) PetscRealPart(a[off+p*dim+d]));CHKERRQ(ierr);
636         }
637         ierr = PetscViewerASCIIPrintf(viewer, ")");CHKERRQ(ierr);
638       }
639       ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
640     }
641     ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
642     ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr);
643   }
644   ierr = VecRestoreArrayRead(coordinates, &a);CHKERRQ(ierr);
645   PetscFunctionReturn(0);
646 }
647 
648 static PetscErrorCode DMPlexView_Ascii(DM dm, PetscViewer viewer)
649 {
650   DM_Plex          *mesh = (DM_Plex*) dm->data;
651   DM                cdm;
652   PetscSection      coordSection;
653   Vec               coordinates;
654   PetscViewerFormat format;
655   PetscErrorCode    ierr;
656 
657   PetscFunctionBegin;
658   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
659   ierr = DMGetLocalSection(cdm, &coordSection);CHKERRQ(ierr);
660   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
661   ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
662   if (format == PETSC_VIEWER_ASCII_INFO_DETAIL) {
663     const char *name;
664     PetscInt    dim, cellHeight, maxConeSize, maxSupportSize;
665     PetscInt    pStart, pEnd, p, numLabels, l;
666     PetscMPIInt rank, size;
667 
668     ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRMPI(ierr);
669     ierr = MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size);CHKERRMPI(ierr);
670     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
671     ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
672     ierr = DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize);CHKERRQ(ierr);
673     ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
674     ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
675     if (name) {ierr = PetscViewerASCIIPrintf(viewer, "%s in %D dimension%s:\n", name, dim, dim == 1 ? "" : "s");CHKERRQ(ierr);}
676     else      {ierr = PetscViewerASCIIPrintf(viewer, "Mesh in %D dimension%s:\n", dim, dim == 1 ? "" : "s");CHKERRQ(ierr);}
677     if (cellHeight) {ierr = PetscViewerASCIIPrintf(viewer, "  Cells are at height %D\n", cellHeight);CHKERRQ(ierr);}
678     ierr = PetscViewerASCIIPrintf(viewer, "Supports:\n", name);CHKERRQ(ierr);
679     ierr = PetscViewerASCIIPushSynchronized(viewer);CHKERRQ(ierr);
680     ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%d] Max support size: %D\n", rank, maxSupportSize);CHKERRQ(ierr);
681     for (p = pStart; p < pEnd; ++p) {
682       PetscInt dof, off, s;
683 
684       ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
685       ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
686       for (s = off; s < off+dof; ++s) {
687         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%d]: %D ----> %D\n", rank, p, mesh->supports[s]);CHKERRQ(ierr);
688       }
689     }
690     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
691     ierr = PetscViewerASCIIPrintf(viewer, "Cones:\n", name);CHKERRQ(ierr);
692     ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%d] Max cone size: %D\n", rank, maxConeSize);CHKERRQ(ierr);
693     for (p = pStart; p < pEnd; ++p) {
694       PetscInt dof, off, c;
695 
696       ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
697       ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
698       for (c = off; c < off+dof; ++c) {
699         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%d]: %D <---- %D (%D)\n", rank, p, mesh->cones[c], mesh->coneOrientations[c]);CHKERRQ(ierr);
700       }
701     }
702     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
703     ierr = PetscViewerASCIIPopSynchronized(viewer);CHKERRQ(ierr);
704     if (coordSection && coordinates) {
705       ierr = PetscSectionVecView(coordSection, coordinates, viewer);CHKERRQ(ierr);
706     }
707     ierr = DMGetNumLabels(dm, &numLabels);CHKERRQ(ierr);
708     if (numLabels) {ierr = PetscViewerASCIIPrintf(viewer, "Labels:\n");CHKERRQ(ierr);}
709     for (l = 0; l < numLabels; ++l) {
710       DMLabel     label;
711       PetscBool   isdepth;
712       const char *name;
713 
714       ierr = DMGetLabelName(dm, l, &name);CHKERRQ(ierr);
715       ierr = PetscStrcmp(name, "depth", &isdepth);CHKERRQ(ierr);
716       if (isdepth) continue;
717       ierr = DMGetLabel(dm, name, &label);CHKERRQ(ierr);
718       ierr = DMLabelView(label, viewer);CHKERRQ(ierr);
719     }
720     if (size > 1) {
721       PetscSF sf;
722 
723       ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr);
724       ierr = PetscSFView(sf, viewer);CHKERRQ(ierr);
725     }
726     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
727   } else if (format == PETSC_VIEWER_ASCII_LATEX) {
728     const char  *name, *color;
729     const char  *defcolors[3]  = {"gray", "orange", "green"};
730     const char  *deflcolors[4] = {"blue", "cyan", "red", "magenta"};
731     char         lname[PETSC_MAX_PATH_LEN];
732     PetscReal    scale         = 2.0;
733     PetscReal    tikzscale     = 1.0;
734     PetscBool    useNumbers    = PETSC_TRUE, useLabels, useColors;
735     double       tcoords[3];
736     PetscScalar *coords;
737     PetscInt     numLabels, l, numColors, numLColors, dim, depth, cStart, cEnd, c, vStart, vEnd, v, eStart = 0, eEnd = 0, e, p;
738     PetscMPIInt  rank, size;
739     char         **names, **colors, **lcolors;
740     PetscBool    plotEdges, flg, lflg;
741     PetscBT      wp = NULL;
742     PetscInt     pEnd, pStart;
743 
744     ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
745     ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
746     ierr = DMGetNumLabels(dm, &numLabels);CHKERRQ(ierr);
747     numLabels  = PetscMax(numLabels, 10);
748     numColors  = 10;
749     numLColors = 10;
750     ierr = PetscCalloc3(numLabels, &names, numColors, &colors, numLColors, &lcolors);CHKERRQ(ierr);
751     ierr = PetscOptionsGetReal(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_scale", &scale, NULL);CHKERRQ(ierr);
752     ierr = PetscOptionsGetReal(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_tikzscale", &tikzscale, NULL);CHKERRQ(ierr);
753     ierr = PetscOptionsGetBool(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_numbers", &useNumbers, NULL);CHKERRQ(ierr);
754     ierr = PetscOptionsGetStringArray(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_labels", names, &numLabels, &useLabels);CHKERRQ(ierr);
755     if (!useLabels) numLabels = 0;
756     ierr = PetscOptionsGetStringArray(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_colors", colors, &numColors, &useColors);CHKERRQ(ierr);
757     if (!useColors) {
758       numColors = 3;
759       for (c = 0; c < numColors; ++c) {ierr = PetscStrallocpy(defcolors[c], &colors[c]);CHKERRQ(ierr);}
760     }
761     ierr = PetscOptionsGetStringArray(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_lcolors", lcolors, &numLColors, &useColors);CHKERRQ(ierr);
762     if (!useColors) {
763       numLColors = 4;
764       for (c = 0; c < numLColors; ++c) {ierr = PetscStrallocpy(deflcolors[c], &lcolors[c]);CHKERRQ(ierr);}
765     }
766     ierr = PetscOptionsGetString(((PetscObject) viewer)->options, ((PetscObject) viewer)->prefix, "-dm_plex_view_label_filter", lname, sizeof(lname), &lflg);CHKERRQ(ierr);
767     plotEdges = (PetscBool)(depth > 1 && useNumbers && dim < 3);
768     ierr = PetscOptionsGetBool(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_edges", &plotEdges, &flg);CHKERRQ(ierr);
769     if (flg && plotEdges && depth < dim) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Mesh must be interpolated");
770     if (depth < dim) plotEdges = PETSC_FALSE;
771 
772     /* filter points with labelvalue != labeldefaultvalue */
773     ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
774     if (lflg) {
775       DMLabel lbl;
776 
777       ierr = DMGetLabel(dm, lname, &lbl);CHKERRQ(ierr);
778       if (lbl) {
779         PetscInt val, defval;
780 
781         ierr = DMLabelGetDefaultValue(lbl, &defval);CHKERRQ(ierr);
782         ierr = PetscBTCreate(pEnd-pStart, &wp);CHKERRQ(ierr);
783         for (c = pStart;  c < pEnd; c++) {
784           PetscInt *closure = NULL;
785           PetscInt  closureSize;
786 
787           ierr = DMLabelGetValue(lbl, c, &val);CHKERRQ(ierr);
788           if (val == defval) continue;
789 
790           ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
791           for (p = 0; p < closureSize*2; p += 2) {
792             ierr = PetscBTSet(wp, closure[p] - pStart);CHKERRQ(ierr);
793           }
794           ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
795         }
796       }
797     }
798 
799     ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRMPI(ierr);
800     ierr = MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size);CHKERRMPI(ierr);
801     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
802     ierr = PetscViewerASCIIPrintf(viewer, "\
803 \\documentclass[tikz]{standalone}\n\n\
804 \\usepackage{pgflibraryshapes}\n\
805 \\usetikzlibrary{backgrounds}\n\
806 \\usetikzlibrary{arrows}\n\
807 \\begin{document}\n");CHKERRQ(ierr);
808     if (size > 1) {
809       ierr = PetscViewerASCIIPrintf(viewer, "%s for process ", name);CHKERRQ(ierr);
810       for (p = 0; p < size; ++p) {
811         if (p > 0 && p == size-1) {
812           ierr = PetscViewerASCIIPrintf(viewer, ", and ", colors[p%numColors], p);CHKERRQ(ierr);
813         } else if (p > 0) {
814           ierr = PetscViewerASCIIPrintf(viewer, ", ", colors[p%numColors], p);CHKERRQ(ierr);
815         }
816         ierr = PetscViewerASCIIPrintf(viewer, "{\\textcolor{%s}%D}", colors[p%numColors], p);CHKERRQ(ierr);
817       }
818       ierr = PetscViewerASCIIPrintf(viewer, ".\n\n\n");CHKERRQ(ierr);
819     }
820     ierr = PetscViewerASCIIPrintf(viewer, "\\begin{tikzpicture}[scale = %g,font=\\fontsize{8}{8}\\selectfont]\n", (double) tikzscale);CHKERRQ(ierr);
821 
822     /* Plot vertices */
823     ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
824     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
825     ierr = PetscViewerASCIIPushSynchronized(viewer);CHKERRQ(ierr);
826     for (v = vStart; v < vEnd; ++v) {
827       PetscInt  off, dof, d;
828       PetscBool isLabeled = PETSC_FALSE;
829 
830       if (wp && !PetscBTLookup(wp,v - pStart)) continue;
831       ierr = PetscSectionGetDof(coordSection, v, &dof);CHKERRQ(ierr);
832       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
833       ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\path (");CHKERRQ(ierr);
834       if (PetscUnlikely(dof > 3)) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"coordSection vertex %D has dof %D > 3",v,dof);
835       for (d = 0; d < dof; ++d) {
836         tcoords[d] = (double) (scale*PetscRealPart(coords[off+d]));
837         tcoords[d] = PetscAbs(tcoords[d]) < 1e-10 ? 0.0 : tcoords[d];
838       }
839       /* Rotate coordinates since PGF makes z point out of the page instead of up */
840       if (dim == 3) {PetscReal tmp = tcoords[1]; tcoords[1] = tcoords[2]; tcoords[2] = -tmp;}
841       for (d = 0; d < dof; ++d) {
842         if (d > 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, ",");CHKERRQ(ierr);}
843         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "%g", (double) tcoords[d]);CHKERRQ(ierr);
844       }
845       color = colors[rank%numColors];
846       for (l = 0; l < numLabels; ++l) {
847         PetscInt val;
848         ierr = DMGetLabelValue(dm, names[l], v, &val);CHKERRQ(ierr);
849         if (val >= 0) {color = lcolors[l%numLColors]; isLabeled = PETSC_TRUE; break;}
850       }
851       if (useNumbers) {
852         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [draw,shape=circle,color=%s] {%D};\n", v, rank, color, v);CHKERRQ(ierr);
853       } else {
854         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [fill,inner sep=%dpt,shape=circle,color=%s] {};\n", v, rank, !isLabeled ? 1 : 2, color);CHKERRQ(ierr);
855       }
856     }
857     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
858     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
859     /* Plot cells */
860     ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
861     ierr = DMPlexGetDepthStratum(dm, 1, &eStart, &eEnd);CHKERRQ(ierr);
862     if (dim == 3 || !useNumbers) {
863       for (e = eStart; e < eEnd; ++e) {
864         const PetscInt *cone;
865 
866         if (wp && !PetscBTLookup(wp,e - pStart)) continue;
867         color = colors[rank%numColors];
868         for (l = 0; l < numLabels; ++l) {
869           PetscInt val;
870           ierr = DMGetLabelValue(dm, names[l], e, &val);CHKERRQ(ierr);
871           if (val >= 0) {color = lcolors[l%numLColors]; break;}
872         }
873         ierr = DMPlexGetCone(dm, e, &cone);CHKERRQ(ierr);
874         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\draw[color=%s] (%D_%d) -- (%D_%d);\n", color, cone[0], rank, cone[1], rank);CHKERRQ(ierr);
875       }
876     } else {
877       for (c = cStart; c < cEnd; ++c) {
878         PetscInt *closure = NULL;
879         PetscInt  closureSize, firstPoint = -1;
880 
881         if (wp && !PetscBTLookup(wp,c - pStart)) continue;
882         ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
883         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\draw[color=%s] ", colors[rank%numColors]);CHKERRQ(ierr);
884         for (p = 0; p < closureSize*2; p += 2) {
885           const PetscInt point = closure[p];
886 
887           if ((point < vStart) || (point >= vEnd)) continue;
888           if (firstPoint >= 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- ");CHKERRQ(ierr);}
889           ierr = PetscViewerASCIISynchronizedPrintf(viewer, "(%D_%d)", point, rank);CHKERRQ(ierr);
890           if (firstPoint < 0) firstPoint = point;
891         }
892         /* Why doesn't this work? ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- cycle;\n");CHKERRQ(ierr); */
893         ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- (%D_%d);\n", firstPoint, rank);CHKERRQ(ierr);
894         ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
895       }
896     }
897     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
898     for (c = cStart; c < cEnd; ++c) {
899       double    ccoords[3] = {0.0, 0.0, 0.0};
900       PetscBool isLabeled  = PETSC_FALSE;
901       PetscInt *closure    = NULL;
902       PetscInt  closureSize, dof, d, n = 0;
903 
904       if (wp && !PetscBTLookup(wp,c - pStart)) continue;
905       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
906       ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\path (");CHKERRQ(ierr);
907       for (p = 0; p < closureSize*2; p += 2) {
908         const PetscInt point = closure[p];
909         PetscInt       off;
910 
911         if ((point < vStart) || (point >= vEnd)) continue;
912         ierr = PetscSectionGetDof(coordSection, point, &dof);CHKERRQ(ierr);
913         ierr = PetscSectionGetOffset(coordSection, point, &off);CHKERRQ(ierr);
914         for (d = 0; d < dof; ++d) {
915           tcoords[d] = (double) (scale*PetscRealPart(coords[off+d]));
916           tcoords[d] = PetscAbs(tcoords[d]) < 1e-10 ? 0.0 : tcoords[d];
917         }
918         /* Rotate coordinates since PGF makes z point out of the page instead of up */
919         if (dof == 3) {PetscReal tmp = tcoords[1]; tcoords[1] = tcoords[2]; tcoords[2] = -tmp;}
920         for (d = 0; d < dof; ++d) {ccoords[d] += tcoords[d];}
921         ++n;
922       }
923       for (d = 0; d < dof; ++d) {ccoords[d] /= n;}
924       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
925       for (d = 0; d < dof; ++d) {
926         if (d > 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, ",");CHKERRQ(ierr);}
927         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "%g", (double) ccoords[d]);CHKERRQ(ierr);
928       }
929       color = colors[rank%numColors];
930       for (l = 0; l < numLabels; ++l) {
931         PetscInt val;
932         ierr = DMGetLabelValue(dm, names[l], c, &val);CHKERRQ(ierr);
933         if (val >= 0) {color = lcolors[l%numLColors]; isLabeled = PETSC_TRUE; break;}
934       }
935       if (useNumbers) {
936         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [draw,shape=circle,color=%s] {%D};\n", c, rank, color, c);CHKERRQ(ierr);
937       } else {
938         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [fill,inner sep=%dpt,shape=circle,color=%s] {};\n", c, rank, !isLabeled ? 1 : 2, color);CHKERRQ(ierr);
939       }
940     }
941     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
942     /* Plot edges */
943     if (plotEdges) {
944       ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
945       ierr = PetscViewerASCIIPrintf(viewer, "\\path\n");CHKERRQ(ierr);
946       for (e = eStart; e < eEnd; ++e) {
947         const PetscInt *cone;
948         PetscInt        coneSize, offA, offB, dof, d;
949 
950         if (wp && !PetscBTLookup(wp,e - pStart)) continue;
951         ierr = DMPlexGetConeSize(dm, e, &coneSize);CHKERRQ(ierr);
952         if (coneSize != 2) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Edge %D cone should have two vertices, not %D", e, coneSize);
953         ierr = DMPlexGetCone(dm, e, &cone);CHKERRQ(ierr);
954         ierr = PetscSectionGetDof(coordSection, cone[0], &dof);CHKERRQ(ierr);
955         ierr = PetscSectionGetOffset(coordSection, cone[0], &offA);CHKERRQ(ierr);
956         ierr = PetscSectionGetOffset(coordSection, cone[1], &offB);CHKERRQ(ierr);
957         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "(");CHKERRQ(ierr);
958         for (d = 0; d < dof; ++d) {
959           tcoords[d] = (double) (0.5*scale*PetscRealPart(coords[offA+d]+coords[offB+d]));
960           tcoords[d] = PetscAbs(tcoords[d]) < 1e-10 ? 0.0 : tcoords[d];
961         }
962         /* Rotate coordinates since PGF makes z point out of the page instead of up */
963         if (dim == 3) {PetscReal tmp = tcoords[1]; tcoords[1] = tcoords[2]; tcoords[2] = -tmp;}
964         for (d = 0; d < dof; ++d) {
965           if (d > 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, ",");CHKERRQ(ierr);}
966           ierr = PetscViewerASCIISynchronizedPrintf(viewer, "%g", (double)tcoords[d]);CHKERRQ(ierr);
967         }
968         color = colors[rank%numColors];
969         for (l = 0; l < numLabels; ++l) {
970           PetscInt val;
971           ierr = DMGetLabelValue(dm, names[l], v, &val);CHKERRQ(ierr);
972           if (val >= 0) {color = lcolors[l%numLColors]; break;}
973         }
974         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [draw,shape=circle,color=%s] {%D} --\n", e, rank, color, e);CHKERRQ(ierr);
975       }
976       ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
977       ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
978       ierr = PetscViewerASCIIPrintf(viewer, "(0,0);\n");CHKERRQ(ierr);
979     }
980     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
981     ierr = PetscViewerASCIIPopSynchronized(viewer);CHKERRQ(ierr);
982     ierr = PetscViewerASCIIPrintf(viewer, "\\end{tikzpicture}\n");CHKERRQ(ierr);
983     ierr = PetscViewerASCIIPrintf(viewer, "\\end{document}\n", name);CHKERRQ(ierr);
984     for (l = 0; l < numLabels;  ++l) {ierr = PetscFree(names[l]);CHKERRQ(ierr);}
985     for (c = 0; c < numColors;  ++c) {ierr = PetscFree(colors[c]);CHKERRQ(ierr);}
986     for (c = 0; c < numLColors; ++c) {ierr = PetscFree(lcolors[c]);CHKERRQ(ierr);}
987     ierr = PetscFree3(names, colors, lcolors);CHKERRQ(ierr);
988     ierr = PetscBTDestroy(&wp);CHKERRQ(ierr);
989   } else if (format == PETSC_VIEWER_LOAD_BALANCE) {
990     Vec                    cown,acown;
991     VecScatter             sct;
992     ISLocalToGlobalMapping g2l;
993     IS                     gid,acis;
994     MPI_Comm               comm,ncomm = MPI_COMM_NULL;
995     MPI_Group              ggroup,ngroup;
996     PetscScalar            *array,nid;
997     const PetscInt         *idxs;
998     PetscInt               *idxs2,*start,*adjacency,*work;
999     PetscInt64             lm[3],gm[3];
1000     PetscInt               i,c,cStart,cEnd,cum,numVertices,ect,ectn,cellHeight;
1001     PetscMPIInt            d1,d2,rank;
1002 
1003     ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
1004     ierr = MPI_Comm_rank(comm,&rank);CHKERRMPI(ierr);
1005 #if defined(PETSC_HAVE_MPI_PROCESS_SHARED_MEMORY)
1006     ierr = MPI_Comm_split_type(comm,MPI_COMM_TYPE_SHARED,rank,MPI_INFO_NULL,&ncomm);CHKERRMPI(ierr);
1007 #endif
1008     if (ncomm != MPI_COMM_NULL) {
1009       ierr = MPI_Comm_group(comm,&ggroup);CHKERRMPI(ierr);
1010       ierr = MPI_Comm_group(ncomm,&ngroup);CHKERRMPI(ierr);
1011       d1   = 0;
1012       ierr = MPI_Group_translate_ranks(ngroup,1,&d1,ggroup,&d2);CHKERRMPI(ierr);
1013       nid  = d2;
1014       ierr = MPI_Group_free(&ggroup);CHKERRMPI(ierr);
1015       ierr = MPI_Group_free(&ngroup);CHKERRMPI(ierr);
1016       ierr = MPI_Comm_free(&ncomm);CHKERRMPI(ierr);
1017     } else nid = 0.0;
1018 
1019     /* Get connectivity */
1020     ierr = DMPlexGetVTKCellHeight(dm,&cellHeight);CHKERRQ(ierr);
1021     ierr = DMPlexCreatePartitionerGraph(dm,cellHeight,&numVertices,&start,&adjacency,&gid);CHKERRQ(ierr);
1022 
1023     /* filter overlapped local cells */
1024     ierr = DMPlexGetHeightStratum(dm,cellHeight,&cStart,&cEnd);CHKERRQ(ierr);
1025     ierr = ISGetIndices(gid,&idxs);CHKERRQ(ierr);
1026     ierr = ISGetLocalSize(gid,&cum);CHKERRQ(ierr);
1027     ierr = PetscMalloc1(cum,&idxs2);CHKERRQ(ierr);
1028     for (c = cStart, cum = 0; c < cEnd; c++) {
1029       if (idxs[c-cStart] < 0) continue;
1030       idxs2[cum++] = idxs[c-cStart];
1031     }
1032     ierr = ISRestoreIndices(gid,&idxs);CHKERRQ(ierr);
1033     if (numVertices != cum) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Unexpected %D != %D",numVertices,cum);
1034     ierr = ISDestroy(&gid);CHKERRQ(ierr);
1035     ierr = ISCreateGeneral(comm,numVertices,idxs2,PETSC_OWN_POINTER,&gid);CHKERRQ(ierr);
1036 
1037     /* support for node-aware cell locality */
1038     ierr = ISCreateGeneral(comm,start[numVertices],adjacency,PETSC_USE_POINTER,&acis);CHKERRQ(ierr);
1039     ierr = VecCreateSeq(PETSC_COMM_SELF,start[numVertices],&acown);CHKERRQ(ierr);
1040     ierr = VecCreateMPI(comm,numVertices,PETSC_DECIDE,&cown);CHKERRQ(ierr);
1041     ierr = VecGetArray(cown,&array);CHKERRQ(ierr);
1042     for (c = 0; c < numVertices; c++) array[c] = nid;
1043     ierr = VecRestoreArray(cown,&array);CHKERRQ(ierr);
1044     ierr = VecScatterCreate(cown,acis,acown,NULL,&sct);CHKERRQ(ierr);
1045     ierr = VecScatterBegin(sct,cown,acown,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
1046     ierr = VecScatterEnd(sct,cown,acown,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
1047     ierr = ISDestroy(&acis);CHKERRQ(ierr);
1048     ierr = VecScatterDestroy(&sct);CHKERRQ(ierr);
1049     ierr = VecDestroy(&cown);CHKERRQ(ierr);
1050 
1051     /* compute edgeCut */
1052     for (c = 0, cum = 0; c < numVertices; c++) cum = PetscMax(cum,start[c+1]-start[c]);
1053     ierr = PetscMalloc1(cum,&work);CHKERRQ(ierr);
1054     ierr = ISLocalToGlobalMappingCreateIS(gid,&g2l);CHKERRQ(ierr);
1055     ierr = ISLocalToGlobalMappingSetType(g2l,ISLOCALTOGLOBALMAPPINGHASH);CHKERRQ(ierr);
1056     ierr = ISDestroy(&gid);CHKERRQ(ierr);
1057     ierr = VecGetArray(acown,&array);CHKERRQ(ierr);
1058     for (c = 0, ect = 0, ectn = 0; c < numVertices; c++) {
1059       PetscInt totl;
1060 
1061       totl = start[c+1]-start[c];
1062       ierr = ISGlobalToLocalMappingApply(g2l,IS_GTOLM_MASK,totl,adjacency+start[c],NULL,work);CHKERRQ(ierr);
1063       for (i = 0; i < totl; i++) {
1064         if (work[i] < 0) {
1065           ect  += 1;
1066           ectn += (array[i + start[c]] != nid) ? 0 : 1;
1067         }
1068       }
1069     }
1070     ierr  = PetscFree(work);CHKERRQ(ierr);
1071     ierr  = VecRestoreArray(acown,&array);CHKERRQ(ierr);
1072     lm[0] = numVertices > 0 ?  numVertices : PETSC_MAX_INT;
1073     lm[1] = -numVertices;
1074     ierr  = MPIU_Allreduce(lm,gm,2,MPIU_INT64,MPI_MIN,comm);CHKERRMPI(ierr);
1075     ierr  = PetscViewerASCIIPrintf(viewer,"  Cell balance: %.2f (max %D, min %D",-((double)gm[1])/((double)gm[0]),-(PetscInt)gm[1],(PetscInt)gm[0]);CHKERRQ(ierr);
1076     lm[0] = ect; /* edgeCut */
1077     lm[1] = ectn; /* node-aware edgeCut */
1078     lm[2] = numVertices > 0 ? 0 : 1; /* empty processes */
1079     ierr  = MPIU_Allreduce(lm,gm,3,MPIU_INT64,MPI_SUM,comm);CHKERRMPI(ierr);
1080     ierr  = PetscViewerASCIIPrintf(viewer,", empty %D)\n",(PetscInt)gm[2]);CHKERRQ(ierr);
1081 #if defined(PETSC_HAVE_MPI_PROCESS_SHARED_MEMORY)
1082     ierr  = PetscViewerASCIIPrintf(viewer,"  Edge Cut: %D (on node %.3f)\n",(PetscInt)(gm[0]/2),gm[0] ? ((double)(gm[1]))/((double)gm[0]) : 1.);CHKERRQ(ierr);
1083 #else
1084     ierr  = PetscViewerASCIIPrintf(viewer,"  Edge Cut: %D (on node %.3f)\n",(PetscInt)(gm[0]/2),0.0);CHKERRQ(ierr);
1085 #endif
1086     ierr  = ISLocalToGlobalMappingDestroy(&g2l);CHKERRQ(ierr);
1087     ierr  = PetscFree(start);CHKERRQ(ierr);
1088     ierr  = PetscFree(adjacency);CHKERRQ(ierr);
1089     ierr  = VecDestroy(&acown);CHKERRQ(ierr);
1090   } else {
1091     const char    *name;
1092     PetscInt      *sizes, *hybsizes, *ghostsizes;
1093     PetscInt       locDepth, depth, cellHeight, dim, d;
1094     PetscInt       pStart, pEnd, p, gcStart, gcEnd, gcNum;
1095     PetscInt       numLabels, l;
1096     DMPolytopeType ct0 = DM_POLYTOPE_UNKNOWN;
1097     MPI_Comm       comm;
1098     PetscMPIInt    size, rank;
1099 
1100     ierr = PetscObjectGetComm((PetscObject) dm, &comm);CHKERRQ(ierr);
1101     ierr = MPI_Comm_size(comm, &size);CHKERRMPI(ierr);
1102     ierr = MPI_Comm_rank(comm, &rank);CHKERRMPI(ierr);
1103     ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
1104     ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
1105     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
1106     if (name) {ierr = PetscViewerASCIIPrintf(viewer, "%s in %D dimension%s:\n", name, dim, dim == 1 ? "" : "s");CHKERRQ(ierr);}
1107     else      {ierr = PetscViewerASCIIPrintf(viewer, "Mesh in %D dimension%s:\n", dim, dim == 1 ? "" : "s");CHKERRQ(ierr);}
1108     if (cellHeight) {ierr = PetscViewerASCIIPrintf(viewer, "  Cells are at height %D\n", cellHeight);CHKERRQ(ierr);}
1109     ierr = DMPlexGetDepth(dm, &locDepth);CHKERRQ(ierr);
1110     ierr = MPIU_Allreduce(&locDepth, &depth, 1, MPIU_INT, MPI_MAX, comm);CHKERRMPI(ierr);
1111     ierr = DMPlexGetGhostCellStratum(dm, &gcStart, &gcEnd);CHKERRQ(ierr);
1112     gcNum = gcEnd - gcStart;
1113     ierr = PetscCalloc3(size,&sizes,size,&hybsizes,size,&ghostsizes);CHKERRQ(ierr);
1114     for (d = 0; d <= depth; d++) {
1115       PetscInt Nc[2] = {0, 0}, ict;
1116 
1117       ierr = DMPlexGetDepthStratum(dm, d, &pStart, &pEnd);CHKERRQ(ierr);
1118       if (pStart < pEnd) {ierr = DMPlexGetCellType(dm, pStart, &ct0);CHKERRQ(ierr);}
1119       ict  = ct0;
1120       ierr = MPI_Bcast(&ict, 1, MPIU_INT, 0, comm);CHKERRMPI(ierr);
1121       ct0  = (DMPolytopeType) ict;
1122       for (p = pStart; p < pEnd; ++p) {
1123         DMPolytopeType ct;
1124 
1125         ierr = DMPlexGetCellType(dm, p, &ct);CHKERRQ(ierr);
1126         if (ct == ct0) ++Nc[0];
1127         else           ++Nc[1];
1128       }
1129       ierr = MPI_Gather(&Nc[0], 1, MPIU_INT, sizes,    1, MPIU_INT, 0, comm);CHKERRMPI(ierr);
1130       ierr = MPI_Gather(&Nc[1], 1, MPIU_INT, hybsizes, 1, MPIU_INT, 0, comm);CHKERRMPI(ierr);
1131       if (d == depth) {ierr = MPI_Gather(&gcNum, 1, MPIU_INT, ghostsizes, 1, MPIU_INT, 0, comm);CHKERRMPI(ierr);}
1132       ierr = PetscViewerASCIIPrintf(viewer, "  %D-cells:", (depth == 1) && d ? dim : d);CHKERRQ(ierr);
1133       for (p = 0; p < size; ++p) {
1134         if (!rank) {
1135           ierr = PetscViewerASCIIPrintf(viewer, " %D", sizes[p]+hybsizes[p]);CHKERRQ(ierr);
1136           if (hybsizes[p]   > 0) {ierr = PetscViewerASCIIPrintf(viewer, " (%D)", hybsizes[p]);CHKERRQ(ierr);}
1137           if (ghostsizes[p] > 0) {ierr = PetscViewerASCIIPrintf(viewer, " [%D]", ghostsizes[p]);CHKERRQ(ierr);}
1138         }
1139       }
1140       ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
1141     }
1142     ierr = PetscFree3(sizes,hybsizes,ghostsizes);CHKERRQ(ierr);
1143     {
1144       const PetscReal      *maxCell;
1145       const PetscReal      *L;
1146       const DMBoundaryType *bd;
1147       PetscBool             per, localized;
1148 
1149       ierr = DMGetPeriodicity(dm, &per, &maxCell, &L, &bd);CHKERRQ(ierr);
1150       ierr = DMGetCoordinatesLocalized(dm, &localized);CHKERRQ(ierr);
1151       if (per) {
1152         ierr = PetscViewerASCIIPrintf(viewer, "Periodic mesh (");CHKERRQ(ierr);
1153         ierr = PetscViewerASCIIUseTabs(viewer, PETSC_FALSE);CHKERRQ(ierr);
1154         for (d = 0; d < dim; ++d) {
1155           if (bd && d > 0) {ierr = PetscViewerASCIIPrintf(viewer, ", ");CHKERRQ(ierr);}
1156           if (bd)    {ierr = PetscViewerASCIIPrintf(viewer, "%s", DMBoundaryTypes[bd[d]]);CHKERRQ(ierr);}
1157         }
1158         ierr = PetscViewerASCIIPrintf(viewer, ") coordinates %s\n", localized ? "localized" : "not localized");CHKERRQ(ierr);
1159         ierr = PetscViewerASCIIUseTabs(viewer, PETSC_TRUE);CHKERRQ(ierr);
1160       }
1161     }
1162     ierr = DMGetNumLabels(dm, &numLabels);CHKERRQ(ierr);
1163     if (numLabels) {ierr = PetscViewerASCIIPrintf(viewer, "Labels:\n");CHKERRQ(ierr);}
1164     for (l = 0; l < numLabels; ++l) {
1165       DMLabel         label;
1166       const char     *name;
1167       IS              valueIS;
1168       const PetscInt *values;
1169       PetscInt        numValues, v;
1170 
1171       ierr = DMGetLabelName(dm, l, &name);CHKERRQ(ierr);
1172       ierr = DMGetLabel(dm, name, &label);CHKERRQ(ierr);
1173       ierr = DMLabelGetNumValues(label, &numValues);CHKERRQ(ierr);
1174       ierr = PetscViewerASCIIPrintf(viewer, "  %s: %D strata with value/size (", name, numValues);CHKERRQ(ierr);
1175       ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
1176       ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
1177       ierr = PetscViewerASCIIUseTabs(viewer, PETSC_FALSE);CHKERRQ(ierr);
1178       for (v = 0; v < numValues; ++v) {
1179         PetscInt size;
1180 
1181         ierr = DMLabelGetStratumSize(label, values[v], &size);CHKERRQ(ierr);
1182         if (v > 0) {ierr = PetscViewerASCIIPrintf(viewer, ", ");CHKERRQ(ierr);}
1183         ierr = PetscViewerASCIIPrintf(viewer, "%D (%D)", values[v], size);CHKERRQ(ierr);
1184       }
1185       ierr = PetscViewerASCIIPrintf(viewer, ")\n");CHKERRQ(ierr);
1186       ierr = PetscViewerASCIIUseTabs(viewer, PETSC_TRUE);CHKERRQ(ierr);
1187       ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
1188       ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
1189     }
1190     {
1191       char    **labelNames;
1192       PetscInt  Nl = numLabels;
1193       PetscBool flg;
1194 
1195       ierr = PetscMalloc1(Nl, &labelNames);CHKERRQ(ierr);
1196       ierr = PetscOptionsGetStringArray(((PetscObject) dm)->options, ((PetscObject) dm)->prefix, "-dm_plex_view_labels", labelNames, &Nl, &flg);CHKERRQ(ierr);
1197       for (l = 0; l < Nl; ++l) {
1198         DMLabel label;
1199 
1200         ierr = DMHasLabel(dm, labelNames[l], &flg);CHKERRQ(ierr);
1201         if (flg) {
1202           ierr = DMGetLabel(dm, labelNames[l], &label);CHKERRQ(ierr);
1203           ierr = DMLabelView(label, viewer);CHKERRQ(ierr);
1204         }
1205         ierr = PetscFree(labelNames[l]);CHKERRQ(ierr);
1206       }
1207       ierr = PetscFree(labelNames);CHKERRQ(ierr);
1208     }
1209     /* If no fields are specified, people do not want to see adjacency */
1210     if (dm->Nf) {
1211       PetscInt f;
1212 
1213       for (f = 0; f < dm->Nf; ++f) {
1214         const char *name;
1215 
1216         ierr = PetscObjectGetName(dm->fields[f].disc, &name);CHKERRQ(ierr);
1217         if (numLabels) {ierr = PetscViewerASCIIPrintf(viewer, "Field %s:\n", name);CHKERRQ(ierr);}
1218         ierr = PetscViewerASCIIPushTab(viewer);CHKERRQ(ierr);
1219         if (dm->fields[f].label) {ierr = DMLabelView(dm->fields[f].label, viewer);CHKERRQ(ierr);}
1220         if (dm->fields[f].adjacency[0]) {
1221           if (dm->fields[f].adjacency[1]) {ierr = PetscViewerASCIIPrintf(viewer, "adjacency FVM++\n");CHKERRQ(ierr);}
1222           else                            {ierr = PetscViewerASCIIPrintf(viewer, "adjacency FVM\n");CHKERRQ(ierr);}
1223         } else {
1224           if (dm->fields[f].adjacency[1]) {ierr = PetscViewerASCIIPrintf(viewer, "adjacency FEM\n");CHKERRQ(ierr);}
1225           else                            {ierr = PetscViewerASCIIPrintf(viewer, "adjacency FUNKY\n");CHKERRQ(ierr);}
1226         }
1227         ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr);
1228       }
1229     }
1230     ierr = DMGetCoarseDM(dm, &cdm);CHKERRQ(ierr);
1231     if (cdm) {
1232       ierr = PetscViewerASCIIPushTab(viewer);CHKERRQ(ierr);
1233       ierr = DMPlexView_Ascii(cdm, viewer);CHKERRQ(ierr);
1234       ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr);
1235     }
1236   }
1237   PetscFunctionReturn(0);
1238 }
1239 
1240 static PetscErrorCode DMPlexDrawCell(DM dm, PetscDraw draw, PetscInt cell, const PetscScalar coords[])
1241 {
1242   DMPolytopeType ct;
1243   PetscMPIInt    rank;
1244   PetscErrorCode ierr;
1245 
1246   PetscFunctionBegin;
1247   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRMPI(ierr);
1248   ierr = DMPlexGetCellType(dm, cell, &ct);CHKERRQ(ierr);
1249   switch (ct) {
1250   case DM_POLYTOPE_TRIANGLE:
1251     ierr = PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]),
1252                              PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1253                              PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1254                              PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2);CHKERRQ(ierr);
1255     ierr = PetscDrawLine(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1256     ierr = PetscDrawLine(draw, PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1257     ierr = PetscDrawLine(draw, PetscRealPart(coords[4]), PetscRealPart(coords[5]), PetscRealPart(coords[0]), PetscRealPart(coords[1]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1258     break;
1259   case DM_POLYTOPE_QUADRILATERAL:
1260     ierr = PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]),
1261                               PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1262                               PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1263                               PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2);CHKERRQ(ierr);
1264     ierr = PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), PetscRealPart(coords[6]), PetscRealPart(coords[7]),
1265                               PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1266                               PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1267                               PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2);CHKERRQ(ierr);
1268     ierr = PetscDrawLine(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1269     ierr = PetscDrawLine(draw, PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1270     ierr = PetscDrawLine(draw, PetscRealPart(coords[4]), PetscRealPart(coords[5]), PetscRealPart(coords[6]), PetscRealPart(coords[7]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1271     ierr = PetscDrawLine(draw, PetscRealPart(coords[6]), PetscRealPart(coords[7]), PetscRealPart(coords[0]), PetscRealPart(coords[1]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1272     break;
1273   default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_SUP, "Cannot draw cells of type %s", DMPolytopeTypes[ct]);
1274   }
1275   PetscFunctionReturn(0);
1276 }
1277 
1278 static PetscErrorCode DMPlexDrawCellHighOrder(DM dm, PetscDraw draw, PetscInt cell, const PetscScalar coords[], PetscInt edgeDiv, PetscReal refCoords[], PetscReal edgeCoords[])
1279 {
1280   DMPolytopeType ct;
1281   PetscReal      centroid[2] = {0., 0.};
1282   PetscMPIInt    rank;
1283   PetscInt       fillColor, v, e, d;
1284   PetscErrorCode ierr;
1285 
1286   PetscFunctionBegin;
1287   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRMPI(ierr);
1288   ierr = DMPlexGetCellType(dm, cell, &ct);CHKERRQ(ierr);
1289   fillColor = PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2;
1290   switch (ct) {
1291   case DM_POLYTOPE_TRIANGLE:
1292     {
1293       PetscReal refVertices[6] = {-1., -1., 1., -1., -1., 1.};
1294 
1295       for (v = 0; v < 3; ++v) {centroid[0] += PetscRealPart(coords[v*2+0])/3.;centroid[1] += PetscRealPart(coords[v*2+1])/3.;}
1296       for (e = 0; e < 3; ++e) {
1297         refCoords[0] = refVertices[e*2+0];
1298         refCoords[1] = refVertices[e*2+1];
1299         for (d = 1; d <= edgeDiv; ++d) {
1300           refCoords[d*2+0] = refCoords[0] + (refVertices[(e+1)%3 * 2 + 0] - refCoords[0])*d/edgeDiv;
1301           refCoords[d*2+1] = refCoords[1] + (refVertices[(e+1)%3 * 2 + 1] - refCoords[1])*d/edgeDiv;
1302         }
1303         ierr = DMPlexReferenceToCoordinates(dm, cell, edgeDiv+1, refCoords, edgeCoords);CHKERRQ(ierr);
1304         for (d = 0; d < edgeDiv; ++d) {
1305           ierr = PetscDrawTriangle(draw, centroid[0], centroid[1], edgeCoords[d*2+0], edgeCoords[d*2+1], edgeCoords[(d+1)*2+0], edgeCoords[(d+1)*2+1], fillColor, fillColor, fillColor);CHKERRQ(ierr);
1306           ierr = PetscDrawLine(draw, edgeCoords[d*2+0], edgeCoords[d*2+1], edgeCoords[(d+1)*2+0], edgeCoords[(d+1)*2+1], PETSC_DRAW_BLACK);CHKERRQ(ierr);
1307         }
1308       }
1309     }
1310     break;
1311   default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_SUP, "Cannot draw cells of type %s", DMPolytopeTypes[ct]);
1312   }
1313   PetscFunctionReturn(0);
1314 }
1315 
1316 static PetscErrorCode DMPlexView_Draw(DM dm, PetscViewer viewer)
1317 {
1318   PetscDraw          draw;
1319   DM                 cdm;
1320   PetscSection       coordSection;
1321   Vec                coordinates;
1322   const PetscScalar *coords;
1323   PetscReal          xyl[2],xyr[2],bound[4] = {PETSC_MAX_REAL, PETSC_MAX_REAL, PETSC_MIN_REAL, PETSC_MIN_REAL};
1324   PetscReal         *refCoords, *edgeCoords;
1325   PetscBool          isnull, drawAffine = PETSC_TRUE;
1326   PetscInt           dim, vStart, vEnd, cStart, cEnd, c, N, edgeDiv = 4;
1327   PetscErrorCode     ierr;
1328 
1329   PetscFunctionBegin;
1330   ierr = DMGetCoordinateDim(dm, &dim);CHKERRQ(ierr);
1331   if (dim != 2) SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Cannot draw meshes of dimension %D", dim);
1332   ierr = PetscOptionsGetBool(((PetscObject) dm)->options, ((PetscObject) dm)->prefix, "-dm_view_draw_affine", &drawAffine, NULL);CHKERRQ(ierr);
1333   if (!drawAffine) {ierr = PetscMalloc2((edgeDiv+1)*dim, &refCoords, (edgeDiv+1)*dim, &edgeCoords);CHKERRQ(ierr);}
1334   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
1335   ierr = DMGetLocalSection(cdm, &coordSection);CHKERRQ(ierr);
1336   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
1337   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
1338   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
1339 
1340   ierr = PetscViewerDrawGetDraw(viewer, 0, &draw);CHKERRQ(ierr);
1341   ierr = PetscDrawIsNull(draw, &isnull);CHKERRQ(ierr);
1342   if (isnull) PetscFunctionReturn(0);
1343   ierr = PetscDrawSetTitle(draw, "Mesh");CHKERRQ(ierr);
1344 
1345   ierr = VecGetLocalSize(coordinates, &N);CHKERRQ(ierr);
1346   ierr = VecGetArrayRead(coordinates, &coords);CHKERRQ(ierr);
1347   for (c = 0; c < N; c += dim) {
1348     bound[0] = PetscMin(bound[0], PetscRealPart(coords[c]));   bound[2] = PetscMax(bound[2], PetscRealPart(coords[c]));
1349     bound[1] = PetscMin(bound[1], PetscRealPart(coords[c+1])); bound[3] = PetscMax(bound[3], PetscRealPart(coords[c+1]));
1350   }
1351   ierr = VecRestoreArrayRead(coordinates, &coords);CHKERRQ(ierr);
1352   ierr = MPIU_Allreduce(&bound[0],xyl,2,MPIU_REAL,MPIU_MIN,PetscObjectComm((PetscObject)dm));CHKERRMPI(ierr);
1353   ierr = MPIU_Allreduce(&bound[2],xyr,2,MPIU_REAL,MPIU_MAX,PetscObjectComm((PetscObject)dm));CHKERRMPI(ierr);
1354   ierr = PetscDrawSetCoordinates(draw, xyl[0], xyl[1], xyr[0], xyr[1]);CHKERRQ(ierr);
1355   ierr = PetscDrawClear(draw);CHKERRQ(ierr);
1356 
1357   for (c = cStart; c < cEnd; ++c) {
1358     PetscScalar *coords = NULL;
1359     PetscInt     numCoords;
1360 
1361     ierr = DMPlexVecGetClosureAtDepth_Internal(dm, coordSection, coordinates, c, 0, &numCoords, &coords);CHKERRQ(ierr);
1362     if (drawAffine) {
1363       ierr = DMPlexDrawCell(dm, draw, c, coords);CHKERRQ(ierr);
1364     } else {
1365       ierr = DMPlexDrawCellHighOrder(dm, draw, c, coords, edgeDiv, refCoords, edgeCoords);CHKERRQ(ierr);
1366     }
1367     ierr = DMPlexVecRestoreClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
1368   }
1369   if (!drawAffine) {ierr = PetscFree2(refCoords, edgeCoords);CHKERRQ(ierr);}
1370   ierr = PetscDrawFlush(draw);CHKERRQ(ierr);
1371   ierr = PetscDrawPause(draw);CHKERRQ(ierr);
1372   ierr = PetscDrawSave(draw);CHKERRQ(ierr);
1373   PetscFunctionReturn(0);
1374 }
1375 
1376 #if defined(PETSC_HAVE_EXODUSII)
1377 #include <exodusII.h>
1378 #include <petscviewerexodusii.h>
1379 #endif
1380 
1381 PetscErrorCode DMView_Plex(DM dm, PetscViewer viewer)
1382 {
1383   PetscBool      iascii, ishdf5, isvtk, isdraw, flg, isglvis, isexodus;
1384   char           name[PETSC_MAX_PATH_LEN];
1385   PetscErrorCode ierr;
1386 
1387   PetscFunctionBegin;
1388   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1389   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1390   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERASCII,    &iascii);CHKERRQ(ierr);
1391   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,      &isvtk);CHKERRQ(ierr);
1392   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,     &ishdf5);CHKERRQ(ierr);
1393   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERDRAW,     &isdraw);CHKERRQ(ierr);
1394   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERGLVIS,    &isglvis);CHKERRQ(ierr);
1395   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWEREXODUSII, &isexodus);CHKERRQ(ierr);
1396   if (iascii) {
1397     PetscViewerFormat format;
1398     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
1399     if (format == PETSC_VIEWER_ASCII_GLVIS) {
1400       ierr = DMPlexView_GLVis(dm, viewer);CHKERRQ(ierr);
1401     } else {
1402       ierr = DMPlexView_Ascii(dm, viewer);CHKERRQ(ierr);
1403     }
1404   } else if (ishdf5) {
1405 #if defined(PETSC_HAVE_HDF5)
1406     ierr = DMPlexView_HDF5_Internal(dm, viewer);CHKERRQ(ierr);
1407 #else
1408     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1409 #endif
1410   } else if (isvtk) {
1411     ierr = DMPlexVTKWriteAll((PetscObject) dm,viewer);CHKERRQ(ierr);
1412   } else if (isdraw) {
1413     ierr = DMPlexView_Draw(dm, viewer);CHKERRQ(ierr);
1414   } else if (isglvis) {
1415     ierr = DMPlexView_GLVis(dm, viewer);CHKERRQ(ierr);
1416 #if defined(PETSC_HAVE_EXODUSII)
1417   } else if (isexodus) {
1418 /*
1419       exodusII requires that all sets be part of exactly one cell set.
1420       If the dm does not have a "Cell Sets" label defined, we create one
1421       with ID 1, containig all cells.
1422       Note that if the Cell Sets label is defined but does not cover all cells,
1423       we may still have a problem. This should probably be checked here or in the viewer;
1424     */
1425     PetscInt numCS;
1426     ierr = DMGetLabelSize(dm,"Cell Sets",&numCS);CHKERRQ(ierr);
1427     if (!numCS) {
1428       PetscInt cStart, cEnd, c;
1429       ierr = DMCreateLabel(dm, "Cell Sets");CHKERRQ(ierr);
1430       ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
1431       for (c = cStart; c < cEnd; ++c) {ierr = DMSetLabelValue(dm, "Cell Sets", c, 1);CHKERRQ(ierr);}
1432     }
1433     ierr = DMView_PlexExodusII(dm, viewer);CHKERRQ(ierr);
1434 #endif
1435   } else {
1436     SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Viewer type %s not yet supported for DMPlex writing", ((PetscObject)viewer)->type_name);
1437   }
1438   /* Optionally view the partition */
1439   ierr = PetscOptionsHasName(((PetscObject) dm)->options, ((PetscObject) dm)->prefix, "-dm_partition_view", &flg);CHKERRQ(ierr);
1440   if (flg) {
1441     Vec ranks;
1442     ierr = DMPlexCreateRankField(dm, &ranks);CHKERRQ(ierr);
1443     ierr = VecView(ranks, viewer);CHKERRQ(ierr);
1444     ierr = VecDestroy(&ranks);CHKERRQ(ierr);
1445   }
1446   /* Optionally view a label */
1447   ierr = PetscOptionsGetString(((PetscObject) dm)->options, ((PetscObject) dm)->prefix, "-dm_label_view", name, sizeof(name), &flg);CHKERRQ(ierr);
1448   if (flg) {
1449     DMLabel label;
1450     Vec     val;
1451 
1452     ierr = DMGetLabel(dm, name, &label);CHKERRQ(ierr);
1453     if (!label) SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Label %s provided to -dm_label_view does not exist in this DM", name);
1454     ierr = DMPlexCreateLabelField(dm, label, &val);CHKERRQ(ierr);
1455     ierr = VecView(val, viewer);CHKERRQ(ierr);
1456     ierr = VecDestroy(&val);CHKERRQ(ierr);
1457   }
1458   PetscFunctionReturn(0);
1459 }
1460 
1461 /*@
1462   DMPlexTopologyView - Saves a DMPlex topology into a file
1463 
1464   Collective on DM
1465 
1466   Input Parameters:
1467 + dm     - The DM whose topology is to be saved
1468 - viewer - The PetscViewer for saving
1469 
1470   Level: advanced
1471 
1472 .seealso: DMView(), DMPlexCoordinatesView(), DMPlexLabelsView(), DMPlexTopologyLoad()
1473 @*/
1474 PetscErrorCode DMPlexTopologyView(DM dm, PetscViewer viewer)
1475 {
1476   PetscBool      ishdf5;
1477   PetscErrorCode ierr;
1478 
1479   PetscFunctionBegin;
1480   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1481   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1482   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
1483   if (ishdf5) {
1484 #if defined(PETSC_HAVE_HDF5)
1485     PetscViewerFormat format;
1486     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
1487     if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
1488       IS globalPointNumbering;
1489 
1490       ierr = DMPlexCreatePointNumbering(dm, &globalPointNumbering);CHKERRQ(ierr);
1491       ierr = DMPlexTopologyView_HDF5_Internal(dm, globalPointNumbering, viewer);CHKERRQ(ierr);
1492       ierr = ISDestroy(&globalPointNumbering);CHKERRQ(ierr);
1493     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 output.", PetscViewerFormats[format]);
1494 #else
1495     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1496 #endif
1497   }
1498   PetscFunctionReturn(0);
1499 }
1500 
1501 /*@
1502   DMPlexCoordinatesView - Saves DMPlex coordinates into a file
1503 
1504   Collective on DM
1505 
1506   Input Parameters:
1507 + dm     - The DM whose coordinates are to be saved
1508 - viewer - The PetscViewer for saving
1509 
1510   Level: advanced
1511 
1512 .seealso: DMView(), DMPlexTopologyView(), DMPlexLabelsView(), DMPlexCoordinatesLoad()
1513 @*/
1514 PetscErrorCode DMPlexCoordinatesView(DM dm, PetscViewer viewer)
1515 {
1516   PetscBool      ishdf5;
1517   PetscErrorCode ierr;
1518 
1519   PetscFunctionBegin;
1520   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1521   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1522   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
1523   if (ishdf5) {
1524 #if defined(PETSC_HAVE_HDF5)
1525     PetscViewerFormat format;
1526     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
1527     if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
1528       ierr = DMPlexCoordinatesView_HDF5_Internal(dm, viewer);CHKERRQ(ierr);
1529     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 input.", PetscViewerFormats[format]);
1530 #else
1531     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1532 #endif
1533   }
1534   PetscFunctionReturn(0);
1535 }
1536 
1537 /*@
1538   DMPlexLabelsView - Saves DMPlex labels into a file
1539 
1540   Collective on DM
1541 
1542   Input Parameters:
1543 + dm     - The DM whose labels are to be saved
1544 - viewer - The PetscViewer for saving
1545 
1546   Level: advanced
1547 
1548 .seealso: DMView(), DMPlexTopologyView(), DMPlexCoordinatesView(), DMPlexLabelsLoad()
1549 @*/
1550 PetscErrorCode DMPlexLabelsView(DM dm, PetscViewer viewer)
1551 {
1552   PetscBool      ishdf5;
1553   PetscErrorCode ierr;
1554 
1555   PetscFunctionBegin;
1556   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1557   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1558   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
1559   if (ishdf5) {
1560 #if defined(PETSC_HAVE_HDF5)
1561     IS                globalPointNumbering;
1562     PetscViewerFormat format;
1563 
1564     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
1565     if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
1566       ierr = DMPlexCreatePointNumbering(dm, &globalPointNumbering);CHKERRQ(ierr);
1567       ierr = DMPlexLabelsView_HDF5_Internal(dm, globalPointNumbering, viewer);CHKERRQ(ierr);
1568       ierr = ISDestroy(&globalPointNumbering);CHKERRQ(ierr);
1569     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 input.", PetscViewerFormats[format]);
1570 #else
1571     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1572 #endif
1573   }
1574   PetscFunctionReturn(0);
1575 }
1576 
1577 /*@
1578   DMPlexSectionView - Saves a section associated with a DMPlex
1579 
1580   Collective on DM
1581 
1582   Input Parameters:
1583 + dm         - The DM that contains the topology on which the section to be saved is defined
1584 . viewer     - The PetscViewer for saving
1585 - sectiondm  - The DM that contains the section to be saved
1586 
1587   Level: advanced
1588 
1589   Notes:
1590   This function is a wrapper around PetscSectionView(); in addition to the raw section, it saves information that associates the section points to the topology (dm) points. When the topology (dm) and the section are later loaded with DMPlexTopologyLoad() and DMPlexSectionLoad(), respectively, this information is used to match section points with topology points.
1591 
1592   In general dm and sectiondm are two different objects, the former carrying the topology and the latter carrying the section, and have been given a topology name and a section name, respectively, with PetscObjectSetName(). In practice, however, they can be the same object if it carries both topology and section; in that case the name of the object is used as both the topology name and the section name.
1593 
1594 .seealso: DMView(), DMPlexTopologyView(), DMPlexCoordinatesView(), DMPlexLabelsView(), DMPlexGlobalVectorView(), DMPlexLocalVectorView(), PetscSectionView(), DMPlexSectionLoad()
1595 @*/
1596 PetscErrorCode DMPlexSectionView(DM dm, PetscViewer viewer, DM sectiondm)
1597 {
1598   PetscBool      ishdf5;
1599   PetscErrorCode ierr;
1600 
1601   PetscFunctionBegin;
1602   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1603   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1604   PetscValidHeaderSpecific(sectiondm, DM_CLASSID, 3);
1605   ierr = PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERHDF5,&ishdf5);CHKERRQ(ierr);
1606   if (ishdf5) {
1607 #if defined(PETSC_HAVE_HDF5)
1608     ierr = DMPlexSectionView_HDF5_Internal(dm, viewer, sectiondm);CHKERRQ(ierr);
1609 #else
1610     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1611 #endif
1612   }
1613   PetscFunctionReturn(0);
1614 }
1615 
1616 /*@
1617   DMPlexGlobalVectorView - Saves a global vector
1618 
1619   Collective on DM
1620 
1621   Input Parameters:
1622 + dm        - The DM that represents the topology
1623 . viewer    - The PetscViewer to save data with
1624 . sectiondm - The DM that contains the global section on which vec is defined
1625 - vec       - The global vector to be saved
1626 
1627   Level: advanced
1628 
1629   Notes:
1630   In general dm and sectiondm are two different objects, the former carrying the topology and the latter carrying the section, and have been given a topology name and a section name, respectively, with PetscObjectSetName(). In practice, however, they can be the same object if it carries both topology and section; in that case the name of the object is used as both the topology name and the section name.
1631 
1632   Typical calling sequence
1633 $       DMCreate(PETSC_COMM_WORLD, &dm);
1634 $       DMSetType(dm, DMPLEX);
1635 $       PetscObjectSetName((PetscObject)dm, "topologydm_name");
1636 $       DMClone(dm, &sectiondm);
1637 $       PetscObjectSetName((PetscObject)sectiondm, "sectiondm_name");
1638 $       PetscSectionCreate(PETSC_COMM_WORLD, &section);
1639 $       DMPlexGetChart(sectiondm, &pStart, &pEnd);
1640 $       PetscSectionSetChart(section, pStart, pEnd);
1641 $       PetscSectionSetUp(section);
1642 $       DMSetLocalSection(sectiondm, section);
1643 $       PetscSectionDestroy(&section);
1644 $       DMGetGlobalVector(sectiondm, &vec);
1645 $       PetscObjectSetName((PetscObject)vec, "vec_name");
1646 $       DMPlexTopologyView(dm, viewer);
1647 $       DMPlexSectionView(dm, viewer, sectiondm);
1648 $       DMPlexGlobalVectorView(dm, viewer, sectiondm, vec);
1649 $       DMRestoreGlobalVector(sectiondm, &vec);
1650 $       DMDestroy(&sectiondm);
1651 $       DMDestroy(&dm);
1652 
1653 .seealso: DMPlexTopologyView(), DMPlexSectionView(), DMPlexLocalVectorView(), DMPlexGlobalVectorLoad(), DMPlexLocalVectorLoad()
1654 @*/
1655 PetscErrorCode DMPlexGlobalVectorView(DM dm, PetscViewer viewer, DM sectiondm, Vec vec)
1656 {
1657   PetscBool       ishdf5;
1658   PetscErrorCode  ierr;
1659 
1660   PetscFunctionBegin;
1661   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1662   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1663   PetscValidHeaderSpecific(sectiondm, DM_CLASSID, 3);
1664   PetscValidHeaderSpecific(vec, VEC_CLASSID, 4);
1665   /* Check consistency */
1666   {
1667     PetscSection  section;
1668     PetscBool     includesConstraints;
1669     PetscInt      m, m1;
1670 
1671     ierr = VecGetLocalSize(vec, &m1);CHKERRQ(ierr);
1672     ierr = DMGetGlobalSection(sectiondm, &section);CHKERRQ(ierr);
1673     ierr = PetscSectionGetIncludesConstraints(section, &includesConstraints);CHKERRQ(ierr);
1674     if (includesConstraints) {ierr = PetscSectionGetStorageSize(section, &m);CHKERRQ(ierr);}
1675     else {ierr = PetscSectionGetConstrainedStorageSize(section, &m);CHKERRQ(ierr);}
1676     if (m1 != m) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Global vector size (%D) != global section storage size (%D)", m1, m);
1677   }
1678   ierr = PetscObjectTypeCompare((PetscObject)viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
1679   if (ishdf5) {
1680 #if defined(PETSC_HAVE_HDF5)
1681     ierr = DMPlexGlobalVectorView_HDF5_Internal(dm, viewer, sectiondm, vec);CHKERRQ(ierr);
1682 #else
1683     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1684 #endif
1685   }
1686   PetscFunctionReturn(0);
1687 }
1688 
1689 /*@
1690   DMPlexLocalVectorView - Saves a local vector
1691 
1692   Collective on DM
1693 
1694   Input Parameters:
1695 + dm        - The DM that represents the topology
1696 . viewer    - The PetscViewer to save data with
1697 . sectiondm - The DM that contains the local section on which vec is defined; may be the same as dm
1698 - vec       - The local vector to be saved
1699 
1700   Level: advanced
1701 
1702   Notes:
1703   In general dm and sectiondm are two different objects, the former carrying the topology and the latter carrying the section, and have been given a topology name and a section name, respectively, with PetscObjectSetName(). In practice, however, they can be the same object if it carries both topology and section; in that case the name of the object is used as both the topology name and the section name.
1704 
1705   Typical calling sequence
1706 $       DMCreate(PETSC_COMM_WORLD, &dm);
1707 $       DMSetType(dm, DMPLEX);
1708 $       PetscObjectSetName((PetscObject)dm, "topologydm_name");
1709 $       DMClone(dm, &sectiondm);
1710 $       PetscObjectSetName((PetscObject)sectiondm, "sectiondm_name");
1711 $       PetscSectionCreate(PETSC_COMM_WORLD, &section);
1712 $       DMPlexGetChart(sectiondm, &pStart, &pEnd);
1713 $       PetscSectionSetChart(section, pStart, pEnd);
1714 $       PetscSectionSetUp(section);
1715 $       DMSetLocalSection(sectiondm, section);
1716 $       DMGetLocalVector(sectiondm, &vec);
1717 $       PetscObjectSetName((PetscObject)vec, "vec_name");
1718 $       DMPlexTopologyView(dm, viewer);
1719 $       DMPlexSectionView(dm, viewer, sectiondm);
1720 $       DMPlexLocalVectorView(dm, viewer, sectiondm, vec);
1721 $       DMRestoreLocalVector(sectiondm, &vec);
1722 $       DMDestroy(&sectiondm);
1723 $       DMDestroy(&dm);
1724 
1725 .seealso: DMPlexTopologyView(), DMPlexSectionView(), DMPlexGlobalVectorView(), DMPlexGlobalVectorLoad(), DMPlexLocalVectorLoad()
1726 @*/
1727 PetscErrorCode DMPlexLocalVectorView(DM dm, PetscViewer viewer, DM sectiondm, Vec vec)
1728 {
1729   PetscBool       ishdf5;
1730   PetscErrorCode  ierr;
1731 
1732   PetscFunctionBegin;
1733   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1734   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1735   PetscValidHeaderSpecific(sectiondm, DM_CLASSID, 3);
1736   PetscValidHeaderSpecific(vec, VEC_CLASSID, 4);
1737   /* Check consistency */
1738   {
1739     PetscSection  section;
1740     PetscBool     includesConstraints;
1741     PetscInt      m, m1;
1742 
1743     ierr = VecGetLocalSize(vec, &m1);CHKERRQ(ierr);
1744     ierr = DMGetLocalSection(sectiondm, &section);CHKERRQ(ierr);
1745     ierr = PetscSectionGetIncludesConstraints(section, &includesConstraints);CHKERRQ(ierr);
1746     if (includesConstraints) {ierr = PetscSectionGetStorageSize(section, &m);CHKERRQ(ierr);}
1747     else {ierr = PetscSectionGetConstrainedStorageSize(section, &m);CHKERRQ(ierr);}
1748     if (m1 != m) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Local vector size (%D) != local section storage size (%D)", m1, m);
1749   }
1750   ierr = PetscObjectTypeCompare((PetscObject)viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
1751   if (ishdf5) {
1752 #if defined(PETSC_HAVE_HDF5)
1753     ierr = DMPlexLocalVectorView_HDF5_Internal(dm, viewer, sectiondm, vec);CHKERRQ(ierr);
1754 #else
1755     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1756 #endif
1757   }
1758   PetscFunctionReturn(0);
1759 }
1760 
1761 PetscErrorCode DMLoad_Plex(DM dm, PetscViewer viewer)
1762 {
1763   PetscBool      ishdf5;
1764   PetscErrorCode ierr;
1765 
1766   PetscFunctionBegin;
1767   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1768   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1769   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,   &ishdf5);CHKERRQ(ierr);
1770   if (ishdf5) {
1771 #if defined(PETSC_HAVE_HDF5)
1772     PetscViewerFormat format;
1773     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
1774     if (format == PETSC_VIEWER_HDF5_XDMF || format == PETSC_VIEWER_HDF5_VIZ) {
1775       ierr = DMPlexLoad_HDF5_Xdmf_Internal(dm, viewer);CHKERRQ(ierr);
1776     } else if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
1777       ierr = DMPlexLoad_HDF5_Internal(dm, viewer);CHKERRQ(ierr);
1778     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 input.", PetscViewerFormats[format]);
1779     PetscFunctionReturn(0);
1780 #else
1781     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1782 #endif
1783   } else SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Viewer type %s not yet supported for DMPlex loading", ((PetscObject)viewer)->type_name);
1784 }
1785 
1786 /*@
1787   DMPlexTopologyLoad - Loads a topology into a DMPlex
1788 
1789   Collective on DM
1790 
1791   Input Parameters:
1792 + dm     - The DM into which the topology is loaded
1793 - viewer - The PetscViewer for the saved topology
1794 
1795   Output Parameters:
1796 . globalToLocalPointSF - The PetscSF that pushes points in [0, N) to the associated points in the loaded plex, where N is the global number of points; NULL if unneeded
1797 
1798   Level: advanced
1799 
1800 .seealso: DMLoad(), DMPlexCoordinatesLoad(), DMPlexLabelsLoad(), DMView(), PetscViewerHDF5Open(), PetscViewerPushFormat()
1801 @*/
1802 PetscErrorCode DMPlexTopologyLoad(DM dm, PetscViewer viewer, PetscSF *globalToLocalPointSF)
1803 {
1804   PetscBool      ishdf5;
1805   PetscErrorCode ierr;
1806 
1807   PetscFunctionBegin;
1808   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1809   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1810   if (globalToLocalPointSF) PetscValidPointer(globalToLocalPointSF, 3);
1811   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
1812   if (ishdf5) {
1813 #if defined(PETSC_HAVE_HDF5)
1814     PetscViewerFormat format;
1815     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
1816     if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
1817       ierr = DMPlexTopologyLoad_HDF5_Internal(dm, viewer, globalToLocalPointSF);CHKERRQ(ierr);
1818     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 input.", PetscViewerFormats[format]);
1819 #else
1820     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1821 #endif
1822   }
1823   PetscFunctionReturn(0);
1824 }
1825 
1826 /*@
1827   DMPlexCoordinatesLoad - Loads coordinates into a DMPlex
1828 
1829   Collective on DM
1830 
1831   Input Parameters:
1832 + dm     - The DM into which the coordinates are loaded
1833 - viewer - The PetscViewer for the saved coordinates
1834 
1835   Level: advanced
1836 
1837 .seealso: DMLoad(), DMPlexTopologyLoad(), DMPlexLabelsLoad(), DMView(), PetscViewerHDF5Open(), PetscViewerPushFormat()
1838 @*/
1839 PetscErrorCode DMPlexCoordinatesLoad(DM dm, PetscViewer viewer)
1840 {
1841   PetscBool      ishdf5;
1842   PetscErrorCode ierr;
1843 
1844   PetscFunctionBegin;
1845   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1846   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1847   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
1848   if (ishdf5) {
1849 #if defined(PETSC_HAVE_HDF5)
1850     PetscViewerFormat format;
1851     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
1852     if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
1853       ierr = DMPlexCoordinatesLoad_HDF5_Internal(dm, viewer);CHKERRQ(ierr);
1854     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 input.", PetscViewerFormats[format]);
1855 #else
1856     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1857 #endif
1858   }
1859   PetscFunctionReturn(0);
1860 }
1861 
1862 /*@
1863   DMPlexLabelsLoad - Loads labels into a DMPlex
1864 
1865   Collective on DM
1866 
1867   Input Parameters:
1868 + dm     - The DM into which the labels are loaded
1869 - viewer - The PetscViewer for the saved labels
1870 
1871   Level: advanced
1872 
1873 .seealso: DMLoad(), DMPlexTopologyLoad(), DMPlexCoordinatesLoad(), DMView(), PetscViewerHDF5Open(), PetscViewerPushFormat()
1874 @*/
1875 PetscErrorCode DMPlexLabelsLoad(DM dm, PetscViewer viewer)
1876 {
1877   PetscBool      ishdf5;
1878   PetscErrorCode ierr;
1879 
1880   PetscFunctionBegin;
1881   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1882   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1883   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
1884   if (ishdf5) {
1885 #if defined(PETSC_HAVE_HDF5)
1886     PetscViewerFormat format;
1887 
1888     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
1889     if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
1890       ierr = DMPlexLabelsLoad_HDF5_Internal(dm, viewer);CHKERRQ(ierr);
1891     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 input.", PetscViewerFormats[format]);
1892 #else
1893     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1894 #endif
1895   }
1896   PetscFunctionReturn(0);
1897 }
1898 
1899 /*@
1900   DMPlexSectionLoad - Loads section into a DMPlex
1901 
1902   Collective on DM
1903 
1904   Input Parameters:
1905 + dm          - The DM that represents the topology
1906 . viewer      - The PetscViewer that represents the on-disk section (sectionA)
1907 . sectiondm   - The DM into which the on-disk section (sectionA) is migrated
1908 - globalToLocalPointSF - The SF returned by DMPlexTopologyLoad() when loading dm from viewer
1909 
1910   Output Parameters
1911 + globalDofSF - The SF that migrates any on-disk Vec data associated with sectionA into a global Vec associated with the sectiondm's global section (NULL if not needed)
1912 - localDofSF  - The SF that migrates any on-disk Vec data associated with sectionA into a local Vec associated with the sectiondm's local section (NULL if not needed)
1913 
1914   Level: advanced
1915 
1916   Notes:
1917   This function is a wrapper around PetscSectionLoad(); it loads, in addition to the raw section, a list of global point numbers that associates each on-disk section point with a global point number in [0, NX), where NX is the number of topology points in dm. Noting that globalToLocalPointSF associates each topology point in dm with a global number in [0, NX), one can readily establish an association of the on-disk section points with the topology points.
1918 
1919   In general dm and sectiondm are two different objects, the former carrying the topology and the latter carrying the section, and have been given a topology name and a section name, respectively, with PetscObjectSetName(). In practice, however, they can be the same object if it carries both topology and section; in that case the name of the object is used as both the topology name and the section name.
1920 
1921   The output parameter, globalDofSF (localDofSF), can later be used with DMPlexGlobalVectorLoad() (DMPlexLocalVectorLoad()) to load on-disk vectors into global (local) vectors associated with sectiondm's global (local) section.
1922 
1923   Example using 2 processes:
1924 $  NX (number of points on dm): 4
1925 $  sectionA                   : the on-disk section
1926 $  vecA                       : a vector associated with sectionA
1927 $  sectionB                   : sectiondm's local section constructed in this function
1928 $  vecB (local)               : a vector associated with sectiondm's local section
1929 $  vecB (global)              : a vector associated with sectiondm's global section
1930 $
1931 $                                     rank 0    rank 1
1932 $  vecA (global)                  : [.0 .4 .1 | .2 .3]        <- to be loaded in DMPlexGlobalVectorLoad() or DMPlexLocalVectorLoad()
1933 $  sectionA->atlasOff             :       0 2 | 1             <- loaded in PetscSectionLoad()
1934 $  sectionA->atlasDof             :       1 3 | 1             <- loaded in PetscSectionLoad()
1935 $  sectionA's global point numbers:       0 2 | 3             <- loaded in DMPlexSectionLoad()
1936 $  [0, NX)                        :       0 1 | 2 3           <- conceptual partition used in globalToLocalPointSF
1937 $  sectionB's global point numbers:     0 1 3 | 3 2           <- associated with [0, NX) by globalToLocalPointSF
1938 $  sectionB->atlasDof             :     1 0 1 | 1 3
1939 $  sectionB->atlasOff (no perm)   :     0 1 1 | 0 1
1940 $  vecB (local)                   :   [.0 .4] | [.4 .1 .2 .3] <- to be constructed by calling DMPlexLocalVectorLoad() with localDofSF
1941 $  vecB (global)                  :    [.0 .4 | .1 .2 .3]     <- to be constructed by calling DMPlexGlobalVectorLoad() with globalDofSF
1942 $
1943 $  where "|" represents a partition of loaded data, and global point 3 is assumed to be owned by rank 0.
1944 
1945 .seealso: DMLoad(), DMPlexTopologyLoad(), DMPlexCoordinatesLoad(), DMPlexLabelsLoad(), DMPlexGlobalVectorLoad(), DMPlexLocalVectorLoad(), PetscSectionLoad(), DMPlexSectionView()
1946 @*/
1947 PetscErrorCode DMPlexSectionLoad(DM dm, PetscViewer viewer, DM sectiondm, PetscSF globalToLocalPointSF, PetscSF *globalDofSF, PetscSF *localDofSF)
1948 {
1949   PetscBool      ishdf5;
1950   PetscErrorCode ierr;
1951 
1952   PetscFunctionBegin;
1953   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1954   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1955   PetscValidHeaderSpecific(sectiondm, DM_CLASSID, 3);
1956   PetscValidHeaderSpecific(globalToLocalPointSF, PETSCSF_CLASSID, 4);
1957   if (globalDofSF) PetscValidPointer(globalDofSF, 5);
1958   if (localDofSF) PetscValidPointer(localDofSF, 6);
1959   ierr = PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERHDF5,&ishdf5);CHKERRQ(ierr);
1960   if (ishdf5) {
1961 #if defined(PETSC_HAVE_HDF5)
1962     ierr = DMPlexSectionLoad_HDF5_Internal(dm, viewer, sectiondm, globalToLocalPointSF, globalDofSF, localDofSF);CHKERRQ(ierr);
1963 #else
1964     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1965 #endif
1966   }
1967   PetscFunctionReturn(0);
1968 }
1969 
1970 /*@
1971   DMPlexGlobalVectorLoad - Loads on-disk vector data into a global vector
1972 
1973   Collective on DM
1974 
1975   Input Parameters:
1976 + dm        - The DM that represents the topology
1977 . viewer    - The PetscViewer that represents the on-disk vector data
1978 . sectiondm - The DM that contains the global section on which vec is defined
1979 . sf        - The SF that migrates the on-disk vector data into vec
1980 - vec       - The global vector to set values of
1981 
1982   Level: advanced
1983 
1984   Notes:
1985   In general dm and sectiondm are two different objects, the former carrying the topology and the latter carrying the section, and have been given a topology name and a section name, respectively, with PetscObjectSetName(). In practice, however, they can be the same object if it carries both topology and section; in that case the name of the object is used as both the topology name and the section name.
1986 
1987   Typical calling sequence
1988 $       DMCreate(PETSC_COMM_WORLD, &dm);
1989 $       DMSetType(dm, DMPLEX);
1990 $       PetscObjectSetName((PetscObject)dm, "topologydm_name");
1991 $       DMPlexTopologyLoad(dm, viewer, &sfX);
1992 $       DMClone(dm, &sectiondm);
1993 $       PetscObjectSetName((PetscObject)sectiondm, "sectiondm_name");
1994 $       DMPlexSectionLoad(dm, viewer, sectiondm, sfX, &gsf, NULL);
1995 $       DMGetGlobalVector(sectiondm, &vec);
1996 $       PetscObjectSetName((PetscObject)vec, "vec_name");
1997 $       DMPlexGlobalVectorLoad(dm, viewer, sectiondm, gsf, vec);
1998 $       DMRestoreGlobalVector(sectiondm, &vec);
1999 $       PetscSFDestroy(&gsf);
2000 $       PetscSFDestroy(&sfX);
2001 $       DMDestroy(&sectiondm);
2002 $       DMDestroy(&dm);
2003 
2004 .seealso: DMPlexTopologyLoad(), DMPlexSectionLoad(), DMPlexLocalVectorLoad(), DMPlexGlobalVectorView(), DMPlexLocalVectorView()
2005 @*/
2006 PetscErrorCode DMPlexGlobalVectorLoad(DM dm, PetscViewer viewer, DM sectiondm, PetscSF sf, Vec vec)
2007 {
2008   PetscBool       ishdf5;
2009   PetscErrorCode  ierr;
2010 
2011   PetscFunctionBegin;
2012   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2013   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
2014   PetscValidHeaderSpecific(sectiondm, DM_CLASSID, 3);
2015   PetscValidHeaderSpecific(sf, PETSCSF_CLASSID, 4);
2016   PetscValidHeaderSpecific(vec, VEC_CLASSID, 5);
2017   /* Check consistency */
2018   {
2019     PetscSection  section;
2020     PetscBool     includesConstraints;
2021     PetscInt      m, m1;
2022 
2023     ierr = VecGetLocalSize(vec, &m1);CHKERRQ(ierr);
2024     ierr = DMGetGlobalSection(sectiondm, &section);CHKERRQ(ierr);
2025     ierr = PetscSectionGetIncludesConstraints(section, &includesConstraints);CHKERRQ(ierr);
2026     if (includesConstraints) {ierr = PetscSectionGetStorageSize(section, &m);CHKERRQ(ierr);}
2027     else {ierr = PetscSectionGetConstrainedStorageSize(section, &m);CHKERRQ(ierr);}
2028     if (m1 != m) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Global vector size (%D) != global section storage size (%D)", m1, m);
2029   }
2030   ierr = PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERHDF5,&ishdf5);CHKERRQ(ierr);
2031   if (ishdf5) {
2032 #if defined(PETSC_HAVE_HDF5)
2033     ierr = DMPlexVecLoad_HDF5_Internal(dm, viewer, sectiondm, sf, vec);CHKERRQ(ierr);
2034 #else
2035     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
2036 #endif
2037   }
2038   PetscFunctionReturn(0);
2039 }
2040 
2041 /*@
2042   DMPlexLocalVectorLoad - Loads on-disk vector data into a local vector
2043 
2044   Collective on DM
2045 
2046   Input Parameters:
2047 + dm        - The DM that represents the topology
2048 . viewer    - The PetscViewer that represents the on-disk vector data
2049 . sectiondm - The DM that contains the local section on which vec is defined
2050 . sf        - The SF that migrates the on-disk vector data into vec
2051 - vec       - The local vector to set values of
2052 
2053   Level: advanced
2054 
2055   Notes:
2056   In general dm and sectiondm are two different objects, the former carrying the topology and the latter carrying the section, and have been given a topology name and a section name, respectively, with PetscObjectSetName(). In practice, however, they can be the same object if it carries both topology and section; in that case the name of the object is used as both the topology name and the section name.
2057 
2058   Typical calling sequence
2059 $       DMCreate(PETSC_COMM_WORLD, &dm);
2060 $       DMSetType(dm, DMPLEX);
2061 $       PetscObjectSetName((PetscObject)dm, "topologydm_name");
2062 $       DMPlexTopologyLoad(dm, viewer, &sfX);
2063 $       DMClone(dm, &sectiondm);
2064 $       PetscObjectSetName((PetscObject)sectiondm, "sectiondm_name");
2065 $       DMPlexSectionLoad(dm, viewer, sectiondm, sfX, NULL, &lsf);
2066 $       DMGetLocalVector(sectiondm, &vec);
2067 $       PetscObjectSetName((PetscObject)vec, "vec_name");
2068 $       DMPlexLocalVectorLoad(dm, viewer, sectiondm, lsf, vec);
2069 $       DMRestoreLocalVector(sectiondm, &vec);
2070 $       PetscSFDestroy(&lsf);
2071 $       PetscSFDestroy(&sfX);
2072 $       DMDestroy(&sectiondm);
2073 $       DMDestroy(&dm);
2074 
2075 .seealso: DMPlexTopologyLoad(), DMPlexSectionLoad(), DMPlexGlobalVectorLoad(), DMPlexGlobalVectorView(), DMPlexLocalVectorView()
2076 @*/
2077 PetscErrorCode DMPlexLocalVectorLoad(DM dm, PetscViewer viewer, DM sectiondm, PetscSF sf, Vec vec)
2078 {
2079   PetscBool       ishdf5;
2080   PetscErrorCode  ierr;
2081 
2082   PetscFunctionBegin;
2083   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2084   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
2085   PetscValidHeaderSpecific(sectiondm, DM_CLASSID, 3);
2086   PetscValidHeaderSpecific(sf, PETSCSF_CLASSID, 4);
2087   PetscValidHeaderSpecific(vec, VEC_CLASSID, 5);
2088   /* Check consistency */
2089   {
2090     PetscSection  section;
2091     PetscBool     includesConstraints;
2092     PetscInt      m, m1;
2093 
2094     ierr = VecGetLocalSize(vec, &m1);CHKERRQ(ierr);
2095     ierr = DMGetLocalSection(sectiondm, &section);CHKERRQ(ierr);
2096     ierr = PetscSectionGetIncludesConstraints(section, &includesConstraints);CHKERRQ(ierr);
2097     if (includesConstraints) {ierr = PetscSectionGetStorageSize(section, &m);CHKERRQ(ierr);}
2098     else {ierr = PetscSectionGetConstrainedStorageSize(section, &m);CHKERRQ(ierr);}
2099     if (m1 != m) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Local vector size (%D) != local section storage size (%D)", m1, m);
2100   }
2101   ierr = PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERHDF5,&ishdf5);CHKERRQ(ierr);
2102   if (ishdf5) {
2103 #if defined(PETSC_HAVE_HDF5)
2104     ierr = DMPlexVecLoad_HDF5_Internal(dm, viewer, sectiondm, sf, vec);CHKERRQ(ierr);
2105 #else
2106     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
2107 #endif
2108   }
2109   PetscFunctionReturn(0);
2110 }
2111 
2112 PetscErrorCode DMDestroy_Plex(DM dm)
2113 {
2114   DM_Plex       *mesh = (DM_Plex*) dm->data;
2115   PetscErrorCode ierr;
2116 
2117   PetscFunctionBegin;
2118   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMSetUpGLVisViewer_C",NULL);CHKERRQ(ierr);
2119   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMPlexInsertBoundaryValues_C", NULL);CHKERRQ(ierr);
2120   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMCreateNeumannOverlap_C", NULL);CHKERRQ(ierr);
2121   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMInterpolateSolution_C", NULL);CHKERRQ(ierr);
2122   if (--mesh->refct > 0) PetscFunctionReturn(0);
2123   ierr = PetscSectionDestroy(&mesh->coneSection);CHKERRQ(ierr);
2124   ierr = PetscFree(mesh->cones);CHKERRQ(ierr);
2125   ierr = PetscFree(mesh->coneOrientations);CHKERRQ(ierr);
2126   ierr = PetscSectionDestroy(&mesh->supportSection);CHKERRQ(ierr);
2127   ierr = PetscSectionDestroy(&mesh->subdomainSection);CHKERRQ(ierr);
2128   ierr = PetscFree(mesh->supports);CHKERRQ(ierr);
2129   ierr = PetscFree(mesh->facesTmp);CHKERRQ(ierr);
2130   ierr = PetscFree(mesh->tetgenOpts);CHKERRQ(ierr);
2131   ierr = PetscFree(mesh->triangleOpts);CHKERRQ(ierr);
2132   ierr = PetscPartitionerDestroy(&mesh->partitioner);CHKERRQ(ierr);
2133   ierr = DMLabelDestroy(&mesh->subpointMap);CHKERRQ(ierr);
2134   ierr = ISDestroy(&mesh->subpointIS);CHKERRQ(ierr);
2135   ierr = ISDestroy(&mesh->globalVertexNumbers);CHKERRQ(ierr);
2136   ierr = ISDestroy(&mesh->globalCellNumbers);CHKERRQ(ierr);
2137   ierr = PetscSectionDestroy(&mesh->anchorSection);CHKERRQ(ierr);
2138   ierr = ISDestroy(&mesh->anchorIS);CHKERRQ(ierr);
2139   ierr = PetscSectionDestroy(&mesh->parentSection);CHKERRQ(ierr);
2140   ierr = PetscFree(mesh->parents);CHKERRQ(ierr);
2141   ierr = PetscFree(mesh->childIDs);CHKERRQ(ierr);
2142   ierr = PetscSectionDestroy(&mesh->childSection);CHKERRQ(ierr);
2143   ierr = PetscFree(mesh->children);CHKERRQ(ierr);
2144   ierr = DMDestroy(&mesh->referenceTree);CHKERRQ(ierr);
2145   ierr = PetscGridHashDestroy(&mesh->lbox);CHKERRQ(ierr);
2146   ierr = PetscFree(mesh->neighbors);CHKERRQ(ierr);
2147   /* This was originally freed in DMDestroy(), but that prevents reference counting of backend objects */
2148   ierr = PetscFree(mesh);CHKERRQ(ierr);
2149   PetscFunctionReturn(0);
2150 }
2151 
2152 PetscErrorCode DMCreateMatrix_Plex(DM dm, Mat *J)
2153 {
2154   PetscSection           sectionGlobal;
2155   PetscInt               bs = -1, mbs;
2156   PetscInt               localSize;
2157   PetscBool              isShell, isBlock, isSeqBlock, isMPIBlock, isSymBlock, isSymSeqBlock, isSymMPIBlock, isMatIS;
2158   PetscErrorCode         ierr;
2159   MatType                mtype;
2160   ISLocalToGlobalMapping ltog;
2161 
2162   PetscFunctionBegin;
2163   ierr = MatInitializePackage();CHKERRQ(ierr);
2164   mtype = dm->mattype;
2165   ierr = DMGetGlobalSection(dm, &sectionGlobal);CHKERRQ(ierr);
2166   /* ierr = PetscSectionGetStorageSize(sectionGlobal, &localSize);CHKERRQ(ierr); */
2167   ierr = PetscSectionGetConstrainedStorageSize(sectionGlobal, &localSize);CHKERRQ(ierr);
2168   ierr = MatCreate(PetscObjectComm((PetscObject)dm), J);CHKERRQ(ierr);
2169   ierr = MatSetSizes(*J, localSize, localSize, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
2170   ierr = MatSetType(*J, mtype);CHKERRQ(ierr);
2171   ierr = MatSetFromOptions(*J);CHKERRQ(ierr);
2172   ierr = MatGetBlockSize(*J, &mbs);CHKERRQ(ierr);
2173   if (mbs > 1) bs = mbs;
2174   ierr = PetscStrcmp(mtype, MATSHELL, &isShell);CHKERRQ(ierr);
2175   ierr = PetscStrcmp(mtype, MATBAIJ, &isBlock);CHKERRQ(ierr);
2176   ierr = PetscStrcmp(mtype, MATSEQBAIJ, &isSeqBlock);CHKERRQ(ierr);
2177   ierr = PetscStrcmp(mtype, MATMPIBAIJ, &isMPIBlock);CHKERRQ(ierr);
2178   ierr = PetscStrcmp(mtype, MATSBAIJ, &isSymBlock);CHKERRQ(ierr);
2179   ierr = PetscStrcmp(mtype, MATSEQSBAIJ, &isSymSeqBlock);CHKERRQ(ierr);
2180   ierr = PetscStrcmp(mtype, MATMPISBAIJ, &isSymMPIBlock);CHKERRQ(ierr);
2181   ierr = PetscStrcmp(mtype, MATIS, &isMatIS);CHKERRQ(ierr);
2182   if (!isShell) {
2183     PetscSection subSection;
2184     PetscBool    fillMatrix = (PetscBool)(!dm->prealloc_only && !isMatIS);
2185     PetscInt    *dnz, *onz, *dnzu, *onzu, bsLocal[2], bsMinMax[2], *ltogidx, lsize;
2186     PetscInt     pStart, pEnd, p, dof, cdof;
2187 
2188     /* Set localtoglobalmapping on the matrix for MatSetValuesLocal() to work (it also creates the local matrices in case of MATIS) */
2189     if (isMatIS) { /* need a different l2g map than the one computed by DMGetLocalToGlobalMapping */
2190       PetscSection section;
2191       PetscInt     size;
2192 
2193       ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);
2194       ierr = PetscSectionGetStorageSize(section, &size);CHKERRQ(ierr);
2195       ierr = PetscMalloc1(size,&ltogidx);CHKERRQ(ierr);
2196       ierr = DMPlexGetSubdomainSection(dm, &subSection);CHKERRQ(ierr);
2197     } else {
2198       ierr = DMGetLocalToGlobalMapping(dm,&ltog);CHKERRQ(ierr);
2199     }
2200     ierr = PetscSectionGetChart(sectionGlobal, &pStart, &pEnd);CHKERRQ(ierr);
2201     for (p = pStart, lsize = 0; p < pEnd; ++p) {
2202       PetscInt bdof;
2203 
2204       ierr = PetscSectionGetDof(sectionGlobal, p, &dof);CHKERRQ(ierr);
2205       ierr = PetscSectionGetConstraintDof(sectionGlobal, p, &cdof);CHKERRQ(ierr);
2206       dof  = dof < 0 ? -(dof+1) : dof;
2207       bdof = cdof && (dof-cdof) ? 1 : dof;
2208       if (dof) {
2209         if (bs < 0)          {bs = bdof;}
2210         else if (bs != bdof) {bs = 1; if (!isMatIS) break;}
2211       }
2212       if (isMatIS) {
2213         PetscInt loff,c,off;
2214         ierr = PetscSectionGetOffset(subSection, p, &loff);CHKERRQ(ierr);
2215         ierr = PetscSectionGetOffset(sectionGlobal, p, &off);CHKERRQ(ierr);
2216         for (c = 0; c < dof-cdof; ++c, ++lsize) ltogidx[loff+c] = off > -1 ? off+c : -(off+1)+c;
2217       }
2218     }
2219     /* Must have same blocksize on all procs (some might have no points) */
2220     bsLocal[0] = bs < 0 ? PETSC_MAX_INT : bs; bsLocal[1] = bs;
2221     ierr = PetscGlobalMinMaxInt(PetscObjectComm((PetscObject) dm), bsLocal, bsMinMax);CHKERRQ(ierr);
2222     if (bsMinMax[0] != bsMinMax[1]) {bs = 1;}
2223     else                            {bs = bsMinMax[0];}
2224     bs = PetscMax(1,bs);
2225     if (isMatIS) { /* Must reduce indices by blocksize */
2226       PetscInt l;
2227 
2228       lsize = lsize/bs;
2229       if (bs > 1) for (l = 0; l < lsize; ++l) ltogidx[l] = ltogidx[l*bs]/bs;
2230       ierr = ISLocalToGlobalMappingCreate(PetscObjectComm((PetscObject)dm), bs, lsize, ltogidx, PETSC_OWN_POINTER, &ltog);CHKERRQ(ierr);
2231     }
2232     ierr = MatSetLocalToGlobalMapping(*J,ltog,ltog);CHKERRQ(ierr);
2233     if (isMatIS) {
2234       ierr = ISLocalToGlobalMappingDestroy(&ltog);CHKERRQ(ierr);
2235     }
2236     ierr = PetscCalloc4(localSize/bs, &dnz, localSize/bs, &onz, localSize/bs, &dnzu, localSize/bs, &onzu);CHKERRQ(ierr);
2237     ierr = DMPlexPreallocateOperator(dm, bs, dnz, onz, dnzu, onzu, *J, fillMatrix);CHKERRQ(ierr);
2238     ierr = PetscFree4(dnz, onz, dnzu, onzu);CHKERRQ(ierr);
2239   }
2240   ierr = MatSetDM(*J, dm);CHKERRQ(ierr);
2241   PetscFunctionReturn(0);
2242 }
2243 
2244 /*@
2245   DMPlexGetSubdomainSection - Returns the section associated with the subdomain
2246 
2247   Not collective
2248 
2249   Input Parameter:
2250 . mesh - The DMPlex
2251 
2252   Output Parameters:
2253 . subsection - The subdomain section
2254 
2255   Level: developer
2256 
2257 .seealso:
2258 @*/
2259 PetscErrorCode DMPlexGetSubdomainSection(DM dm, PetscSection *subsection)
2260 {
2261   DM_Plex       *mesh = (DM_Plex*) dm->data;
2262   PetscErrorCode ierr;
2263 
2264   PetscFunctionBegin;
2265   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2266   if (!mesh->subdomainSection) {
2267     PetscSection section;
2268     PetscSF      sf;
2269 
2270     ierr = PetscSFCreate(PETSC_COMM_SELF,&sf);CHKERRQ(ierr);
2271     ierr = DMGetLocalSection(dm,&section);CHKERRQ(ierr);
2272     ierr = PetscSectionCreateGlobalSection(section,sf,PETSC_FALSE,PETSC_TRUE,&mesh->subdomainSection);CHKERRQ(ierr);
2273     ierr = PetscSFDestroy(&sf);CHKERRQ(ierr);
2274   }
2275   *subsection = mesh->subdomainSection;
2276   PetscFunctionReturn(0);
2277 }
2278 
2279 /*@
2280   DMPlexGetChart - Return the interval for all mesh points [pStart, pEnd)
2281 
2282   Not collective
2283 
2284   Input Parameter:
2285 . mesh - The DMPlex
2286 
2287   Output Parameters:
2288 + pStart - The first mesh point
2289 - pEnd   - The upper bound for mesh points
2290 
2291   Level: beginner
2292 
2293 .seealso: DMPlexCreate(), DMPlexSetChart()
2294 @*/
2295 PetscErrorCode DMPlexGetChart(DM dm, PetscInt *pStart, PetscInt *pEnd)
2296 {
2297   DM_Plex       *mesh = (DM_Plex*) dm->data;
2298   PetscErrorCode ierr;
2299 
2300   PetscFunctionBegin;
2301   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2302   ierr = PetscSectionGetChart(mesh->coneSection, pStart, pEnd);CHKERRQ(ierr);
2303   PetscFunctionReturn(0);
2304 }
2305 
2306 /*@
2307   DMPlexSetChart - Set the interval for all mesh points [pStart, pEnd)
2308 
2309   Not collective
2310 
2311   Input Parameters:
2312 + mesh - The DMPlex
2313 . pStart - The first mesh point
2314 - pEnd   - The upper bound for mesh points
2315 
2316   Output Parameters:
2317 
2318   Level: beginner
2319 
2320 .seealso: DMPlexCreate(), DMPlexGetChart()
2321 @*/
2322 PetscErrorCode DMPlexSetChart(DM dm, PetscInt pStart, PetscInt pEnd)
2323 {
2324   DM_Plex       *mesh = (DM_Plex*) dm->data;
2325   PetscErrorCode ierr;
2326 
2327   PetscFunctionBegin;
2328   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2329   ierr = PetscSectionSetChart(mesh->coneSection, pStart, pEnd);CHKERRQ(ierr);
2330   ierr = PetscSectionSetChart(mesh->supportSection, pStart, pEnd);CHKERRQ(ierr);
2331   PetscFunctionReturn(0);
2332 }
2333 
2334 /*@
2335   DMPlexGetConeSize - Return the number of in-edges for this point in the DAG
2336 
2337   Not collective
2338 
2339   Input Parameters:
2340 + mesh - The DMPlex
2341 - p - The point, which must lie in the chart set with DMPlexSetChart()
2342 
2343   Output Parameter:
2344 . size - The cone size for point p
2345 
2346   Level: beginner
2347 
2348 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart()
2349 @*/
2350 PetscErrorCode DMPlexGetConeSize(DM dm, PetscInt p, PetscInt *size)
2351 {
2352   DM_Plex       *mesh = (DM_Plex*) dm->data;
2353   PetscErrorCode ierr;
2354 
2355   PetscFunctionBegin;
2356   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2357   PetscValidPointer(size, 3);
2358   ierr = PetscSectionGetDof(mesh->coneSection, p, size);CHKERRQ(ierr);
2359   PetscFunctionReturn(0);
2360 }
2361 
2362 /*@
2363   DMPlexSetConeSize - Set the number of in-edges for this point in the DAG
2364 
2365   Not collective
2366 
2367   Input Parameters:
2368 + mesh - The DMPlex
2369 . p - The point, which must lie in the chart set with DMPlexSetChart()
2370 - size - The cone size for point p
2371 
2372   Output Parameter:
2373 
2374   Note:
2375   This should be called after DMPlexSetChart().
2376 
2377   Level: beginner
2378 
2379 .seealso: DMPlexCreate(), DMPlexGetConeSize(), DMPlexSetChart()
2380 @*/
2381 PetscErrorCode DMPlexSetConeSize(DM dm, PetscInt p, PetscInt size)
2382 {
2383   DM_Plex       *mesh = (DM_Plex*) dm->data;
2384   PetscErrorCode ierr;
2385 
2386   PetscFunctionBegin;
2387   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2388   ierr = PetscSectionSetDof(mesh->coneSection, p, size);CHKERRQ(ierr);
2389 
2390   mesh->maxConeSize = PetscMax(mesh->maxConeSize, size);
2391   PetscFunctionReturn(0);
2392 }
2393 
2394 /*@
2395   DMPlexAddConeSize - Add the given number of in-edges to this point in the DAG
2396 
2397   Not collective
2398 
2399   Input Parameters:
2400 + mesh - The DMPlex
2401 . p - The point, which must lie in the chart set with DMPlexSetChart()
2402 - size - The additional cone size for point p
2403 
2404   Output Parameter:
2405 
2406   Note:
2407   This should be called after DMPlexSetChart().
2408 
2409   Level: beginner
2410 
2411 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexGetConeSize(), DMPlexSetChart()
2412 @*/
2413 PetscErrorCode DMPlexAddConeSize(DM dm, PetscInt p, PetscInt size)
2414 {
2415   DM_Plex       *mesh = (DM_Plex*) dm->data;
2416   PetscInt       csize;
2417   PetscErrorCode ierr;
2418 
2419   PetscFunctionBegin;
2420   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2421   ierr = PetscSectionAddDof(mesh->coneSection, p, size);CHKERRQ(ierr);
2422   ierr = PetscSectionGetDof(mesh->coneSection, p, &csize);CHKERRQ(ierr);
2423 
2424   mesh->maxConeSize = PetscMax(mesh->maxConeSize, csize);
2425   PetscFunctionReturn(0);
2426 }
2427 
2428 /*@C
2429   DMPlexGetCone - Return the points on the in-edges for this point in the DAG
2430 
2431   Not collective
2432 
2433   Input Parameters:
2434 + dm - The DMPlex
2435 - p - The point, which must lie in the chart set with DMPlexSetChart()
2436 
2437   Output Parameter:
2438 . cone - An array of points which are on the in-edges for point p
2439 
2440   Level: beginner
2441 
2442   Fortran Notes:
2443   Since it returns an array, this routine is only available in Fortran 90, and you must
2444   include petsc.h90 in your code.
2445   You must also call DMPlexRestoreCone() after you finish using the returned array.
2446   DMPlexRestoreCone() is not needed/available in C.
2447 
2448 .seealso: DMPlexGetConeSize(), DMPlexSetCone(), DMPlexGetConeTuple(), DMPlexSetChart()
2449 @*/
2450 PetscErrorCode DMPlexGetCone(DM dm, PetscInt p, const PetscInt *cone[])
2451 {
2452   DM_Plex       *mesh = (DM_Plex*) dm->data;
2453   PetscInt       off;
2454   PetscErrorCode ierr;
2455 
2456   PetscFunctionBegin;
2457   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2458   PetscValidPointer(cone, 3);
2459   ierr  = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2460   *cone = &mesh->cones[off];
2461   PetscFunctionReturn(0);
2462 }
2463 
2464 /*@C
2465   DMPlexGetConeTuple - Return the points on the in-edges of several points in the DAG
2466 
2467   Not collective
2468 
2469   Input Parameters:
2470 + dm - The DMPlex
2471 - p - The IS of points, which must lie in the chart set with DMPlexSetChart()
2472 
2473   Output Parameter:
2474 + pConesSection - PetscSection describing the layout of pCones
2475 - pCones - An array of points which are on the in-edges for the point set p
2476 
2477   Level: intermediate
2478 
2479 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexGetConeRecursive(), DMPlexSetChart()
2480 @*/
2481 PetscErrorCode DMPlexGetConeTuple(DM dm, IS p, PetscSection *pConesSection, IS *pCones)
2482 {
2483   PetscSection        cs, newcs;
2484   PetscInt            *cones;
2485   PetscInt            *newarr=NULL;
2486   PetscInt            n;
2487   PetscErrorCode      ierr;
2488 
2489   PetscFunctionBegin;
2490   ierr = DMPlexGetCones(dm, &cones);CHKERRQ(ierr);
2491   ierr = DMPlexGetConeSection(dm, &cs);CHKERRQ(ierr);
2492   ierr = PetscSectionExtractDofsFromArray(cs, MPIU_INT, cones, p, &newcs, pCones ? ((void**)&newarr) : NULL);CHKERRQ(ierr);
2493   if (pConesSection) *pConesSection = newcs;
2494   if (pCones) {
2495     ierr = PetscSectionGetStorageSize(newcs, &n);CHKERRQ(ierr);
2496     ierr = ISCreateGeneral(PetscObjectComm((PetscObject)p), n, newarr, PETSC_OWN_POINTER, pCones);CHKERRQ(ierr);
2497   }
2498   PetscFunctionReturn(0);
2499 }
2500 
2501 /*@
2502   DMPlexGetConeRecursiveVertices - Expand each given point into its cone points and do that recursively until we end up just with vertices.
2503 
2504   Not collective
2505 
2506   Input Parameters:
2507 + dm - The DMPlex
2508 - points - The IS of points, which must lie in the chart set with DMPlexSetChart()
2509 
2510   Output Parameter:
2511 . expandedPoints - An array of vertices recursively expanded from input points
2512 
2513   Level: advanced
2514 
2515   Notes:
2516   Like DMPlexGetConeRecursive but returns only the 0-depth IS (i.e. vertices only) and no sections.
2517   There is no corresponding Restore function, just call ISDestroy() on the returned IS to deallocate.
2518 
2519 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexGetConeTuple(), DMPlexGetConeRecursive(), DMPlexRestoreConeRecursive(), DMPlexGetDepth()
2520 @*/
2521 PetscErrorCode DMPlexGetConeRecursiveVertices(DM dm, IS points, IS *expandedPoints)
2522 {
2523   IS                  *expandedPointsAll;
2524   PetscInt            depth;
2525   PetscErrorCode      ierr;
2526 
2527   PetscFunctionBegin;
2528   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2529   PetscValidHeaderSpecific(points, IS_CLASSID, 2);
2530   PetscValidPointer(expandedPoints, 3);
2531   ierr = DMPlexGetConeRecursive(dm, points, &depth, &expandedPointsAll, NULL);CHKERRQ(ierr);
2532   *expandedPoints = expandedPointsAll[0];
2533   ierr = PetscObjectReference((PetscObject)expandedPointsAll[0]);CHKERRQ(ierr);
2534   ierr = DMPlexRestoreConeRecursive(dm, points, &depth, &expandedPointsAll, NULL);CHKERRQ(ierr);
2535   PetscFunctionReturn(0);
2536 }
2537 
2538 /*@
2539   DMPlexGetConeRecursive - Expand each given point into its cone points and do that recursively until we end up just with vertices (DAG points of depth 0, i.e. without cones).
2540 
2541   Not collective
2542 
2543   Input Parameters:
2544 + dm - The DMPlex
2545 - points - The IS of points, which must lie in the chart set with DMPlexSetChart()
2546 
2547   Output Parameter:
2548 + depth - (optional) Size of the output arrays, equal to DMPlex depth, returned by DMPlexGetDepth()
2549 . expandedPoints - (optional) An array of index sets with recursively expanded cones
2550 - sections - (optional) An array of sections which describe mappings from points to their cone points
2551 
2552   Level: advanced
2553 
2554   Notes:
2555   Like DMPlexGetConeTuple() but recursive.
2556 
2557   Array expandedPoints has size equal to depth. Each expandedPoints[d] contains DAG points with maximum depth d, recursively cone-wise expanded from the input points.
2558   For example, for d=0 it contains only vertices, for d=1 it can contain vertices and edges, etc.
2559 
2560   Array section has size equal to depth.  Each PetscSection sections[d] realizes mapping from expandedPoints[d+1] (section points) to expandedPoints[d] (section dofs) as follows:
2561   (1) DAG points in expandedPoints[d+1] with depth d+1 to their cone points in expandedPoints[d];
2562   (2) DAG points in expandedPoints[d+1] with depth in [0,d] to the same points in expandedPoints[d].
2563 
2564 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexGetConeTuple(), DMPlexRestoreConeRecursive(), DMPlexGetConeRecursiveVertices(), DMPlexGetDepth()
2565 @*/
2566 PetscErrorCode DMPlexGetConeRecursive(DM dm, IS points, PetscInt *depth, IS *expandedPoints[], PetscSection *sections[])
2567 {
2568   const PetscInt      *arr0=NULL, *cone=NULL;
2569   PetscInt            *arr=NULL, *newarr=NULL;
2570   PetscInt            d, depth_, i, n, newn, cn, co, start, end;
2571   IS                  *expandedPoints_;
2572   PetscSection        *sections_;
2573   PetscErrorCode      ierr;
2574 
2575   PetscFunctionBegin;
2576   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2577   PetscValidHeaderSpecific(points, IS_CLASSID, 2);
2578   if (depth) PetscValidIntPointer(depth, 3);
2579   if (expandedPoints) PetscValidPointer(expandedPoints, 4);
2580   if (sections) PetscValidPointer(sections, 5);
2581   ierr = ISGetLocalSize(points, &n);CHKERRQ(ierr);
2582   ierr = ISGetIndices(points, &arr0);CHKERRQ(ierr);
2583   ierr = DMPlexGetDepth(dm, &depth_);CHKERRQ(ierr);
2584   ierr = PetscCalloc1(depth_, &expandedPoints_);CHKERRQ(ierr);
2585   ierr = PetscCalloc1(depth_, &sections_);CHKERRQ(ierr);
2586   arr = (PetscInt*) arr0; /* this is ok because first generation of arr is not modified */
2587   for (d=depth_-1; d>=0; d--) {
2588     ierr = PetscSectionCreate(PETSC_COMM_SELF, &sections_[d]);CHKERRQ(ierr);
2589     ierr = PetscSectionSetChart(sections_[d], 0, n);CHKERRQ(ierr);
2590     for (i=0; i<n; i++) {
2591       ierr = DMPlexGetDepthStratum(dm, d+1, &start, &end);CHKERRQ(ierr);
2592       if (arr[i] >= start && arr[i] < end) {
2593         ierr = DMPlexGetConeSize(dm, arr[i], &cn);CHKERRQ(ierr);
2594         ierr = PetscSectionSetDof(sections_[d], i, cn);CHKERRQ(ierr);
2595       } else {
2596         ierr = PetscSectionSetDof(sections_[d], i, 1);CHKERRQ(ierr);
2597       }
2598     }
2599     ierr = PetscSectionSetUp(sections_[d]);CHKERRQ(ierr);
2600     ierr = PetscSectionGetStorageSize(sections_[d], &newn);CHKERRQ(ierr);
2601     ierr = PetscMalloc1(newn, &newarr);CHKERRQ(ierr);
2602     for (i=0; i<n; i++) {
2603       ierr = PetscSectionGetDof(sections_[d], i, &cn);CHKERRQ(ierr);
2604       ierr = PetscSectionGetOffset(sections_[d], i, &co);CHKERRQ(ierr);
2605       if (cn > 1) {
2606         ierr = DMPlexGetCone(dm, arr[i], &cone);CHKERRQ(ierr);
2607         ierr = PetscMemcpy(&newarr[co], cone, cn*sizeof(PetscInt));CHKERRQ(ierr);
2608       } else {
2609         newarr[co] = arr[i];
2610       }
2611     }
2612     ierr = ISCreateGeneral(PETSC_COMM_SELF, newn, newarr, PETSC_OWN_POINTER, &expandedPoints_[d]);CHKERRQ(ierr);
2613     arr = newarr;
2614     n = newn;
2615   }
2616   ierr = ISRestoreIndices(points, &arr0);CHKERRQ(ierr);
2617   *depth = depth_;
2618   if (expandedPoints) *expandedPoints = expandedPoints_;
2619   else {
2620     for (d=0; d<depth_; d++) {ierr = ISDestroy(&expandedPoints_[d]);CHKERRQ(ierr);}
2621     ierr = PetscFree(expandedPoints_);CHKERRQ(ierr);
2622   }
2623   if (sections) *sections = sections_;
2624   else {
2625     for (d=0; d<depth_; d++) {ierr = PetscSectionDestroy(&sections_[d]);CHKERRQ(ierr);}
2626     ierr = PetscFree(sections_);CHKERRQ(ierr);
2627   }
2628   PetscFunctionReturn(0);
2629 }
2630 
2631 /*@
2632   DMPlexRestoreConeRecursive - Deallocates arrays created by DMPlexGetConeRecursive
2633 
2634   Not collective
2635 
2636   Input Parameters:
2637 + dm - The DMPlex
2638 - points - The IS of points, which must lie in the chart set with DMPlexSetChart()
2639 
2640   Output Parameter:
2641 + depth - (optional) Size of the output arrays, equal to DMPlex depth, returned by DMPlexGetDepth()
2642 . expandedPoints - (optional) An array of recursively expanded cones
2643 - sections - (optional) An array of sections which describe mappings from points to their cone points
2644 
2645   Level: advanced
2646 
2647   Notes:
2648   See DMPlexGetConeRecursive() for details.
2649 
2650 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexGetConeTuple(), DMPlexGetConeRecursive(), DMPlexGetConeRecursiveVertices(), DMPlexGetDepth()
2651 @*/
2652 PetscErrorCode DMPlexRestoreConeRecursive(DM dm, IS points, PetscInt *depth, IS *expandedPoints[], PetscSection *sections[])
2653 {
2654   PetscInt            d, depth_;
2655   PetscErrorCode      ierr;
2656 
2657   PetscFunctionBegin;
2658   ierr = DMPlexGetDepth(dm, &depth_);CHKERRQ(ierr);
2659   if (depth && *depth != depth_) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "depth changed since last call to DMPlexGetConeRecursive");
2660   if (depth) *depth = 0;
2661   if (expandedPoints) {
2662     for (d=0; d<depth_; d++) {ierr = ISDestroy(&((*expandedPoints)[d]));CHKERRQ(ierr);}
2663     ierr = PetscFree(*expandedPoints);CHKERRQ(ierr);
2664   }
2665   if (sections)  {
2666     for (d=0; d<depth_; d++) {ierr = PetscSectionDestroy(&((*sections)[d]));CHKERRQ(ierr);}
2667     ierr = PetscFree(*sections);CHKERRQ(ierr);
2668   }
2669   PetscFunctionReturn(0);
2670 }
2671 
2672 /*@
2673   DMPlexSetCone - Set the points on the in-edges for this point in the DAG; that is these are the points that cover the specific point
2674 
2675   Not collective
2676 
2677   Input Parameters:
2678 + mesh - The DMPlex
2679 . p - The point, which must lie in the chart set with DMPlexSetChart()
2680 - cone - An array of points which are on the in-edges for point p
2681 
2682   Output Parameter:
2683 
2684   Note:
2685   This should be called after all calls to DMPlexSetConeSize() and DMSetUp().
2686 
2687   Developer Note: Why not call this DMPlexSetCover()
2688 
2689   Level: beginner
2690 
2691 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp(), DMPlexSetSupport(), DMPlexSetSupportSize()
2692 @*/
2693 PetscErrorCode DMPlexSetCone(DM dm, PetscInt p, const PetscInt cone[])
2694 {
2695   DM_Plex       *mesh = (DM_Plex*) dm->data;
2696   PetscInt       pStart, pEnd;
2697   PetscInt       dof, off, c;
2698   PetscErrorCode ierr;
2699 
2700   PetscFunctionBegin;
2701   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2702   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
2703   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2704   if (dof) PetscValidPointer(cone, 3);
2705   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2706   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
2707   for (c = 0; c < dof; ++c) {
2708     if ((cone[c] < pStart) || (cone[c] >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone point %D is not in the valid range [%D, %D)", cone[c], pStart, pEnd);
2709     mesh->cones[off+c] = cone[c];
2710   }
2711   PetscFunctionReturn(0);
2712 }
2713 
2714 /*@C
2715   DMPlexGetConeOrientation - Return the orientations on the in-edges for this point in the DAG
2716 
2717   Not collective
2718 
2719   Input Parameters:
2720 + mesh - The DMPlex
2721 - p - The point, which must lie in the chart set with DMPlexSetChart()
2722 
2723   Output Parameter:
2724 . coneOrientation - An array of orientations which are on the in-edges for point p. An orientation is an
2725                     integer giving the prescription for cone traversal. If it is negative, the cone is
2726                     traversed in the opposite direction. Its value 'o', or if negative '-(o+1)', gives
2727                     the index of the cone point on which to start.
2728 
2729   Level: beginner
2730 
2731   Fortran Notes:
2732   Since it returns an array, this routine is only available in Fortran 90, and you must
2733   include petsc.h90 in your code.
2734   You must also call DMPlexRestoreConeOrientation() after you finish using the returned array.
2735   DMPlexRestoreConeOrientation() is not needed/available in C.
2736 
2737 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetCone(), DMPlexSetChart()
2738 @*/
2739 PetscErrorCode DMPlexGetConeOrientation(DM dm, PetscInt p, const PetscInt *coneOrientation[])
2740 {
2741   DM_Plex       *mesh = (DM_Plex*) dm->data;
2742   PetscInt       off;
2743   PetscErrorCode ierr;
2744 
2745   PetscFunctionBegin;
2746   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2747   if (PetscDefined(USE_DEBUG)) {
2748     PetscInt dof;
2749     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2750     if (dof) PetscValidPointer(coneOrientation, 3);
2751   }
2752   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2753 
2754   *coneOrientation = &mesh->coneOrientations[off];
2755   PetscFunctionReturn(0);
2756 }
2757 
2758 /*@
2759   DMPlexSetConeOrientation - Set the orientations on the in-edges for this point in the DAG
2760 
2761   Not collective
2762 
2763   Input Parameters:
2764 + mesh - The DMPlex
2765 . p - The point, which must lie in the chart set with DMPlexSetChart()
2766 - coneOrientation - An array of orientations which are on the in-edges for point p. An orientation is an
2767                     integer giving the prescription for cone traversal. If it is negative, the cone is
2768                     traversed in the opposite direction. Its value 'o', or if negative '-(o+1)', gives
2769                     the index of the cone point on which to start.
2770 
2771   Output Parameter:
2772 
2773   Note:
2774   This should be called after all calls to DMPlexSetConeSize() and DMSetUp().
2775 
2776   Level: beginner
2777 
2778 .seealso: DMPlexCreate(), DMPlexGetConeOrientation(), DMPlexSetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
2779 @*/
2780 PetscErrorCode DMPlexSetConeOrientation(DM dm, PetscInt p, const PetscInt coneOrientation[])
2781 {
2782   DM_Plex       *mesh = (DM_Plex*) dm->data;
2783   PetscInt       pStart, pEnd;
2784   PetscInt       dof, off, c;
2785   PetscErrorCode ierr;
2786 
2787   PetscFunctionBegin;
2788   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2789   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
2790   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2791   if (dof) PetscValidPointer(coneOrientation, 3);
2792   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2793   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
2794   for (c = 0; c < dof; ++c) {
2795     PetscInt cdof, o = coneOrientation[c];
2796 
2797     ierr = PetscSectionGetDof(mesh->coneSection, mesh->cones[off+c], &cdof);CHKERRQ(ierr);
2798     if (o && ((o < -(cdof+1)) || (o >= cdof))) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone orientation %D is not in the valid range [%D. %D)", o, -(cdof+1), cdof);
2799     mesh->coneOrientations[off+c] = o;
2800   }
2801   PetscFunctionReturn(0);
2802 }
2803 
2804 /*@
2805   DMPlexInsertCone - Insert a point into the in-edges for the point p in the DAG
2806 
2807   Not collective
2808 
2809   Input Parameters:
2810 + mesh - The DMPlex
2811 . p - The point, which must lie in the chart set with DMPlexSetChart()
2812 . conePos - The local index in the cone where the point should be put
2813 - conePoint - The mesh point to insert
2814 
2815   Level: beginner
2816 
2817 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
2818 @*/
2819 PetscErrorCode DMPlexInsertCone(DM dm, PetscInt p, PetscInt conePos, PetscInt conePoint)
2820 {
2821   DM_Plex       *mesh = (DM_Plex*) dm->data;
2822   PetscInt       pStart, pEnd;
2823   PetscInt       dof, off;
2824   PetscErrorCode ierr;
2825 
2826   PetscFunctionBegin;
2827   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2828   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
2829   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
2830   if ((conePoint < pStart) || (conePoint >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone point %D is not in the valid range [%D, %D)", conePoint, pStart, pEnd);
2831   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2832   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2833   if ((conePos < 0) || (conePos >= dof)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone position %D of point %D is not in the valid range [0, %D)", conePos, p, dof);
2834   mesh->cones[off+conePos] = conePoint;
2835   PetscFunctionReturn(0);
2836 }
2837 
2838 /*@
2839   DMPlexInsertConeOrientation - Insert a point orientation for the in-edge for the point p in the DAG
2840 
2841   Not collective
2842 
2843   Input Parameters:
2844 + mesh - The DMPlex
2845 . p - The point, which must lie in the chart set with DMPlexSetChart()
2846 . conePos - The local index in the cone where the point should be put
2847 - coneOrientation - The point orientation to insert
2848 
2849   Level: beginner
2850 
2851 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
2852 @*/
2853 PetscErrorCode DMPlexInsertConeOrientation(DM dm, PetscInt p, PetscInt conePos, PetscInt coneOrientation)
2854 {
2855   DM_Plex       *mesh = (DM_Plex*) dm->data;
2856   PetscInt       pStart, pEnd;
2857   PetscInt       dof, off;
2858   PetscErrorCode ierr;
2859 
2860   PetscFunctionBegin;
2861   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2862   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
2863   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
2864   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2865   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2866   if ((conePos < 0) || (conePos >= dof)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone position %D of point %D is not in the valid range [0, %D)", conePos, p, dof);
2867   mesh->coneOrientations[off+conePos] = coneOrientation;
2868   PetscFunctionReturn(0);
2869 }
2870 
2871 /*@
2872   DMPlexGetSupportSize - Return the number of out-edges for this point in the DAG
2873 
2874   Not collective
2875 
2876   Input Parameters:
2877 + mesh - The DMPlex
2878 - p - The point, which must lie in the chart set with DMPlexSetChart()
2879 
2880   Output Parameter:
2881 . size - The support size for point p
2882 
2883   Level: beginner
2884 
2885 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart(), DMPlexGetConeSize()
2886 @*/
2887 PetscErrorCode DMPlexGetSupportSize(DM dm, PetscInt p, PetscInt *size)
2888 {
2889   DM_Plex       *mesh = (DM_Plex*) dm->data;
2890   PetscErrorCode ierr;
2891 
2892   PetscFunctionBegin;
2893   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2894   PetscValidPointer(size, 3);
2895   ierr = PetscSectionGetDof(mesh->supportSection, p, size);CHKERRQ(ierr);
2896   PetscFunctionReturn(0);
2897 }
2898 
2899 /*@
2900   DMPlexSetSupportSize - Set the number of out-edges for this point in the DAG
2901 
2902   Not collective
2903 
2904   Input Parameters:
2905 + mesh - The DMPlex
2906 . p - The point, which must lie in the chart set with DMPlexSetChart()
2907 - size - The support size for point p
2908 
2909   Output Parameter:
2910 
2911   Note:
2912   This should be called after DMPlexSetChart().
2913 
2914   Level: beginner
2915 
2916 .seealso: DMPlexCreate(), DMPlexGetSupportSize(), DMPlexSetChart()
2917 @*/
2918 PetscErrorCode DMPlexSetSupportSize(DM dm, PetscInt p, PetscInt size)
2919 {
2920   DM_Plex       *mesh = (DM_Plex*) dm->data;
2921   PetscErrorCode ierr;
2922 
2923   PetscFunctionBegin;
2924   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2925   ierr = PetscSectionSetDof(mesh->supportSection, p, size);CHKERRQ(ierr);
2926 
2927   mesh->maxSupportSize = PetscMax(mesh->maxSupportSize, size);
2928   PetscFunctionReturn(0);
2929 }
2930 
2931 /*@C
2932   DMPlexGetSupport - Return the points on the out-edges for this point in the DAG
2933 
2934   Not collective
2935 
2936   Input Parameters:
2937 + mesh - The DMPlex
2938 - p - The point, which must lie in the chart set with DMPlexSetChart()
2939 
2940   Output Parameter:
2941 . support - An array of points which are on the out-edges for point p
2942 
2943   Level: beginner
2944 
2945   Fortran Notes:
2946   Since it returns an array, this routine is only available in Fortran 90, and you must
2947   include petsc.h90 in your code.
2948   You must also call DMPlexRestoreSupport() after you finish using the returned array.
2949   DMPlexRestoreSupport() is not needed/available in C.
2950 
2951 .seealso: DMPlexGetSupportSize(), DMPlexSetSupport(), DMPlexGetCone(), DMPlexSetChart()
2952 @*/
2953 PetscErrorCode DMPlexGetSupport(DM dm, PetscInt p, const PetscInt *support[])
2954 {
2955   DM_Plex       *mesh = (DM_Plex*) dm->data;
2956   PetscInt       off;
2957   PetscErrorCode ierr;
2958 
2959   PetscFunctionBegin;
2960   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2961   PetscValidPointer(support, 3);
2962   ierr     = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
2963   *support = &mesh->supports[off];
2964   PetscFunctionReturn(0);
2965 }
2966 
2967 /*@
2968   DMPlexSetSupport - Set the points on the out-edges for this point in the DAG, that is the list of points that this point covers
2969 
2970   Not collective
2971 
2972   Input Parameters:
2973 + mesh - The DMPlex
2974 . p - The point, which must lie in the chart set with DMPlexSetChart()
2975 - support - An array of points which are on the out-edges for point p
2976 
2977   Output Parameter:
2978 
2979   Note:
2980   This should be called after all calls to DMPlexSetSupportSize() and DMSetUp().
2981 
2982   Level: beginner
2983 
2984 .seealso: DMPlexSetCone(), DMPlexSetConeSize(), DMPlexCreate(), DMPlexGetSupport(), DMPlexSetChart(), DMPlexSetSupportSize(), DMSetUp()
2985 @*/
2986 PetscErrorCode DMPlexSetSupport(DM dm, PetscInt p, const PetscInt support[])
2987 {
2988   DM_Plex       *mesh = (DM_Plex*) dm->data;
2989   PetscInt       pStart, pEnd;
2990   PetscInt       dof, off, c;
2991   PetscErrorCode ierr;
2992 
2993   PetscFunctionBegin;
2994   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2995   ierr = PetscSectionGetChart(mesh->supportSection, &pStart, &pEnd);CHKERRQ(ierr);
2996   ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
2997   if (dof) PetscValidPointer(support, 3);
2998   ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
2999   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
3000   for (c = 0; c < dof; ++c) {
3001     if ((support[c] < pStart) || (support[c] >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support point %D is not in the valid range [%D, %D)", support[c], pStart, pEnd);
3002     mesh->supports[off+c] = support[c];
3003   }
3004   PetscFunctionReturn(0);
3005 }
3006 
3007 /*@
3008   DMPlexInsertSupport - Insert a point into the out-edges for the point p in the DAG
3009 
3010   Not collective
3011 
3012   Input Parameters:
3013 + mesh - The DMPlex
3014 . p - The point, which must lie in the chart set with DMPlexSetChart()
3015 . supportPos - The local index in the cone where the point should be put
3016 - supportPoint - The mesh point to insert
3017 
3018   Level: beginner
3019 
3020 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
3021 @*/
3022 PetscErrorCode DMPlexInsertSupport(DM dm, PetscInt p, PetscInt supportPos, PetscInt supportPoint)
3023 {
3024   DM_Plex       *mesh = (DM_Plex*) dm->data;
3025   PetscInt       pStart, pEnd;
3026   PetscInt       dof, off;
3027   PetscErrorCode ierr;
3028 
3029   PetscFunctionBegin;
3030   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3031   ierr = PetscSectionGetChart(mesh->supportSection, &pStart, &pEnd);CHKERRQ(ierr);
3032   ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
3033   ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
3034   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
3035   if ((supportPoint < pStart) || (supportPoint >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support point %D is not in the valid range [%D, %D)", supportPoint, pStart, pEnd);
3036   if (supportPos >= dof) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support position %D of point %D is not in the valid range [0, %D)", supportPos, p, dof);
3037   mesh->supports[off+supportPos] = supportPoint;
3038   PetscFunctionReturn(0);
3039 }
3040 
3041 /*@C
3042   DMPlexGetTransitiveClosure - Return the points on the transitive closure of the in-edges or out-edges for this point in the DAG
3043 
3044   Not collective
3045 
3046   Input Parameters:
3047 + mesh - The DMPlex
3048 . p - The point, which must lie in the chart set with DMPlexSetChart()
3049 . useCone - PETSC_TRUE for in-edges,  otherwise use out-edges
3050 - points - If points is NULL on input, internal storage will be returned, otherwise the provided array is used
3051 
3052   Output Parameters:
3053 + numPoints - The number of points in the closure, so points[] is of size 2*numPoints
3054 - points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...]
3055 
3056   Note:
3057   If using internal storage (points is NULL on input), each call overwrites the last output.
3058 
3059   Fortran Notes:
3060   Since it returns an array, this routine is only available in Fortran 90, and you must
3061   include petsc.h90 in your code.
3062 
3063   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3064 
3065   Level: beginner
3066 
3067 .seealso: DMPlexRestoreTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
3068 @*/
3069 PetscErrorCode DMPlexGetTransitiveClosure(DM dm, PetscInt p, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
3070 {
3071   DM_Plex        *mesh = (DM_Plex*) dm->data;
3072   PetscInt       *closure, *fifo;
3073   const PetscInt *tmp = NULL, *tmpO = NULL;
3074   PetscInt        tmpSize, t;
3075   PetscInt        depth       = 0, maxSize;
3076   PetscInt        closureSize = 2, fifoSize = 0, fifoStart = 0;
3077   PetscErrorCode  ierr;
3078 
3079   PetscFunctionBegin;
3080   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3081   ierr    = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3082   /* This is only 1-level */
3083   if (useCone) {
3084     ierr = DMPlexGetConeSize(dm, p, &tmpSize);CHKERRQ(ierr);
3085     ierr = DMPlexGetCone(dm, p, &tmp);CHKERRQ(ierr);
3086     ierr = DMPlexGetConeOrientation(dm, p, &tmpO);CHKERRQ(ierr);
3087   } else {
3088     ierr = DMPlexGetSupportSize(dm, p, &tmpSize);CHKERRQ(ierr);
3089     ierr = DMPlexGetSupport(dm, p, &tmp);CHKERRQ(ierr);
3090   }
3091   if (depth == 1) {
3092     if (*points) {
3093       closure = *points;
3094     } else {
3095       maxSize = 2*(PetscMax(mesh->maxConeSize, mesh->maxSupportSize)+1);
3096       ierr = DMGetWorkArray(dm, maxSize, MPIU_INT, &closure);CHKERRQ(ierr);
3097     }
3098     closure[0] = p; closure[1] = 0;
3099     for (t = 0; t < tmpSize; ++t, closureSize += 2) {
3100       closure[closureSize]   = tmp[t];
3101       closure[closureSize+1] = tmpO ? tmpO[t] : 0;
3102     }
3103     if (numPoints) *numPoints = closureSize/2;
3104     if (points)    *points    = closure;
3105     PetscFunctionReturn(0);
3106   }
3107   {
3108     PetscInt c, coneSeries, s,supportSeries;
3109 
3110     c = mesh->maxConeSize;
3111     coneSeries = (c > 1) ? ((PetscPowInt(c,depth+1)-1)/(c-1)) : depth+1;
3112     s = mesh->maxSupportSize;
3113     supportSeries = (s > 1) ? ((PetscPowInt(s,depth+1)-1)/(s-1)) : depth+1;
3114     maxSize = 2*PetscMax(coneSeries,supportSeries);
3115   }
3116   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &fifo);CHKERRQ(ierr);
3117   if (*points) {
3118     closure = *points;
3119   } else {
3120     ierr = DMGetWorkArray(dm, maxSize, MPIU_INT, &closure);CHKERRQ(ierr);
3121   }
3122   closure[0] = p; closure[1] = 0;
3123   for (t = 0; t < tmpSize; ++t, closureSize += 2, fifoSize += 2) {
3124     const PetscInt cp = tmp[t];
3125     const PetscInt co = tmpO ? tmpO[t] : 0;
3126 
3127     closure[closureSize]   = cp;
3128     closure[closureSize+1] = co;
3129     fifo[fifoSize]         = cp;
3130     fifo[fifoSize+1]       = co;
3131   }
3132   /* Should kick out early when depth is reached, rather than checking all vertices for empty cones */
3133   while (fifoSize - fifoStart) {
3134     const PetscInt q   = fifo[fifoStart];
3135     const PetscInt o   = fifo[fifoStart+1];
3136     const PetscInt rev = o >= 0 ? 0 : 1;
3137     const PetscInt off = rev ? -(o+1) : o;
3138 
3139     if (useCone) {
3140       ierr = DMPlexGetConeSize(dm, q, &tmpSize);CHKERRQ(ierr);
3141       ierr = DMPlexGetCone(dm, q, &tmp);CHKERRQ(ierr);
3142       ierr = DMPlexGetConeOrientation(dm, q, &tmpO);CHKERRQ(ierr);
3143     } else {
3144       ierr = DMPlexGetSupportSize(dm, q, &tmpSize);CHKERRQ(ierr);
3145       ierr = DMPlexGetSupport(dm, q, &tmp);CHKERRQ(ierr);
3146       tmpO = NULL;
3147     }
3148     for (t = 0; t < tmpSize; ++t) {
3149       const PetscInt i  = ((rev ? tmpSize-t : t) + off)%tmpSize;
3150       const PetscInt cp = tmp[i];
3151       /* Must propogate orientation: When we reverse orientation, we both reverse the direction of iteration and start at the other end of the chain. */
3152       /* HACK: It is worse to get the size here, than to change the interpretation of -(*+1)
3153        const PetscInt co = tmpO ? (rev ? -(tmpO[i]+1) : tmpO[i]) : 0; */
3154       PetscInt       co = tmpO ? tmpO[i] : 0;
3155       PetscInt       c;
3156 
3157       if (rev) {
3158         PetscInt childSize, coff;
3159         ierr = DMPlexGetConeSize(dm, cp, &childSize);CHKERRQ(ierr);
3160         coff = tmpO[i] < 0 ? -(tmpO[i]+1) : tmpO[i];
3161         co   = childSize ? -(((coff+childSize-1)%childSize)+1) : 0;
3162       }
3163       /* Check for duplicate */
3164       for (c = 0; c < closureSize; c += 2) {
3165         if (closure[c] == cp) break;
3166       }
3167       if (c == closureSize) {
3168         closure[closureSize]   = cp;
3169         closure[closureSize+1] = co;
3170         fifo[fifoSize]         = cp;
3171         fifo[fifoSize+1]       = co;
3172         closureSize           += 2;
3173         fifoSize              += 2;
3174       }
3175     }
3176     fifoStart += 2;
3177   }
3178   if (numPoints) *numPoints = closureSize/2;
3179   if (points)    *points    = closure;
3180   ierr = DMRestoreWorkArray(dm, maxSize, MPIU_INT, &fifo);CHKERRQ(ierr);
3181   PetscFunctionReturn(0);
3182 }
3183 
3184 /*@C
3185   DMPlexGetTransitiveClosure_Internal - Return the points on the transitive closure of the in-edges or out-edges for this point in the DAG with a specified initial orientation
3186 
3187   Not collective
3188 
3189   Input Parameters:
3190 + mesh - The DMPlex
3191 . p - The point, which must lie in the chart set with DMPlexSetChart()
3192 . orientation - The orientation of the point
3193 . useCone - PETSC_TRUE for in-edges,  otherwise use out-edges
3194 - points - If points is NULL on input, internal storage will be returned, otherwise the provided array is used
3195 
3196   Output Parameters:
3197 + numPoints - The number of points in the closure, so points[] is of size 2*numPoints
3198 - points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...]
3199 
3200   Note:
3201   If using internal storage (points is NULL on input), each call overwrites the last output.
3202 
3203   Fortran Notes:
3204   Since it returns an array, this routine is only available in Fortran 90, and you must
3205   include petsc.h90 in your code.
3206 
3207   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3208 
3209   Level: beginner
3210 
3211 .seealso: DMPlexRestoreTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
3212 @*/
3213 PetscErrorCode DMPlexGetTransitiveClosure_Internal(DM dm, PetscInt p, PetscInt ornt, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
3214 {
3215   DM_Plex        *mesh = (DM_Plex*) dm->data;
3216   PetscInt       *closure, *fifo;
3217   const PetscInt *tmp = NULL, *tmpO = NULL;
3218   PetscInt        tmpSize, t;
3219   PetscInt        depth       = 0, maxSize;
3220   PetscInt        closureSize = 2, fifoSize = 0, fifoStart = 0;
3221   PetscErrorCode  ierr;
3222 
3223   PetscFunctionBegin;
3224   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3225   ierr    = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3226   /* This is only 1-level */
3227   if (useCone) {
3228     ierr = DMPlexGetConeSize(dm, p, &tmpSize);CHKERRQ(ierr);
3229     ierr = DMPlexGetCone(dm, p, &tmp);CHKERRQ(ierr);
3230     ierr = DMPlexGetConeOrientation(dm, p, &tmpO);CHKERRQ(ierr);
3231   } else {
3232     ierr = DMPlexGetSupportSize(dm, p, &tmpSize);CHKERRQ(ierr);
3233     ierr = DMPlexGetSupport(dm, p, &tmp);CHKERRQ(ierr);
3234   }
3235   if (depth == 1) {
3236     if (*points) {
3237       closure = *points;
3238     } else {
3239       maxSize = 2*(PetscMax(mesh->maxConeSize, mesh->maxSupportSize)+1);
3240       ierr = DMGetWorkArray(dm, maxSize, MPIU_INT, &closure);CHKERRQ(ierr);
3241     }
3242     closure[0] = p; closure[1] = ornt;
3243     for (t = 0; t < tmpSize; ++t, closureSize += 2) {
3244       const PetscInt i = ornt >= 0 ? (t+ornt)%tmpSize : (-(ornt+1) + tmpSize-t)%tmpSize;
3245       closure[closureSize]   = tmp[i];
3246       closure[closureSize+1] = tmpO ? tmpO[i] : 0;
3247     }
3248     if (numPoints) *numPoints = closureSize/2;
3249     if (points)    *points    = closure;
3250     PetscFunctionReturn(0);
3251   }
3252   {
3253     PetscInt c, coneSeries, s,supportSeries;
3254 
3255     c = mesh->maxConeSize;
3256     coneSeries = (c > 1) ? ((PetscPowInt(c,depth+1)-1)/(c-1)) : depth+1;
3257     s = mesh->maxSupportSize;
3258     supportSeries = (s > 1) ? ((PetscPowInt(s,depth+1)-1)/(s-1)) : depth+1;
3259     maxSize = 2*PetscMax(coneSeries,supportSeries);
3260   }
3261   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &fifo);CHKERRQ(ierr);
3262   if (*points) {
3263     closure = *points;
3264   } else {
3265     ierr = DMGetWorkArray(dm, maxSize, MPIU_INT, &closure);CHKERRQ(ierr);
3266   }
3267   closure[0] = p; closure[1] = ornt;
3268   for (t = 0; t < tmpSize; ++t, closureSize += 2, fifoSize += 2) {
3269     const PetscInt i  = ornt >= 0 ? (t+ornt)%tmpSize : (-(ornt+1) + tmpSize-t)%tmpSize;
3270     const PetscInt cp = tmp[i];
3271     PetscInt       co = tmpO ? tmpO[i] : 0;
3272 
3273     if (ornt < 0) {
3274       PetscInt childSize, coff;
3275       ierr = DMPlexGetConeSize(dm, cp, &childSize);CHKERRQ(ierr);
3276       coff = co < 0 ? -(tmpO[i]+1) : tmpO[i];
3277       co   = childSize ? -(((coff+childSize-1)%childSize)+1) : 0;
3278     }
3279     closure[closureSize]   = cp;
3280     closure[closureSize+1] = co;
3281     fifo[fifoSize]         = cp;
3282     fifo[fifoSize+1]       = co;
3283   }
3284   /* Should kick out early when depth is reached, rather than checking all vertices for empty cones */
3285   while (fifoSize - fifoStart) {
3286     const PetscInt q   = fifo[fifoStart];
3287     const PetscInt o   = fifo[fifoStart+1];
3288     const PetscInt rev = o >= 0 ? 0 : 1;
3289     const PetscInt off = rev ? -(o+1) : o;
3290 
3291     if (useCone) {
3292       ierr = DMPlexGetConeSize(dm, q, &tmpSize);CHKERRQ(ierr);
3293       ierr = DMPlexGetCone(dm, q, &tmp);CHKERRQ(ierr);
3294       ierr = DMPlexGetConeOrientation(dm, q, &tmpO);CHKERRQ(ierr);
3295     } else {
3296       ierr = DMPlexGetSupportSize(dm, q, &tmpSize);CHKERRQ(ierr);
3297       ierr = DMPlexGetSupport(dm, q, &tmp);CHKERRQ(ierr);
3298       tmpO = NULL;
3299     }
3300     for (t = 0; t < tmpSize; ++t) {
3301       const PetscInt i  = ((rev ? tmpSize-t : t) + off)%tmpSize;
3302       const PetscInt cp = tmp[i];
3303       /* Must propogate orientation: When we reverse orientation, we both reverse the direction of iteration and start at the other end of the chain. */
3304       /* HACK: It is worse to get the size here, than to change the interpretation of -(*+1)
3305        const PetscInt co = tmpO ? (rev ? -(tmpO[i]+1) : tmpO[i]) : 0; */
3306       PetscInt       co = tmpO ? tmpO[i] : 0;
3307       PetscInt       c;
3308 
3309       if (rev) {
3310         PetscInt childSize, coff;
3311         ierr = DMPlexGetConeSize(dm, cp, &childSize);CHKERRQ(ierr);
3312         coff = tmpO[i] < 0 ? -(tmpO[i]+1) : tmpO[i];
3313         co   = childSize ? -(((coff+childSize-1)%childSize)+1) : 0;
3314       }
3315       /* Check for duplicate */
3316       for (c = 0; c < closureSize; c += 2) {
3317         if (closure[c] == cp) break;
3318       }
3319       if (c == closureSize) {
3320         closure[closureSize]   = cp;
3321         closure[closureSize+1] = co;
3322         fifo[fifoSize]         = cp;
3323         fifo[fifoSize+1]       = co;
3324         closureSize           += 2;
3325         fifoSize              += 2;
3326       }
3327     }
3328     fifoStart += 2;
3329   }
3330   if (numPoints) *numPoints = closureSize/2;
3331   if (points)    *points    = closure;
3332   ierr = DMRestoreWorkArray(dm, maxSize, MPIU_INT, &fifo);CHKERRQ(ierr);
3333   PetscFunctionReturn(0);
3334 }
3335 
3336 /*@C
3337   DMPlexRestoreTransitiveClosure - Restore the array of points on the transitive closure of the in-edges or out-edges for this point in the DAG
3338 
3339   Not collective
3340 
3341   Input Parameters:
3342 + mesh - The DMPlex
3343 . p - The point, which must lie in the chart set with DMPlexSetChart()
3344 . useCone - PETSC_TRUE for in-edges,  otherwise use out-edges
3345 . numPoints - The number of points in the closure, so points[] is of size 2*numPoints, zeroed on exit
3346 - points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...], zeroed on exit
3347 
3348   Note:
3349   If not using internal storage (points is not NULL on input), this call is unnecessary
3350 
3351   Fortran Notes:
3352   Since it returns an array, this routine is only available in Fortran 90, and you must
3353   include petsc.h90 in your code.
3354 
3355   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3356 
3357   Level: beginner
3358 
3359 .seealso: DMPlexGetTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
3360 @*/
3361 PetscErrorCode DMPlexRestoreTransitiveClosure(DM dm, PetscInt p, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
3362 {
3363   PetscErrorCode ierr;
3364 
3365   PetscFunctionBegin;
3366   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3367   if (numPoints) PetscValidIntPointer(numPoints,4);
3368   if (points) PetscValidPointer(points,5);
3369   ierr = DMRestoreWorkArray(dm, 0, MPIU_INT, points);CHKERRQ(ierr);
3370   if (numPoints) *numPoints = 0;
3371   PetscFunctionReturn(0);
3372 }
3373 
3374 /*@
3375   DMPlexGetMaxSizes - Return the maximum number of in-edges (cone) and out-edges (support) for any point in the DAG
3376 
3377   Not collective
3378 
3379   Input Parameter:
3380 . mesh - The DMPlex
3381 
3382   Output Parameters:
3383 + maxConeSize - The maximum number of in-edges
3384 - maxSupportSize - The maximum number of out-edges
3385 
3386   Level: beginner
3387 
3388 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart()
3389 @*/
3390 PetscErrorCode DMPlexGetMaxSizes(DM dm, PetscInt *maxConeSize, PetscInt *maxSupportSize)
3391 {
3392   DM_Plex *mesh = (DM_Plex*) dm->data;
3393 
3394   PetscFunctionBegin;
3395   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3396   if (maxConeSize)    *maxConeSize    = mesh->maxConeSize;
3397   if (maxSupportSize) *maxSupportSize = mesh->maxSupportSize;
3398   PetscFunctionReturn(0);
3399 }
3400 
3401 PetscErrorCode DMSetUp_Plex(DM dm)
3402 {
3403   DM_Plex       *mesh = (DM_Plex*) dm->data;
3404   PetscInt       size;
3405   PetscErrorCode ierr;
3406 
3407   PetscFunctionBegin;
3408   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3409   ierr = PetscSectionSetUp(mesh->coneSection);CHKERRQ(ierr);
3410   ierr = PetscSectionGetStorageSize(mesh->coneSection, &size);CHKERRQ(ierr);
3411   ierr = PetscMalloc1(size, &mesh->cones);CHKERRQ(ierr);
3412   ierr = PetscCalloc1(size, &mesh->coneOrientations);CHKERRQ(ierr);
3413   ierr = PetscLogObjectMemory((PetscObject) dm, size*2*sizeof(PetscInt));CHKERRQ(ierr);
3414   if (mesh->maxSupportSize) {
3415     ierr = PetscSectionSetUp(mesh->supportSection);CHKERRQ(ierr);
3416     ierr = PetscSectionGetStorageSize(mesh->supportSection, &size);CHKERRQ(ierr);
3417     ierr = PetscMalloc1(size, &mesh->supports);CHKERRQ(ierr);
3418     ierr = PetscLogObjectMemory((PetscObject) dm, size*sizeof(PetscInt));CHKERRQ(ierr);
3419   }
3420   PetscFunctionReturn(0);
3421 }
3422 
3423 PetscErrorCode DMCreateSubDM_Plex(DM dm, PetscInt numFields, const PetscInt fields[], IS *is, DM *subdm)
3424 {
3425   PetscErrorCode ierr;
3426 
3427   PetscFunctionBegin;
3428   if (subdm) {ierr = DMClone(dm, subdm);CHKERRQ(ierr);}
3429   ierr = DMCreateSectionSubDM(dm, numFields, fields, is, subdm);CHKERRQ(ierr);
3430   if (subdm) {(*subdm)->useNatural = dm->useNatural;}
3431   if (dm->useNatural && dm->sfMigration) {
3432     PetscSF        sfMigrationInv,sfNatural;
3433     PetscSection   section, sectionSeq;
3434 
3435     (*subdm)->sfMigration = dm->sfMigration;
3436     ierr = PetscObjectReference((PetscObject) dm->sfMigration);CHKERRQ(ierr);
3437     ierr = DMGetLocalSection((*subdm), &section);CHKERRQ(ierr);
3438     ierr = PetscSFCreateInverseSF((*subdm)->sfMigration, &sfMigrationInv);CHKERRQ(ierr);
3439     ierr = PetscSectionCreate(PetscObjectComm((PetscObject) (*subdm)), &sectionSeq);CHKERRQ(ierr);
3440     ierr = PetscSFDistributeSection(sfMigrationInv, section, NULL, sectionSeq);CHKERRQ(ierr);
3441 
3442     ierr = DMPlexCreateGlobalToNaturalSF(*subdm, sectionSeq, (*subdm)->sfMigration, &sfNatural);CHKERRQ(ierr);
3443     (*subdm)->sfNatural = sfNatural;
3444     ierr = PetscSectionDestroy(&sectionSeq);CHKERRQ(ierr);
3445     ierr = PetscSFDestroy(&sfMigrationInv);CHKERRQ(ierr);
3446   }
3447   PetscFunctionReturn(0);
3448 }
3449 
3450 PetscErrorCode DMCreateSuperDM_Plex(DM dms[], PetscInt len, IS **is, DM *superdm)
3451 {
3452   PetscErrorCode ierr;
3453   PetscInt       i = 0;
3454 
3455   PetscFunctionBegin;
3456   ierr = DMClone(dms[0], superdm);CHKERRQ(ierr);
3457   ierr = DMCreateSectionSuperDM(dms, len, is, superdm);CHKERRQ(ierr);
3458   (*superdm)->useNatural = PETSC_FALSE;
3459   for (i = 0; i < len; i++) {
3460     if (dms[i]->useNatural && dms[i]->sfMigration) {
3461       PetscSF        sfMigrationInv,sfNatural;
3462       PetscSection   section, sectionSeq;
3463 
3464       (*superdm)->sfMigration = dms[i]->sfMigration;
3465       ierr = PetscObjectReference((PetscObject) dms[i]->sfMigration);CHKERRQ(ierr);
3466       (*superdm)->useNatural = PETSC_TRUE;
3467       ierr = DMGetLocalSection((*superdm), &section);CHKERRQ(ierr);
3468       ierr = PetscSFCreateInverseSF((*superdm)->sfMigration, &sfMigrationInv);CHKERRQ(ierr);
3469       ierr = PetscSectionCreate(PetscObjectComm((PetscObject) (*superdm)), &sectionSeq);CHKERRQ(ierr);
3470       ierr = PetscSFDistributeSection(sfMigrationInv, section, NULL, sectionSeq);CHKERRQ(ierr);
3471 
3472       ierr = DMPlexCreateGlobalToNaturalSF(*superdm, sectionSeq, (*superdm)->sfMigration, &sfNatural);CHKERRQ(ierr);
3473       (*superdm)->sfNatural = sfNatural;
3474       ierr = PetscSectionDestroy(&sectionSeq);CHKERRQ(ierr);
3475       ierr = PetscSFDestroy(&sfMigrationInv);CHKERRQ(ierr);
3476       break;
3477     }
3478   }
3479   PetscFunctionReturn(0);
3480 }
3481 
3482 /*@
3483   DMPlexSymmetrize - Create support (out-edge) information from cone (in-edge) information
3484 
3485   Not collective
3486 
3487   Input Parameter:
3488 . mesh - The DMPlex
3489 
3490   Output Parameter:
3491 
3492   Note:
3493   This should be called after all calls to DMPlexSetCone()
3494 
3495   Level: beginner
3496 
3497 .seealso: DMPlexCreate(), DMPlexSetChart(), DMPlexSetConeSize(), DMPlexSetCone()
3498 @*/
3499 PetscErrorCode DMPlexSymmetrize(DM dm)
3500 {
3501   DM_Plex       *mesh = (DM_Plex*) dm->data;
3502   PetscInt      *offsets;
3503   PetscInt       supportSize;
3504   PetscInt       pStart, pEnd, p;
3505   PetscErrorCode ierr;
3506 
3507   PetscFunctionBegin;
3508   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3509   if (mesh->supports) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONGSTATE, "Supports were already setup in this DMPlex");
3510   ierr = PetscLogEventBegin(DMPLEX_Symmetrize,dm,0,0,0);CHKERRQ(ierr);
3511   /* Calculate support sizes */
3512   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
3513   for (p = pStart; p < pEnd; ++p) {
3514     PetscInt dof, off, c;
3515 
3516     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
3517     ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
3518     for (c = off; c < off+dof; ++c) {
3519       ierr = PetscSectionAddDof(mesh->supportSection, mesh->cones[c], 1);CHKERRQ(ierr);
3520     }
3521   }
3522   for (p = pStart; p < pEnd; ++p) {
3523     PetscInt dof;
3524 
3525     ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
3526 
3527     mesh->maxSupportSize = PetscMax(mesh->maxSupportSize, dof);
3528   }
3529   ierr = PetscSectionSetUp(mesh->supportSection);CHKERRQ(ierr);
3530   /* Calculate supports */
3531   ierr = PetscSectionGetStorageSize(mesh->supportSection, &supportSize);CHKERRQ(ierr);
3532   ierr = PetscMalloc1(supportSize, &mesh->supports);CHKERRQ(ierr);
3533   ierr = PetscCalloc1(pEnd - pStart, &offsets);CHKERRQ(ierr);
3534   for (p = pStart; p < pEnd; ++p) {
3535     PetscInt dof, off, c;
3536 
3537     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
3538     ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
3539     for (c = off; c < off+dof; ++c) {
3540       const PetscInt q = mesh->cones[c];
3541       PetscInt       offS;
3542 
3543       ierr = PetscSectionGetOffset(mesh->supportSection, q, &offS);CHKERRQ(ierr);
3544 
3545       mesh->supports[offS+offsets[q]] = p;
3546       ++offsets[q];
3547     }
3548   }
3549   ierr = PetscFree(offsets);CHKERRQ(ierr);
3550   ierr = PetscLogEventEnd(DMPLEX_Symmetrize,dm,0,0,0);CHKERRQ(ierr);
3551   PetscFunctionReturn(0);
3552 }
3553 
3554 static PetscErrorCode DMPlexCreateDepthStratum(DM dm, DMLabel label, PetscInt depth, PetscInt pStart, PetscInt pEnd)
3555 {
3556   IS             stratumIS;
3557   PetscErrorCode ierr;
3558 
3559   PetscFunctionBegin;
3560   if (pStart >= pEnd) PetscFunctionReturn(0);
3561   if (PetscDefined(USE_DEBUG)) {
3562     PetscInt  qStart, qEnd, numLevels, level;
3563     PetscBool overlap = PETSC_FALSE;
3564     ierr = DMLabelGetNumValues(label, &numLevels);CHKERRQ(ierr);
3565     for (level = 0; level < numLevels; level++) {
3566       ierr = DMLabelGetStratumBounds(label, level, &qStart, &qEnd);CHKERRQ(ierr);
3567       if ((pStart >= qStart && pStart < qEnd) || (pEnd > qStart && pEnd <= qEnd)) {overlap = PETSC_TRUE; break;}
3568     }
3569     if (overlap) SETERRQ6(PETSC_COMM_SELF, PETSC_ERR_PLIB, "New depth %D range [%D,%D) overlaps with depth %D range [%D,%D)", depth, pStart, pEnd, level, qStart, qEnd);
3570   }
3571   ierr = ISCreateStride(PETSC_COMM_SELF, pEnd-pStart, pStart, 1, &stratumIS);CHKERRQ(ierr);
3572   ierr = DMLabelSetStratumIS(label, depth, stratumIS);CHKERRQ(ierr);
3573   ierr = ISDestroy(&stratumIS);CHKERRQ(ierr);
3574   PetscFunctionReturn(0);
3575 }
3576 
3577 /*@
3578   DMPlexStratify - The DAG for most topologies is a graded poset (https://en.wikipedia.org/wiki/Graded_poset), and
3579   can be illustrated by a Hasse Diagram (https://en.wikipedia.org/wiki/Hasse_diagram). The strata group all points of the
3580   same grade, and this function calculates the strata. This grade can be seen as the height (or depth) of the point in
3581   the DAG.
3582 
3583   Collective on dm
3584 
3585   Input Parameter:
3586 . mesh - The DMPlex
3587 
3588   Output Parameter:
3589 
3590   Notes:
3591   Concretely, DMPlexStratify() creates a new label named "depth" containing the depth in the DAG of each point. For cell-vertex
3592   meshes, vertices are depth 0 and cells are depth 1. For fully interpolated meshes, depth 0 for vertices, 1 for edges, and so on
3593   until cells have depth equal to the dimension of the mesh. The depth label can be accessed through DMPlexGetDepthLabel() or DMPlexGetDepthStratum(), or
3594   manually via DMGetLabel().  The height is defined implicitly by height = maxDimension - depth, and can be accessed
3595   via DMPlexGetHeightStratum().  For example, cells have height 0 and faces have height 1.
3596 
3597   The depth of a point is calculated by executing a breadth-first search (BFS) on the DAG. This could produce surprising results
3598   if run on a partially interpolated mesh, meaning one that had some edges and faces, but not others. For example, suppose that
3599   we had a mesh consisting of one triangle (c0) and three vertices (v0, v1, v2), and only one edge is on the boundary so we choose
3600   to interpolate only that one (e0), so that
3601 $  cone(c0) = {e0, v2}
3602 $  cone(e0) = {v0, v1}
3603   If DMPlexStratify() is run on this mesh, it will give depths
3604 $  depth 0 = {v0, v1, v2}
3605 $  depth 1 = {e0, c0}
3606   where the triangle has been given depth 1, instead of 2, because it is reachable from vertex v2.
3607 
3608   DMPlexStratify() should be called after all calls to DMPlexSymmetrize()
3609 
3610   Level: beginner
3611 
3612 .seealso: DMPlexCreate(), DMPlexSymmetrize(), DMPlexComputeCellTypes()
3613 @*/
3614 PetscErrorCode DMPlexStratify(DM dm)
3615 {
3616   DM_Plex       *mesh = (DM_Plex*) dm->data;
3617   DMLabel        label;
3618   PetscInt       pStart, pEnd, p;
3619   PetscInt       numRoots = 0, numLeaves = 0;
3620   PetscErrorCode ierr;
3621 
3622   PetscFunctionBegin;
3623   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3624   ierr = PetscLogEventBegin(DMPLEX_Stratify,dm,0,0,0);CHKERRQ(ierr);
3625 
3626   /* Create depth label */
3627   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
3628   ierr = DMCreateLabel(dm, "depth");CHKERRQ(ierr);
3629   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
3630 
3631   {
3632     /* Initialize roots and count leaves */
3633     PetscInt sMin = PETSC_MAX_INT;
3634     PetscInt sMax = PETSC_MIN_INT;
3635     PetscInt coneSize, supportSize;
3636 
3637     for (p = pStart; p < pEnd; ++p) {
3638       ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
3639       ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
3640       if (!coneSize && supportSize) {
3641         sMin = PetscMin(p, sMin);
3642         sMax = PetscMax(p, sMax);
3643         ++numRoots;
3644       } else if (!supportSize && coneSize) {
3645         ++numLeaves;
3646       } else if (!supportSize && !coneSize) {
3647         /* Isolated points */
3648         sMin = PetscMin(p, sMin);
3649         sMax = PetscMax(p, sMax);
3650       }
3651     }
3652     ierr = DMPlexCreateDepthStratum(dm, label, 0, sMin, sMax+1);CHKERRQ(ierr);
3653   }
3654 
3655   if (numRoots + numLeaves == (pEnd - pStart)) {
3656     PetscInt sMin = PETSC_MAX_INT;
3657     PetscInt sMax = PETSC_MIN_INT;
3658     PetscInt coneSize, supportSize;
3659 
3660     for (p = pStart; p < pEnd; ++p) {
3661       ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
3662       ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
3663       if (!supportSize && coneSize) {
3664         sMin = PetscMin(p, sMin);
3665         sMax = PetscMax(p, sMax);
3666       }
3667     }
3668     ierr = DMPlexCreateDepthStratum(dm, label, 1, sMin, sMax+1);CHKERRQ(ierr);
3669   } else {
3670     PetscInt level = 0;
3671     PetscInt qStart, qEnd, q;
3672 
3673     ierr = DMLabelGetStratumBounds(label, level, &qStart, &qEnd);CHKERRQ(ierr);
3674     while (qEnd > qStart) {
3675       PetscInt sMin = PETSC_MAX_INT;
3676       PetscInt sMax = PETSC_MIN_INT;
3677 
3678       for (q = qStart; q < qEnd; ++q) {
3679         const PetscInt *support;
3680         PetscInt        supportSize, s;
3681 
3682         ierr = DMPlexGetSupportSize(dm, q, &supportSize);CHKERRQ(ierr);
3683         ierr = DMPlexGetSupport(dm, q, &support);CHKERRQ(ierr);
3684         for (s = 0; s < supportSize; ++s) {
3685           sMin = PetscMin(support[s], sMin);
3686           sMax = PetscMax(support[s], sMax);
3687         }
3688       }
3689       ierr = DMLabelGetNumValues(label, &level);CHKERRQ(ierr);
3690       ierr = DMPlexCreateDepthStratum(dm, label, level, sMin, sMax+1);CHKERRQ(ierr);
3691       ierr = DMLabelGetStratumBounds(label, level, &qStart, &qEnd);CHKERRQ(ierr);
3692     }
3693   }
3694   { /* just in case there is an empty process */
3695     PetscInt numValues, maxValues = 0, v;
3696 
3697     ierr = DMLabelGetNumValues(label, &numValues);CHKERRQ(ierr);
3698     ierr = MPI_Allreduce(&numValues,&maxValues,1,MPIU_INT,MPI_MAX,PetscObjectComm((PetscObject)dm));CHKERRMPI(ierr);
3699     for (v = numValues; v < maxValues; v++) {
3700       ierr = DMLabelAddStratum(label, v);CHKERRQ(ierr);
3701     }
3702   }
3703   ierr = PetscObjectStateGet((PetscObject) label, &mesh->depthState);CHKERRQ(ierr);
3704   ierr = PetscLogEventEnd(DMPLEX_Stratify,dm,0,0,0);CHKERRQ(ierr);
3705   PetscFunctionReturn(0);
3706 }
3707 
3708 PetscErrorCode DMPlexComputeCellType_Internal(DM dm, PetscInt p, PetscInt pdepth, DMPolytopeType *pt)
3709 {
3710   DMPolytopeType ct = DM_POLYTOPE_UNKNOWN;
3711   PetscInt       dim, depth, pheight, coneSize;
3712   PetscErrorCode ierr;
3713 
3714   PetscFunctionBeginHot;
3715   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
3716   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3717   ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
3718   pheight = depth - pdepth;
3719   if (depth <= 1) {
3720     switch (pdepth) {
3721       case 0: ct = DM_POLYTOPE_POINT;break;
3722       case 1:
3723         switch (coneSize) {
3724           case 2: ct = DM_POLYTOPE_SEGMENT;break;
3725           case 3: ct = DM_POLYTOPE_TRIANGLE;break;
3726           case 4:
3727           switch (dim) {
3728             case 2: ct = DM_POLYTOPE_QUADRILATERAL;break;
3729             case 3: ct = DM_POLYTOPE_TETRAHEDRON;break;
3730             default: break;
3731           }
3732           break;
3733         case 5: ct = DM_POLYTOPE_PYRAMID;break;
3734         case 6: ct = DM_POLYTOPE_TRI_PRISM_TENSOR;break;
3735         case 8: ct = DM_POLYTOPE_HEXAHEDRON;break;
3736         default: break;
3737       }
3738     }
3739   } else {
3740     if (pdepth == 0) {
3741       ct = DM_POLYTOPE_POINT;
3742     } else if (pheight == 0) {
3743       switch (dim) {
3744         case 1:
3745           switch (coneSize) {
3746             case 2: ct = DM_POLYTOPE_SEGMENT;break;
3747             default: break;
3748           }
3749           break;
3750         case 2:
3751           switch (coneSize) {
3752             case 3: ct = DM_POLYTOPE_TRIANGLE;break;
3753             case 4: ct = DM_POLYTOPE_QUADRILATERAL;break;
3754             default: break;
3755           }
3756           break;
3757         case 3:
3758           switch (coneSize) {
3759             case 4: ct = DM_POLYTOPE_TETRAHEDRON;break;
3760             case 5:
3761             {
3762               const PetscInt *cone;
3763               PetscInt        faceConeSize;
3764 
3765               ierr = DMPlexGetCone(dm, p, &cone);CHKERRQ(ierr);
3766               ierr = DMPlexGetConeSize(dm, cone[0], &faceConeSize);CHKERRQ(ierr);
3767               switch (faceConeSize) {
3768                 case 3: ct = DM_POLYTOPE_TRI_PRISM_TENSOR;break;
3769                 case 4: ct = DM_POLYTOPE_PYRAMID;break;
3770               }
3771             }
3772             break;
3773             case 6: ct = DM_POLYTOPE_HEXAHEDRON;break;
3774             default: break;
3775           }
3776           break;
3777         default: break;
3778       }
3779     } else if (pheight > 0) {
3780       switch (coneSize) {
3781         case 2: ct = DM_POLYTOPE_SEGMENT;break;
3782         case 3: ct = DM_POLYTOPE_TRIANGLE;break;
3783         case 4: ct = DM_POLYTOPE_QUADRILATERAL;break;
3784         default: break;
3785       }
3786     }
3787   }
3788   *pt = ct;
3789   PetscFunctionReturn(0);
3790 }
3791 
3792 /*@
3793   DMPlexComputeCellTypes - Infer the polytope type of every cell using its dimension and cone size.
3794 
3795   Collective on dm
3796 
3797   Input Parameter:
3798 . mesh - The DMPlex
3799 
3800   DMPlexComputeCellTypes() should be called after all calls to DMPlexSymmetrize() and DMPlexStratify()
3801 
3802   Level: developer
3803 
3804   Note: This function is normally called automatically by Plex when a cell type is requested. It creates an
3805   internal DMLabel named "celltype" which can be directly accessed using DMGetLabel(). A user may disable
3806   automatic creation by creating the label manually, using DMCreateLabel(dm, "celltype").
3807 
3808 .seealso: DMPlexCreate(), DMPlexSymmetrize(), DMPlexStratify(), DMGetLabel(), DMCreateLabel()
3809 @*/
3810 PetscErrorCode DMPlexComputeCellTypes(DM dm)
3811 {
3812   DM_Plex       *mesh;
3813   DMLabel        ctLabel;
3814   PetscInt       pStart, pEnd, p;
3815   PetscErrorCode ierr;
3816 
3817   PetscFunctionBegin;
3818   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3819   mesh = (DM_Plex *) dm->data;
3820   ierr = DMCreateLabel(dm, "celltype");CHKERRQ(ierr);
3821   ierr = DMPlexGetCellTypeLabel(dm, &ctLabel);CHKERRQ(ierr);
3822   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
3823   for (p = pStart; p < pEnd; ++p) {
3824     DMPolytopeType ct = DM_POLYTOPE_UNKNOWN;
3825     PetscInt       pdepth;
3826 
3827     ierr = DMPlexGetPointDepth(dm, p, &pdepth);CHKERRQ(ierr);
3828     ierr = DMPlexComputeCellType_Internal(dm, p, pdepth, &ct);CHKERRQ(ierr);
3829     if (ct == DM_POLYTOPE_UNKNOWN) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_SUP, "Point %D is screwed up", p);
3830     ierr = DMLabelSetValue(ctLabel, p, ct);CHKERRQ(ierr);
3831   }
3832   ierr = PetscObjectStateGet((PetscObject) ctLabel, &mesh->celltypeState);CHKERRQ(ierr);
3833   ierr = PetscObjectViewFromOptions((PetscObject) ctLabel, NULL, "-dm_plex_celltypes_view");CHKERRQ(ierr);
3834   PetscFunctionReturn(0);
3835 }
3836 
3837 /*@C
3838   DMPlexGetJoin - Get an array for the join of the set of points
3839 
3840   Not Collective
3841 
3842   Input Parameters:
3843 + dm - The DMPlex object
3844 . numPoints - The number of input points for the join
3845 - points - The input points
3846 
3847   Output Parameters:
3848 + numCoveredPoints - The number of points in the join
3849 - coveredPoints - The points in the join
3850 
3851   Level: intermediate
3852 
3853   Note: Currently, this is restricted to a single level join
3854 
3855   Fortran Notes:
3856   Since it returns an array, this routine is only available in Fortran 90, and you must
3857   include petsc.h90 in your code.
3858 
3859   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3860 
3861 .seealso: DMPlexRestoreJoin(), DMPlexGetMeet()
3862 @*/
3863 PetscErrorCode DMPlexGetJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
3864 {
3865   DM_Plex       *mesh = (DM_Plex*) dm->data;
3866   PetscInt      *join[2];
3867   PetscInt       joinSize, i = 0;
3868   PetscInt       dof, off, p, c, m;
3869   PetscErrorCode ierr;
3870 
3871   PetscFunctionBegin;
3872   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3873   PetscValidIntPointer(points, 3);
3874   PetscValidIntPointer(numCoveredPoints, 4);
3875   PetscValidPointer(coveredPoints, 5);
3876   ierr = DMGetWorkArray(dm, mesh->maxSupportSize, MPIU_INT, &join[0]);CHKERRQ(ierr);
3877   ierr = DMGetWorkArray(dm, mesh->maxSupportSize, MPIU_INT, &join[1]);CHKERRQ(ierr);
3878   /* Copy in support of first point */
3879   ierr = PetscSectionGetDof(mesh->supportSection, points[0], &dof);CHKERRQ(ierr);
3880   ierr = PetscSectionGetOffset(mesh->supportSection, points[0], &off);CHKERRQ(ierr);
3881   for (joinSize = 0; joinSize < dof; ++joinSize) {
3882     join[i][joinSize] = mesh->supports[off+joinSize];
3883   }
3884   /* Check each successive support */
3885   for (p = 1; p < numPoints; ++p) {
3886     PetscInt newJoinSize = 0;
3887 
3888     ierr = PetscSectionGetDof(mesh->supportSection, points[p], &dof);CHKERRQ(ierr);
3889     ierr = PetscSectionGetOffset(mesh->supportSection, points[p], &off);CHKERRQ(ierr);
3890     for (c = 0; c < dof; ++c) {
3891       const PetscInt point = mesh->supports[off+c];
3892 
3893       for (m = 0; m < joinSize; ++m) {
3894         if (point == join[i][m]) {
3895           join[1-i][newJoinSize++] = point;
3896           break;
3897         }
3898       }
3899     }
3900     joinSize = newJoinSize;
3901     i        = 1-i;
3902   }
3903   *numCoveredPoints = joinSize;
3904   *coveredPoints    = join[i];
3905   ierr              = DMRestoreWorkArray(dm, mesh->maxSupportSize, MPIU_INT, &join[1-i]);CHKERRQ(ierr);
3906   PetscFunctionReturn(0);
3907 }
3908 
3909 /*@C
3910   DMPlexRestoreJoin - Restore an array for the join of the set of points
3911 
3912   Not Collective
3913 
3914   Input Parameters:
3915 + dm - The DMPlex object
3916 . numPoints - The number of input points for the join
3917 - points - The input points
3918 
3919   Output Parameters:
3920 + numCoveredPoints - The number of points in the join
3921 - coveredPoints - The points in the join
3922 
3923   Fortran Notes:
3924   Since it returns an array, this routine is only available in Fortran 90, and you must
3925   include petsc.h90 in your code.
3926 
3927   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3928 
3929   Level: intermediate
3930 
3931 .seealso: DMPlexGetJoin(), DMPlexGetFullJoin(), DMPlexGetMeet()
3932 @*/
3933 PetscErrorCode DMPlexRestoreJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
3934 {
3935   PetscErrorCode ierr;
3936 
3937   PetscFunctionBegin;
3938   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3939   if (points) PetscValidIntPointer(points,3);
3940   if (numCoveredPoints) PetscValidIntPointer(numCoveredPoints,4);
3941   PetscValidPointer(coveredPoints, 5);
3942   ierr = DMRestoreWorkArray(dm, 0, MPIU_INT, (void*) coveredPoints);CHKERRQ(ierr);
3943   if (numCoveredPoints) *numCoveredPoints = 0;
3944   PetscFunctionReturn(0);
3945 }
3946 
3947 /*@C
3948   DMPlexGetFullJoin - Get an array for the join of the set of points
3949 
3950   Not Collective
3951 
3952   Input Parameters:
3953 + dm - The DMPlex object
3954 . numPoints - The number of input points for the join
3955 - points - The input points
3956 
3957   Output Parameters:
3958 + numCoveredPoints - The number of points in the join
3959 - coveredPoints - The points in the join
3960 
3961   Fortran Notes:
3962   Since it returns an array, this routine is only available in Fortran 90, and you must
3963   include petsc.h90 in your code.
3964 
3965   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3966 
3967   Level: intermediate
3968 
3969 .seealso: DMPlexGetJoin(), DMPlexRestoreJoin(), DMPlexGetMeet()
3970 @*/
3971 PetscErrorCode DMPlexGetFullJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
3972 {
3973   DM_Plex       *mesh = (DM_Plex*) dm->data;
3974   PetscInt      *offsets, **closures;
3975   PetscInt      *join[2];
3976   PetscInt       depth = 0, maxSize, joinSize = 0, i = 0;
3977   PetscInt       p, d, c, m, ms;
3978   PetscErrorCode ierr;
3979 
3980   PetscFunctionBegin;
3981   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3982   PetscValidIntPointer(points, 3);
3983   PetscValidIntPointer(numCoveredPoints, 4);
3984   PetscValidPointer(coveredPoints, 5);
3985 
3986   ierr    = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3987   ierr    = PetscCalloc1(numPoints, &closures);CHKERRQ(ierr);
3988   ierr    = DMGetWorkArray(dm, numPoints*(depth+2), MPIU_INT, &offsets);CHKERRQ(ierr);
3989   ms      = mesh->maxSupportSize;
3990   maxSize = (ms > 1) ? ((PetscPowInt(ms,depth+1)-1)/(ms-1)) : depth + 1;
3991   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &join[0]);CHKERRQ(ierr);
3992   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &join[1]);CHKERRQ(ierr);
3993 
3994   for (p = 0; p < numPoints; ++p) {
3995     PetscInt closureSize;
3996 
3997     ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_FALSE, &closureSize, &closures[p]);CHKERRQ(ierr);
3998 
3999     offsets[p*(depth+2)+0] = 0;
4000     for (d = 0; d < depth+1; ++d) {
4001       PetscInt pStart, pEnd, i;
4002 
4003       ierr = DMPlexGetDepthStratum(dm, d, &pStart, &pEnd);CHKERRQ(ierr);
4004       for (i = offsets[p*(depth+2)+d]; i < closureSize; ++i) {
4005         if ((pStart > closures[p][i*2]) || (pEnd <= closures[p][i*2])) {
4006           offsets[p*(depth+2)+d+1] = i;
4007           break;
4008         }
4009       }
4010       if (i == closureSize) offsets[p*(depth+2)+d+1] = i;
4011     }
4012     if (offsets[p*(depth+2)+depth+1] != closureSize) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Total size of closure %D should be %D", offsets[p*(depth+2)+depth+1], closureSize);
4013   }
4014   for (d = 0; d < depth+1; ++d) {
4015     PetscInt dof;
4016 
4017     /* Copy in support of first point */
4018     dof = offsets[d+1] - offsets[d];
4019     for (joinSize = 0; joinSize < dof; ++joinSize) {
4020       join[i][joinSize] = closures[0][(offsets[d]+joinSize)*2];
4021     }
4022     /* Check each successive cone */
4023     for (p = 1; p < numPoints && joinSize; ++p) {
4024       PetscInt newJoinSize = 0;
4025 
4026       dof = offsets[p*(depth+2)+d+1] - offsets[p*(depth+2)+d];
4027       for (c = 0; c < dof; ++c) {
4028         const PetscInt point = closures[p][(offsets[p*(depth+2)+d]+c)*2];
4029 
4030         for (m = 0; m < joinSize; ++m) {
4031           if (point == join[i][m]) {
4032             join[1-i][newJoinSize++] = point;
4033             break;
4034           }
4035         }
4036       }
4037       joinSize = newJoinSize;
4038       i        = 1-i;
4039     }
4040     if (joinSize) break;
4041   }
4042   *numCoveredPoints = joinSize;
4043   *coveredPoints    = join[i];
4044   for (p = 0; p < numPoints; ++p) {
4045     ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_FALSE, NULL, &closures[p]);CHKERRQ(ierr);
4046   }
4047   ierr = PetscFree(closures);CHKERRQ(ierr);
4048   ierr = DMRestoreWorkArray(dm, numPoints*(depth+2), MPIU_INT, &offsets);CHKERRQ(ierr);
4049   ierr = DMRestoreWorkArray(dm, mesh->maxSupportSize, MPIU_INT, &join[1-i]);CHKERRQ(ierr);
4050   PetscFunctionReturn(0);
4051 }
4052 
4053 /*@C
4054   DMPlexGetMeet - Get an array for the meet of the set of points
4055 
4056   Not Collective
4057 
4058   Input Parameters:
4059 + dm - The DMPlex object
4060 . numPoints - The number of input points for the meet
4061 - points - The input points
4062 
4063   Output Parameters:
4064 + numCoveredPoints - The number of points in the meet
4065 - coveredPoints - The points in the meet
4066 
4067   Level: intermediate
4068 
4069   Note: Currently, this is restricted to a single level meet
4070 
4071   Fortran Notes:
4072   Since it returns an array, this routine is only available in Fortran 90, and you must
4073   include petsc.h90 in your code.
4074 
4075   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
4076 
4077 .seealso: DMPlexRestoreMeet(), DMPlexGetJoin()
4078 @*/
4079 PetscErrorCode DMPlexGetMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveringPoints, const PetscInt **coveringPoints)
4080 {
4081   DM_Plex       *mesh = (DM_Plex*) dm->data;
4082   PetscInt      *meet[2];
4083   PetscInt       meetSize, i = 0;
4084   PetscInt       dof, off, p, c, m;
4085   PetscErrorCode ierr;
4086 
4087   PetscFunctionBegin;
4088   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4089   PetscValidPointer(points, 3);
4090   PetscValidPointer(numCoveringPoints, 4);
4091   PetscValidPointer(coveringPoints, 5);
4092   ierr = DMGetWorkArray(dm, mesh->maxConeSize, MPIU_INT, &meet[0]);CHKERRQ(ierr);
4093   ierr = DMGetWorkArray(dm, mesh->maxConeSize, MPIU_INT, &meet[1]);CHKERRQ(ierr);
4094   /* Copy in cone of first point */
4095   ierr = PetscSectionGetDof(mesh->coneSection, points[0], &dof);CHKERRQ(ierr);
4096   ierr = PetscSectionGetOffset(mesh->coneSection, points[0], &off);CHKERRQ(ierr);
4097   for (meetSize = 0; meetSize < dof; ++meetSize) {
4098     meet[i][meetSize] = mesh->cones[off+meetSize];
4099   }
4100   /* Check each successive cone */
4101   for (p = 1; p < numPoints; ++p) {
4102     PetscInt newMeetSize = 0;
4103 
4104     ierr = PetscSectionGetDof(mesh->coneSection, points[p], &dof);CHKERRQ(ierr);
4105     ierr = PetscSectionGetOffset(mesh->coneSection, points[p], &off);CHKERRQ(ierr);
4106     for (c = 0; c < dof; ++c) {
4107       const PetscInt point = mesh->cones[off+c];
4108 
4109       for (m = 0; m < meetSize; ++m) {
4110         if (point == meet[i][m]) {
4111           meet[1-i][newMeetSize++] = point;
4112           break;
4113         }
4114       }
4115     }
4116     meetSize = newMeetSize;
4117     i        = 1-i;
4118   }
4119   *numCoveringPoints = meetSize;
4120   *coveringPoints    = meet[i];
4121   ierr               = DMRestoreWorkArray(dm, mesh->maxConeSize, MPIU_INT, &meet[1-i]);CHKERRQ(ierr);
4122   PetscFunctionReturn(0);
4123 }
4124 
4125 /*@C
4126   DMPlexRestoreMeet - Restore an array for the meet of the set of points
4127 
4128   Not Collective
4129 
4130   Input Parameters:
4131 + dm - The DMPlex object
4132 . numPoints - The number of input points for the meet
4133 - points - The input points
4134 
4135   Output Parameters:
4136 + numCoveredPoints - The number of points in the meet
4137 - coveredPoints - The points in the meet
4138 
4139   Level: intermediate
4140 
4141   Fortran Notes:
4142   Since it returns an array, this routine is only available in Fortran 90, and you must
4143   include petsc.h90 in your code.
4144 
4145   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
4146 
4147 .seealso: DMPlexGetMeet(), DMPlexGetFullMeet(), DMPlexGetJoin()
4148 @*/
4149 PetscErrorCode DMPlexRestoreMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
4150 {
4151   PetscErrorCode ierr;
4152 
4153   PetscFunctionBegin;
4154   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4155   if (points) PetscValidIntPointer(points,3);
4156   if (numCoveredPoints) PetscValidIntPointer(numCoveredPoints,4);
4157   PetscValidPointer(coveredPoints,5);
4158   ierr = DMRestoreWorkArray(dm, 0, MPIU_INT, (void*) coveredPoints);CHKERRQ(ierr);
4159   if (numCoveredPoints) *numCoveredPoints = 0;
4160   PetscFunctionReturn(0);
4161 }
4162 
4163 /*@C
4164   DMPlexGetFullMeet - Get an array for the meet of the set of points
4165 
4166   Not Collective
4167 
4168   Input Parameters:
4169 + dm - The DMPlex object
4170 . numPoints - The number of input points for the meet
4171 - points - The input points
4172 
4173   Output Parameters:
4174 + numCoveredPoints - The number of points in the meet
4175 - coveredPoints - The points in the meet
4176 
4177   Level: intermediate
4178 
4179   Fortran Notes:
4180   Since it returns an array, this routine is only available in Fortran 90, and you must
4181   include petsc.h90 in your code.
4182 
4183   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
4184 
4185 .seealso: DMPlexGetMeet(), DMPlexRestoreMeet(), DMPlexGetJoin()
4186 @*/
4187 PetscErrorCode DMPlexGetFullMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
4188 {
4189   DM_Plex       *mesh = (DM_Plex*) dm->data;
4190   PetscInt      *offsets, **closures;
4191   PetscInt      *meet[2];
4192   PetscInt       height = 0, maxSize, meetSize = 0, i = 0;
4193   PetscInt       p, h, c, m, mc;
4194   PetscErrorCode ierr;
4195 
4196   PetscFunctionBegin;
4197   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4198   PetscValidPointer(points, 3);
4199   PetscValidPointer(numCoveredPoints, 4);
4200   PetscValidPointer(coveredPoints, 5);
4201 
4202   ierr    = DMPlexGetDepth(dm, &height);CHKERRQ(ierr);
4203   ierr    = PetscMalloc1(numPoints, &closures);CHKERRQ(ierr);
4204   ierr    = DMGetWorkArray(dm, numPoints*(height+2), MPIU_INT, &offsets);CHKERRQ(ierr);
4205   mc      = mesh->maxConeSize;
4206   maxSize = (mc > 1) ? ((PetscPowInt(mc,height+1)-1)/(mc-1)) : height + 1;
4207   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &meet[0]);CHKERRQ(ierr);
4208   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &meet[1]);CHKERRQ(ierr);
4209 
4210   for (p = 0; p < numPoints; ++p) {
4211     PetscInt closureSize;
4212 
4213     ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_TRUE, &closureSize, &closures[p]);CHKERRQ(ierr);
4214 
4215     offsets[p*(height+2)+0] = 0;
4216     for (h = 0; h < height+1; ++h) {
4217       PetscInt pStart, pEnd, i;
4218 
4219       ierr = DMPlexGetHeightStratum(dm, h, &pStart, &pEnd);CHKERRQ(ierr);
4220       for (i = offsets[p*(height+2)+h]; i < closureSize; ++i) {
4221         if ((pStart > closures[p][i*2]) || (pEnd <= closures[p][i*2])) {
4222           offsets[p*(height+2)+h+1] = i;
4223           break;
4224         }
4225       }
4226       if (i == closureSize) offsets[p*(height+2)+h+1] = i;
4227     }
4228     if (offsets[p*(height+2)+height+1] != closureSize) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Total size of closure %D should be %D", offsets[p*(height+2)+height+1], closureSize);
4229   }
4230   for (h = 0; h < height+1; ++h) {
4231     PetscInt dof;
4232 
4233     /* Copy in cone of first point */
4234     dof = offsets[h+1] - offsets[h];
4235     for (meetSize = 0; meetSize < dof; ++meetSize) {
4236       meet[i][meetSize] = closures[0][(offsets[h]+meetSize)*2];
4237     }
4238     /* Check each successive cone */
4239     for (p = 1; p < numPoints && meetSize; ++p) {
4240       PetscInt newMeetSize = 0;
4241 
4242       dof = offsets[p*(height+2)+h+1] - offsets[p*(height+2)+h];
4243       for (c = 0; c < dof; ++c) {
4244         const PetscInt point = closures[p][(offsets[p*(height+2)+h]+c)*2];
4245 
4246         for (m = 0; m < meetSize; ++m) {
4247           if (point == meet[i][m]) {
4248             meet[1-i][newMeetSize++] = point;
4249             break;
4250           }
4251         }
4252       }
4253       meetSize = newMeetSize;
4254       i        = 1-i;
4255     }
4256     if (meetSize) break;
4257   }
4258   *numCoveredPoints = meetSize;
4259   *coveredPoints    = meet[i];
4260   for (p = 0; p < numPoints; ++p) {
4261     ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_TRUE, NULL, &closures[p]);CHKERRQ(ierr);
4262   }
4263   ierr = PetscFree(closures);CHKERRQ(ierr);
4264   ierr = DMRestoreWorkArray(dm, numPoints*(height+2), MPIU_INT, &offsets);CHKERRQ(ierr);
4265   ierr = DMRestoreWorkArray(dm, mesh->maxConeSize, MPIU_INT, &meet[1-i]);CHKERRQ(ierr);
4266   PetscFunctionReturn(0);
4267 }
4268 
4269 /*@C
4270   DMPlexEqual - Determine if two DMs have the same topology
4271 
4272   Not Collective
4273 
4274   Input Parameters:
4275 + dmA - A DMPlex object
4276 - dmB - A DMPlex object
4277 
4278   Output Parameters:
4279 . equal - PETSC_TRUE if the topologies are identical
4280 
4281   Level: intermediate
4282 
4283   Notes:
4284   We are not solving graph isomorphism, so we do not permutation.
4285 
4286 .seealso: DMPlexGetCone()
4287 @*/
4288 PetscErrorCode DMPlexEqual(DM dmA, DM dmB, PetscBool *equal)
4289 {
4290   PetscInt       depth, depthB, pStart, pEnd, pStartB, pEndB, p;
4291   PetscErrorCode ierr;
4292 
4293   PetscFunctionBegin;
4294   PetscValidHeaderSpecific(dmA, DM_CLASSID, 1);
4295   PetscValidHeaderSpecific(dmB, DM_CLASSID, 2);
4296   PetscValidPointer(equal, 3);
4297 
4298   *equal = PETSC_FALSE;
4299   ierr = DMPlexGetDepth(dmA, &depth);CHKERRQ(ierr);
4300   ierr = DMPlexGetDepth(dmB, &depthB);CHKERRQ(ierr);
4301   if (depth != depthB) PetscFunctionReturn(0);
4302   ierr = DMPlexGetChart(dmA, &pStart,  &pEnd);CHKERRQ(ierr);
4303   ierr = DMPlexGetChart(dmB, &pStartB, &pEndB);CHKERRQ(ierr);
4304   if ((pStart != pStartB) || (pEnd != pEndB)) PetscFunctionReturn(0);
4305   for (p = pStart; p < pEnd; ++p) {
4306     const PetscInt *cone, *coneB, *ornt, *orntB, *support, *supportB;
4307     PetscInt        coneSize, coneSizeB, c, supportSize, supportSizeB, s;
4308 
4309     ierr = DMPlexGetConeSize(dmA, p, &coneSize);CHKERRQ(ierr);
4310     ierr = DMPlexGetCone(dmA, p, &cone);CHKERRQ(ierr);
4311     ierr = DMPlexGetConeOrientation(dmA, p, &ornt);CHKERRQ(ierr);
4312     ierr = DMPlexGetConeSize(dmB, p, &coneSizeB);CHKERRQ(ierr);
4313     ierr = DMPlexGetCone(dmB, p, &coneB);CHKERRQ(ierr);
4314     ierr = DMPlexGetConeOrientation(dmB, p, &orntB);CHKERRQ(ierr);
4315     if (coneSize != coneSizeB) PetscFunctionReturn(0);
4316     for (c = 0; c < coneSize; ++c) {
4317       if (cone[c] != coneB[c]) PetscFunctionReturn(0);
4318       if (ornt[c] != orntB[c]) PetscFunctionReturn(0);
4319     }
4320     ierr = DMPlexGetSupportSize(dmA, p, &supportSize);CHKERRQ(ierr);
4321     ierr = DMPlexGetSupport(dmA, p, &support);CHKERRQ(ierr);
4322     ierr = DMPlexGetSupportSize(dmB, p, &supportSizeB);CHKERRQ(ierr);
4323     ierr = DMPlexGetSupport(dmB, p, &supportB);CHKERRQ(ierr);
4324     if (supportSize != supportSizeB) PetscFunctionReturn(0);
4325     for (s = 0; s < supportSize; ++s) {
4326       if (support[s] != supportB[s]) PetscFunctionReturn(0);
4327     }
4328   }
4329   *equal = PETSC_TRUE;
4330   PetscFunctionReturn(0);
4331 }
4332 
4333 /*@C
4334   DMPlexGetNumFaceVertices - Returns the number of vertices on a face
4335 
4336   Not Collective
4337 
4338   Input Parameters:
4339 + dm         - The DMPlex
4340 . cellDim    - The cell dimension
4341 - numCorners - The number of vertices on a cell
4342 
4343   Output Parameters:
4344 . numFaceVertices - The number of vertices on a face
4345 
4346   Level: developer
4347 
4348   Notes:
4349   Of course this can only work for a restricted set of symmetric shapes
4350 
4351 .seealso: DMPlexGetCone()
4352 @*/
4353 PetscErrorCode DMPlexGetNumFaceVertices(DM dm, PetscInt cellDim, PetscInt numCorners, PetscInt *numFaceVertices)
4354 {
4355   MPI_Comm       comm;
4356   PetscErrorCode ierr;
4357 
4358   PetscFunctionBegin;
4359   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
4360   PetscValidPointer(numFaceVertices,4);
4361   switch (cellDim) {
4362   case 0:
4363     *numFaceVertices = 0;
4364     break;
4365   case 1:
4366     *numFaceVertices = 1;
4367     break;
4368   case 2:
4369     switch (numCorners) {
4370     case 3: /* triangle */
4371       *numFaceVertices = 2; /* Edge has 2 vertices */
4372       break;
4373     case 4: /* quadrilateral */
4374       *numFaceVertices = 2; /* Edge has 2 vertices */
4375       break;
4376     case 6: /* quadratic triangle, tri and quad cohesive Lagrange cells */
4377       *numFaceVertices = 3; /* Edge has 3 vertices */
4378       break;
4379     case 9: /* quadratic quadrilateral, quadratic quad cohesive Lagrange cells */
4380       *numFaceVertices = 3; /* Edge has 3 vertices */
4381       break;
4382     default:
4383       SETERRQ2(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid number of face corners %D for dimension %D", numCorners, cellDim);
4384     }
4385     break;
4386   case 3:
4387     switch (numCorners) {
4388     case 4: /* tetradehdron */
4389       *numFaceVertices = 3; /* Face has 3 vertices */
4390       break;
4391     case 6: /* tet cohesive cells */
4392       *numFaceVertices = 4; /* Face has 4 vertices */
4393       break;
4394     case 8: /* hexahedron */
4395       *numFaceVertices = 4; /* Face has 4 vertices */
4396       break;
4397     case 9: /* tet cohesive Lagrange cells */
4398       *numFaceVertices = 6; /* Face has 6 vertices */
4399       break;
4400     case 10: /* quadratic tetrahedron */
4401       *numFaceVertices = 6; /* Face has 6 vertices */
4402       break;
4403     case 12: /* hex cohesive Lagrange cells */
4404       *numFaceVertices = 6; /* Face has 6 vertices */
4405       break;
4406     case 18: /* quadratic tet cohesive Lagrange cells */
4407       *numFaceVertices = 6; /* Face has 6 vertices */
4408       break;
4409     case 27: /* quadratic hexahedron, quadratic hex cohesive Lagrange cells */
4410       *numFaceVertices = 9; /* Face has 9 vertices */
4411       break;
4412     default:
4413       SETERRQ2(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid number of face corners %D for dimension %D", numCorners, cellDim);
4414     }
4415     break;
4416   default:
4417     SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid cell dimension %D", cellDim);
4418   }
4419   PetscFunctionReturn(0);
4420 }
4421 
4422 /*@
4423   DMPlexGetDepthLabel - Get the DMLabel recording the depth of each point
4424 
4425   Not Collective
4426 
4427   Input Parameter:
4428 . dm    - The DMPlex object
4429 
4430   Output Parameter:
4431 . depthLabel - The DMLabel recording point depth
4432 
4433   Level: developer
4434 
4435 .seealso: DMPlexGetDepth(), DMPlexGetHeightStratum(), DMPlexGetDepthStratum(), DMPlexGetPointDepth(),
4436 @*/
4437 PetscErrorCode DMPlexGetDepthLabel(DM dm, DMLabel *depthLabel)
4438 {
4439   PetscFunctionBegin;
4440   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4441   PetscValidPointer(depthLabel, 2);
4442   *depthLabel = dm->depthLabel;
4443   PetscFunctionReturn(0);
4444 }
4445 
4446 /*@
4447   DMPlexGetDepth - Get the depth of the DAG representing this mesh
4448 
4449   Not Collective
4450 
4451   Input Parameter:
4452 . dm    - The DMPlex object
4453 
4454   Output Parameter:
4455 . depth - The number of strata (breadth first levels) in the DAG
4456 
4457   Level: developer
4458 
4459   Notes:
4460   This returns maximum of point depths over all points, i.e. maximum value of the label returned by DMPlexGetDepthLabel().
4461   The point depth is described more in detail in DMPlexGetDepthStratum().
4462   An empty mesh gives -1.
4463 
4464 .seealso: DMPlexGetDepthLabel(), DMPlexGetDepthStratum(), DMPlexGetPointDepth(), DMPlexSymmetrize()
4465 @*/
4466 PetscErrorCode DMPlexGetDepth(DM dm, PetscInt *depth)
4467 {
4468   DMLabel        label;
4469   PetscInt       d = 0;
4470   PetscErrorCode ierr;
4471 
4472   PetscFunctionBegin;
4473   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4474   PetscValidPointer(depth, 2);
4475   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
4476   if (label) {ierr = DMLabelGetNumValues(label, &d);CHKERRQ(ierr);}
4477   *depth = d-1;
4478   PetscFunctionReturn(0);
4479 }
4480 
4481 /*@
4482   DMPlexGetDepthStratum - Get the bounds [start, end) for all points at a certain depth.
4483 
4484   Not Collective
4485 
4486   Input Parameters:
4487 + dm           - The DMPlex object
4488 - stratumValue - The requested depth
4489 
4490   Output Parameters:
4491 + start - The first point at this depth
4492 - end   - One beyond the last point at this depth
4493 
4494   Notes:
4495   Depth indexing is related to topological dimension.  Depth stratum 0 contains the lowest topological dimension points,
4496   often "vertices".  If the mesh is "interpolated" (see DMPlexInterpolate()), then depth stratum 1 contains the next
4497   higher dimension, e.g., "edges".
4498 
4499   Level: developer
4500 
4501 .seealso: DMPlexGetHeightStratum(), DMPlexGetDepth(), DMPlexGetDepthLabel(), DMPlexGetPointDepth(), DMPlexSymmetrize(), DMPlexInterpolate()
4502 @*/
4503 PetscErrorCode DMPlexGetDepthStratum(DM dm, PetscInt stratumValue, PetscInt *start, PetscInt *end)
4504 {
4505   DMLabel        label;
4506   PetscInt       pStart, pEnd;
4507   PetscErrorCode ierr;
4508 
4509   PetscFunctionBegin;
4510   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4511   if (start) {PetscValidPointer(start, 3); *start = 0;}
4512   if (end)   {PetscValidPointer(end,   4); *end   = 0;}
4513   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
4514   if (pStart == pEnd) PetscFunctionReturn(0);
4515   if (stratumValue < 0) {
4516     if (start) *start = pStart;
4517     if (end)   *end   = pEnd;
4518     PetscFunctionReturn(0);
4519   }
4520   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
4521   if (!label) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "No label named depth was found");
4522   ierr = DMLabelGetStratumBounds(label, stratumValue, start, end);CHKERRQ(ierr);
4523   PetscFunctionReturn(0);
4524 }
4525 
4526 /*@
4527   DMPlexGetHeightStratum - Get the bounds [start, end) for all points at a certain height.
4528 
4529   Not Collective
4530 
4531   Input Parameters:
4532 + dm           - The DMPlex object
4533 - stratumValue - The requested height
4534 
4535   Output Parameters:
4536 + start - The first point at this height
4537 - end   - One beyond the last point at this height
4538 
4539   Notes:
4540   Height indexing is related to topological codimension.  Height stratum 0 contains the highest topological dimension
4541   points, often called "cells" or "elements".  If the mesh is "interpolated" (see DMPlexInterpolate()), then height
4542   stratum 1 contains the boundary of these "cells", often called "faces" or "facets".
4543 
4544   Level: developer
4545 
4546 .seealso: DMPlexGetDepthStratum(), DMPlexGetDepth(), DMPlexGetPointHeight()
4547 @*/
4548 PetscErrorCode DMPlexGetHeightStratum(DM dm, PetscInt stratumValue, PetscInt *start, PetscInt *end)
4549 {
4550   DMLabel        label;
4551   PetscInt       depth, pStart, pEnd;
4552   PetscErrorCode ierr;
4553 
4554   PetscFunctionBegin;
4555   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4556   if (start) {PetscValidPointer(start, 3); *start = 0;}
4557   if (end)   {PetscValidPointer(end,   4); *end   = 0;}
4558   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
4559   if (pStart == pEnd) PetscFunctionReturn(0);
4560   if (stratumValue < 0) {
4561     if (start) *start = pStart;
4562     if (end)   *end   = pEnd;
4563     PetscFunctionReturn(0);
4564   }
4565   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
4566   if (!label) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "No label named depth was found");
4567   ierr = DMLabelGetNumValues(label, &depth);CHKERRQ(ierr);
4568   ierr = DMLabelGetStratumBounds(label, depth-1-stratumValue, start, end);CHKERRQ(ierr);
4569   PetscFunctionReturn(0);
4570 }
4571 
4572 /*@
4573   DMPlexGetPointDepth - Get the depth of a given point
4574 
4575   Not Collective
4576 
4577   Input Parameter:
4578 + dm    - The DMPlex object
4579 - point - The point
4580 
4581   Output Parameter:
4582 . depth - The depth of the point
4583 
4584   Level: intermediate
4585 
4586 .seealso: DMPlexGetCellType(), DMPlexGetDepthLabel(), DMPlexGetDepth(), DMPlexGetPointHeight()
4587 @*/
4588 PetscErrorCode DMPlexGetPointDepth(DM dm, PetscInt point, PetscInt *depth)
4589 {
4590   PetscErrorCode ierr;
4591 
4592   PetscFunctionBegin;
4593   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4594   PetscValidIntPointer(depth, 3);
4595   ierr = DMLabelGetValue(dm->depthLabel, point, depth);CHKERRQ(ierr);
4596   PetscFunctionReturn(0);
4597 }
4598 
4599 /*@
4600   DMPlexGetPointHeight - Get the height of a given point
4601 
4602   Not Collective
4603 
4604   Input Parameter:
4605 + dm    - The DMPlex object
4606 - point - The point
4607 
4608   Output Parameter:
4609 . height - The height of the point
4610 
4611   Level: intermediate
4612 
4613 .seealso: DMPlexGetCellType(), DMPlexGetDepthLabel(), DMPlexGetDepth(), DMPlexGetPointDepth()
4614 @*/
4615 PetscErrorCode DMPlexGetPointHeight(DM dm, PetscInt point, PetscInt *height)
4616 {
4617   PetscInt       n, pDepth;
4618   PetscErrorCode ierr;
4619 
4620   PetscFunctionBegin;
4621   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4622   PetscValidIntPointer(height, 3);
4623   ierr = DMLabelGetNumValues(dm->depthLabel, &n);CHKERRQ(ierr);
4624   ierr = DMLabelGetValue(dm->depthLabel, point, &pDepth);CHKERRQ(ierr);
4625   *height = n - 1 - pDepth;  /* DAG depth is n-1 */
4626   PetscFunctionReturn(0);
4627 }
4628 
4629 /*@
4630   DMPlexGetCellTypeLabel - Get the DMLabel recording the polytope type of each cell
4631 
4632   Not Collective
4633 
4634   Input Parameter:
4635 . dm - The DMPlex object
4636 
4637   Output Parameter:
4638 . celltypeLabel - The DMLabel recording cell polytope type
4639 
4640   Note: This function will trigger automatica computation of cell types. This can be disabled by calling
4641   DMCreateLabel(dm, "celltype") beforehand.
4642 
4643   Level: developer
4644 
4645 .seealso: DMPlexGetCellType(), DMPlexGetDepthLabel(), DMCreateLabel()
4646 @*/
4647 PetscErrorCode DMPlexGetCellTypeLabel(DM dm, DMLabel *celltypeLabel)
4648 {
4649   PetscErrorCode ierr;
4650 
4651   PetscFunctionBegin;
4652   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4653   PetscValidPointer(celltypeLabel, 2);
4654   if (!dm->celltypeLabel) {ierr = DMPlexComputeCellTypes(dm);CHKERRQ(ierr);}
4655   *celltypeLabel = dm->celltypeLabel;
4656   PetscFunctionReturn(0);
4657 }
4658 
4659 /*@
4660   DMPlexGetCellType - Get the polytope type of a given cell
4661 
4662   Not Collective
4663 
4664   Input Parameter:
4665 + dm   - The DMPlex object
4666 - cell - The cell
4667 
4668   Output Parameter:
4669 . celltype - The polytope type of the cell
4670 
4671   Level: intermediate
4672 
4673 .seealso: DMPlexGetCellTypeLabel(), DMPlexGetDepthLabel(), DMPlexGetDepth()
4674 @*/
4675 PetscErrorCode DMPlexGetCellType(DM dm, PetscInt cell, DMPolytopeType *celltype)
4676 {
4677   DMLabel        label;
4678   PetscInt       ct;
4679   PetscErrorCode ierr;
4680 
4681   PetscFunctionBegin;
4682   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4683   PetscValidPointer(celltype, 3);
4684   ierr = DMPlexGetCellTypeLabel(dm, &label);CHKERRQ(ierr);
4685   ierr = DMLabelGetValue(label, cell, &ct);CHKERRQ(ierr);
4686   if (ct < 0) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Cell %D has not been assigned a cell type", cell);
4687   *celltype = (DMPolytopeType) ct;
4688   PetscFunctionReturn(0);
4689 }
4690 
4691 /*@
4692   DMPlexSetCellType - Set the polytope type of a given cell
4693 
4694   Not Collective
4695 
4696   Input Parameters:
4697 + dm   - The DMPlex object
4698 . cell - The cell
4699 - celltype - The polytope type of the cell
4700 
4701   Note: By default, cell types will be automatically computed using DMPlexComputeCellTypes() before this function
4702   is executed. This function will override the computed type. However, if automatic classification will not succeed
4703   and a user wants to manually specify all types, the classification must be disabled by calling
4704   DMCreaateLabel(dm, "celltype") before getting or setting any cell types.
4705 
4706   Level: advanced
4707 
4708 .seealso: DMPlexGetCellTypeLabel(), DMPlexGetDepthLabel(), DMPlexGetDepth(), DMPlexComputeCellTypes(), DMCreateLabel()
4709 @*/
4710 PetscErrorCode DMPlexSetCellType(DM dm, PetscInt cell, DMPolytopeType celltype)
4711 {
4712   DMLabel        label;
4713   PetscErrorCode ierr;
4714 
4715   PetscFunctionBegin;
4716   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4717   ierr = DMPlexGetCellTypeLabel(dm, &label);CHKERRQ(ierr);
4718   ierr = DMLabelSetValue(label, cell, celltype);CHKERRQ(ierr);
4719   PetscFunctionReturn(0);
4720 }
4721 
4722 PetscErrorCode DMCreateCoordinateDM_Plex(DM dm, DM *cdm)
4723 {
4724   PetscSection   section, s;
4725   Mat            m;
4726   PetscInt       maxHeight;
4727   PetscErrorCode ierr;
4728 
4729   PetscFunctionBegin;
4730   ierr = DMClone(dm, cdm);CHKERRQ(ierr);
4731   ierr = DMPlexGetMaxProjectionHeight(dm, &maxHeight);CHKERRQ(ierr);
4732   ierr = DMPlexSetMaxProjectionHeight(*cdm, maxHeight);CHKERRQ(ierr);
4733   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &section);CHKERRQ(ierr);
4734   ierr = DMSetLocalSection(*cdm, section);CHKERRQ(ierr);
4735   ierr = PetscSectionDestroy(&section);CHKERRQ(ierr);
4736   ierr = PetscSectionCreate(PETSC_COMM_SELF, &s);CHKERRQ(ierr);
4737   ierr = MatCreate(PETSC_COMM_SELF, &m);CHKERRQ(ierr);
4738   ierr = DMSetDefaultConstraints(*cdm, s, m);CHKERRQ(ierr);
4739   ierr = PetscSectionDestroy(&s);CHKERRQ(ierr);
4740   ierr = MatDestroy(&m);CHKERRQ(ierr);
4741 
4742   ierr = DMSetNumFields(*cdm, 1);CHKERRQ(ierr);
4743   ierr = DMCreateDS(*cdm);CHKERRQ(ierr);
4744   PetscFunctionReturn(0);
4745 }
4746 
4747 PetscErrorCode DMCreateCoordinateField_Plex(DM dm, DMField *field)
4748 {
4749   Vec            coordsLocal;
4750   DM             coordsDM;
4751   PetscErrorCode ierr;
4752 
4753   PetscFunctionBegin;
4754   *field = NULL;
4755   ierr = DMGetCoordinatesLocal(dm,&coordsLocal);CHKERRQ(ierr);
4756   ierr = DMGetCoordinateDM(dm,&coordsDM);CHKERRQ(ierr);
4757   if (coordsLocal && coordsDM) {
4758     ierr = DMFieldCreateDS(coordsDM, 0, coordsLocal, field);CHKERRQ(ierr);
4759   }
4760   PetscFunctionReturn(0);
4761 }
4762 
4763 /*@C
4764   DMPlexGetConeSection - Return a section which describes the layout of cone data
4765 
4766   Not Collective
4767 
4768   Input Parameters:
4769 . dm        - The DMPlex object
4770 
4771   Output Parameter:
4772 . section - The PetscSection object
4773 
4774   Level: developer
4775 
4776 .seealso: DMPlexGetSupportSection(), DMPlexGetCones(), DMPlexGetConeOrientations()
4777 @*/
4778 PetscErrorCode DMPlexGetConeSection(DM dm, PetscSection *section)
4779 {
4780   DM_Plex *mesh = (DM_Plex*) dm->data;
4781 
4782   PetscFunctionBegin;
4783   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4784   if (section) *section = mesh->coneSection;
4785   PetscFunctionReturn(0);
4786 }
4787 
4788 /*@C
4789   DMPlexGetSupportSection - Return a section which describes the layout of support data
4790 
4791   Not Collective
4792 
4793   Input Parameters:
4794 . dm        - The DMPlex object
4795 
4796   Output Parameter:
4797 . section - The PetscSection object
4798 
4799   Level: developer
4800 
4801 .seealso: DMPlexGetConeSection()
4802 @*/
4803 PetscErrorCode DMPlexGetSupportSection(DM dm, PetscSection *section)
4804 {
4805   DM_Plex *mesh = (DM_Plex*) dm->data;
4806 
4807   PetscFunctionBegin;
4808   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4809   if (section) *section = mesh->supportSection;
4810   PetscFunctionReturn(0);
4811 }
4812 
4813 /*@C
4814   DMPlexGetCones - Return cone data
4815 
4816   Not Collective
4817 
4818   Input Parameters:
4819 . dm        - The DMPlex object
4820 
4821   Output Parameter:
4822 . cones - The cone for each point
4823 
4824   Level: developer
4825 
4826 .seealso: DMPlexGetConeSection()
4827 @*/
4828 PetscErrorCode DMPlexGetCones(DM dm, PetscInt *cones[])
4829 {
4830   DM_Plex *mesh = (DM_Plex*) dm->data;
4831 
4832   PetscFunctionBegin;
4833   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4834   if (cones) *cones = mesh->cones;
4835   PetscFunctionReturn(0);
4836 }
4837 
4838 /*@C
4839   DMPlexGetConeOrientations - Return cone orientation data
4840 
4841   Not Collective
4842 
4843   Input Parameters:
4844 . dm        - The DMPlex object
4845 
4846   Output Parameter:
4847 . coneOrientations - The cone orientation for each point
4848 
4849   Level: developer
4850 
4851 .seealso: DMPlexGetConeSection()
4852 @*/
4853 PetscErrorCode DMPlexGetConeOrientations(DM dm, PetscInt *coneOrientations[])
4854 {
4855   DM_Plex *mesh = (DM_Plex*) dm->data;
4856 
4857   PetscFunctionBegin;
4858   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4859   if (coneOrientations) *coneOrientations = mesh->coneOrientations;
4860   PetscFunctionReturn(0);
4861 }
4862 
4863 /******************************** FEM Support **********************************/
4864 
4865 /*
4866  Returns number of components and tensor degree for the field.  For interpolated meshes, line should be a point
4867  representing a line in the section.
4868 */
4869 static PetscErrorCode PetscSectionFieldGetTensorDegree_Private(PetscSection section,PetscInt field,PetscInt line,PetscBool vertexchart,PetscInt *Nc,PetscInt *k)
4870 {
4871   PetscErrorCode ierr;
4872 
4873   PetscFunctionBeginHot;
4874   ierr = PetscSectionGetFieldComponents(section, field, Nc);CHKERRQ(ierr);
4875   if (line < 0) {
4876     *k = 0;
4877     *Nc = 0;
4878   } else if (vertexchart) {            /* If we only have a vertex chart, we must have degree k=1 */
4879     *k = 1;
4880   } else {                      /* Assume the full interpolated mesh is in the chart; lines in particular */
4881     /* An order k SEM disc has k-1 dofs on an edge */
4882     ierr = PetscSectionGetFieldDof(section, line, field, k);CHKERRQ(ierr);
4883     *k = *k / *Nc + 1;
4884   }
4885   PetscFunctionReturn(0);
4886 }
4887 
4888 /*@
4889 
4890   DMPlexSetClosurePermutationTensor - Create a permutation from the default (BFS) point ordering in the closure, to a
4891   lexicographic ordering over the tensor product cell (i.e., line, quad, hex, etc.), and set this permutation in the
4892   section provided (or the section of the DM).
4893 
4894   Input Parameters:
4895 + dm      - The DM
4896 . point   - Either a cell (highest dim point) or an edge (dim 1 point), or PETSC_DETERMINE
4897 - section - The PetscSection to reorder, or NULL for the default section
4898 
4899   Note: The point is used to determine the number of dofs/field on an edge. For SEM, this is related to the polynomial
4900   degree of the basis.
4901 
4902   Example:
4903   A typical interpolated single-quad mesh might order points as
4904 .vb
4905   [c0, v1, v2, v3, v4, e5, e6, e7, e8]
4906 
4907   v4 -- e6 -- v3
4908   |           |
4909   e7    c0    e8
4910   |           |
4911   v1 -- e5 -- v2
4912 .ve
4913 
4914   (There is no significance to the ordering described here.)  The default section for a Q3 quad might typically assign
4915   dofs in the order of points, e.g.,
4916 .vb
4917     c0 -> [0,1,2,3]
4918     v1 -> [4]
4919     ...
4920     e5 -> [8, 9]
4921 .ve
4922 
4923   which corresponds to the dofs
4924 .vb
4925     6   10  11  7
4926     13  2   3   15
4927     12  0   1   14
4928     4   8   9   5
4929 .ve
4930 
4931   The closure in BFS ordering works through height strata (cells, edges, vertices) to produce the ordering
4932 .vb
4933   0 1 2 3 8 9 14 15 11 10 13 12 4 5 7 6
4934 .ve
4935 
4936   After calling DMPlexSetClosurePermutationTensor(), the closure will be ordered lexicographically,
4937 .vb
4938    4 8 9 5 12 0 1 14 13 2 3 15 6 10 11 7
4939 .ve
4940 
4941   Level: developer
4942 
4943 .seealso: DMGetLocalSection(), PetscSectionSetClosurePermutation(), DMSetGlobalSection()
4944 @*/
4945 PetscErrorCode DMPlexSetClosurePermutationTensor(DM dm, PetscInt point, PetscSection section)
4946 {
4947   DMLabel        label;
4948   PetscInt       dim, depth = -1, eStart = -1, Nf;
4949   PetscBool      vertexchart;
4950   PetscErrorCode ierr;
4951 
4952   PetscFunctionBegin;
4953   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
4954   if (dim < 1) PetscFunctionReturn(0);
4955   if (point < 0) {
4956     PetscInt sStart,sEnd;
4957 
4958     ierr = DMPlexGetDepthStratum(dm, 1, &sStart, &sEnd);CHKERRQ(ierr);
4959     point = sEnd-sStart ? sStart : point;
4960   }
4961   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
4962   if (point >= 0) { ierr = DMLabelGetValue(label, point, &depth);CHKERRQ(ierr); }
4963   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
4964   if (depth == 1) {eStart = point;}
4965   else if  (depth == dim) {
4966     const PetscInt *cone;
4967 
4968     ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
4969     if (dim == 2) eStart = cone[0];
4970     else if (dim == 3) {
4971       const PetscInt *cone2;
4972       ierr = DMPlexGetCone(dm, cone[0], &cone2);CHKERRQ(ierr);
4973       eStart = cone2[0];
4974     } else SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Point %D of depth %D cannot be used to bootstrap spectral ordering for dim %D", point, depth, dim);
4975   } else if (depth >= 0) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Point %D of depth %D cannot be used to bootstrap spectral ordering for dim %D", point, depth, dim);
4976   {                             /* Determine whether the chart covers all points or just vertices. */
4977     PetscInt pStart,pEnd,cStart,cEnd;
4978     ierr = DMPlexGetDepthStratum(dm,0,&pStart,&pEnd);CHKERRQ(ierr);
4979     ierr = PetscSectionGetChart(section,&cStart,&cEnd);CHKERRQ(ierr);
4980     if (pStart == cStart && pEnd == cEnd) vertexchart = PETSC_TRUE; /* Just vertices */
4981     else vertexchart = PETSC_FALSE;                                 /* Assume all interpolated points are in chart */
4982   }
4983   ierr = PetscSectionGetNumFields(section, &Nf);CHKERRQ(ierr);
4984   for (PetscInt d=1; d<=dim; d++) {
4985     PetscInt k, f, Nc, c, i, j, size = 0, offset = 0, foffset = 0;
4986     PetscInt *perm;
4987 
4988     for (f = 0; f < Nf; ++f) {
4989       ierr = PetscSectionFieldGetTensorDegree_Private(section,f,eStart,vertexchart,&Nc,&k);CHKERRQ(ierr);
4990       size += PetscPowInt(k+1, d)*Nc;
4991     }
4992     ierr = PetscMalloc1(size, &perm);CHKERRQ(ierr);
4993     for (f = 0; f < Nf; ++f) {
4994       switch (d) {
4995       case 1:
4996         ierr = PetscSectionFieldGetTensorDegree_Private(section,f,eStart,vertexchart,&Nc,&k);CHKERRQ(ierr);
4997         /*
4998          Original ordering is [ edge of length k-1; vtx0; vtx1 ]
4999          We want              [ vtx0; edge of length k-1; vtx1 ]
5000          */
5001         for (c=0; c<Nc; c++,offset++) perm[offset] = (k-1)*Nc + c + foffset;
5002         for (i=0; i<k-1; i++) for (c=0; c<Nc; c++,offset++) perm[offset] = i*Nc + c + foffset;
5003         for (c=0; c<Nc; c++,offset++) perm[offset] = k*Nc + c + foffset;
5004         foffset = offset;
5005         break;
5006       case 2:
5007         /* The original quad closure is oriented clockwise, {f, e_b, e_r, e_t, e_l, v_lb, v_rb, v_tr, v_tl} */
5008         ierr = PetscSectionFieldGetTensorDegree_Private(section,f,eStart,vertexchart,&Nc,&k);CHKERRQ(ierr);
5009         /* The SEM order is
5010 
5011          v_lb, {e_b}, v_rb,
5012          e^{(k-1)-i}_l, {f^{i*(k-1)}}, e^i_r,
5013          v_lt, reverse {e_t}, v_rt
5014          */
5015         {
5016           const PetscInt of   = 0;
5017           const PetscInt oeb  = of   + PetscSqr(k-1);
5018           const PetscInt oer  = oeb  + (k-1);
5019           const PetscInt oet  = oer  + (k-1);
5020           const PetscInt oel  = oet  + (k-1);
5021           const PetscInt ovlb = oel  + (k-1);
5022           const PetscInt ovrb = ovlb + 1;
5023           const PetscInt ovrt = ovrb + 1;
5024           const PetscInt ovlt = ovrt + 1;
5025           PetscInt       o;
5026 
5027           /* bottom */
5028           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovlb*Nc + c + foffset;
5029           for (o = oeb; o < oer; ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5030           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovrb*Nc + c + foffset;
5031           /* middle */
5032           for (i = 0; i < k-1; ++i) {
5033             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oel+(k-2)-i)*Nc + c + foffset;
5034             for (o = of+(k-1)*i; o < of+(k-1)*(i+1); ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5035             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oer+i)*Nc + c + foffset;
5036           }
5037           /* top */
5038           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovlt*Nc + c + foffset;
5039           for (o = oel-1; o >= oet; --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5040           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovrt*Nc + c + foffset;
5041           foffset = offset;
5042         }
5043         break;
5044       case 3:
5045         /* The original hex closure is
5046 
5047          {c,
5048          f_b, f_t, f_f, f_b, f_r, f_l,
5049          e_bl, e_bb, e_br, e_bf,  e_tf, e_tr, e_tb, e_tl,  e_rf, e_lf, e_lb, e_rb,
5050          v_blf, v_blb, v_brb, v_brf, v_tlf, v_trf, v_trb, v_tlb}
5051          */
5052         ierr = PetscSectionFieldGetTensorDegree_Private(section,f,eStart,vertexchart,&Nc,&k);CHKERRQ(ierr);
5053         /* The SEM order is
5054          Bottom Slice
5055          v_blf, {e^{(k-1)-n}_bf}, v_brf,
5056          e^{i}_bl, f^{n*(k-1)+(k-1)-i}_b, e^{(k-1)-i}_br,
5057          v_blb, {e_bb}, v_brb,
5058 
5059          Middle Slice (j)
5060          {e^{(k-1)-j}_lf}, {f^{j*(k-1)+n}_f}, e^j_rf,
5061          f^{i*(k-1)+j}_l, {c^{(j*(k-1) + i)*(k-1)+n}_t}, f^{j*(k-1)+i}_r,
5062          e^j_lb, {f^{j*(k-1)+(k-1)-n}_b}, e^{(k-1)-j}_rb,
5063 
5064          Top Slice
5065          v_tlf, {e_tf}, v_trf,
5066          e^{(k-1)-i}_tl, {f^{i*(k-1)}_t}, e^{i}_tr,
5067          v_tlb, {e^{(k-1)-n}_tb}, v_trb,
5068          */
5069         {
5070           const PetscInt oc    = 0;
5071           const PetscInt ofb   = oc    + PetscSqr(k-1)*(k-1);
5072           const PetscInt oft   = ofb   + PetscSqr(k-1);
5073           const PetscInt off   = oft   + PetscSqr(k-1);
5074           const PetscInt ofk   = off   + PetscSqr(k-1);
5075           const PetscInt ofr   = ofk   + PetscSqr(k-1);
5076           const PetscInt ofl   = ofr   + PetscSqr(k-1);
5077           const PetscInt oebl  = ofl   + PetscSqr(k-1);
5078           const PetscInt oebb  = oebl  + (k-1);
5079           const PetscInt oebr  = oebb  + (k-1);
5080           const PetscInt oebf  = oebr  + (k-1);
5081           const PetscInt oetf  = oebf  + (k-1);
5082           const PetscInt oetr  = oetf  + (k-1);
5083           const PetscInt oetb  = oetr  + (k-1);
5084           const PetscInt oetl  = oetb  + (k-1);
5085           const PetscInt oerf  = oetl  + (k-1);
5086           const PetscInt oelf  = oerf  + (k-1);
5087           const PetscInt oelb  = oelf  + (k-1);
5088           const PetscInt oerb  = oelb  + (k-1);
5089           const PetscInt ovblf = oerb  + (k-1);
5090           const PetscInt ovblb = ovblf + 1;
5091           const PetscInt ovbrb = ovblb + 1;
5092           const PetscInt ovbrf = ovbrb + 1;
5093           const PetscInt ovtlf = ovbrf + 1;
5094           const PetscInt ovtrf = ovtlf + 1;
5095           const PetscInt ovtrb = ovtrf + 1;
5096           const PetscInt ovtlb = ovtrb + 1;
5097           PetscInt       o, n;
5098 
5099           /* Bottom Slice */
5100           /*   bottom */
5101           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovblf*Nc + c + foffset;
5102           for (o = oetf-1; o >= oebf; --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5103           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovbrf*Nc + c + foffset;
5104           /*   middle */
5105           for (i = 0; i < k-1; ++i) {
5106             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oebl+i)*Nc + c + foffset;
5107             for (n = 0; n < k-1; ++n) {o = ofb+n*(k-1)+i; for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;}
5108             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oebr+(k-2)-i)*Nc + c + foffset;
5109           }
5110           /*   top */
5111           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovblb*Nc + c + foffset;
5112           for (o = oebb; o < oebr; ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5113           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovbrb*Nc + c + foffset;
5114 
5115           /* Middle Slice */
5116           for (j = 0; j < k-1; ++j) {
5117             /*   bottom */
5118             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oelf+(k-2)-j)*Nc + c + foffset;
5119             for (o = off+j*(k-1); o < off+(j+1)*(k-1); ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5120             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oerf+j)*Nc + c + foffset;
5121             /*   middle */
5122             for (i = 0; i < k-1; ++i) {
5123               for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (ofl+i*(k-1)+j)*Nc + c + foffset;
5124               for (n = 0; n < k-1; ++n) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oc+(j*(k-1)+i)*(k-1)+n)*Nc + c + foffset;
5125               for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (ofr+j*(k-1)+i)*Nc + c + foffset;
5126             }
5127             /*   top */
5128             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oelb+j)*Nc + c + foffset;
5129             for (o = ofk+j*(k-1)+(k-2); o >= ofk+j*(k-1); --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5130             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oerb+(k-2)-j)*Nc + c + foffset;
5131           }
5132 
5133           /* Top Slice */
5134           /*   bottom */
5135           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtlf*Nc + c + foffset;
5136           for (o = oetf; o < oetr; ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5137           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtrf*Nc + c + foffset;
5138           /*   middle */
5139           for (i = 0; i < k-1; ++i) {
5140             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oetl+(k-2)-i)*Nc + c + foffset;
5141             for (n = 0; n < k-1; ++n) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oft+i*(k-1)+n)*Nc + c + foffset;
5142             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oetr+i)*Nc + c + foffset;
5143           }
5144           /*   top */
5145           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtlb*Nc + c + foffset;
5146           for (o = oetl-1; o >= oetb; --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5147           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtrb*Nc + c + foffset;
5148 
5149           foffset = offset;
5150         }
5151         break;
5152       default: SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_OUTOFRANGE, "No spectral ordering for dimension %D", d);
5153       }
5154     }
5155     if (offset != size) SETERRQ2(PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Number of permutation entries %D != %D", offset, size);
5156     /* Check permutation */
5157     {
5158       PetscInt *check;
5159 
5160       ierr = PetscMalloc1(size, &check);CHKERRQ(ierr);
5161       for (i = 0; i < size; ++i) {check[i] = -1; if (perm[i] < 0 || perm[i] >= size) SETERRQ2(PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Invalid permutation index p[%D] = %D", i, perm[i]);}
5162       for (i = 0; i < size; ++i) check[perm[i]] = i;
5163       for (i = 0; i < size; ++i) {if (check[i] < 0) SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Missing permutation index %D", i);}
5164       ierr = PetscFree(check);CHKERRQ(ierr);
5165     }
5166     ierr = PetscSectionSetClosurePermutation_Internal(section, (PetscObject) dm, d, size, PETSC_OWN_POINTER, perm);CHKERRQ(ierr);
5167   }
5168   PetscFunctionReturn(0);
5169 }
5170 
5171 PetscErrorCode DMPlexGetPointDualSpaceFEM(DM dm, PetscInt point, PetscInt field, PetscDualSpace *dspace)
5172 {
5173   PetscDS        prob;
5174   PetscInt       depth, Nf, h;
5175   DMLabel        label;
5176   PetscErrorCode ierr;
5177 
5178   PetscFunctionBeginHot;
5179   ierr = DMGetDS(dm, &prob);CHKERRQ(ierr);
5180   Nf      = prob->Nf;
5181   label   = dm->depthLabel;
5182   *dspace = NULL;
5183   if (field < Nf) {
5184     PetscObject disc = prob->disc[field];
5185 
5186     if (disc->classid == PETSCFE_CLASSID) {
5187       PetscDualSpace dsp;
5188 
5189       ierr = PetscFEGetDualSpace((PetscFE)disc,&dsp);CHKERRQ(ierr);
5190       ierr = DMLabelGetNumValues(label,&depth);CHKERRQ(ierr);
5191       ierr = DMLabelGetValue(label,point,&h);CHKERRQ(ierr);
5192       h    = depth - 1 - h;
5193       if (h) {
5194         ierr = PetscDualSpaceGetHeightSubspace(dsp,h,dspace);CHKERRQ(ierr);
5195       } else {
5196         *dspace = dsp;
5197       }
5198     }
5199   }
5200   PetscFunctionReturn(0);
5201 }
5202 
5203 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecGetClosure_Depth1_Static(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
5204 {
5205   PetscScalar    *array, *vArray;
5206   const PetscInt *cone, *coneO;
5207   PetscInt        pStart, pEnd, p, numPoints, size = 0, offset = 0;
5208   PetscErrorCode  ierr;
5209 
5210   PetscFunctionBeginHot;
5211   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
5212   ierr = DMPlexGetConeSize(dm, point, &numPoints);CHKERRQ(ierr);
5213   ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
5214   ierr = DMPlexGetConeOrientation(dm, point, &coneO);CHKERRQ(ierr);
5215   if (!values || !*values) {
5216     if ((point >= pStart) && (point < pEnd)) {
5217       PetscInt dof;
5218 
5219       ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
5220       size += dof;
5221     }
5222     for (p = 0; p < numPoints; ++p) {
5223       const PetscInt cp = cone[p];
5224       PetscInt       dof;
5225 
5226       if ((cp < pStart) || (cp >= pEnd)) continue;
5227       ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
5228       size += dof;
5229     }
5230     if (!values) {
5231       if (csize) *csize = size;
5232       PetscFunctionReturn(0);
5233     }
5234     ierr = DMGetWorkArray(dm, size, MPIU_SCALAR, &array);CHKERRQ(ierr);
5235   } else {
5236     array = *values;
5237   }
5238   size = 0;
5239   ierr = VecGetArray(v, &vArray);CHKERRQ(ierr);
5240   if ((point >= pStart) && (point < pEnd)) {
5241     PetscInt     dof, off, d;
5242     PetscScalar *varr;
5243 
5244     ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
5245     ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
5246     varr = &vArray[off];
5247     for (d = 0; d < dof; ++d, ++offset) {
5248       array[offset] = varr[d];
5249     }
5250     size += dof;
5251   }
5252   for (p = 0; p < numPoints; ++p) {
5253     const PetscInt cp = cone[p];
5254     PetscInt       o  = coneO[p];
5255     PetscInt       dof, off, d;
5256     PetscScalar   *varr;
5257 
5258     if ((cp < pStart) || (cp >= pEnd)) continue;
5259     ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
5260     ierr = PetscSectionGetOffset(section, cp, &off);CHKERRQ(ierr);
5261     varr = &vArray[off];
5262     if (o >= 0) {
5263       for (d = 0; d < dof; ++d, ++offset) {
5264         array[offset] = varr[d];
5265       }
5266     } else {
5267       for (d = dof-1; d >= 0; --d, ++offset) {
5268         array[offset] = varr[d];
5269       }
5270     }
5271     size += dof;
5272   }
5273   ierr = VecRestoreArray(v, &vArray);CHKERRQ(ierr);
5274   if (!*values) {
5275     if (csize) *csize = size;
5276     *values = array;
5277   } else {
5278     if (size > *csize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Size of input array %D < actual size %D", *csize, size);
5279     *csize = size;
5280   }
5281   PetscFunctionReturn(0);
5282 }
5283 
5284 /* Compress out points not in the section */
5285 PETSC_STATIC_INLINE PetscErrorCode CompressPoints_Private(PetscSection section, PetscInt *numPoints, PetscInt points[])
5286 {
5287   const PetscInt np = *numPoints;
5288   PetscInt       pStart, pEnd, p, q;
5289   PetscErrorCode ierr;
5290 
5291   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
5292   for (p = 0, q = 0; p < np; ++p) {
5293     const PetscInt r = points[p*2];
5294     if ((r >= pStart) && (r < pEnd)) {
5295       points[q*2]   = r;
5296       points[q*2+1] = points[p*2+1];
5297       ++q;
5298     }
5299   }
5300   *numPoints = q;
5301   return 0;
5302 }
5303 
5304 static PetscErrorCode DMPlexTransitiveClosure_Hybrid_Internal(DM dm, PetscInt point, PetscInt np, PetscInt *numPoints, PetscInt **points)
5305 {
5306   const PetscInt *cone, *ornt;
5307   PetscInt       *pts,  *closure = NULL;
5308   PetscInt        dim, coneSize, c, d, clSize, cl;
5309   PetscErrorCode  ierr;
5310 
5311   PetscFunctionBeginHot;
5312   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
5313   ierr = DMPlexGetConeSize(dm, point, &coneSize);CHKERRQ(ierr);
5314   ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
5315   ierr = DMPlexGetConeOrientation(dm, point, &ornt);CHKERRQ(ierr);
5316   ierr = DMPlexGetTransitiveClosure(dm, cone[0], PETSC_TRUE, &clSize, &closure);CHKERRQ(ierr);
5317   ierr = DMGetWorkArray(dm, np*2, MPIU_INT, &pts);CHKERRQ(ierr);
5318   c    = 0;
5319   pts[c*2+0] = point;
5320   pts[c*2+1] = 0;
5321   ++c;
5322   for (cl = 0; cl < clSize*2; cl += 2, ++c) {pts[c*2+0] = closure[cl]; pts[c*2+1] = closure[cl+1];}
5323   ierr = DMPlexGetTransitiveClosure(dm, cone[1], PETSC_TRUE, &clSize, &closure);CHKERRQ(ierr);
5324   for (cl = 0; cl < clSize*2; cl += 2, ++c) {pts[c*2+0] = closure[cl]; pts[c*2+1] = closure[cl+1];}
5325   ierr = DMPlexRestoreTransitiveClosure(dm, cone[0], PETSC_TRUE, &clSize, &closure);CHKERRQ(ierr);
5326   if (dim >= 2) {
5327     for (d = 2; d < coneSize; ++d, ++c) {pts[c*2+0] = cone[d]; pts[c*2+1] = ornt[d];}
5328   }
5329   if (dim >= 3) {
5330     for (d = 2; d < coneSize; ++d) {
5331       const PetscInt  fpoint = cone[d];
5332       const PetscInt *fcone;
5333       PetscInt        fconeSize, fc, i;
5334 
5335       ierr = DMPlexGetConeSize(dm, fpoint, &fconeSize);CHKERRQ(ierr);
5336       ierr = DMPlexGetCone(dm, fpoint, &fcone);CHKERRQ(ierr);
5337       for (fc = 0; fc < fconeSize; ++fc) {
5338         for (i = 0; i < c; ++i) if (pts[i*2] == fcone[fc]) break;
5339         if (i == c) {pts[c*2+0] = fcone[fc]; pts[c*2+1] = 0; ++c;}
5340       }
5341     }
5342   }
5343   if (c != np) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid closure for hybrid point %D, size %D != %D", point, c, np);
5344   *numPoints = np;
5345   *points    = pts;
5346   PetscFunctionReturn(0);
5347 }
5348 
5349 /* Compressed closure does not apply closure permutation */
5350 PetscErrorCode DMPlexGetCompressedClosure(DM dm, PetscSection section, PetscInt point, PetscInt *numPoints, PetscInt **points, PetscSection *clSec, IS *clPoints, const PetscInt **clp)
5351 {
5352   const PetscInt *cla = NULL;
5353   PetscInt       np, *pts = NULL;
5354   PetscErrorCode ierr;
5355 
5356   PetscFunctionBeginHot;
5357   ierr = PetscSectionGetClosureIndex(section, (PetscObject) dm, clSec, clPoints);CHKERRQ(ierr);
5358   if (*clPoints) {
5359     PetscInt dof, off;
5360 
5361     ierr = PetscSectionGetDof(*clSec, point, &dof);CHKERRQ(ierr);
5362     ierr = PetscSectionGetOffset(*clSec, point, &off);CHKERRQ(ierr);
5363     ierr = ISGetIndices(*clPoints, &cla);CHKERRQ(ierr);
5364     np   = dof/2;
5365     pts  = (PetscInt *) &cla[off];
5366   } else {
5367     DMPolytopeType ct;
5368 
5369     /* Do not make the label if it does not exist */
5370     if (!dm->celltypeLabel) {ct = DM_POLYTOPE_POINT;}
5371     else                    {ierr = DMPlexGetCellType(dm, point, &ct);CHKERRQ(ierr);}
5372     switch (ct) {
5373       case DM_POLYTOPE_SEG_PRISM_TENSOR:
5374         ierr = DMPlexTransitiveClosure_Hybrid_Internal(dm, point, 9, &np, &pts);CHKERRQ(ierr);
5375         break;
5376       case DM_POLYTOPE_TRI_PRISM_TENSOR:
5377         ierr = DMPlexTransitiveClosure_Hybrid_Internal(dm, point, 21, &np, &pts);CHKERRQ(ierr);
5378         break;
5379       case DM_POLYTOPE_QUAD_PRISM_TENSOR:
5380         ierr = DMPlexTransitiveClosure_Hybrid_Internal(dm, point, 27, &np, &pts);CHKERRQ(ierr);
5381         break;
5382       default:
5383         ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &np, &pts);CHKERRQ(ierr);
5384     }
5385     ierr = CompressPoints_Private(section, &np, pts);CHKERRQ(ierr);
5386   }
5387   *numPoints = np;
5388   *points    = pts;
5389   *clp       = cla;
5390   PetscFunctionReturn(0);
5391 }
5392 
5393 PetscErrorCode DMPlexRestoreCompressedClosure(DM dm, PetscSection section, PetscInt point, PetscInt *numPoints, PetscInt **points, PetscSection *clSec, IS *clPoints, const PetscInt **clp)
5394 {
5395   PetscErrorCode ierr;
5396 
5397   PetscFunctionBeginHot;
5398   if (!*clPoints) {
5399     ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, numPoints, points);CHKERRQ(ierr);
5400   } else {
5401     ierr = ISRestoreIndices(*clPoints, clp);CHKERRQ(ierr);
5402   }
5403   *numPoints = 0;
5404   *points    = NULL;
5405   *clSec     = NULL;
5406   *clPoints  = NULL;
5407   *clp       = NULL;
5408   PetscFunctionReturn(0);
5409 }
5410 
5411 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecGetClosure_Static(DM dm, PetscSection section, PetscInt numPoints, const PetscInt points[], const PetscInt clperm[], const PetscScalar vArray[], PetscInt *size, PetscScalar array[])
5412 {
5413   PetscInt          offset = 0, p;
5414   const PetscInt    **perms = NULL;
5415   const PetscScalar **flips = NULL;
5416   PetscErrorCode    ierr;
5417 
5418   PetscFunctionBeginHot;
5419   *size = 0;
5420   ierr = PetscSectionGetPointSyms(section,numPoints,points,&perms,&flips);CHKERRQ(ierr);
5421   for (p = 0; p < numPoints; p++) {
5422     const PetscInt    point = points[2*p];
5423     const PetscInt    *perm = perms ? perms[p] : NULL;
5424     const PetscScalar *flip = flips ? flips[p] : NULL;
5425     PetscInt          dof, off, d;
5426     const PetscScalar *varr;
5427 
5428     ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
5429     ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
5430     varr = &vArray[off];
5431     if (clperm) {
5432       if (perm) {
5433         for (d = 0; d < dof; d++) array[clperm[offset + perm[d]]]  = varr[d];
5434       } else {
5435         for (d = 0; d < dof; d++) array[clperm[offset +      d ]]  = varr[d];
5436       }
5437       if (flip) {
5438         for (d = 0; d < dof; d++) array[clperm[offset +      d ]] *= flip[d];
5439       }
5440     } else {
5441       if (perm) {
5442         for (d = 0; d < dof; d++) array[offset + perm[d]]  = varr[d];
5443       } else {
5444         for (d = 0; d < dof; d++) array[offset +      d ]  = varr[d];
5445       }
5446       if (flip) {
5447         for (d = 0; d < dof; d++) array[offset +      d ] *= flip[d];
5448       }
5449     }
5450     offset += dof;
5451   }
5452   ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms,&flips);CHKERRQ(ierr);
5453   *size = offset;
5454   PetscFunctionReturn(0);
5455 }
5456 
5457 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecGetClosure_Fields_Static(DM dm, PetscSection section, PetscInt numPoints, const PetscInt points[], PetscInt numFields, const PetscInt clperm[], const PetscScalar vArray[], PetscInt *size, PetscScalar array[])
5458 {
5459   PetscInt          offset = 0, f;
5460   PetscErrorCode    ierr;
5461 
5462   PetscFunctionBeginHot;
5463   *size = 0;
5464   for (f = 0; f < numFields; ++f) {
5465     PetscInt          p;
5466     const PetscInt    **perms = NULL;
5467     const PetscScalar **flips = NULL;
5468 
5469     ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
5470     for (p = 0; p < numPoints; p++) {
5471       const PetscInt    point = points[2*p];
5472       PetscInt          fdof, foff, b;
5473       const PetscScalar *varr;
5474       const PetscInt    *perm = perms ? perms[p] : NULL;
5475       const PetscScalar *flip = flips ? flips[p] : NULL;
5476 
5477       ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
5478       ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
5479       varr = &vArray[foff];
5480       if (clperm) {
5481         if (perm) {for (b = 0; b < fdof; b++) {array[clperm[offset + perm[b]]]  = varr[b];}}
5482         else      {for (b = 0; b < fdof; b++) {array[clperm[offset +      b ]]  = varr[b];}}
5483         if (flip) {for (b = 0; b < fdof; b++) {array[clperm[offset +      b ]] *= flip[b];}}
5484       } else {
5485         if (perm) {for (b = 0; b < fdof; b++) {array[offset + perm[b]]  = varr[b];}}
5486         else      {for (b = 0; b < fdof; b++) {array[offset +      b ]  = varr[b];}}
5487         if (flip) {for (b = 0; b < fdof; b++) {array[offset +      b ] *= flip[b];}}
5488       }
5489       offset += fdof;
5490     }
5491     ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
5492   }
5493   *size = offset;
5494   PetscFunctionReturn(0);
5495 }
5496 
5497 /*@C
5498   DMPlexVecGetClosure - Get an array of the values on the closure of 'point'
5499 
5500   Not collective
5501 
5502   Input Parameters:
5503 + dm - The DM
5504 . section - The section describing the layout in v, or NULL to use the default section
5505 . v - The local vector
5506 . point - The point in the DM
5507 . csize - The size of the input values array, or NULL
5508 - values - An array to use for the values, or NULL to have it allocated automatically
5509 
5510   Output Parameters:
5511 + csize - The number of values in the closure
5512 - values - The array of values. If the user provided NULL, it is a borrowed array and should not be freed
5513 
5514 $ Note that DMPlexVecGetClosure/DMPlexVecRestoreClosure only allocates the values array if it set to NULL in the
5515 $ calling function. This is because DMPlexVecGetClosure() is typically called in the inner loop of a Vec or Mat
5516 $ assembly function, and a user may already have allocated storage for this operation.
5517 $
5518 $ A typical use could be
5519 $
5520 $  values = NULL;
5521 $  ierr = DMPlexVecGetClosure(dm, NULL, v, p, &clSize, &values);CHKERRQ(ierr);
5522 $  for (cl = 0; cl < clSize; ++cl) {
5523 $    <Compute on closure>
5524 $  }
5525 $  ierr = DMPlexVecRestoreClosure(dm, NULL, v, p, &clSize, &values);CHKERRQ(ierr);
5526 $
5527 $ or
5528 $
5529 $  PetscMalloc1(clMaxSize, &values);
5530 $  for (p = pStart; p < pEnd; ++p) {
5531 $    clSize = clMaxSize;
5532 $    ierr = DMPlexVecGetClosure(dm, NULL, v, p, &clSize, &values);CHKERRQ(ierr);
5533 $    for (cl = 0; cl < clSize; ++cl) {
5534 $      <Compute on closure>
5535 $    }
5536 $  }
5537 $  PetscFree(values);
5538 
5539   Fortran Notes:
5540   Since it returns an array, this routine is only available in Fortran 90, and you must
5541   include petsc.h90 in your code.
5542 
5543   The csize argument is not present in the Fortran 90 binding since it is internal to the array.
5544 
5545   Level: intermediate
5546 
5547 .seealso DMPlexVecRestoreClosure(), DMPlexVecSetClosure(), DMPlexMatSetClosure()
5548 @*/
5549 PetscErrorCode DMPlexVecGetClosure(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
5550 {
5551   PetscSection       clSection;
5552   IS                 clPoints;
5553   PetscInt          *points = NULL;
5554   const PetscInt    *clp, *perm;
5555   PetscInt           depth, numFields, numPoints, asize;
5556   PetscErrorCode     ierr;
5557 
5558   PetscFunctionBeginHot;
5559   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5560   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
5561   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
5562   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
5563   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
5564   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5565   if (depth == 1 && numFields < 2) {
5566     ierr = DMPlexVecGetClosure_Depth1_Static(dm, section, v, point, csize, values);CHKERRQ(ierr);
5567     PetscFunctionReturn(0);
5568   }
5569   /* Get points */
5570   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5571   /* Get sizes */
5572   asize = 0;
5573   for (PetscInt p = 0; p < numPoints*2; p += 2) {
5574     PetscInt dof;
5575     ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5576     asize += dof;
5577   }
5578   if (values) {
5579     const PetscScalar *vArray;
5580     PetscInt          size;
5581 
5582     if (*values) {
5583       if (PetscUnlikely(*csize < asize)) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Provided array size %D not sufficient to hold closure size %D", *csize, asize);
5584     } else {ierr = DMGetWorkArray(dm, asize, MPIU_SCALAR, values);CHKERRQ(ierr);}
5585     ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, depth, asize, &perm);CHKERRQ(ierr);
5586     ierr = VecGetArrayRead(v, &vArray);CHKERRQ(ierr);
5587     /* Get values */
5588     if (numFields > 0) {ierr = DMPlexVecGetClosure_Fields_Static(dm, section, numPoints, points, numFields, perm, vArray, &size, *values);CHKERRQ(ierr);}
5589     else               {ierr = DMPlexVecGetClosure_Static(dm, section, numPoints, points, perm, vArray, &size, *values);CHKERRQ(ierr);}
5590     if (PetscUnlikely(asize != size)) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Section size %D does not match Vec closure size %D", asize, size);
5591     /* Cleanup array */
5592     ierr = VecRestoreArrayRead(v, &vArray);CHKERRQ(ierr);
5593   }
5594   if (csize) *csize = asize;
5595   /* Cleanup points */
5596   ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5597   PetscFunctionReturn(0);
5598 }
5599 
5600 PetscErrorCode DMPlexVecGetClosureAtDepth_Internal(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt depth, PetscInt *csize, PetscScalar *values[])
5601 {
5602   DMLabel            depthLabel;
5603   PetscSection       clSection;
5604   IS                 clPoints;
5605   PetscScalar       *array;
5606   const PetscScalar *vArray;
5607   PetscInt          *points = NULL;
5608   const PetscInt    *clp, *perm = NULL;
5609   PetscInt           mdepth, numFields, numPoints, Np = 0, p, clsize, size;
5610   PetscErrorCode     ierr;
5611 
5612   PetscFunctionBeginHot;
5613   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5614   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
5615   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
5616   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
5617   ierr = DMPlexGetDepth(dm, &mdepth);CHKERRQ(ierr);
5618   ierr = DMPlexGetDepthLabel(dm, &depthLabel);CHKERRQ(ierr);
5619   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5620   if (mdepth == 1 && numFields < 2) {
5621     ierr = DMPlexVecGetClosure_Depth1_Static(dm, section, v, point, csize, values);CHKERRQ(ierr);
5622     PetscFunctionReturn(0);
5623   }
5624   /* Get points */
5625   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5626   for (clsize=0,p=0; p<Np; p++) {
5627     PetscInt dof;
5628     ierr = PetscSectionGetDof(section, points[2*p], &dof);CHKERRQ(ierr);
5629     clsize += dof;
5630   }
5631   ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, depth, clsize, &perm);CHKERRQ(ierr);
5632   /* Filter points */
5633   for (p = 0; p < numPoints*2; p += 2) {
5634     PetscInt dep;
5635 
5636     ierr = DMLabelGetValue(depthLabel, points[p], &dep);CHKERRQ(ierr);
5637     if (dep != depth) continue;
5638     points[Np*2+0] = points[p];
5639     points[Np*2+1] = points[p+1];
5640     ++Np;
5641   }
5642   /* Get array */
5643   if (!values || !*values) {
5644     PetscInt asize = 0, dof;
5645 
5646     for (p = 0; p < Np*2; p += 2) {
5647       ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5648       asize += dof;
5649     }
5650     if (!values) {
5651       ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5652       if (csize) *csize = asize;
5653       PetscFunctionReturn(0);
5654     }
5655     ierr = DMGetWorkArray(dm, asize, MPIU_SCALAR, &array);CHKERRQ(ierr);
5656   } else {
5657     array = *values;
5658   }
5659   ierr = VecGetArrayRead(v, &vArray);CHKERRQ(ierr);
5660   /* Get values */
5661   if (numFields > 0) {ierr = DMPlexVecGetClosure_Fields_Static(dm, section, Np, points, numFields, perm, vArray, &size, array);CHKERRQ(ierr);}
5662   else               {ierr = DMPlexVecGetClosure_Static(dm, section, Np, points, perm, vArray, &size, array);CHKERRQ(ierr);}
5663   /* Cleanup points */
5664   ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5665   /* Cleanup array */
5666   ierr = VecRestoreArrayRead(v, &vArray);CHKERRQ(ierr);
5667   if (!*values) {
5668     if (csize) *csize = size;
5669     *values = array;
5670   } else {
5671     if (size > *csize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Size of input array %D < actual size %D", *csize, size);
5672     *csize = size;
5673   }
5674   PetscFunctionReturn(0);
5675 }
5676 
5677 /*@C
5678   DMPlexVecRestoreClosure - Restore the array of the values on the closure of 'point'
5679 
5680   Not collective
5681 
5682   Input Parameters:
5683 + dm - The DM
5684 . section - The section describing the layout in v, or NULL to use the default section
5685 . v - The local vector
5686 . point - The point in the DM
5687 . csize - The number of values in the closure, or NULL
5688 - values - The array of values, which is a borrowed array and should not be freed
5689 
5690   Note that the array values are discarded and not copied back into v. In order to copy values back to v, use DMPlexVecSetClosure()
5691 
5692   Fortran Notes:
5693   Since it returns an array, this routine is only available in Fortran 90, and you must
5694   include petsc.h90 in your code.
5695 
5696   The csize argument is not present in the Fortran 90 binding since it is internal to the array.
5697 
5698   Level: intermediate
5699 
5700 .seealso DMPlexVecGetClosure(), DMPlexVecSetClosure(), DMPlexMatSetClosure()
5701 @*/
5702 PetscErrorCode DMPlexVecRestoreClosure(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
5703 {
5704   PetscInt       size = 0;
5705   PetscErrorCode ierr;
5706 
5707   PetscFunctionBegin;
5708   /* Should work without recalculating size */
5709   ierr = DMRestoreWorkArray(dm, size, MPIU_SCALAR, (void*) values);CHKERRQ(ierr);
5710   *values = NULL;
5711   PetscFunctionReturn(0);
5712 }
5713 
5714 PETSC_STATIC_INLINE void add   (PetscScalar *x, PetscScalar y) {*x += y;}
5715 PETSC_STATIC_INLINE void insert(PetscScalar *x, PetscScalar y) {*x  = y;}
5716 
5717 PETSC_STATIC_INLINE PetscErrorCode updatePoint_private(PetscSection section, PetscInt point, PetscInt dof, void (*fuse)(PetscScalar*, PetscScalar), PetscBool setBC, const PetscInt perm[], const PetscScalar flip[], const PetscInt clperm[], const PetscScalar values[], PetscInt offset, PetscScalar array[])
5718 {
5719   PetscInt        cdof;   /* The number of constraints on this point */
5720   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
5721   PetscScalar    *a;
5722   PetscInt        off, cind = 0, k;
5723   PetscErrorCode  ierr;
5724 
5725   PetscFunctionBegin;
5726   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
5727   ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
5728   a    = &array[off];
5729   if (!cdof || setBC) {
5730     if (clperm) {
5731       if (perm) {for (k = 0; k < dof; ++k) {fuse(&a[k], values[clperm[offset+perm[k]]] * (flip ? flip[perm[k]] : 1.));}}
5732       else      {for (k = 0; k < dof; ++k) {fuse(&a[k], values[clperm[offset+     k ]] * (flip ? flip[     k ] : 1.));}}
5733     } else {
5734       if (perm) {for (k = 0; k < dof; ++k) {fuse(&a[k], values[offset+perm[k]] * (flip ? flip[perm[k]] : 1.));}}
5735       else      {for (k = 0; k < dof; ++k) {fuse(&a[k], values[offset+     k ] * (flip ? flip[     k ] : 1.));}}
5736     }
5737   } else {
5738     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
5739     if (clperm) {
5740       if (perm) {for (k = 0; k < dof; ++k) {
5741           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
5742           fuse(&a[k], values[clperm[offset+perm[k]]] * (flip ? flip[perm[k]] : 1.));
5743         }
5744       } else {
5745         for (k = 0; k < dof; ++k) {
5746           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
5747           fuse(&a[k], values[clperm[offset+     k ]] * (flip ? flip[     k ] : 1.));
5748         }
5749       }
5750     } else {
5751       if (perm) {
5752         for (k = 0; k < dof; ++k) {
5753           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
5754           fuse(&a[k], values[offset+perm[k]] * (flip ? flip[perm[k]] : 1.));
5755         }
5756       } else {
5757         for (k = 0; k < dof; ++k) {
5758           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
5759           fuse(&a[k], values[offset+     k ] * (flip ? flip[     k ] : 1.));
5760         }
5761       }
5762     }
5763   }
5764   PetscFunctionReturn(0);
5765 }
5766 
5767 PETSC_STATIC_INLINE PetscErrorCode updatePointBC_private(PetscSection section, PetscInt point, PetscInt dof, void (*fuse)(PetscScalar*, PetscScalar), const PetscInt perm[], const PetscScalar flip[], const PetscInt clperm[], const PetscScalar values[], PetscInt offset, PetscScalar array[])
5768 {
5769   PetscInt        cdof;   /* The number of constraints on this point */
5770   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
5771   PetscScalar    *a;
5772   PetscInt        off, cind = 0, k;
5773   PetscErrorCode  ierr;
5774 
5775   PetscFunctionBegin;
5776   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
5777   ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
5778   a    = &array[off];
5779   if (cdof) {
5780     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
5781     if (clperm) {
5782       if (perm) {
5783         for (k = 0; k < dof; ++k) {
5784           if ((cind < cdof) && (k == cdofs[cind])) {
5785             fuse(&a[k], values[clperm[offset+perm[k]]] * (flip ? flip[perm[k]] : 1.));
5786             cind++;
5787           }
5788         }
5789       } else {
5790         for (k = 0; k < dof; ++k) {
5791           if ((cind < cdof) && (k == cdofs[cind])) {
5792             fuse(&a[k], values[clperm[offset+     k ]] * (flip ? flip[     k ] : 1.));
5793             cind++;
5794           }
5795         }
5796       }
5797     } else {
5798       if (perm) {
5799         for (k = 0; k < dof; ++k) {
5800           if ((cind < cdof) && (k == cdofs[cind])) {
5801             fuse(&a[k], values[offset+perm[k]] * (flip ? flip[perm[k]] : 1.));
5802             cind++;
5803           }
5804         }
5805       } else {
5806         for (k = 0; k < dof; ++k) {
5807           if ((cind < cdof) && (k == cdofs[cind])) {
5808             fuse(&a[k], values[offset+     k ] * (flip ? flip[     k ] : 1.));
5809             cind++;
5810           }
5811         }
5812       }
5813     }
5814   }
5815   PetscFunctionReturn(0);
5816 }
5817 
5818 PETSC_STATIC_INLINE PetscErrorCode updatePointFields_private(PetscSection section, PetscInt point, const PetscInt *perm, const PetscScalar *flip, PetscInt f, void (*fuse)(PetscScalar*, PetscScalar), PetscBool setBC, const PetscInt clperm[], const PetscScalar values[], PetscInt *offset, PetscScalar array[])
5819 {
5820   PetscScalar    *a;
5821   PetscInt        fdof, foff, fcdof, foffset = *offset;
5822   const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
5823   PetscInt        cind = 0, b;
5824   PetscErrorCode  ierr;
5825 
5826   PetscFunctionBegin;
5827   ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
5828   ierr = PetscSectionGetFieldConstraintDof(section, point, f, &fcdof);CHKERRQ(ierr);
5829   ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
5830   a    = &array[foff];
5831   if (!fcdof || setBC) {
5832     if (clperm) {
5833       if (perm) {for (b = 0; b < fdof; b++) {fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));}}
5834       else      {for (b = 0; b < fdof; b++) {fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));}}
5835     } else {
5836       if (perm) {for (b = 0; b < fdof; b++) {fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));}}
5837       else      {for (b = 0; b < fdof; b++) {fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));}}
5838     }
5839   } else {
5840     ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
5841     if (clperm) {
5842       if (perm) {
5843         for (b = 0; b < fdof; b++) {
5844           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
5845           fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));
5846         }
5847       } else {
5848         for (b = 0; b < fdof; b++) {
5849           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
5850           fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));
5851         }
5852       }
5853     } else {
5854       if (perm) {
5855         for (b = 0; b < fdof; b++) {
5856           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
5857           fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));
5858         }
5859       } else {
5860         for (b = 0; b < fdof; b++) {
5861           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
5862           fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));
5863         }
5864       }
5865     }
5866   }
5867   *offset += fdof;
5868   PetscFunctionReturn(0);
5869 }
5870 
5871 PETSC_STATIC_INLINE PetscErrorCode updatePointFieldsBC_private(PetscSection section, PetscInt point, const PetscInt perm[], const PetscScalar flip[], PetscInt f, PetscInt Ncc, const PetscInt comps[], void (*fuse)(PetscScalar*, PetscScalar), const PetscInt clperm[], const PetscScalar values[], PetscInt *offset, PetscScalar array[])
5872 {
5873   PetscScalar    *a;
5874   PetscInt        fdof, foff, fcdof, foffset = *offset;
5875   const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
5876   PetscInt        Nc, cind = 0, ncind = 0, b;
5877   PetscBool       ncSet, fcSet;
5878   PetscErrorCode  ierr;
5879 
5880   PetscFunctionBegin;
5881   ierr = PetscSectionGetFieldComponents(section, f, &Nc);CHKERRQ(ierr);
5882   ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
5883   ierr = PetscSectionGetFieldConstraintDof(section, point, f, &fcdof);CHKERRQ(ierr);
5884   ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
5885   a    = &array[foff];
5886   if (fcdof) {
5887     /* We just override fcdof and fcdofs with Ncc and comps */
5888     ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
5889     if (clperm) {
5890       if (perm) {
5891         if (comps) {
5892           for (b = 0; b < fdof; b++) {
5893             ncSet = fcSet = PETSC_FALSE;
5894             if (b%Nc == comps[ncind]) {ncind = (ncind+1)%Ncc; ncSet = PETSC_TRUE;}
5895             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
5896             if (ncSet && fcSet) {fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));}
5897           }
5898         } else {
5899           for (b = 0; b < fdof; b++) {
5900             if ((cind < fcdof) && (b == fcdofs[cind])) {
5901               fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));
5902               ++cind;
5903             }
5904           }
5905         }
5906       } else {
5907         if (comps) {
5908           for (b = 0; b < fdof; b++) {
5909             ncSet = fcSet = PETSC_FALSE;
5910             if (b%Nc == comps[ncind]) {ncind = (ncind+1)%Ncc; ncSet = PETSC_TRUE;}
5911             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
5912             if (ncSet && fcSet) {fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));}
5913           }
5914         } else {
5915           for (b = 0; b < fdof; b++) {
5916             if ((cind < fcdof) && (b == fcdofs[cind])) {
5917               fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));
5918               ++cind;
5919             }
5920           }
5921         }
5922       }
5923     } else {
5924       if (perm) {
5925         if (comps) {
5926           for (b = 0; b < fdof; b++) {
5927             ncSet = fcSet = PETSC_FALSE;
5928             if (b%Nc == comps[ncind]) {ncind = (ncind+1)%Ncc; ncSet = PETSC_TRUE;}
5929             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
5930             if (ncSet && fcSet) {fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));}
5931           }
5932         } else {
5933           for (b = 0; b < fdof; b++) {
5934             if ((cind < fcdof) && (b == fcdofs[cind])) {
5935               fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));
5936               ++cind;
5937             }
5938           }
5939         }
5940       } else {
5941         if (comps) {
5942           for (b = 0; b < fdof; b++) {
5943             ncSet = fcSet = PETSC_FALSE;
5944             if (b%Nc == comps[ncind]) {ncind = (ncind+1)%Ncc; ncSet = PETSC_TRUE;}
5945             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
5946             if (ncSet && fcSet) {fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));}
5947           }
5948         } else {
5949           for (b = 0; b < fdof; b++) {
5950             if ((cind < fcdof) && (b == fcdofs[cind])) {
5951               fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));
5952               ++cind;
5953             }
5954           }
5955         }
5956       }
5957     }
5958   }
5959   *offset += fdof;
5960   PetscFunctionReturn(0);
5961 }
5962 
5963 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecSetClosure_Depth1_Static(DM dm, PetscSection section, Vec v, PetscInt point, const PetscScalar values[], InsertMode mode)
5964 {
5965   PetscScalar    *array;
5966   const PetscInt *cone, *coneO;
5967   PetscInt        pStart, pEnd, p, numPoints, off, dof;
5968   PetscErrorCode  ierr;
5969 
5970   PetscFunctionBeginHot;
5971   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
5972   ierr = DMPlexGetConeSize(dm, point, &numPoints);CHKERRQ(ierr);
5973   ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
5974   ierr = DMPlexGetConeOrientation(dm, point, &coneO);CHKERRQ(ierr);
5975   ierr = VecGetArray(v, &array);CHKERRQ(ierr);
5976   for (p = 0, off = 0; p <= numPoints; ++p, off += dof) {
5977     const PetscInt cp = !p ? point : cone[p-1];
5978     const PetscInt o  = !p ? 0     : coneO[p-1];
5979 
5980     if ((cp < pStart) || (cp >= pEnd)) {dof = 0; continue;}
5981     ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
5982     /* ADD_VALUES */
5983     {
5984       const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
5985       PetscScalar    *a;
5986       PetscInt        cdof, coff, cind = 0, k;
5987 
5988       ierr = PetscSectionGetConstraintDof(section, cp, &cdof);CHKERRQ(ierr);
5989       ierr = PetscSectionGetOffset(section, cp, &coff);CHKERRQ(ierr);
5990       a    = &array[coff];
5991       if (!cdof) {
5992         if (o >= 0) {
5993           for (k = 0; k < dof; ++k) {
5994             a[k] += values[off+k];
5995           }
5996         } else {
5997           for (k = 0; k < dof; ++k) {
5998             a[k] += values[off+dof-k-1];
5999           }
6000         }
6001       } else {
6002         ierr = PetscSectionGetConstraintIndices(section, cp, &cdofs);CHKERRQ(ierr);
6003         if (o >= 0) {
6004           for (k = 0; k < dof; ++k) {
6005             if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
6006             a[k] += values[off+k];
6007           }
6008         } else {
6009           for (k = 0; k < dof; ++k) {
6010             if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
6011             a[k] += values[off+dof-k-1];
6012           }
6013         }
6014       }
6015     }
6016   }
6017   ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
6018   PetscFunctionReturn(0);
6019 }
6020 
6021 /*@C
6022   DMPlexVecSetClosure - Set an array of the values on the closure of 'point'
6023 
6024   Not collective
6025 
6026   Input Parameters:
6027 + dm - The DM
6028 . section - The section describing the layout in v, or NULL to use the default section
6029 . v - The local vector
6030 . point - The point in the DM
6031 . values - The array of values
6032 - mode - The insert mode. One of INSERT_ALL_VALUES, ADD_ALL_VALUES, INSERT_VALUES, ADD_VALUES, INSERT_BC_VALUES, and ADD_BC_VALUES,
6033          where INSERT_ALL_VALUES and ADD_ALL_VALUES also overwrite boundary conditions.
6034 
6035   Fortran Notes:
6036   This routine is only available in Fortran 90, and you must include petsc.h90 in your code.
6037 
6038   Level: intermediate
6039 
6040 .seealso DMPlexVecGetClosure(), DMPlexMatSetClosure()
6041 @*/
6042 PetscErrorCode DMPlexVecSetClosure(DM dm, PetscSection section, Vec v, PetscInt point, const PetscScalar values[], InsertMode mode)
6043 {
6044   PetscSection    clSection;
6045   IS              clPoints;
6046   PetscScalar    *array;
6047   PetscInt       *points = NULL;
6048   const PetscInt *clp, *clperm = NULL;
6049   PetscInt        depth, numFields, numPoints, p, clsize;
6050   PetscErrorCode  ierr;
6051 
6052   PetscFunctionBeginHot;
6053   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6054   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
6055   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
6056   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
6057   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
6058   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
6059   if (depth == 1 && numFields < 2 && mode == ADD_VALUES) {
6060     ierr = DMPlexVecSetClosure_Depth1_Static(dm, section, v, point, values, mode);CHKERRQ(ierr);
6061     PetscFunctionReturn(0);
6062   }
6063   /* Get points */
6064   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
6065   for (clsize=0,p=0; p<numPoints; p++) {
6066     PetscInt dof;
6067     ierr = PetscSectionGetDof(section, points[2*p], &dof);CHKERRQ(ierr);
6068     clsize += dof;
6069   }
6070   ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, depth, clsize, &clperm);CHKERRQ(ierr);
6071   /* Get array */
6072   ierr = VecGetArray(v, &array);CHKERRQ(ierr);
6073   /* Get values */
6074   if (numFields > 0) {
6075     PetscInt offset = 0, f;
6076     for (f = 0; f < numFields; ++f) {
6077       const PetscInt    **perms = NULL;
6078       const PetscScalar **flips = NULL;
6079 
6080       ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
6081       switch (mode) {
6082       case INSERT_VALUES:
6083         for (p = 0; p < numPoints; p++) {
6084           const PetscInt    point = points[2*p];
6085           const PetscInt    *perm = perms ? perms[p] : NULL;
6086           const PetscScalar *flip = flips ? flips[p] : NULL;
6087           updatePointFields_private(section, point, perm, flip, f, insert, PETSC_FALSE, clperm, values, &offset, array);
6088         } break;
6089       case INSERT_ALL_VALUES:
6090         for (p = 0; p < numPoints; p++) {
6091           const PetscInt    point = points[2*p];
6092           const PetscInt    *perm = perms ? perms[p] : NULL;
6093           const PetscScalar *flip = flips ? flips[p] : NULL;
6094           updatePointFields_private(section, point, perm, flip, f, insert, PETSC_TRUE, clperm, values, &offset, array);
6095         } break;
6096       case INSERT_BC_VALUES:
6097         for (p = 0; p < numPoints; p++) {
6098           const PetscInt    point = points[2*p];
6099           const PetscInt    *perm = perms ? perms[p] : NULL;
6100           const PetscScalar *flip = flips ? flips[p] : NULL;
6101           updatePointFieldsBC_private(section, point, perm, flip, f, -1, NULL, insert, clperm, values, &offset, array);
6102         } break;
6103       case ADD_VALUES:
6104         for (p = 0; p < numPoints; p++) {
6105           const PetscInt    point = points[2*p];
6106           const PetscInt    *perm = perms ? perms[p] : NULL;
6107           const PetscScalar *flip = flips ? flips[p] : NULL;
6108           updatePointFields_private(section, point, perm, flip, f, add, PETSC_FALSE, clperm, values, &offset, array);
6109         } break;
6110       case ADD_ALL_VALUES:
6111         for (p = 0; p < numPoints; p++) {
6112           const PetscInt    point = points[2*p];
6113           const PetscInt    *perm = perms ? perms[p] : NULL;
6114           const PetscScalar *flip = flips ? flips[p] : NULL;
6115           updatePointFields_private(section, point, perm, flip, f, add, PETSC_TRUE, clperm, values, &offset, array);
6116         } break;
6117       case ADD_BC_VALUES:
6118         for (p = 0; p < numPoints; p++) {
6119           const PetscInt    point = points[2*p];
6120           const PetscInt    *perm = perms ? perms[p] : NULL;
6121           const PetscScalar *flip = flips ? flips[p] : NULL;
6122           updatePointFieldsBC_private(section, point, perm, flip, f, -1, NULL, add, clperm, values, &offset, array);
6123         } break;
6124       default:
6125         SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %d", mode);
6126       }
6127       ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
6128     }
6129   } else {
6130     PetscInt dof, off;
6131     const PetscInt    **perms = NULL;
6132     const PetscScalar **flips = NULL;
6133 
6134     ierr = PetscSectionGetPointSyms(section,numPoints,points,&perms,&flips);CHKERRQ(ierr);
6135     switch (mode) {
6136     case INSERT_VALUES:
6137       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
6138         const PetscInt    point = points[2*p];
6139         const PetscInt    *perm = perms ? perms[p] : NULL;
6140         const PetscScalar *flip = flips ? flips[p] : NULL;
6141         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
6142         updatePoint_private(section, point, dof, insert, PETSC_FALSE, perm, flip, clperm, values, off, array);
6143       } break;
6144     case INSERT_ALL_VALUES:
6145       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
6146         const PetscInt    point = points[2*p];
6147         const PetscInt    *perm = perms ? perms[p] : NULL;
6148         const PetscScalar *flip = flips ? flips[p] : NULL;
6149         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
6150         updatePoint_private(section, point, dof, insert, PETSC_TRUE,  perm, flip, clperm, values, off, array);
6151       } break;
6152     case INSERT_BC_VALUES:
6153       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
6154         const PetscInt    point = points[2*p];
6155         const PetscInt    *perm = perms ? perms[p] : NULL;
6156         const PetscScalar *flip = flips ? flips[p] : NULL;
6157         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
6158         updatePointBC_private(section, point, dof, insert,  perm, flip, clperm, values, off, array);
6159       } break;
6160     case ADD_VALUES:
6161       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
6162         const PetscInt    point = points[2*p];
6163         const PetscInt    *perm = perms ? perms[p] : NULL;
6164         const PetscScalar *flip = flips ? flips[p] : NULL;
6165         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
6166         updatePoint_private(section, point, dof, add,    PETSC_FALSE, perm, flip, clperm, values, off, array);
6167       } break;
6168     case ADD_ALL_VALUES:
6169       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
6170         const PetscInt    point = points[2*p];
6171         const PetscInt    *perm = perms ? perms[p] : NULL;
6172         const PetscScalar *flip = flips ? flips[p] : NULL;
6173         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
6174         updatePoint_private(section, point, dof, add,    PETSC_TRUE,  perm, flip, clperm, values, off, array);
6175       } break;
6176     case ADD_BC_VALUES:
6177       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
6178         const PetscInt    point = points[2*p];
6179         const PetscInt    *perm = perms ? perms[p] : NULL;
6180         const PetscScalar *flip = flips ? flips[p] : NULL;
6181         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
6182         updatePointBC_private(section, point, dof, add,  perm, flip, clperm, values, off, array);
6183       } break;
6184     default:
6185       SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %d", mode);
6186     }
6187     ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms,&flips);CHKERRQ(ierr);
6188   }
6189   /* Cleanup points */
6190   ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
6191   /* Cleanup array */
6192   ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
6193   PetscFunctionReturn(0);
6194 }
6195 
6196 /* Check whether the given point is in the label. If not, update the offset to skip this point */
6197 PETSC_STATIC_INLINE PetscErrorCode CheckPoint_Private(DMLabel label, PetscInt labelId, PetscSection section, PetscInt point, PetscInt f, PetscInt *offset)
6198 {
6199   PetscFunctionBegin;
6200   if (label) {
6201     PetscInt       val, fdof;
6202     PetscErrorCode ierr;
6203 
6204     /* There is a problem with this:
6205          Suppose we have two label values, defining surfaces, interecting along a line in 3D. When we add cells to the label, the cells that
6206        touch both surfaces must pick a label value. Thus we miss setting values for the surface with that other value intersecting that cell.
6207        Thus I am only going to check val != -1, not val != labelId
6208     */
6209     ierr = DMLabelGetValue(label, point, &val);CHKERRQ(ierr);
6210     if (val < 0) {
6211       ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
6212       *offset += fdof;
6213       PetscFunctionReturn(1);
6214     }
6215   }
6216   PetscFunctionReturn(0);
6217 }
6218 
6219 /* Unlike DMPlexVecSetClosure(), this uses plex-native closure permutation, not a user-specified permutation such as DMPlexSetClosurePermutationTensor(). */
6220 PetscErrorCode DMPlexVecSetFieldClosure_Internal(DM dm, PetscSection section, Vec v, PetscBool fieldActive[], PetscInt point, PetscInt Ncc, const PetscInt comps[], DMLabel label, PetscInt labelId, const PetscScalar values[], InsertMode mode)
6221 {
6222   PetscSection      clSection;
6223   IS                clPoints;
6224   PetscScalar       *array;
6225   PetscInt          *points = NULL;
6226   const PetscInt    *clp;
6227   PetscInt          numFields, numPoints, p;
6228   PetscInt          offset = 0, f;
6229   PetscErrorCode    ierr;
6230 
6231   PetscFunctionBeginHot;
6232   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6233   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
6234   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
6235   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
6236   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
6237   /* Get points */
6238   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
6239   /* Get array */
6240   ierr = VecGetArray(v, &array);CHKERRQ(ierr);
6241   /* Get values */
6242   for (f = 0; f < numFields; ++f) {
6243     const PetscInt    **perms = NULL;
6244     const PetscScalar **flips = NULL;
6245 
6246     if (!fieldActive[f]) {
6247       for (p = 0; p < numPoints*2; p += 2) {
6248         PetscInt fdof;
6249         ierr = PetscSectionGetFieldDof(section, points[p], f, &fdof);CHKERRQ(ierr);
6250         offset += fdof;
6251       }
6252       continue;
6253     }
6254     ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
6255     switch (mode) {
6256     case INSERT_VALUES:
6257       for (p = 0; p < numPoints; p++) {
6258         const PetscInt    point = points[2*p];
6259         const PetscInt    *perm = perms ? perms[p] : NULL;
6260         const PetscScalar *flip = flips ? flips[p] : NULL;
6261         ierr = CheckPoint_Private(label, labelId, section, point, f, &offset); if (ierr) continue;
6262         updatePointFields_private(section, point, perm, flip, f, insert, PETSC_FALSE, NULL, values, &offset, array);
6263       } break;
6264     case INSERT_ALL_VALUES:
6265       for (p = 0; p < numPoints; p++) {
6266         const PetscInt    point = points[2*p];
6267         const PetscInt    *perm = perms ? perms[p] : NULL;
6268         const PetscScalar *flip = flips ? flips[p] : NULL;
6269         ierr = CheckPoint_Private(label, labelId, section, point, f, &offset); if (ierr) continue;
6270         updatePointFields_private(section, point, perm, flip, f, insert, PETSC_TRUE, NULL, values, &offset, array);
6271       } break;
6272     case INSERT_BC_VALUES:
6273       for (p = 0; p < numPoints; p++) {
6274         const PetscInt    point = points[2*p];
6275         const PetscInt    *perm = perms ? perms[p] : NULL;
6276         const PetscScalar *flip = flips ? flips[p] : NULL;
6277         ierr = CheckPoint_Private(label, labelId, section, point, f, &offset); if (ierr) continue;
6278         updatePointFieldsBC_private(section, point, perm, flip, f, Ncc, comps, insert, NULL, values, &offset, array);
6279       } break;
6280     case ADD_VALUES:
6281       for (p = 0; p < numPoints; p++) {
6282         const PetscInt    point = points[2*p];
6283         const PetscInt    *perm = perms ? perms[p] : NULL;
6284         const PetscScalar *flip = flips ? flips[p] : NULL;
6285         ierr = CheckPoint_Private(label, labelId, section, point, f, &offset); if (ierr) continue;
6286         updatePointFields_private(section, point, perm, flip, f, add, PETSC_FALSE, NULL, values, &offset, array);
6287       } break;
6288     case ADD_ALL_VALUES:
6289       for (p = 0; p < numPoints; p++) {
6290         const PetscInt    point = points[2*p];
6291         const PetscInt    *perm = perms ? perms[p] : NULL;
6292         const PetscScalar *flip = flips ? flips[p] : NULL;
6293         ierr = CheckPoint_Private(label, labelId, section, point, f, &offset); if (ierr) continue;
6294         updatePointFields_private(section, point, perm, flip, f, add, PETSC_TRUE, NULL, values, &offset, array);
6295       } break;
6296     default:
6297       SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %d", mode);
6298     }
6299     ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
6300   }
6301   /* Cleanup points */
6302   ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
6303   /* Cleanup array */
6304   ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
6305   PetscFunctionReturn(0);
6306 }
6307 
6308 static PetscErrorCode DMPlexPrintMatSetValues(PetscViewer viewer, Mat A, PetscInt point, PetscInt numRIndices, const PetscInt rindices[], PetscInt numCIndices, const PetscInt cindices[], const PetscScalar values[])
6309 {
6310   PetscMPIInt    rank;
6311   PetscInt       i, j;
6312   PetscErrorCode ierr;
6313 
6314   PetscFunctionBegin;
6315   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRMPI(ierr);
6316   ierr = PetscViewerASCIIPrintf(viewer, "[%d]mat for point %D\n", rank, point);CHKERRQ(ierr);
6317   for (i = 0; i < numRIndices; i++) {ierr = PetscViewerASCIIPrintf(viewer, "[%d]mat row indices[%D] = %D\n", rank, i, rindices[i]);CHKERRQ(ierr);}
6318   for (i = 0; i < numCIndices; i++) {ierr = PetscViewerASCIIPrintf(viewer, "[%d]mat col indices[%D] = %D\n", rank, i, cindices[i]);CHKERRQ(ierr);}
6319   numCIndices = numCIndices ? numCIndices : numRIndices;
6320   if (!values) PetscFunctionReturn(0);
6321   for (i = 0; i < numRIndices; i++) {
6322     ierr = PetscViewerASCIIPrintf(viewer, "[%d]", rank);CHKERRQ(ierr);
6323     for (j = 0; j < numCIndices; j++) {
6324 #if defined(PETSC_USE_COMPLEX)
6325       ierr = PetscViewerASCIIPrintf(viewer, " (%g,%g)", (double)PetscRealPart(values[i*numCIndices+j]), (double)PetscImaginaryPart(values[i*numCIndices+j]));CHKERRQ(ierr);
6326 #else
6327       ierr = PetscViewerASCIIPrintf(viewer, " %g", (double)values[i*numCIndices+j]);CHKERRQ(ierr);
6328 #endif
6329     }
6330     ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
6331   }
6332   PetscFunctionReturn(0);
6333 }
6334 
6335 /*
6336   DMPlexGetIndicesPoint_Internal - Add the indices for dofs on a point to an index array
6337 
6338   Input Parameters:
6339 + section - The section for this data layout
6340 . islocal - Is the section (and thus indices being requested) local or global?
6341 . point   - The point contributing dofs with these indices
6342 . off     - The global offset of this point
6343 . loff    - The local offset of each field
6344 . setBC   - The flag determining whether to include indices of bounsary values
6345 . perm    - A permutation of the dofs on this point, or NULL
6346 - indperm - A permutation of the entire indices array, or NULL
6347 
6348   Output Parameter:
6349 . indices - Indices for dofs on this point
6350 
6351   Level: developer
6352 
6353   Note: The indices could be local or global, depending on the value of 'off'.
6354 */
6355 PetscErrorCode DMPlexGetIndicesPoint_Internal(PetscSection section, PetscBool islocal,PetscInt point, PetscInt off, PetscInt *loff, PetscBool setBC, const PetscInt perm[], const PetscInt indperm[], PetscInt indices[])
6356 {
6357   PetscInt        dof;   /* The number of unknowns on this point */
6358   PetscInt        cdof;  /* The number of constraints on this point */
6359   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
6360   PetscInt        cind = 0, k;
6361   PetscErrorCode  ierr;
6362 
6363   PetscFunctionBegin;
6364   if (!islocal && setBC) SETERRQ(PetscObjectComm((PetscObject)section),PETSC_ERR_ARG_INCOMP,"setBC incompatible with global indices; use a local section or disable setBC");
6365   ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
6366   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
6367   if (!cdof || setBC) {
6368     for (k = 0; k < dof; ++k) {
6369       const PetscInt preind = perm ? *loff+perm[k] : *loff+k;
6370       const PetscInt ind    = indperm ? indperm[preind] : preind;
6371 
6372       indices[ind] = off + k;
6373     }
6374   } else {
6375     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
6376     for (k = 0; k < dof; ++k) {
6377       const PetscInt preind = perm ? *loff+perm[k] : *loff+k;
6378       const PetscInt ind    = indperm ? indperm[preind] : preind;
6379 
6380       if ((cind < cdof) && (k == cdofs[cind])) {
6381         /* Insert check for returning constrained indices */
6382         indices[ind] = -(off+k+1);
6383         ++cind;
6384       } else {
6385         indices[ind] = off + k - (islocal ? 0 : cind);
6386       }
6387     }
6388   }
6389   *loff += dof;
6390   PetscFunctionReturn(0);
6391 }
6392 
6393 /*
6394  DMPlexGetIndicesPointFields_Internal - gets section indices for a point in its canonical ordering.
6395 
6396  Input Parameters:
6397 + section - a section (global or local)
6398 - islocal - PETSC_TRUE if requesting local indices (i.e., section is local); PETSC_FALSE for global
6399 . point - point within section
6400 . off - The offset of this point in the (local or global) indexed space - should match islocal and (usually) the section
6401 . foffs - array of length numFields containing the offset in canonical point ordering (the location in indices) of each field
6402 . setBC - identify constrained (boundary condition) points via involution.
6403 . perms - perms[f][permsoff][:] is a permutation of dofs within each field
6404 . permsoff - offset
6405 - indperm - index permutation
6406 
6407  Output Parameter:
6408 . foffs - each entry is incremented by the number of (unconstrained if setBC=FALSE) dofs in that field
6409 . indices - array to hold indices (as defined by section) of each dof associated with point
6410 
6411  Notes:
6412  If section is local and setBC=true, there is no distinction between constrained and unconstrained dofs.
6413  If section is local and setBC=false, the indices for constrained points are the involution -(i+1) of their position
6414  in the local vector.
6415 
6416  If section is global and setBC=false, the indices for constrained points are negative (and their value is not
6417  significant).  It is invalid to call with a global section and setBC=true.
6418 
6419  Developer Note:
6420  The section is only used for field layout, so islocal is technically a statement about the offset (off).  At some point
6421  in the future, global sections may have fields set, in which case we could pass the global section and obtain the
6422  offset could be obtained from the section instead of passing it explicitly as we do now.
6423 
6424  Example:
6425  Suppose a point contains one field with three components, and for which the unconstrained indices are {10, 11, 12}.
6426  When the middle component is constrained, we get the array {10, -12, 12} for (islocal=TRUE, setBC=FALSE).
6427  Note that -12 is the involution of 11, so the user can involute negative indices to recover local indices.
6428  The global vector does not store constrained dofs, so when this function returns global indices, say {110, -112, 111}, the value of -112 is an arbitrary flag that should not be interpreted beyond its sign.
6429 
6430  Level: developer
6431 */
6432 PetscErrorCode DMPlexGetIndicesPointFields_Internal(PetscSection section, PetscBool islocal, PetscInt point, PetscInt off, PetscInt foffs[], PetscBool setBC, const PetscInt ***perms, PetscInt permsoff, const PetscInt indperm[], PetscInt indices[])
6433 {
6434   PetscInt       numFields, foff, f;
6435   PetscErrorCode ierr;
6436 
6437   PetscFunctionBegin;
6438   if (!islocal && setBC) SETERRQ(PetscObjectComm((PetscObject)section),PETSC_ERR_ARG_INCOMP,"setBC incompatible with global indices; use a local section or disable setBC");
6439   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
6440   for (f = 0, foff = 0; f < numFields; ++f) {
6441     PetscInt        fdof, cfdof;
6442     const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
6443     PetscInt        cind = 0, b;
6444     const PetscInt  *perm = (perms && perms[f]) ? perms[f][permsoff] : NULL;
6445 
6446     ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
6447     ierr = PetscSectionGetFieldConstraintDof(section, point, f, &cfdof);CHKERRQ(ierr);
6448     if (!cfdof || setBC) {
6449       for (b = 0; b < fdof; ++b) {
6450         const PetscInt preind = perm ? foffs[f]+perm[b] : foffs[f]+b;
6451         const PetscInt ind    = indperm ? indperm[preind] : preind;
6452 
6453         indices[ind] = off+foff+b;
6454       }
6455     } else {
6456       ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
6457       for (b = 0; b < fdof; ++b) {
6458         const PetscInt preind = perm ? foffs[f]+perm[b] : foffs[f]+b;
6459         const PetscInt ind    = indperm ? indperm[preind] : preind;
6460 
6461         if ((cind < cfdof) && (b == fcdofs[cind])) {
6462           indices[ind] = -(off+foff+b+1);
6463           ++cind;
6464         } else {
6465           indices[ind] = off + foff + b - (islocal ? 0 : cind);
6466         }
6467       }
6468     }
6469     foff     += (setBC || islocal ? fdof : (fdof - cfdof));
6470     foffs[f] += fdof;
6471   }
6472   PetscFunctionReturn(0);
6473 }
6474 
6475 /*
6476   This version believes the globalSection offsets for each field, rather than just the point offset
6477 
6478  . foffs - The offset into 'indices' for each field, since it is segregated by field
6479 
6480  Notes:
6481  The semantics of this function relate to that of setBC=FALSE in DMPlexGetIndicesPointFields_Internal.
6482  Since this function uses global indices, setBC=TRUE would be invalid, so no such argument exists.
6483 */
6484 static PetscErrorCode DMPlexGetIndicesPointFieldsSplit_Internal(PetscSection section, PetscSection globalSection, PetscInt point, PetscInt foffs[], const PetscInt ***perms, PetscInt permsoff, const PetscInt indperm[], PetscInt indices[])
6485 {
6486   PetscInt       numFields, foff, f;
6487   PetscErrorCode ierr;
6488 
6489   PetscFunctionBegin;
6490   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
6491   for (f = 0; f < numFields; ++f) {
6492     PetscInt        fdof, cfdof;
6493     const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
6494     PetscInt        cind = 0, b;
6495     const PetscInt  *perm = (perms && perms[f]) ? perms[f][permsoff] : NULL;
6496 
6497     ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
6498     ierr = PetscSectionGetFieldConstraintDof(section, point, f, &cfdof);CHKERRQ(ierr);
6499     ierr = PetscSectionGetFieldOffset(globalSection, point, f, &foff);CHKERRQ(ierr);
6500     if (!cfdof) {
6501       for (b = 0; b < fdof; ++b) {
6502         const PetscInt preind = perm ? foffs[f]+perm[b] : foffs[f]+b;
6503         const PetscInt ind    = indperm ? indperm[preind] : preind;
6504 
6505         indices[ind] = foff+b;
6506       }
6507     } else {
6508       ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
6509       for (b = 0; b < fdof; ++b) {
6510         const PetscInt preind = perm ? foffs[f]+perm[b] : foffs[f]+b;
6511         const PetscInt ind    = indperm ? indperm[preind] : preind;
6512 
6513         if ((cind < cfdof) && (b == fcdofs[cind])) {
6514           indices[ind] = -(foff+b+1);
6515           ++cind;
6516         } else {
6517           indices[ind] = foff+b-cind;
6518         }
6519       }
6520     }
6521     foffs[f] += fdof;
6522   }
6523   PetscFunctionReturn(0);
6524 }
6525 
6526 PetscErrorCode DMPlexAnchorsModifyMat(DM dm, PetscSection section, PetscInt numPoints, PetscInt numIndices, const PetscInt points[], const PetscInt ***perms, const PetscScalar values[], PetscInt *outNumPoints, PetscInt *outNumIndices, PetscInt *outPoints[], PetscScalar *outValues[], PetscInt offsets[], PetscBool multiplyLeft)
6527 {
6528   Mat             cMat;
6529   PetscSection    aSec, cSec;
6530   IS              aIS;
6531   PetscInt        aStart = -1, aEnd = -1;
6532   const PetscInt  *anchors;
6533   PetscInt        numFields, f, p, q, newP = 0;
6534   PetscInt        newNumPoints = 0, newNumIndices = 0;
6535   PetscInt        *newPoints, *indices, *newIndices;
6536   PetscInt        maxAnchor, maxDof;
6537   PetscInt        newOffsets[32];
6538   PetscInt        *pointMatOffsets[32];
6539   PetscInt        *newPointOffsets[32];
6540   PetscScalar     *pointMat[32];
6541   PetscScalar     *newValues=NULL,*tmpValues;
6542   PetscBool       anyConstrained = PETSC_FALSE;
6543   PetscErrorCode  ierr;
6544 
6545   PetscFunctionBegin;
6546   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6547   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
6548   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
6549 
6550   ierr = DMPlexGetAnchors(dm,&aSec,&aIS);CHKERRQ(ierr);
6551   /* if there are point-to-point constraints */
6552   if (aSec) {
6553     ierr = PetscArrayzero(newOffsets, 32);CHKERRQ(ierr);
6554     ierr = ISGetIndices(aIS,&anchors);CHKERRQ(ierr);
6555     ierr = PetscSectionGetChart(aSec,&aStart,&aEnd);CHKERRQ(ierr);
6556     /* figure out how many points are going to be in the new element matrix
6557      * (we allow double counting, because it's all just going to be summed
6558      * into the global matrix anyway) */
6559     for (p = 0; p < 2*numPoints; p+=2) {
6560       PetscInt b    = points[p];
6561       PetscInt bDof = 0, bSecDof;
6562 
6563       ierr = PetscSectionGetDof(section,b,&bSecDof);CHKERRQ(ierr);
6564       if (!bSecDof) {
6565         continue;
6566       }
6567       if (b >= aStart && b < aEnd) {
6568         ierr = PetscSectionGetDof(aSec,b,&bDof);CHKERRQ(ierr);
6569       }
6570       if (bDof) {
6571         /* this point is constrained */
6572         /* it is going to be replaced by its anchors */
6573         PetscInt bOff, q;
6574 
6575         anyConstrained = PETSC_TRUE;
6576         newNumPoints  += bDof;
6577         ierr = PetscSectionGetOffset(aSec,b,&bOff);CHKERRQ(ierr);
6578         for (q = 0; q < bDof; q++) {
6579           PetscInt a = anchors[bOff + q];
6580           PetscInt aDof;
6581 
6582           ierr           = PetscSectionGetDof(section,a,&aDof);CHKERRQ(ierr);
6583           newNumIndices += aDof;
6584           for (f = 0; f < numFields; ++f) {
6585             PetscInt fDof;
6586 
6587             ierr             = PetscSectionGetFieldDof(section, a, f, &fDof);CHKERRQ(ierr);
6588             newOffsets[f+1] += fDof;
6589           }
6590         }
6591       }
6592       else {
6593         /* this point is not constrained */
6594         newNumPoints++;
6595         newNumIndices += bSecDof;
6596         for (f = 0; f < numFields; ++f) {
6597           PetscInt fDof;
6598 
6599           ierr = PetscSectionGetFieldDof(section, b, f, &fDof);CHKERRQ(ierr);
6600           newOffsets[f+1] += fDof;
6601         }
6602       }
6603     }
6604   }
6605   if (!anyConstrained) {
6606     if (outNumPoints)  *outNumPoints  = 0;
6607     if (outNumIndices) *outNumIndices = 0;
6608     if (outPoints)     *outPoints     = NULL;
6609     if (outValues)     *outValues     = NULL;
6610     if (aSec) {ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);}
6611     PetscFunctionReturn(0);
6612   }
6613 
6614   if (outNumPoints)  *outNumPoints  = newNumPoints;
6615   if (outNumIndices) *outNumIndices = newNumIndices;
6616 
6617   for (f = 0; f < numFields; ++f) newOffsets[f+1] += newOffsets[f];
6618 
6619   if (!outPoints && !outValues) {
6620     if (offsets) {
6621       for (f = 0; f <= numFields; f++) {
6622         offsets[f] = newOffsets[f];
6623       }
6624     }
6625     if (aSec) {ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);}
6626     PetscFunctionReturn(0);
6627   }
6628 
6629   if (numFields && newOffsets[numFields] != newNumIndices) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", newOffsets[numFields], newNumIndices);
6630 
6631   ierr = DMGetDefaultConstraints(dm, &cSec, &cMat);CHKERRQ(ierr);
6632 
6633   /* workspaces */
6634   if (numFields) {
6635     for (f = 0; f < numFields; f++) {
6636       ierr = DMGetWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[f]);CHKERRQ(ierr);
6637       ierr = DMGetWorkArray(dm,numPoints+1,MPIU_INT,&newPointOffsets[f]);CHKERRQ(ierr);
6638     }
6639   }
6640   else {
6641     ierr = DMGetWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[0]);CHKERRQ(ierr);
6642     ierr = DMGetWorkArray(dm,numPoints,MPIU_INT,&newPointOffsets[0]);CHKERRQ(ierr);
6643   }
6644 
6645   /* get workspaces for the point-to-point matrices */
6646   if (numFields) {
6647     PetscInt totalOffset, totalMatOffset;
6648 
6649     for (p = 0; p < numPoints; p++) {
6650       PetscInt b    = points[2*p];
6651       PetscInt bDof = 0, bSecDof;
6652 
6653       ierr = PetscSectionGetDof(section,b,&bSecDof);CHKERRQ(ierr);
6654       if (!bSecDof) {
6655         for (f = 0; f < numFields; f++) {
6656           newPointOffsets[f][p + 1] = 0;
6657           pointMatOffsets[f][p + 1] = 0;
6658         }
6659         continue;
6660       }
6661       if (b >= aStart && b < aEnd) {
6662         ierr = PetscSectionGetDof(aSec, b, &bDof);CHKERRQ(ierr);
6663       }
6664       if (bDof) {
6665         for (f = 0; f < numFields; f++) {
6666           PetscInt fDof, q, bOff, allFDof = 0;
6667 
6668           ierr = PetscSectionGetFieldDof(section, b, f, &fDof);CHKERRQ(ierr);
6669           ierr = PetscSectionGetOffset(aSec, b, &bOff);CHKERRQ(ierr);
6670           for (q = 0; q < bDof; q++) {
6671             PetscInt a = anchors[bOff + q];
6672             PetscInt aFDof;
6673 
6674             ierr     = PetscSectionGetFieldDof(section, a, f, &aFDof);CHKERRQ(ierr);
6675             allFDof += aFDof;
6676           }
6677           newPointOffsets[f][p+1] = allFDof;
6678           pointMatOffsets[f][p+1] = fDof * allFDof;
6679         }
6680       }
6681       else {
6682         for (f = 0; f < numFields; f++) {
6683           PetscInt fDof;
6684 
6685           ierr = PetscSectionGetFieldDof(section, b, f, &fDof);CHKERRQ(ierr);
6686           newPointOffsets[f][p+1] = fDof;
6687           pointMatOffsets[f][p+1] = 0;
6688         }
6689       }
6690     }
6691     for (f = 0, totalOffset = 0, totalMatOffset = 0; f < numFields; f++) {
6692       newPointOffsets[f][0] = totalOffset;
6693       pointMatOffsets[f][0] = totalMatOffset;
6694       for (p = 0; p < numPoints; p++) {
6695         newPointOffsets[f][p+1] += newPointOffsets[f][p];
6696         pointMatOffsets[f][p+1] += pointMatOffsets[f][p];
6697       }
6698       totalOffset    = newPointOffsets[f][numPoints];
6699       totalMatOffset = pointMatOffsets[f][numPoints];
6700       ierr = DMGetWorkArray(dm,pointMatOffsets[f][numPoints],MPIU_SCALAR,&pointMat[f]);CHKERRQ(ierr);
6701     }
6702   }
6703   else {
6704     for (p = 0; p < numPoints; p++) {
6705       PetscInt b    = points[2*p];
6706       PetscInt bDof = 0, bSecDof;
6707 
6708       ierr = PetscSectionGetDof(section,b,&bSecDof);CHKERRQ(ierr);
6709       if (!bSecDof) {
6710         newPointOffsets[0][p + 1] = 0;
6711         pointMatOffsets[0][p + 1] = 0;
6712         continue;
6713       }
6714       if (b >= aStart && b < aEnd) {
6715         ierr = PetscSectionGetDof(aSec, b, &bDof);CHKERRQ(ierr);
6716       }
6717       if (bDof) {
6718         PetscInt bOff, q, allDof = 0;
6719 
6720         ierr = PetscSectionGetOffset(aSec, b, &bOff);CHKERRQ(ierr);
6721         for (q = 0; q < bDof; q++) {
6722           PetscInt a = anchors[bOff + q], aDof;
6723 
6724           ierr    = PetscSectionGetDof(section, a, &aDof);CHKERRQ(ierr);
6725           allDof += aDof;
6726         }
6727         newPointOffsets[0][p+1] = allDof;
6728         pointMatOffsets[0][p+1] = bSecDof * allDof;
6729       }
6730       else {
6731         newPointOffsets[0][p+1] = bSecDof;
6732         pointMatOffsets[0][p+1] = 0;
6733       }
6734     }
6735     newPointOffsets[0][0] = 0;
6736     pointMatOffsets[0][0] = 0;
6737     for (p = 0; p < numPoints; p++) {
6738       newPointOffsets[0][p+1] += newPointOffsets[0][p];
6739       pointMatOffsets[0][p+1] += pointMatOffsets[0][p];
6740     }
6741     ierr = DMGetWorkArray(dm,pointMatOffsets[0][numPoints],MPIU_SCALAR,&pointMat[0]);CHKERRQ(ierr);
6742   }
6743 
6744   /* output arrays */
6745   ierr = DMGetWorkArray(dm,2*newNumPoints,MPIU_INT,&newPoints);CHKERRQ(ierr);
6746 
6747   /* get the point-to-point matrices; construct newPoints */
6748   ierr = PetscSectionGetMaxDof(aSec, &maxAnchor);CHKERRQ(ierr);
6749   ierr = PetscSectionGetMaxDof(section, &maxDof);CHKERRQ(ierr);
6750   ierr = DMGetWorkArray(dm,maxDof,MPIU_INT,&indices);CHKERRQ(ierr);
6751   ierr = DMGetWorkArray(dm,maxAnchor*maxDof,MPIU_INT,&newIndices);CHKERRQ(ierr);
6752   if (numFields) {
6753     for (p = 0, newP = 0; p < numPoints; p++) {
6754       PetscInt b    = points[2*p];
6755       PetscInt o    = points[2*p+1];
6756       PetscInt bDof = 0, bSecDof;
6757 
6758       ierr = PetscSectionGetDof(section, b, &bSecDof);CHKERRQ(ierr);
6759       if (!bSecDof) {
6760         continue;
6761       }
6762       if (b >= aStart && b < aEnd) {
6763         ierr = PetscSectionGetDof(aSec, b, &bDof);CHKERRQ(ierr);
6764       }
6765       if (bDof) {
6766         PetscInt fStart[32], fEnd[32], fAnchorStart[32], fAnchorEnd[32], bOff, q;
6767 
6768         fStart[0] = 0;
6769         fEnd[0]   = 0;
6770         for (f = 0; f < numFields; f++) {
6771           PetscInt fDof;
6772 
6773           ierr        = PetscSectionGetFieldDof(cSec, b, f, &fDof);CHKERRQ(ierr);
6774           fStart[f+1] = fStart[f] + fDof;
6775           fEnd[f+1]   = fStart[f+1];
6776         }
6777         ierr = PetscSectionGetOffset(cSec, b, &bOff);CHKERRQ(ierr);
6778         ierr = DMPlexGetIndicesPointFields_Internal(cSec, PETSC_TRUE, b, bOff, fEnd, PETSC_TRUE, perms, p, NULL, indices);CHKERRQ(ierr);
6779 
6780         fAnchorStart[0] = 0;
6781         fAnchorEnd[0]   = 0;
6782         for (f = 0; f < numFields; f++) {
6783           PetscInt fDof = newPointOffsets[f][p + 1] - newPointOffsets[f][p];
6784 
6785           fAnchorStart[f+1] = fAnchorStart[f] + fDof;
6786           fAnchorEnd[f+1]   = fAnchorStart[f + 1];
6787         }
6788         ierr = PetscSectionGetOffset(aSec, b, &bOff);CHKERRQ(ierr);
6789         for (q = 0; q < bDof; q++) {
6790           PetscInt a = anchors[bOff + q], aOff;
6791 
6792           /* we take the orientation of ap into account in the order that we constructed the indices above: the newly added points have no orientation */
6793           newPoints[2*(newP + q)]     = a;
6794           newPoints[2*(newP + q) + 1] = 0;
6795           ierr = PetscSectionGetOffset(section, a, &aOff);CHKERRQ(ierr);
6796           ierr = DMPlexGetIndicesPointFields_Internal(section, PETSC_TRUE, a, aOff, fAnchorEnd, PETSC_TRUE, NULL, -1, NULL, newIndices);CHKERRQ(ierr);
6797         }
6798         newP += bDof;
6799 
6800         if (outValues) {
6801           /* get the point-to-point submatrix */
6802           for (f = 0; f < numFields; f++) {
6803             ierr = MatGetValues(cMat,fEnd[f]-fStart[f],indices + fStart[f],fAnchorEnd[f] - fAnchorStart[f],newIndices + fAnchorStart[f],pointMat[f] + pointMatOffsets[f][p]);CHKERRQ(ierr);
6804           }
6805         }
6806       }
6807       else {
6808         newPoints[2 * newP]     = b;
6809         newPoints[2 * newP + 1] = o;
6810         newP++;
6811       }
6812     }
6813   } else {
6814     for (p = 0; p < numPoints; p++) {
6815       PetscInt b    = points[2*p];
6816       PetscInt o    = points[2*p+1];
6817       PetscInt bDof = 0, bSecDof;
6818 
6819       ierr = PetscSectionGetDof(section, b, &bSecDof);CHKERRQ(ierr);
6820       if (!bSecDof) {
6821         continue;
6822       }
6823       if (b >= aStart && b < aEnd) {
6824         ierr = PetscSectionGetDof(aSec, b, &bDof);CHKERRQ(ierr);
6825       }
6826       if (bDof) {
6827         PetscInt bEnd = 0, bAnchorEnd = 0, bOff;
6828 
6829         ierr = PetscSectionGetOffset(cSec, b, &bOff);CHKERRQ(ierr);
6830         ierr = DMPlexGetIndicesPoint_Internal(cSec, PETSC_TRUE, b, bOff, &bEnd, PETSC_TRUE, (perms && perms[0]) ? perms[0][p] : NULL, NULL, indices);CHKERRQ(ierr);
6831 
6832         ierr = PetscSectionGetOffset (aSec, b, &bOff);CHKERRQ(ierr);
6833         for (q = 0; q < bDof; q++) {
6834           PetscInt a = anchors[bOff + q], aOff;
6835 
6836           /* we take the orientation of ap into account in the order that we constructed the indices above: the newly added points have no orientation */
6837 
6838           newPoints[2*(newP + q)]     = a;
6839           newPoints[2*(newP + q) + 1] = 0;
6840           ierr = PetscSectionGetOffset(section, a, &aOff);CHKERRQ(ierr);
6841           ierr = DMPlexGetIndicesPoint_Internal(section, PETSC_TRUE, a, aOff, &bAnchorEnd, PETSC_TRUE, NULL, NULL, newIndices);CHKERRQ(ierr);
6842         }
6843         newP += bDof;
6844 
6845         /* get the point-to-point submatrix */
6846         if (outValues) {
6847           ierr = MatGetValues(cMat,bEnd,indices,bAnchorEnd,newIndices,pointMat[0] + pointMatOffsets[0][p]);CHKERRQ(ierr);
6848         }
6849       }
6850       else {
6851         newPoints[2 * newP]     = b;
6852         newPoints[2 * newP + 1] = o;
6853         newP++;
6854       }
6855     }
6856   }
6857 
6858   if (outValues) {
6859     ierr = DMGetWorkArray(dm,newNumIndices*numIndices,MPIU_SCALAR,&tmpValues);CHKERRQ(ierr);
6860     ierr = PetscArrayzero(tmpValues,newNumIndices*numIndices);CHKERRQ(ierr);
6861     /* multiply constraints on the right */
6862     if (numFields) {
6863       for (f = 0; f < numFields; f++) {
6864         PetscInt oldOff = offsets[f];
6865 
6866         for (p = 0; p < numPoints; p++) {
6867           PetscInt cStart = newPointOffsets[f][p];
6868           PetscInt b      = points[2 * p];
6869           PetscInt c, r, k;
6870           PetscInt dof;
6871 
6872           ierr = PetscSectionGetFieldDof(section,b,f,&dof);CHKERRQ(ierr);
6873           if (!dof) {
6874             continue;
6875           }
6876           if (pointMatOffsets[f][p] < pointMatOffsets[f][p + 1]) {
6877             PetscInt nCols         = newPointOffsets[f][p+1]-cStart;
6878             const PetscScalar *mat = pointMat[f] + pointMatOffsets[f][p];
6879 
6880             for (r = 0; r < numIndices; r++) {
6881               for (c = 0; c < nCols; c++) {
6882                 for (k = 0; k < dof; k++) {
6883                   tmpValues[r * newNumIndices + cStart + c] += values[r * numIndices + oldOff + k] * mat[k * nCols + c];
6884                 }
6885               }
6886             }
6887           }
6888           else {
6889             /* copy this column as is */
6890             for (r = 0; r < numIndices; r++) {
6891               for (c = 0; c < dof; c++) {
6892                 tmpValues[r * newNumIndices + cStart + c] = values[r * numIndices + oldOff + c];
6893               }
6894             }
6895           }
6896           oldOff += dof;
6897         }
6898       }
6899     }
6900     else {
6901       PetscInt oldOff = 0;
6902       for (p = 0; p < numPoints; p++) {
6903         PetscInt cStart = newPointOffsets[0][p];
6904         PetscInt b      = points[2 * p];
6905         PetscInt c, r, k;
6906         PetscInt dof;
6907 
6908         ierr = PetscSectionGetDof(section,b,&dof);CHKERRQ(ierr);
6909         if (!dof) {
6910           continue;
6911         }
6912         if (pointMatOffsets[0][p] < pointMatOffsets[0][p + 1]) {
6913           PetscInt nCols         = newPointOffsets[0][p+1]-cStart;
6914           const PetscScalar *mat = pointMat[0] + pointMatOffsets[0][p];
6915 
6916           for (r = 0; r < numIndices; r++) {
6917             for (c = 0; c < nCols; c++) {
6918               for (k = 0; k < dof; k++) {
6919                 tmpValues[r * newNumIndices + cStart + c] += mat[k * nCols + c] * values[r * numIndices + oldOff + k];
6920               }
6921             }
6922           }
6923         }
6924         else {
6925           /* copy this column as is */
6926           for (r = 0; r < numIndices; r++) {
6927             for (c = 0; c < dof; c++) {
6928               tmpValues[r * newNumIndices + cStart + c] = values[r * numIndices + oldOff + c];
6929             }
6930           }
6931         }
6932         oldOff += dof;
6933       }
6934     }
6935 
6936     if (multiplyLeft) {
6937       ierr = DMGetWorkArray(dm,newNumIndices*newNumIndices,MPIU_SCALAR,&newValues);CHKERRQ(ierr);
6938       ierr = PetscArrayzero(newValues,newNumIndices*newNumIndices);CHKERRQ(ierr);
6939       /* multiply constraints transpose on the left */
6940       if (numFields) {
6941         for (f = 0; f < numFields; f++) {
6942           PetscInt oldOff = offsets[f];
6943 
6944           for (p = 0; p < numPoints; p++) {
6945             PetscInt rStart = newPointOffsets[f][p];
6946             PetscInt b      = points[2 * p];
6947             PetscInt c, r, k;
6948             PetscInt dof;
6949 
6950             ierr = PetscSectionGetFieldDof(section,b,f,&dof);CHKERRQ(ierr);
6951             if (pointMatOffsets[f][p] < pointMatOffsets[f][p + 1]) {
6952               PetscInt nRows                        = newPointOffsets[f][p+1]-rStart;
6953               const PetscScalar *PETSC_RESTRICT mat = pointMat[f] + pointMatOffsets[f][p];
6954 
6955               for (r = 0; r < nRows; r++) {
6956                 for (c = 0; c < newNumIndices; c++) {
6957                   for (k = 0; k < dof; k++) {
6958                     newValues[(rStart + r) * newNumIndices + c] += mat[k * nRows + r] * tmpValues[(oldOff + k) * newNumIndices + c];
6959                   }
6960                 }
6961               }
6962             }
6963             else {
6964               /* copy this row as is */
6965               for (r = 0; r < dof; r++) {
6966                 for (c = 0; c < newNumIndices; c++) {
6967                   newValues[(rStart + r) * newNumIndices + c] = tmpValues[(oldOff + r) * newNumIndices + c];
6968                 }
6969               }
6970             }
6971             oldOff += dof;
6972           }
6973         }
6974       }
6975       else {
6976         PetscInt oldOff = 0;
6977 
6978         for (p = 0; p < numPoints; p++) {
6979           PetscInt rStart = newPointOffsets[0][p];
6980           PetscInt b      = points[2 * p];
6981           PetscInt c, r, k;
6982           PetscInt dof;
6983 
6984           ierr = PetscSectionGetDof(section,b,&dof);CHKERRQ(ierr);
6985           if (pointMatOffsets[0][p] < pointMatOffsets[0][p + 1]) {
6986             PetscInt nRows                        = newPointOffsets[0][p+1]-rStart;
6987             const PetscScalar *PETSC_RESTRICT mat = pointMat[0] + pointMatOffsets[0][p];
6988 
6989             for (r = 0; r < nRows; r++) {
6990               for (c = 0; c < newNumIndices; c++) {
6991                 for (k = 0; k < dof; k++) {
6992                   newValues[(rStart + r) * newNumIndices + c] += mat[k * nRows + r] * tmpValues[(oldOff + k) * newNumIndices + c];
6993                 }
6994               }
6995             }
6996           }
6997           else {
6998             /* copy this row as is */
6999             for (r = 0; r < dof; r++) {
7000               for (c = 0; c < newNumIndices; c++) {
7001                 newValues[(rStart + r) * newNumIndices + c] = tmpValues[(oldOff + r) * newNumIndices + c];
7002               }
7003             }
7004           }
7005           oldOff += dof;
7006         }
7007       }
7008 
7009       ierr = DMRestoreWorkArray(dm,newNumIndices*numIndices,MPIU_SCALAR,&tmpValues);CHKERRQ(ierr);
7010     }
7011     else {
7012       newValues = tmpValues;
7013     }
7014   }
7015 
7016   /* clean up */
7017   ierr = DMRestoreWorkArray(dm,maxDof,MPIU_INT,&indices);CHKERRQ(ierr);
7018   ierr = DMRestoreWorkArray(dm,maxAnchor*maxDof,MPIU_INT,&newIndices);CHKERRQ(ierr);
7019 
7020   if (numFields) {
7021     for (f = 0; f < numFields; f++) {
7022       ierr = DMRestoreWorkArray(dm,pointMatOffsets[f][numPoints],MPIU_SCALAR,&pointMat[f]);CHKERRQ(ierr);
7023       ierr = DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[f]);CHKERRQ(ierr);
7024       ierr = DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&newPointOffsets[f]);CHKERRQ(ierr);
7025     }
7026   }
7027   else {
7028     ierr = DMRestoreWorkArray(dm,pointMatOffsets[0][numPoints],MPIU_SCALAR,&pointMat[0]);CHKERRQ(ierr);
7029     ierr = DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[0]);CHKERRQ(ierr);
7030     ierr = DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&newPointOffsets[0]);CHKERRQ(ierr);
7031   }
7032   ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);
7033 
7034   /* output */
7035   if (outPoints) {
7036     *outPoints = newPoints;
7037   }
7038   else {
7039     ierr = DMRestoreWorkArray(dm,2*newNumPoints,MPIU_INT,&newPoints);CHKERRQ(ierr);
7040   }
7041   if (outValues) {
7042     *outValues = newValues;
7043   }
7044   for (f = 0; f <= numFields; f++) {
7045     offsets[f] = newOffsets[f];
7046   }
7047   PetscFunctionReturn(0);
7048 }
7049 
7050 /*@C
7051   DMPlexGetClosureIndices - Gets the global dof indices associated with the closure of the given point within the provided sections.
7052 
7053   Not collective
7054 
7055   Input Parameters:
7056 + dm         - The DM
7057 . section    - The PetscSection describing the points (a local section)
7058 . idxSection - The PetscSection from which to obtain indices (may be local or global)
7059 . point      - The point defining the closure
7060 - useClPerm  - Use the closure point permutation if available
7061 
7062   Output Parameters:
7063 + numIndices - The number of dof indices in the closure of point with the input sections
7064 . indices    - The dof indices
7065 . outOffsets - Array to write the field offsets into, or NULL
7066 - values     - The input values, which may be modified if sign flips are induced by the point symmetries, or NULL
7067 
7068   Notes:
7069   Must call DMPlexRestoreClosureIndices() to free allocated memory
7070 
7071   If idxSection is global, any constrained dofs (see DMAddBoundary(), for example) will get negative indices.  The value
7072   of those indices is not significant.  If idxSection is local, the constrained dofs will yield the involution -(idx+1)
7073   of their index in a local vector.  A caller who does not wish to distinguish those points may recover the nonnegative
7074   indices via involution, -(-(idx+1)+1)==idx.  Local indices are provided when idxSection == section, otherwise global
7075   indices (with the above semantics) are implied.
7076 
7077   Level: advanced
7078 
7079 .seealso DMPlexRestoreClosureIndices(), DMPlexVecGetClosure(), DMPlexMatSetClosure(), DMGetLocalSection(), DMGetGlobalSection()
7080 @*/
7081 PetscErrorCode DMPlexGetClosureIndices(DM dm, PetscSection section, PetscSection idxSection, PetscInt point, PetscBool useClPerm,
7082                                        PetscInt *numIndices, PetscInt *indices[], PetscInt outOffsets[], PetscScalar *values[])
7083 {
7084   /* Closure ordering */
7085   PetscSection        clSection;
7086   IS                  clPoints;
7087   const PetscInt     *clp;
7088   PetscInt           *points;
7089   const PetscInt     *clperm = NULL;
7090   /* Dof permutation and sign flips */
7091   const PetscInt    **perms[32] = {NULL};
7092   const PetscScalar **flips[32] = {NULL};
7093   PetscScalar        *valCopy   = NULL;
7094   /* Hanging node constraints */
7095   PetscInt           *pointsC = NULL;
7096   PetscScalar        *valuesC = NULL;
7097   PetscInt            NclC, NiC;
7098 
7099   PetscInt           *idx;
7100   PetscInt            Nf, Ncl, Ni = 0, offsets[32], p, f;
7101   PetscBool           isLocal = (section == idxSection) ? PETSC_TRUE : PETSC_FALSE;
7102   PetscErrorCode      ierr;
7103 
7104   PetscFunctionBeginHot;
7105   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7106   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
7107   PetscValidHeaderSpecific(idxSection, PETSC_SECTION_CLASSID, 3);
7108   if (numIndices) PetscValidPointer(numIndices, 6);
7109   if (indices)    PetscValidPointer(indices, 7);
7110   if (outOffsets) PetscValidPointer(outOffsets, 8);
7111   if (values)     PetscValidPointer(values, 9);
7112   ierr = PetscSectionGetNumFields(section, &Nf);CHKERRQ(ierr);
7113   if (Nf > 31) SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", Nf);
7114   ierr = PetscArrayzero(offsets, 32);CHKERRQ(ierr);
7115   /* 1) Get points in closure */
7116   ierr = DMPlexGetCompressedClosure(dm, section, point, &Ncl, &points, &clSection, &clPoints, &clp);CHKERRQ(ierr);
7117   if (useClPerm) {
7118     PetscInt depth, clsize;
7119     ierr = DMPlexGetPointDepth(dm, point, &depth);CHKERRQ(ierr);
7120     for (clsize=0,p=0; p<Ncl; p++) {
7121       PetscInt dof;
7122       ierr = PetscSectionGetDof(section, points[2*p], &dof);CHKERRQ(ierr);
7123       clsize += dof;
7124     }
7125     ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, depth, clsize, &clperm);CHKERRQ(ierr);
7126   }
7127   /* 2) Get number of indices on these points and field offsets from section */
7128   for (p = 0; p < Ncl*2; p += 2) {
7129     PetscInt dof, fdof;
7130 
7131     ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
7132     for (f = 0; f < Nf; ++f) {
7133       ierr = PetscSectionGetFieldDof(section, points[p], f, &fdof);CHKERRQ(ierr);
7134       offsets[f+1] += fdof;
7135     }
7136     Ni += dof;
7137   }
7138   for (f = 1; f < Nf; ++f) offsets[f+1] += offsets[f];
7139   if (Nf && offsets[Nf] != Ni) SETERRQ2(PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", offsets[Nf], Ni);
7140   /* 3) Get symmetries and sign flips. Apply sign flips to values if passed in (only works for square values matrix) */
7141   for (f = 0; f < PetscMax(1, Nf); ++f) {
7142     if (Nf) {ierr = PetscSectionGetFieldPointSyms(section, f, Ncl, points, &perms[f], &flips[f]);CHKERRQ(ierr);}
7143     else    {ierr = PetscSectionGetPointSyms(section, Ncl, points, &perms[f], &flips[f]);CHKERRQ(ierr);}
7144     /* may need to apply sign changes to the element matrix */
7145     if (values && flips[f]) {
7146       PetscInt foffset = offsets[f];
7147 
7148       for (p = 0; p < Ncl; ++p) {
7149         PetscInt           pnt  = points[2*p], fdof;
7150         const PetscScalar *flip = flips[f] ? flips[f][p] : NULL;
7151 
7152         if (!Nf) {ierr = PetscSectionGetDof(section, pnt, &fdof);CHKERRQ(ierr);}
7153         else     {ierr = PetscSectionGetFieldDof(section, pnt, f, &fdof);CHKERRQ(ierr);}
7154         if (flip) {
7155           PetscInt i, j, k;
7156 
7157           if (!valCopy) {
7158             ierr = DMGetWorkArray(dm, Ni*Ni, MPIU_SCALAR, &valCopy);CHKERRQ(ierr);
7159             for (j = 0; j < Ni * Ni; ++j) valCopy[j] = (*values)[j];
7160             *values = valCopy;
7161           }
7162           for (i = 0; i < fdof; ++i) {
7163             PetscScalar fval = flip[i];
7164 
7165             for (k = 0; k < Ni; ++k) {
7166               valCopy[Ni * (foffset + i) + k] *= fval;
7167               valCopy[Ni * k + (foffset + i)] *= fval;
7168             }
7169           }
7170         }
7171         foffset += fdof;
7172       }
7173     }
7174   }
7175   /* 4) Apply hanging node constraints. Get new symmetries and replace all storage with constrained storage */
7176   ierr = DMPlexAnchorsModifyMat(dm, section, Ncl, Ni, points, perms, values ? *values : NULL, &NclC, &NiC, &pointsC, values ? &valuesC : NULL, offsets, PETSC_TRUE);CHKERRQ(ierr);
7177   if (NclC) {
7178     if (valCopy) {ierr = DMRestoreWorkArray(dm, Ni*Ni, MPIU_SCALAR, &valCopy);CHKERRQ(ierr);}
7179     for (f = 0; f < PetscMax(1, Nf); ++f) {
7180       if (Nf) {ierr = PetscSectionRestoreFieldPointSyms(section, f, Ncl, points, &perms[f], &flips[f]);CHKERRQ(ierr);}
7181       else    {ierr = PetscSectionRestorePointSyms(section, Ncl, points, &perms[f], &flips[f]);CHKERRQ(ierr);}
7182     }
7183     for (f = 0; f < PetscMax(1, Nf); ++f) {
7184       if (Nf) {ierr = PetscSectionGetFieldPointSyms(section, f, NclC, pointsC, &perms[f], &flips[f]);CHKERRQ(ierr);}
7185       else    {ierr = PetscSectionGetPointSyms(section, NclC, pointsC, &perms[f], &flips[f]);CHKERRQ(ierr);}
7186     }
7187     ierr = DMPlexRestoreCompressedClosure(dm, section, point, &Ncl, &points, &clSection, &clPoints, &clp);CHKERRQ(ierr);
7188     Ncl     = NclC;
7189     Ni      = NiC;
7190     points  = pointsC;
7191     if (values) *values = valuesC;
7192   }
7193   /* 5) Calculate indices */
7194   ierr = DMGetWorkArray(dm, Ni, MPIU_INT, &idx);CHKERRQ(ierr);
7195   if (Nf) {
7196     PetscInt  idxOff;
7197     PetscBool useFieldOffsets;
7198 
7199     if (outOffsets) {for (f = 0; f <= Nf; f++) outOffsets[f] = offsets[f];}
7200     ierr = PetscSectionGetUseFieldOffsets(idxSection, &useFieldOffsets);CHKERRQ(ierr);
7201     if (useFieldOffsets) {
7202       for (p = 0; p < Ncl; ++p) {
7203         const PetscInt pnt = points[p*2];
7204 
7205         ierr = DMPlexGetIndicesPointFieldsSplit_Internal(section, idxSection, pnt, offsets, perms, p, clperm, idx);CHKERRQ(ierr);
7206       }
7207     } else {
7208       for (p = 0; p < Ncl; ++p) {
7209         const PetscInt pnt = points[p*2];
7210 
7211         ierr = PetscSectionGetOffset(idxSection, pnt, &idxOff);CHKERRQ(ierr);
7212         /* Note that we pass a local section even though we're using global offsets.  This is because global sections do
7213          * not (at the time of this writing) have fields set. They probably should, in which case we would pass the
7214          * global section. */
7215         ierr = DMPlexGetIndicesPointFields_Internal(section, isLocal, pnt, idxOff < 0 ? -(idxOff+1) : idxOff, offsets, PETSC_FALSE, perms, p, clperm, idx);CHKERRQ(ierr);
7216       }
7217     }
7218   } else {
7219     PetscInt off = 0, idxOff;
7220 
7221     for (p = 0; p < Ncl; ++p) {
7222       const PetscInt  pnt  = points[p*2];
7223       const PetscInt *perm = perms[0] ? perms[0][p] : NULL;
7224 
7225       ierr = PetscSectionGetOffset(idxSection, pnt, &idxOff);CHKERRQ(ierr);
7226       /* Note that we pass a local section even though we're using global offsets.  This is because global sections do
7227        * not (at the time of this writing) have fields set. They probably should, in which case we would pass the global section. */
7228       ierr = DMPlexGetIndicesPoint_Internal(section, isLocal, pnt, idxOff < 0 ? -(idxOff+1) : idxOff, &off, PETSC_FALSE, perm, clperm, idx);CHKERRQ(ierr);
7229     }
7230   }
7231   /* 6) Cleanup */
7232   for (f = 0; f < PetscMax(1, Nf); ++f) {
7233     if (Nf) {ierr = PetscSectionRestoreFieldPointSyms(section, f, Ncl, points, &perms[f], &flips[f]);CHKERRQ(ierr);}
7234     else    {ierr = PetscSectionRestorePointSyms(section, Ncl, points, &perms[f], &flips[f]);CHKERRQ(ierr);}
7235   }
7236   if (NclC) {
7237     ierr = DMRestoreWorkArray(dm, NclC*2, MPIU_INT, &pointsC);CHKERRQ(ierr);
7238   } else {
7239     ierr = DMPlexRestoreCompressedClosure(dm, section, point, &Ncl, &points, &clSection, &clPoints, &clp);CHKERRQ(ierr);
7240   }
7241 
7242   if (numIndices) *numIndices = Ni;
7243   if (indices)    *indices    = idx;
7244   PetscFunctionReturn(0);
7245 }
7246 
7247 /*@C
7248   DMPlexRestoreClosureIndices - Restores the global dof indices associated with the closure of the given point within the provided sections.
7249 
7250   Not collective
7251 
7252   Input Parameters:
7253 + dm         - The DM
7254 . section    - The PetscSection describing the points (a local section)
7255 . idxSection - The PetscSection from which to obtain indices (may be local or global)
7256 . point      - The point defining the closure
7257 - useClPerm  - Use the closure point permutation if available
7258 
7259   Output Parameters:
7260 + numIndices - The number of dof indices in the closure of point with the input sections
7261 . indices    - The dof indices
7262 . outOffsets - Array to write the field offsets into, or NULL
7263 - values     - The input values, which may be modified if sign flips are induced by the point symmetries, or NULL
7264 
7265   Notes:
7266   If values were modified, the user is responsible for calling DMRestoreWorkArray(dm, 0, MPIU_SCALAR, &values).
7267 
7268   If idxSection is global, any constrained dofs (see DMAddBoundary(), for example) will get negative indices.  The value
7269   of those indices is not significant.  If idxSection is local, the constrained dofs will yield the involution -(idx+1)
7270   of their index in a local vector.  A caller who does not wish to distinguish those points may recover the nonnegative
7271   indices via involution, -(-(idx+1)+1)==idx.  Local indices are provided when idxSection == section, otherwise global
7272   indices (with the above semantics) are implied.
7273 
7274   Level: advanced
7275 
7276 .seealso DMPlexGetClosureIndices(), DMPlexVecGetClosure(), DMPlexMatSetClosure(), DMGetLocalSection(), DMGetGlobalSection()
7277 @*/
7278 PetscErrorCode DMPlexRestoreClosureIndices(DM dm, PetscSection section, PetscSection idxSection, PetscInt point, PetscBool useClPerm,
7279                                            PetscInt *numIndices, PetscInt *indices[], PetscInt outOffsets[], PetscScalar *values[])
7280 {
7281   PetscErrorCode ierr;
7282 
7283   PetscFunctionBegin;
7284   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7285   PetscValidPointer(indices, 7);
7286   ierr = DMRestoreWorkArray(dm, 0, MPIU_INT, indices);CHKERRQ(ierr);
7287   PetscFunctionReturn(0);
7288 }
7289 
7290 /*@C
7291   DMPlexMatSetClosure - Set an array of the values on the closure of 'point'
7292 
7293   Not collective
7294 
7295   Input Parameters:
7296 + dm - The DM
7297 . section - The section describing the layout in v, or NULL to use the default section
7298 . globalSection - The section describing the layout in v, or NULL to use the default global section
7299 . A - The matrix
7300 . point - The point in the DM
7301 . values - The array of values
7302 - mode - The insert mode, where INSERT_ALL_VALUES and ADD_ALL_VALUES also overwrite boundary conditions
7303 
7304   Fortran Notes:
7305   This routine is only available in Fortran 90, and you must include petsc.h90 in your code.
7306 
7307   Level: intermediate
7308 
7309 .seealso DMPlexMatSetClosureGeneral(), DMPlexVecGetClosure(), DMPlexVecSetClosure()
7310 @*/
7311 PetscErrorCode DMPlexMatSetClosure(DM dm, PetscSection section, PetscSection globalSection, Mat A, PetscInt point, const PetscScalar values[], InsertMode mode)
7312 {
7313   DM_Plex           *mesh = (DM_Plex*) dm->data;
7314   PetscInt          *indices;
7315   PetscInt           numIndices;
7316   const PetscScalar *valuesOrig = values;
7317   PetscErrorCode     ierr;
7318 
7319   PetscFunctionBegin;
7320   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7321   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
7322   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
7323   if (!globalSection) {ierr = DMGetGlobalSection(dm, &globalSection);CHKERRQ(ierr);}
7324   PetscValidHeaderSpecific(globalSection, PETSC_SECTION_CLASSID, 3);
7325   PetscValidHeaderSpecific(A, MAT_CLASSID, 4);
7326 
7327   ierr = DMPlexGetClosureIndices(dm, section, globalSection, point, PETSC_TRUE, &numIndices, &indices, NULL, (PetscScalar **) &values);CHKERRQ(ierr);
7328 
7329   if (mesh->printSetValues) {ierr = DMPlexPrintMatSetValues(PETSC_VIEWER_STDOUT_SELF, A, point, numIndices, indices, 0, NULL, values);CHKERRQ(ierr);}
7330   ierr = MatSetValues(A, numIndices, indices, numIndices, indices, values, mode);
7331   if (ierr) {
7332     PetscMPIInt    rank;
7333     PetscErrorCode ierr2;
7334 
7335     ierr2 = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRMPI(ierr2);
7336     ierr2 = (*PetscErrorPrintf)("[%d]ERROR in DMPlexMatSetClosure\n", rank);CHKERRQ(ierr2);
7337     ierr2 = DMPlexPrintMatSetValues(PETSC_VIEWER_STDERR_SELF, A, point, numIndices, indices, 0, NULL, values);CHKERRQ(ierr2);
7338     ierr2 = DMPlexRestoreClosureIndices(dm, section, globalSection, point, PETSC_TRUE, &numIndices, &indices, NULL, (PetscScalar **) &values);CHKERRQ(ierr2);
7339     if (values != valuesOrig) {ierr2 = DMRestoreWorkArray(dm, 0, MPIU_SCALAR, &values);CHKERRQ(ierr2);}
7340     CHKERRQ(ierr);
7341   }
7342   if (mesh->printFEM > 1) {
7343     PetscInt i;
7344     ierr = PetscPrintf(PETSC_COMM_SELF, "  Indices:");CHKERRQ(ierr);
7345     for (i = 0; i < numIndices; ++i) {ierr = PetscPrintf(PETSC_COMM_SELF, " %D", indices[i]);CHKERRQ(ierr);}
7346     ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
7347   }
7348 
7349   ierr = DMPlexRestoreClosureIndices(dm, section, globalSection, point, PETSC_TRUE, &numIndices, &indices, NULL, (PetscScalar **) &values);CHKERRQ(ierr);
7350   if (values != valuesOrig) {ierr = DMRestoreWorkArray(dm, 0, MPIU_SCALAR, &values);CHKERRQ(ierr);}
7351   PetscFunctionReturn(0);
7352 }
7353 
7354 /*@C
7355   DMPlexMatSetClosure - Set an array of the values on the closure of 'point' using a different row and column section
7356 
7357   Not collective
7358 
7359   Input Parameters:
7360 + dmRow - The DM for the row fields
7361 . sectionRow - The section describing the layout, or NULL to use the default section in dmRow
7362 . globalSectionRow - The section describing the layout, or NULL to use the default global section in dmRow
7363 . dmCol - The DM for the column fields
7364 . sectionCol - The section describing the layout, or NULL to use the default section in dmCol
7365 . globalSectionCol - The section describing the layout, or NULL to use the default global section in dmCol
7366 . A - The matrix
7367 . point - The point in the DMs
7368 . values - The array of values
7369 - mode - The insert mode, where INSERT_ALL_VALUES and ADD_ALL_VALUES also overwrite boundary conditions
7370 
7371   Level: intermediate
7372 
7373 .seealso DMPlexMatSetClosure(), DMPlexVecGetClosure(), DMPlexVecSetClosure()
7374 @*/
7375 PetscErrorCode DMPlexMatSetClosureGeneral(DM dmRow, PetscSection sectionRow, PetscSection globalSectionRow, DM dmCol, PetscSection sectionCol, PetscSection globalSectionCol, Mat A, PetscInt point, const PetscScalar values[], InsertMode mode)
7376 {
7377   DM_Plex           *mesh = (DM_Plex*) dmRow->data;
7378   PetscInt          *indicesRow, *indicesCol;
7379   PetscInt           numIndicesRow, numIndicesCol;
7380   const PetscScalar *valuesOrig = values;
7381   PetscErrorCode     ierr;
7382 
7383   PetscFunctionBegin;
7384   PetscValidHeaderSpecific(dmRow, DM_CLASSID, 1);
7385   if (!sectionRow) {ierr = DMGetLocalSection(dmRow, &sectionRow);CHKERRQ(ierr);}
7386   PetscValidHeaderSpecific(sectionRow, PETSC_SECTION_CLASSID, 2);
7387   if (!globalSectionRow) {ierr = DMGetGlobalSection(dmRow, &globalSectionRow);CHKERRQ(ierr);}
7388   PetscValidHeaderSpecific(globalSectionRow, PETSC_SECTION_CLASSID, 3);
7389   PetscValidHeaderSpecific(dmCol, DM_CLASSID, 4);
7390   if (!sectionCol) {ierr = DMGetLocalSection(dmCol, &sectionCol);CHKERRQ(ierr);}
7391   PetscValidHeaderSpecific(sectionCol, PETSC_SECTION_CLASSID, 5);
7392   if (!globalSectionCol) {ierr = DMGetGlobalSection(dmCol, &globalSectionCol);CHKERRQ(ierr);}
7393   PetscValidHeaderSpecific(globalSectionCol, PETSC_SECTION_CLASSID, 6);
7394   PetscValidHeaderSpecific(A, MAT_CLASSID, 7);
7395 
7396   ierr = DMPlexGetClosureIndices(dmRow, sectionRow, globalSectionRow, point, PETSC_TRUE, &numIndicesRow, &indicesRow, NULL, (PetscScalar **) &values);CHKERRQ(ierr);
7397   ierr = DMPlexGetClosureIndices(dmCol, sectionCol, globalSectionCol, point, PETSC_TRUE, &numIndicesCol, &indicesCol, NULL, (PetscScalar **) &values);CHKERRQ(ierr);
7398 
7399   if (mesh->printSetValues) {ierr = DMPlexPrintMatSetValues(PETSC_VIEWER_STDOUT_SELF, A, point, numIndicesRow, indicesRow, numIndicesCol, indicesCol, values);CHKERRQ(ierr);}
7400   ierr = MatSetValues(A, numIndicesRow, indicesRow, numIndicesCol, indicesCol, values, mode);
7401   if (ierr) {
7402     PetscMPIInt    rank;
7403     PetscErrorCode ierr2;
7404 
7405     ierr2 = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRMPI(ierr2);
7406     ierr2 = (*PetscErrorPrintf)("[%d]ERROR in DMPlexMatSetClosure\n", rank);CHKERRQ(ierr2);
7407     ierr2 = DMPlexPrintMatSetValues(PETSC_VIEWER_STDERR_SELF, A, point, numIndicesRow, indicesRow, numIndicesCol, indicesCol, values);CHKERRQ(ierr2);
7408     ierr2 = DMPlexRestoreClosureIndices(dmRow, sectionRow, globalSectionRow, point, PETSC_TRUE, &numIndicesRow, &indicesRow, NULL, (PetscScalar **) &values);CHKERRQ(ierr2);
7409     ierr2 = DMPlexRestoreClosureIndices(dmCol, sectionCol, globalSectionCol, point, PETSC_TRUE, &numIndicesCol, &indicesRow, NULL, (PetscScalar **) &values);CHKERRQ(ierr2);
7410     if (values != valuesOrig) {ierr2 = DMRestoreWorkArray(dmRow, 0, MPIU_SCALAR, &values);CHKERRQ(ierr2);}
7411     CHKERRQ(ierr);
7412   }
7413 
7414   ierr = DMPlexRestoreClosureIndices(dmRow, sectionRow, globalSectionRow, point, PETSC_TRUE, &numIndicesRow, &indicesRow, NULL, (PetscScalar **) &values);CHKERRQ(ierr);
7415   ierr = DMPlexRestoreClosureIndices(dmCol, sectionCol, globalSectionCol, point, PETSC_TRUE, &numIndicesCol, &indicesCol, NULL, (PetscScalar **) &values);CHKERRQ(ierr);
7416   if (values != valuesOrig) {ierr = DMRestoreWorkArray(dmRow, 0, MPIU_SCALAR, &values);CHKERRQ(ierr);}
7417   PetscFunctionReturn(0);
7418 }
7419 
7420 PetscErrorCode DMPlexMatSetClosureRefined(DM dmf, PetscSection fsection, PetscSection globalFSection, DM dmc, PetscSection csection, PetscSection globalCSection, Mat A, PetscInt point, const PetscScalar values[], InsertMode mode)
7421 {
7422   DM_Plex        *mesh   = (DM_Plex*) dmf->data;
7423   PetscInt       *fpoints = NULL, *ftotpoints = NULL;
7424   PetscInt       *cpoints = NULL;
7425   PetscInt       *findices, *cindices;
7426   const PetscInt *fclperm = NULL, *cclperm = NULL; /* Closure permutations cannot work here */
7427   PetscInt        foffsets[32], coffsets[32];
7428   DMPolytopeType  ct;
7429   PetscInt        numFields, numSubcells, maxFPoints, numFPoints, numCPoints, numFIndices, numCIndices, dof, off, globalOff, pStart, pEnd, p, q, r, s, f;
7430   PetscErrorCode  ierr;
7431 
7432   PetscFunctionBegin;
7433   PetscValidHeaderSpecific(dmf, DM_CLASSID, 1);
7434   PetscValidHeaderSpecific(dmc, DM_CLASSID, 4);
7435   if (!fsection) {ierr = DMGetLocalSection(dmf, &fsection);CHKERRQ(ierr);}
7436   PetscValidHeaderSpecific(fsection, PETSC_SECTION_CLASSID, 2);
7437   if (!csection) {ierr = DMGetLocalSection(dmc, &csection);CHKERRQ(ierr);}
7438   PetscValidHeaderSpecific(csection, PETSC_SECTION_CLASSID, 5);
7439   if (!globalFSection) {ierr = DMGetGlobalSection(dmf, &globalFSection);CHKERRQ(ierr);}
7440   PetscValidHeaderSpecific(globalFSection, PETSC_SECTION_CLASSID, 3);
7441   if (!globalCSection) {ierr = DMGetGlobalSection(dmc, &globalCSection);CHKERRQ(ierr);}
7442   PetscValidHeaderSpecific(globalCSection, PETSC_SECTION_CLASSID, 6);
7443   PetscValidHeaderSpecific(A, MAT_CLASSID, 7);
7444   ierr = PetscSectionGetNumFields(fsection, &numFields);CHKERRQ(ierr);
7445   if (numFields > 31) SETERRQ1(PetscObjectComm((PetscObject)dmf), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", numFields);
7446   ierr = PetscArrayzero(foffsets, 32);CHKERRQ(ierr);
7447   ierr = PetscArrayzero(coffsets, 32);CHKERRQ(ierr);
7448   /* Column indices */
7449   ierr = DMPlexGetTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
7450   maxFPoints = numCPoints;
7451   /* Compress out points not in the section */
7452   /*   TODO: Squeeze out points with 0 dof as well */
7453   ierr = PetscSectionGetChart(csection, &pStart, &pEnd);CHKERRQ(ierr);
7454   for (p = 0, q = 0; p < numCPoints*2; p += 2) {
7455     if ((cpoints[p] >= pStart) && (cpoints[p] < pEnd)) {
7456       cpoints[q*2]   = cpoints[p];
7457       cpoints[q*2+1] = cpoints[p+1];
7458       ++q;
7459     }
7460   }
7461   numCPoints = q;
7462   for (p = 0, numCIndices = 0; p < numCPoints*2; p += 2) {
7463     PetscInt fdof;
7464 
7465     ierr = PetscSectionGetDof(csection, cpoints[p], &dof);CHKERRQ(ierr);
7466     if (!dof) continue;
7467     for (f = 0; f < numFields; ++f) {
7468       ierr           = PetscSectionGetFieldDof(csection, cpoints[p], f, &fdof);CHKERRQ(ierr);
7469       coffsets[f+1] += fdof;
7470     }
7471     numCIndices += dof;
7472   }
7473   for (f = 1; f < numFields; ++f) coffsets[f+1] += coffsets[f];
7474   /* Row indices */
7475   ierr = DMPlexGetCellType(dmc, point, &ct);CHKERRQ(ierr);
7476   {
7477     DMPlexCellRefiner cr;
7478     ierr = DMPlexCellRefinerCreate(dmc, &cr);CHKERRQ(ierr);
7479     ierr = DMPlexCellRefinerGetAffineTransforms(cr, ct, &numSubcells, NULL, NULL, NULL);CHKERRQ(ierr);
7480     ierr = DMPlexCellRefinerDestroy(&cr);CHKERRQ(ierr);
7481   }
7482   ierr = DMGetWorkArray(dmf, maxFPoints*2*numSubcells, MPIU_INT, &ftotpoints);CHKERRQ(ierr);
7483   for (r = 0, q = 0; r < numSubcells; ++r) {
7484     /* TODO Map from coarse to fine cells */
7485     ierr = DMPlexGetTransitiveClosure(dmf, point*numSubcells + r, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
7486     /* Compress out points not in the section */
7487     ierr = PetscSectionGetChart(fsection, &pStart, &pEnd);CHKERRQ(ierr);
7488     for (p = 0; p < numFPoints*2; p += 2) {
7489       if ((fpoints[p] >= pStart) && (fpoints[p] < pEnd)) {
7490         ierr = PetscSectionGetDof(fsection, fpoints[p], &dof);CHKERRQ(ierr);
7491         if (!dof) continue;
7492         for (s = 0; s < q; ++s) if (fpoints[p] == ftotpoints[s*2]) break;
7493         if (s < q) continue;
7494         ftotpoints[q*2]   = fpoints[p];
7495         ftotpoints[q*2+1] = fpoints[p+1];
7496         ++q;
7497       }
7498     }
7499     ierr = DMPlexRestoreTransitiveClosure(dmf, point, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
7500   }
7501   numFPoints = q;
7502   for (p = 0, numFIndices = 0; p < numFPoints*2; p += 2) {
7503     PetscInt fdof;
7504 
7505     ierr = PetscSectionGetDof(fsection, ftotpoints[p], &dof);CHKERRQ(ierr);
7506     if (!dof) continue;
7507     for (f = 0; f < numFields; ++f) {
7508       ierr           = PetscSectionGetFieldDof(fsection, ftotpoints[p], f, &fdof);CHKERRQ(ierr);
7509       foffsets[f+1] += fdof;
7510     }
7511     numFIndices += dof;
7512   }
7513   for (f = 1; f < numFields; ++f) foffsets[f+1] += foffsets[f];
7514 
7515   if (numFields && foffsets[numFields] != numFIndices) SETERRQ2(PetscObjectComm((PetscObject)dmf), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", foffsets[numFields], numFIndices);
7516   if (numFields && coffsets[numFields] != numCIndices) SETERRQ2(PetscObjectComm((PetscObject)dmc), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", coffsets[numFields], numCIndices);
7517   ierr = DMGetWorkArray(dmf, numFIndices, MPIU_INT, &findices);CHKERRQ(ierr);
7518   ierr = DMGetWorkArray(dmc, numCIndices, MPIU_INT, &cindices);CHKERRQ(ierr);
7519   if (numFields) {
7520     const PetscInt **permsF[32] = {NULL};
7521     const PetscInt **permsC[32] = {NULL};
7522 
7523     for (f = 0; f < numFields; f++) {
7524       ierr = PetscSectionGetFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL);CHKERRQ(ierr);
7525       ierr = PetscSectionGetFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL);CHKERRQ(ierr);
7526     }
7527     for (p = 0; p < numFPoints; p++) {
7528       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff);CHKERRQ(ierr);
7529       ierr = DMPlexGetIndicesPointFields_Internal(fsection, PETSC_FALSE, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, foffsets, PETSC_FALSE, permsF, p, fclperm, findices);CHKERRQ(ierr);
7530     }
7531     for (p = 0; p < numCPoints; p++) {
7532       ierr = PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff);CHKERRQ(ierr);
7533       ierr = DMPlexGetIndicesPointFields_Internal(csection, PETSC_FALSE, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, coffsets, PETSC_FALSE, permsC, p, cclperm, cindices);CHKERRQ(ierr);
7534     }
7535     for (f = 0; f < numFields; f++) {
7536       ierr = PetscSectionRestoreFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL);CHKERRQ(ierr);
7537       ierr = PetscSectionRestoreFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL);CHKERRQ(ierr);
7538     }
7539   } else {
7540     const PetscInt **permsF = NULL;
7541     const PetscInt **permsC = NULL;
7542 
7543     ierr = PetscSectionGetPointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL);CHKERRQ(ierr);
7544     ierr = PetscSectionGetPointSyms(csection,numCPoints,cpoints,&permsC,NULL);CHKERRQ(ierr);
7545     for (p = 0, off = 0; p < numFPoints; p++) {
7546       const PetscInt *perm = permsF ? permsF[p] : NULL;
7547 
7548       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff);CHKERRQ(ierr);
7549       ierr = DMPlexGetIndicesPoint_Internal(fsection, PETSC_FALSE, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, fclperm, findices);CHKERRQ(ierr);
7550     }
7551     for (p = 0, off = 0; p < numCPoints; p++) {
7552       const PetscInt *perm = permsC ? permsC[p] : NULL;
7553 
7554       ierr = PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff);CHKERRQ(ierr);
7555       ierr = DMPlexGetIndicesPoint_Internal(csection, PETSC_FALSE, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, cclperm, cindices);CHKERRQ(ierr);
7556     }
7557     ierr = PetscSectionRestorePointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL);CHKERRQ(ierr);
7558     ierr = PetscSectionRestorePointSyms(csection,numCPoints,cpoints,&permsC,NULL);CHKERRQ(ierr);
7559   }
7560   if (mesh->printSetValues) {ierr = DMPlexPrintMatSetValues(PETSC_VIEWER_STDOUT_SELF, A, point, numFIndices, findices, numCIndices, cindices, values);CHKERRQ(ierr);}
7561   /* TODO: flips */
7562   ierr = MatSetValues(A, numFIndices, findices, numCIndices, cindices, values, mode);
7563   if (ierr) {
7564     PetscMPIInt    rank;
7565     PetscErrorCode ierr2;
7566 
7567     ierr2 = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRMPI(ierr2);
7568     ierr2 = (*PetscErrorPrintf)("[%d]ERROR in DMPlexMatSetClosure\n", rank);CHKERRQ(ierr2);
7569     ierr2 = DMPlexPrintMatSetValues(PETSC_VIEWER_STDERR_SELF, A, point, numFIndices, findices, numCIndices, cindices, values);CHKERRQ(ierr2);
7570     ierr2 = DMRestoreWorkArray(dmf, numFIndices, MPIU_INT, &findices);CHKERRQ(ierr2);
7571     ierr2 = DMRestoreWorkArray(dmc, numCIndices, MPIU_INT, &cindices);CHKERRQ(ierr2);
7572     CHKERRQ(ierr);
7573   }
7574   ierr = DMRestoreWorkArray(dmf, numCPoints*2*4, MPIU_INT, &ftotpoints);CHKERRQ(ierr);
7575   ierr = DMPlexRestoreTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
7576   ierr = DMRestoreWorkArray(dmf, numFIndices, MPIU_INT, &findices);CHKERRQ(ierr);
7577   ierr = DMRestoreWorkArray(dmc, numCIndices, MPIU_INT, &cindices);CHKERRQ(ierr);
7578   PetscFunctionReturn(0);
7579 }
7580 
7581 PetscErrorCode DMPlexMatGetClosureIndicesRefined(DM dmf, PetscSection fsection, PetscSection globalFSection, DM dmc, PetscSection csection, PetscSection globalCSection, PetscInt point, PetscInt cindices[], PetscInt findices[])
7582 {
7583   PetscInt      *fpoints = NULL, *ftotpoints = NULL;
7584   PetscInt      *cpoints = NULL;
7585   PetscInt       foffsets[32], coffsets[32];
7586   const PetscInt *fclperm = NULL, *cclperm = NULL; /* Closure permutations cannot work here */
7587   DMPolytopeType ct;
7588   PetscInt       numFields, numSubcells, maxFPoints, numFPoints, numCPoints, numFIndices, numCIndices, dof, off, globalOff, pStart, pEnd, p, q, r, s, f;
7589   PetscErrorCode ierr;
7590 
7591   PetscFunctionBegin;
7592   PetscValidHeaderSpecific(dmf, DM_CLASSID, 1);
7593   PetscValidHeaderSpecific(dmc, DM_CLASSID, 4);
7594   if (!fsection) {ierr = DMGetLocalSection(dmf, &fsection);CHKERRQ(ierr);}
7595   PetscValidHeaderSpecific(fsection, PETSC_SECTION_CLASSID, 2);
7596   if (!csection) {ierr = DMGetLocalSection(dmc, &csection);CHKERRQ(ierr);}
7597   PetscValidHeaderSpecific(csection, PETSC_SECTION_CLASSID, 5);
7598   if (!globalFSection) {ierr = DMGetGlobalSection(dmf, &globalFSection);CHKERRQ(ierr);}
7599   PetscValidHeaderSpecific(globalFSection, PETSC_SECTION_CLASSID, 3);
7600   if (!globalCSection) {ierr = DMGetGlobalSection(dmc, &globalCSection);CHKERRQ(ierr);}
7601   PetscValidHeaderSpecific(globalCSection, PETSC_SECTION_CLASSID, 6);
7602   ierr = PetscSectionGetNumFields(fsection, &numFields);CHKERRQ(ierr);
7603   if (numFields > 31) SETERRQ1(PetscObjectComm((PetscObject)dmf), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", numFields);
7604   ierr = PetscArrayzero(foffsets, 32);CHKERRQ(ierr);
7605   ierr = PetscArrayzero(coffsets, 32);CHKERRQ(ierr);
7606   /* Column indices */
7607   ierr = DMPlexGetTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
7608   maxFPoints = numCPoints;
7609   /* Compress out points not in the section */
7610   /*   TODO: Squeeze out points with 0 dof as well */
7611   ierr = PetscSectionGetChart(csection, &pStart, &pEnd);CHKERRQ(ierr);
7612   for (p = 0, q = 0; p < numCPoints*2; p += 2) {
7613     if ((cpoints[p] >= pStart) && (cpoints[p] < pEnd)) {
7614       cpoints[q*2]   = cpoints[p];
7615       cpoints[q*2+1] = cpoints[p+1];
7616       ++q;
7617     }
7618   }
7619   numCPoints = q;
7620   for (p = 0, numCIndices = 0; p < numCPoints*2; p += 2) {
7621     PetscInt fdof;
7622 
7623     ierr = PetscSectionGetDof(csection, cpoints[p], &dof);CHKERRQ(ierr);
7624     if (!dof) continue;
7625     for (f = 0; f < numFields; ++f) {
7626       ierr           = PetscSectionGetFieldDof(csection, cpoints[p], f, &fdof);CHKERRQ(ierr);
7627       coffsets[f+1] += fdof;
7628     }
7629     numCIndices += dof;
7630   }
7631   for (f = 1; f < numFields; ++f) coffsets[f+1] += coffsets[f];
7632   /* Row indices */
7633   ierr = DMPlexGetCellType(dmc, point, &ct);CHKERRQ(ierr);
7634   {
7635     DMPlexCellRefiner cr;
7636     ierr = DMPlexCellRefinerCreate(dmc, &cr);CHKERRQ(ierr);
7637     ierr = DMPlexCellRefinerGetAffineTransforms(cr, ct, &numSubcells, NULL, NULL, NULL);CHKERRQ(ierr);
7638     ierr = DMPlexCellRefinerDestroy(&cr);CHKERRQ(ierr);
7639   }
7640   ierr = DMGetWorkArray(dmf, maxFPoints*2*numSubcells, MPIU_INT, &ftotpoints);CHKERRQ(ierr);
7641   for (r = 0, q = 0; r < numSubcells; ++r) {
7642     /* TODO Map from coarse to fine cells */
7643     ierr = DMPlexGetTransitiveClosure(dmf, point*numSubcells + r, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
7644     /* Compress out points not in the section */
7645     ierr = PetscSectionGetChart(fsection, &pStart, &pEnd);CHKERRQ(ierr);
7646     for (p = 0; p < numFPoints*2; p += 2) {
7647       if ((fpoints[p] >= pStart) && (fpoints[p] < pEnd)) {
7648         ierr = PetscSectionGetDof(fsection, fpoints[p], &dof);CHKERRQ(ierr);
7649         if (!dof) continue;
7650         for (s = 0; s < q; ++s) if (fpoints[p] == ftotpoints[s*2]) break;
7651         if (s < q) continue;
7652         ftotpoints[q*2]   = fpoints[p];
7653         ftotpoints[q*2+1] = fpoints[p+1];
7654         ++q;
7655       }
7656     }
7657     ierr = DMPlexRestoreTransitiveClosure(dmf, point, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
7658   }
7659   numFPoints = q;
7660   for (p = 0, numFIndices = 0; p < numFPoints*2; p += 2) {
7661     PetscInt fdof;
7662 
7663     ierr = PetscSectionGetDof(fsection, ftotpoints[p], &dof);CHKERRQ(ierr);
7664     if (!dof) continue;
7665     for (f = 0; f < numFields; ++f) {
7666       ierr           = PetscSectionGetFieldDof(fsection, ftotpoints[p], f, &fdof);CHKERRQ(ierr);
7667       foffsets[f+1] += fdof;
7668     }
7669     numFIndices += dof;
7670   }
7671   for (f = 1; f < numFields; ++f) foffsets[f+1] += foffsets[f];
7672 
7673   if (numFields && foffsets[numFields] != numFIndices) SETERRQ2(PetscObjectComm((PetscObject)dmf), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", foffsets[numFields], numFIndices);
7674   if (numFields && coffsets[numFields] != numCIndices) SETERRQ2(PetscObjectComm((PetscObject)dmc), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", coffsets[numFields], numCIndices);
7675   if (numFields) {
7676     const PetscInt **permsF[32] = {NULL};
7677     const PetscInt **permsC[32] = {NULL};
7678 
7679     for (f = 0; f < numFields; f++) {
7680       ierr = PetscSectionGetFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL);CHKERRQ(ierr);
7681       ierr = PetscSectionGetFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL);CHKERRQ(ierr);
7682     }
7683     for (p = 0; p < numFPoints; p++) {
7684       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff);CHKERRQ(ierr);
7685       ierr = DMPlexGetIndicesPointFields_Internal(fsection, PETSC_FALSE, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, foffsets, PETSC_FALSE, permsF, p, fclperm, findices);CHKERRQ(ierr);
7686     }
7687     for (p = 0; p < numCPoints; p++) {
7688       ierr = PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff);CHKERRQ(ierr);
7689       ierr = DMPlexGetIndicesPointFields_Internal(csection, PETSC_FALSE, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, coffsets, PETSC_FALSE, permsC, p, cclperm, cindices);CHKERRQ(ierr);
7690     }
7691     for (f = 0; f < numFields; f++) {
7692       ierr = PetscSectionRestoreFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL);CHKERRQ(ierr);
7693       ierr = PetscSectionRestoreFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL);CHKERRQ(ierr);
7694     }
7695   } else {
7696     const PetscInt **permsF = NULL;
7697     const PetscInt **permsC = NULL;
7698 
7699     ierr = PetscSectionGetPointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL);CHKERRQ(ierr);
7700     ierr = PetscSectionGetPointSyms(csection,numCPoints,cpoints,&permsC,NULL);CHKERRQ(ierr);
7701     for (p = 0, off = 0; p < numFPoints; p++) {
7702       const PetscInt *perm = permsF ? permsF[p] : NULL;
7703 
7704       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff);CHKERRQ(ierr);
7705       ierr = DMPlexGetIndicesPoint_Internal(fsection, PETSC_FALSE, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, fclperm, findices);CHKERRQ(ierr);
7706     }
7707     for (p = 0, off = 0; p < numCPoints; p++) {
7708       const PetscInt *perm = permsC ? permsC[p] : NULL;
7709 
7710       ierr = PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff);CHKERRQ(ierr);
7711       ierr = DMPlexGetIndicesPoint_Internal(csection, PETSC_FALSE, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, cclperm, cindices);CHKERRQ(ierr);
7712     }
7713     ierr = PetscSectionRestorePointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL);CHKERRQ(ierr);
7714     ierr = PetscSectionRestorePointSyms(csection,numCPoints,cpoints,&permsC,NULL);CHKERRQ(ierr);
7715   }
7716   ierr = DMRestoreWorkArray(dmf, numCPoints*2*4, MPIU_INT, &ftotpoints);CHKERRQ(ierr);
7717   ierr = DMPlexRestoreTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
7718   PetscFunctionReturn(0);
7719 }
7720 
7721 /*@C
7722   DMPlexGetVTKCellHeight - Returns the height in the DAG used to determine which points are cells (normally 0)
7723 
7724   Input Parameter:
7725 . dm   - The DMPlex object
7726 
7727   Output Parameter:
7728 . cellHeight - The height of a cell
7729 
7730   Level: developer
7731 
7732 .seealso DMPlexSetVTKCellHeight()
7733 @*/
7734 PetscErrorCode DMPlexGetVTKCellHeight(DM dm, PetscInt *cellHeight)
7735 {
7736   DM_Plex *mesh = (DM_Plex*) dm->data;
7737 
7738   PetscFunctionBegin;
7739   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7740   PetscValidPointer(cellHeight, 2);
7741   *cellHeight = mesh->vtkCellHeight;
7742   PetscFunctionReturn(0);
7743 }
7744 
7745 /*@C
7746   DMPlexSetVTKCellHeight - Sets the height in the DAG used to determine which points are cells (normally 0)
7747 
7748   Input Parameters:
7749 + dm   - The DMPlex object
7750 - cellHeight - The height of a cell
7751 
7752   Level: developer
7753 
7754 .seealso DMPlexGetVTKCellHeight()
7755 @*/
7756 PetscErrorCode DMPlexSetVTKCellHeight(DM dm, PetscInt cellHeight)
7757 {
7758   DM_Plex *mesh = (DM_Plex*) dm->data;
7759 
7760   PetscFunctionBegin;
7761   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7762   mesh->vtkCellHeight = cellHeight;
7763   PetscFunctionReturn(0);
7764 }
7765 
7766 /*@
7767   DMPlexGetGhostCellStratum - Get the range of cells which are used to enforce FV boundary conditions
7768 
7769   Input Parameter:
7770 . dm - The DMPlex object
7771 
7772   Output Parameters:
7773 + gcStart - The first ghost cell, or NULL
7774 - gcEnd   - The upper bound on ghost cells, or NULL
7775 
7776   Level: advanced
7777 
7778 .seealso DMPlexConstructGhostCells(), DMPlexGetGhostCellStratum()
7779 @*/
7780 PetscErrorCode DMPlexGetGhostCellStratum(DM dm, PetscInt *gcStart, PetscInt *gcEnd)
7781 {
7782   DMLabel        ctLabel;
7783   PetscErrorCode ierr;
7784 
7785   PetscFunctionBegin;
7786   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7787   ierr = DMPlexGetCellTypeLabel(dm, &ctLabel);CHKERRQ(ierr);
7788   ierr = DMLabelGetStratumBounds(ctLabel, DM_POLYTOPE_FV_GHOST, gcStart, gcEnd);CHKERRQ(ierr);
7789   PetscFunctionReturn(0);
7790 }
7791 
7792 /* We can easily have a form that takes an IS instead */
7793 PetscErrorCode DMPlexCreateNumbering_Plex(DM dm, PetscInt pStart, PetscInt pEnd, PetscInt shift, PetscInt *globalSize, PetscSF sf, IS *numbering)
7794 {
7795   PetscSection   section, globalSection;
7796   PetscInt      *numbers, p;
7797   PetscErrorCode ierr;
7798 
7799   PetscFunctionBegin;
7800   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &section);CHKERRQ(ierr);
7801   ierr = PetscSectionSetChart(section, pStart, pEnd);CHKERRQ(ierr);
7802   for (p = pStart; p < pEnd; ++p) {
7803     ierr = PetscSectionSetDof(section, p, 1);CHKERRQ(ierr);
7804   }
7805   ierr = PetscSectionSetUp(section);CHKERRQ(ierr);
7806   ierr = PetscSectionCreateGlobalSection(section, sf, PETSC_FALSE, PETSC_FALSE, &globalSection);CHKERRQ(ierr);
7807   ierr = PetscMalloc1(pEnd - pStart, &numbers);CHKERRQ(ierr);
7808   for (p = pStart; p < pEnd; ++p) {
7809     ierr = PetscSectionGetOffset(globalSection, p, &numbers[p-pStart]);CHKERRQ(ierr);
7810     if (numbers[p-pStart] < 0) numbers[p-pStart] -= shift;
7811     else                       numbers[p-pStart] += shift;
7812   }
7813   ierr = ISCreateGeneral(PetscObjectComm((PetscObject) dm), pEnd - pStart, numbers, PETSC_OWN_POINTER, numbering);CHKERRQ(ierr);
7814   if (globalSize) {
7815     PetscLayout layout;
7816     ierr = PetscSectionGetPointLayout(PetscObjectComm((PetscObject) dm), globalSection, &layout);CHKERRQ(ierr);
7817     ierr = PetscLayoutGetSize(layout, globalSize);CHKERRQ(ierr);
7818     ierr = PetscLayoutDestroy(&layout);CHKERRQ(ierr);
7819   }
7820   ierr = PetscSectionDestroy(&section);CHKERRQ(ierr);
7821   ierr = PetscSectionDestroy(&globalSection);CHKERRQ(ierr);
7822   PetscFunctionReturn(0);
7823 }
7824 
7825 PetscErrorCode DMPlexCreateCellNumbering_Internal(DM dm, PetscBool includeHybrid, IS *globalCellNumbers)
7826 {
7827   PetscInt       cellHeight, cStart, cEnd;
7828   PetscErrorCode ierr;
7829 
7830   PetscFunctionBegin;
7831   ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
7832   if (includeHybrid) {ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);}
7833   else               {ierr = DMPlexGetSimplexOrBoxCells(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);}
7834   ierr = DMPlexCreateNumbering_Plex(dm, cStart, cEnd, 0, NULL, dm->sf, globalCellNumbers);CHKERRQ(ierr);
7835   PetscFunctionReturn(0);
7836 }
7837 
7838 /*@
7839   DMPlexGetCellNumbering - Get a global cell numbering for all cells on this process
7840 
7841   Input Parameter:
7842 . dm   - The DMPlex object
7843 
7844   Output Parameter:
7845 . globalCellNumbers - Global cell numbers for all cells on this process
7846 
7847   Level: developer
7848 
7849 .seealso DMPlexGetVertexNumbering()
7850 @*/
7851 PetscErrorCode DMPlexGetCellNumbering(DM dm, IS *globalCellNumbers)
7852 {
7853   DM_Plex       *mesh = (DM_Plex*) dm->data;
7854   PetscErrorCode ierr;
7855 
7856   PetscFunctionBegin;
7857   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7858   if (!mesh->globalCellNumbers) {ierr = DMPlexCreateCellNumbering_Internal(dm, PETSC_FALSE, &mesh->globalCellNumbers);CHKERRQ(ierr);}
7859   *globalCellNumbers = mesh->globalCellNumbers;
7860   PetscFunctionReturn(0);
7861 }
7862 
7863 PetscErrorCode DMPlexCreateVertexNumbering_Internal(DM dm, PetscBool includeHybrid, IS *globalVertexNumbers)
7864 {
7865   PetscInt       vStart, vEnd;
7866   PetscErrorCode ierr;
7867 
7868   PetscFunctionBegin;
7869   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7870   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
7871   ierr = DMPlexCreateNumbering_Plex(dm, vStart, vEnd, 0, NULL, dm->sf, globalVertexNumbers);CHKERRQ(ierr);
7872   PetscFunctionReturn(0);
7873 }
7874 
7875 /*@
7876   DMPlexGetVertexNumbering - Get a global vertex numbering for all vertices on this process
7877 
7878   Input Parameter:
7879 . dm   - The DMPlex object
7880 
7881   Output Parameter:
7882 . globalVertexNumbers - Global vertex numbers for all vertices on this process
7883 
7884   Level: developer
7885 
7886 .seealso DMPlexGetCellNumbering()
7887 @*/
7888 PetscErrorCode DMPlexGetVertexNumbering(DM dm, IS *globalVertexNumbers)
7889 {
7890   DM_Plex       *mesh = (DM_Plex*) dm->data;
7891   PetscErrorCode ierr;
7892 
7893   PetscFunctionBegin;
7894   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7895   if (!mesh->globalVertexNumbers) {ierr = DMPlexCreateVertexNumbering_Internal(dm, PETSC_FALSE, &mesh->globalVertexNumbers);CHKERRQ(ierr);}
7896   *globalVertexNumbers = mesh->globalVertexNumbers;
7897   PetscFunctionReturn(0);
7898 }
7899 
7900 /*@
7901   DMPlexCreatePointNumbering - Create a global numbering for all points on this process
7902 
7903   Input Parameter:
7904 . dm   - The DMPlex object
7905 
7906   Output Parameter:
7907 . globalPointNumbers - Global numbers for all points on this process
7908 
7909   Level: developer
7910 
7911 .seealso DMPlexGetCellNumbering()
7912 @*/
7913 PetscErrorCode DMPlexCreatePointNumbering(DM dm, IS *globalPointNumbers)
7914 {
7915   IS             nums[4];
7916   PetscInt       depths[4], gdepths[4], starts[4];
7917   PetscInt       depth, d, shift = 0;
7918   PetscErrorCode ierr;
7919 
7920   PetscFunctionBegin;
7921   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7922   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
7923   /* For unstratified meshes use dim instead of depth */
7924   if (depth < 0) {ierr = DMGetDimension(dm, &depth);CHKERRQ(ierr);}
7925   for (d = 0; d <= depth; ++d) {
7926     PetscInt end;
7927 
7928     depths[d] = depth-d;
7929     ierr = DMPlexGetDepthStratum(dm, depths[d], &starts[d], &end);CHKERRQ(ierr);
7930     if (!(starts[d]-end)) { starts[d] = depths[d] = -1; }
7931   }
7932   ierr = PetscSortIntWithArray(depth+1, starts, depths);CHKERRQ(ierr);
7933   ierr = MPIU_Allreduce(depths, gdepths, depth+1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject) dm));CHKERRMPI(ierr);
7934   for (d = 0; d <= depth; ++d) {
7935     if (starts[d] >= 0 && depths[d] != gdepths[d]) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Expected depth %D, found %D",depths[d],gdepths[d]);
7936   }
7937   for (d = 0; d <= depth; ++d) {
7938     PetscInt pStart, pEnd, gsize;
7939 
7940     ierr = DMPlexGetDepthStratum(dm, gdepths[d], &pStart, &pEnd);CHKERRQ(ierr);
7941     ierr = DMPlexCreateNumbering_Plex(dm, pStart, pEnd, shift, &gsize, dm->sf, &nums[d]);CHKERRQ(ierr);
7942     shift += gsize;
7943   }
7944   ierr = ISConcatenate(PetscObjectComm((PetscObject) dm), depth+1, nums, globalPointNumbers);CHKERRQ(ierr);
7945   for (d = 0; d <= depth; ++d) {ierr = ISDestroy(&nums[d]);CHKERRQ(ierr);}
7946   PetscFunctionReturn(0);
7947 }
7948 
7949 /*@
7950   DMPlexCreateRankField - Create a cell field whose value is the rank of the owner
7951 
7952   Input Parameter:
7953 . dm - The DMPlex object
7954 
7955   Output Parameter:
7956 . ranks - The rank field
7957 
7958   Options Database Keys:
7959 . -dm_partition_view - Adds the rank field into the DM output from -dm_view using the same viewer
7960 
7961   Level: intermediate
7962 
7963 .seealso: DMView()
7964 @*/
7965 PetscErrorCode DMPlexCreateRankField(DM dm, Vec *ranks)
7966 {
7967   DM             rdm;
7968   PetscFE        fe;
7969   PetscScalar   *r;
7970   PetscMPIInt    rank;
7971   DMPolytopeType ct;
7972   PetscInt       dim, cStart, cEnd, c;
7973   PetscBool      simplex;
7974   PetscErrorCode ierr;
7975 
7976   PetscFunctionBeginUser;
7977   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7978   PetscValidPointer(ranks, 2);
7979   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRMPI(ierr);
7980   ierr = DMClone(dm, &rdm);CHKERRQ(ierr);
7981   ierr = DMGetDimension(rdm, &dim);CHKERRQ(ierr);
7982   ierr = DMPlexGetHeightStratum(rdm, 0, &cStart, &cEnd);CHKERRQ(ierr);
7983   ierr = DMPlexGetCellType(dm, cStart, &ct);CHKERRQ(ierr);
7984   simplex = DMPolytopeTypeGetNumVertices(ct) == DMPolytopeTypeGetDim(ct)+1 ? PETSC_TRUE : PETSC_FALSE;
7985   ierr = PetscFECreateDefault(PETSC_COMM_SELF, dim, 1, simplex, "PETSc___rank_", -1, &fe);CHKERRQ(ierr);
7986   ierr = PetscObjectSetName((PetscObject) fe, "rank");CHKERRQ(ierr);
7987   ierr = DMSetField(rdm, 0, NULL, (PetscObject) fe);CHKERRQ(ierr);
7988   ierr = PetscFEDestroy(&fe);CHKERRQ(ierr);
7989   ierr = DMCreateDS(rdm);CHKERRQ(ierr);
7990   ierr = DMCreateGlobalVector(rdm, ranks);CHKERRQ(ierr);
7991   ierr = PetscObjectSetName((PetscObject) *ranks, "partition");CHKERRQ(ierr);
7992   ierr = VecGetArray(*ranks, &r);CHKERRQ(ierr);
7993   for (c = cStart; c < cEnd; ++c) {
7994     PetscScalar *lr;
7995 
7996     ierr = DMPlexPointGlobalRef(rdm, c, r, &lr);CHKERRQ(ierr);
7997     if (lr) *lr = rank;
7998   }
7999   ierr = VecRestoreArray(*ranks, &r);CHKERRQ(ierr);
8000   ierr = DMDestroy(&rdm);CHKERRQ(ierr);
8001   PetscFunctionReturn(0);
8002 }
8003 
8004 /*@
8005   DMPlexCreateLabelField - Create a cell field whose value is the label value for that cell
8006 
8007   Input Parameters:
8008 + dm    - The DMPlex
8009 - label - The DMLabel
8010 
8011   Output Parameter:
8012 . val - The label value field
8013 
8014   Options Database Keys:
8015 . -dm_label_view - Adds the label value field into the DM output from -dm_view using the same viewer
8016 
8017   Level: intermediate
8018 
8019 .seealso: DMView()
8020 @*/
8021 PetscErrorCode DMPlexCreateLabelField(DM dm, DMLabel label, Vec *val)
8022 {
8023   DM             rdm;
8024   PetscFE        fe;
8025   PetscScalar   *v;
8026   PetscInt       dim, cStart, cEnd, c;
8027   PetscErrorCode ierr;
8028 
8029   PetscFunctionBeginUser;
8030   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8031   PetscValidPointer(label, 2);
8032   PetscValidPointer(val, 3);
8033   ierr = DMClone(dm, &rdm);CHKERRQ(ierr);
8034   ierr = DMGetDimension(rdm, &dim);CHKERRQ(ierr);
8035   ierr = PetscFECreateDefault(PetscObjectComm((PetscObject) rdm), dim, 1, PETSC_TRUE, "PETSc___label_value_", -1, &fe);CHKERRQ(ierr);
8036   ierr = PetscObjectSetName((PetscObject) fe, "label_value");CHKERRQ(ierr);
8037   ierr = DMSetField(rdm, 0, NULL, (PetscObject) fe);CHKERRQ(ierr);
8038   ierr = PetscFEDestroy(&fe);CHKERRQ(ierr);
8039   ierr = DMCreateDS(rdm);CHKERRQ(ierr);
8040   ierr = DMPlexGetHeightStratum(rdm, 0, &cStart, &cEnd);CHKERRQ(ierr);
8041   ierr = DMCreateGlobalVector(rdm, val);CHKERRQ(ierr);
8042   ierr = PetscObjectSetName((PetscObject) *val, "label_value");CHKERRQ(ierr);
8043   ierr = VecGetArray(*val, &v);CHKERRQ(ierr);
8044   for (c = cStart; c < cEnd; ++c) {
8045     PetscScalar *lv;
8046     PetscInt     cval;
8047 
8048     ierr = DMPlexPointGlobalRef(rdm, c, v, &lv);CHKERRQ(ierr);
8049     ierr = DMLabelGetValue(label, c, &cval);CHKERRQ(ierr);
8050     *lv = cval;
8051   }
8052   ierr = VecRestoreArray(*val, &v);CHKERRQ(ierr);
8053   ierr = DMDestroy(&rdm);CHKERRQ(ierr);
8054   PetscFunctionReturn(0);
8055 }
8056 
8057 /*@
8058   DMPlexCheckSymmetry - Check that the adjacency information in the mesh is symmetric.
8059 
8060   Input Parameter:
8061 . dm - The DMPlex object
8062 
8063   Notes:
8064   This is a useful diagnostic when creating meshes programmatically.
8065 
8066   For the complete list of DMPlexCheck* functions, see DMSetFromOptions().
8067 
8068   Level: developer
8069 
8070 .seealso: DMCreate(), DMSetFromOptions()
8071 @*/
8072 PetscErrorCode DMPlexCheckSymmetry(DM dm)
8073 {
8074   PetscSection    coneSection, supportSection;
8075   const PetscInt *cone, *support;
8076   PetscInt        coneSize, c, supportSize, s;
8077   PetscInt        pStart, pEnd, p, pp, csize, ssize;
8078   PetscBool       storagecheck = PETSC_TRUE;
8079   PetscErrorCode  ierr;
8080 
8081   PetscFunctionBegin;
8082   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8083   ierr = DMViewFromOptions(dm, NULL, "-sym_dm_view");CHKERRQ(ierr);
8084   ierr = DMPlexGetConeSection(dm, &coneSection);CHKERRQ(ierr);
8085   ierr = DMPlexGetSupportSection(dm, &supportSection);CHKERRQ(ierr);
8086   /* Check that point p is found in the support of its cone points, and vice versa */
8087   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
8088   for (p = pStart; p < pEnd; ++p) {
8089     ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
8090     ierr = DMPlexGetCone(dm, p, &cone);CHKERRQ(ierr);
8091     for (c = 0; c < coneSize; ++c) {
8092       PetscBool dup = PETSC_FALSE;
8093       PetscInt  d;
8094       for (d = c-1; d >= 0; --d) {
8095         if (cone[c] == cone[d]) {dup = PETSC_TRUE; break;}
8096       }
8097       ierr = DMPlexGetSupportSize(dm, cone[c], &supportSize);CHKERRQ(ierr);
8098       ierr = DMPlexGetSupport(dm, cone[c], &support);CHKERRQ(ierr);
8099       for (s = 0; s < supportSize; ++s) {
8100         if (support[s] == p) break;
8101       }
8102       if ((s >= supportSize) || (dup && (support[s+1] != p))) {
8103         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %D cone: ", p);CHKERRQ(ierr);
8104         for (s = 0; s < coneSize; ++s) {
8105           ierr = PetscPrintf(PETSC_COMM_SELF, "%D, ", cone[s]);CHKERRQ(ierr);
8106         }
8107         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
8108         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %D support: ", cone[c]);CHKERRQ(ierr);
8109         for (s = 0; s < supportSize; ++s) {
8110           ierr = PetscPrintf(PETSC_COMM_SELF, "%D, ", support[s]);CHKERRQ(ierr);
8111         }
8112         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
8113         if (dup) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %D not repeatedly found in support of repeated cone point %D", p, cone[c]);
8114         else SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %D not found in support of cone point %D", p, cone[c]);
8115       }
8116     }
8117     ierr = DMPlexGetTreeParent(dm, p, &pp, NULL);CHKERRQ(ierr);
8118     if (p != pp) { storagecheck = PETSC_FALSE; continue; }
8119     ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
8120     ierr = DMPlexGetSupport(dm, p, &support);CHKERRQ(ierr);
8121     for (s = 0; s < supportSize; ++s) {
8122       ierr = DMPlexGetConeSize(dm, support[s], &coneSize);CHKERRQ(ierr);
8123       ierr = DMPlexGetCone(dm, support[s], &cone);CHKERRQ(ierr);
8124       for (c = 0; c < coneSize; ++c) {
8125         ierr = DMPlexGetTreeParent(dm, cone[c], &pp, NULL);CHKERRQ(ierr);
8126         if (cone[c] != pp) { c = 0; break; }
8127         if (cone[c] == p) break;
8128       }
8129       if (c >= coneSize) {
8130         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %D support: ", p);CHKERRQ(ierr);
8131         for (c = 0; c < supportSize; ++c) {
8132           ierr = PetscPrintf(PETSC_COMM_SELF, "%D, ", support[c]);CHKERRQ(ierr);
8133         }
8134         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
8135         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %D cone: ", support[s]);CHKERRQ(ierr);
8136         for (c = 0; c < coneSize; ++c) {
8137           ierr = PetscPrintf(PETSC_COMM_SELF, "%D, ", cone[c]);CHKERRQ(ierr);
8138         }
8139         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
8140         SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %D not found in cone of support point %D", p, support[s]);
8141       }
8142     }
8143   }
8144   if (storagecheck) {
8145     ierr = PetscSectionGetStorageSize(coneSection, &csize);CHKERRQ(ierr);
8146     ierr = PetscSectionGetStorageSize(supportSection, &ssize);CHKERRQ(ierr);
8147     if (csize != ssize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "Total cone size %D != Total support size %D", csize, ssize);
8148   }
8149   PetscFunctionReturn(0);
8150 }
8151 
8152 /*
8153   For submeshes with cohesive cells (see DMPlexConstructCohesiveCells()), we allow a special case where some of the boundary of a face (edges and vertices) are not duplicated. We call these special boundary points "unsplit", since the same edge or vertex appears in both copies of the face. These unsplit points throw off our counting, so we have to explicitly account for them here.
8154 */
8155 static PetscErrorCode DMPlexCellUnsplitVertices_Private(DM dm, PetscInt c, DMPolytopeType ct, PetscInt *unsplit)
8156 {
8157   DMPolytopeType  cct;
8158   PetscInt        ptpoints[4];
8159   const PetscInt *cone, *ccone, *ptcone;
8160   PetscInt        coneSize, cp, cconeSize, ccp, npt = 0, pt;
8161   PetscErrorCode  ierr;
8162 
8163   PetscFunctionBegin;
8164   *unsplit = 0;
8165   switch (ct) {
8166     case DM_POLYTOPE_SEG_PRISM_TENSOR:
8167       ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
8168       ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
8169       for (cp = 0; cp < coneSize; ++cp) {
8170         ierr = DMPlexGetCellType(dm, cone[cp], &cct);CHKERRQ(ierr);
8171         if (cct == DM_POLYTOPE_POINT_PRISM_TENSOR) ptpoints[npt++] = cone[cp];
8172       }
8173       break;
8174     case DM_POLYTOPE_TRI_PRISM_TENSOR:
8175     case DM_POLYTOPE_QUAD_PRISM_TENSOR:
8176       ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
8177       ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
8178       for (cp = 0; cp < coneSize; ++cp) {
8179         ierr = DMPlexGetCone(dm, cone[cp], &ccone);CHKERRQ(ierr);
8180         ierr = DMPlexGetConeSize(dm, cone[cp], &cconeSize);CHKERRQ(ierr);
8181         for (ccp = 0; ccp < cconeSize; ++ccp) {
8182           ierr = DMPlexGetCellType(dm, ccone[ccp], &cct);CHKERRQ(ierr);
8183           if (cct == DM_POLYTOPE_POINT_PRISM_TENSOR) {
8184             PetscInt p;
8185             for (p = 0; p < npt; ++p) if (ptpoints[p] == ccone[ccp]) break;
8186             if (p == npt) ptpoints[npt++] = ccone[ccp];
8187           }
8188         }
8189       }
8190       break;
8191     default: break;
8192   }
8193   for (pt = 0; pt < npt; ++pt) {
8194     ierr = DMPlexGetCone(dm, ptpoints[pt], &ptcone);CHKERRQ(ierr);
8195     if (ptcone[0] == ptcone[1]) ++(*unsplit);
8196   }
8197   PetscFunctionReturn(0);
8198 }
8199 
8200 /*@
8201   DMPlexCheckSkeleton - Check that each cell has the correct number of vertices
8202 
8203   Input Parameters:
8204 + dm - The DMPlex object
8205 - cellHeight - Normally 0
8206 
8207   Notes:
8208   This is a useful diagnostic when creating meshes programmatically.
8209   Currently applicable only to homogeneous simplex or tensor meshes.
8210 
8211   For the complete list of DMPlexCheck* functions, see DMSetFromOptions().
8212 
8213   Level: developer
8214 
8215 .seealso: DMCreate(), DMSetFromOptions()
8216 @*/
8217 PetscErrorCode DMPlexCheckSkeleton(DM dm, PetscInt cellHeight)
8218 {
8219   DMPlexInterpolatedFlag interp;
8220   DMPolytopeType         ct;
8221   PetscInt               vStart, vEnd, cStart, cEnd, c;
8222   PetscErrorCode         ierr;
8223 
8224   PetscFunctionBegin;
8225   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8226   ierr = DMPlexIsInterpolated(dm, &interp);CHKERRQ(ierr);
8227   ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
8228   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
8229   for (c = cStart; c < cEnd; ++c) {
8230     PetscInt *closure = NULL;
8231     PetscInt  coneSize, closureSize, cl, Nv = 0;
8232 
8233     ierr = DMPlexGetCellType(dm, c, &ct);CHKERRQ(ierr);
8234     if ((PetscInt) ct < 0) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %D has no cell type", c);
8235     if (ct == DM_POLYTOPE_UNKNOWN) continue;
8236     if (interp == DMPLEX_INTERPOLATED_FULL) {
8237       ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
8238       if (coneSize != DMPolytopeTypeGetConeSize(ct)) SETERRQ4(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %D of type %s has cone size %D != %D", c, DMPolytopeTypes[ct], coneSize, DMPolytopeTypeGetConeSize(ct));
8239     }
8240     ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
8241     for (cl = 0; cl < closureSize*2; cl += 2) {
8242       const PetscInt p = closure[cl];
8243       if ((p >= vStart) && (p < vEnd)) ++Nv;
8244     }
8245     ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
8246     /* Special Case: Tensor faces with identified vertices */
8247     if (Nv < DMPolytopeTypeGetNumVertices(ct)) {
8248       PetscInt unsplit;
8249 
8250       ierr = DMPlexCellUnsplitVertices_Private(dm, c, ct, &unsplit);CHKERRQ(ierr);
8251       if (Nv + unsplit == DMPolytopeTypeGetNumVertices(ct)) continue;
8252     }
8253     if (Nv != DMPolytopeTypeGetNumVertices(ct)) SETERRQ4(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %D of type %s has %D vertices != %D", c, DMPolytopeTypes[ct], Nv, DMPolytopeTypeGetNumVertices(ct));
8254   }
8255   PetscFunctionReturn(0);
8256 }
8257 
8258 /*@
8259   DMPlexCheckFaces - Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type
8260 
8261   Not Collective
8262 
8263   Input Parameters:
8264 + dm - The DMPlex object
8265 - cellHeight - Normally 0
8266 
8267   Notes:
8268   This is a useful diagnostic when creating meshes programmatically.
8269   This routine is only relevant for meshes that are fully interpolated across all ranks.
8270   It will error out if a partially interpolated mesh is given on some rank.
8271   It will do nothing for locally uninterpolated mesh (as there is nothing to check).
8272 
8273   For the complete list of DMPlexCheck* functions, see DMSetFromOptions().
8274 
8275   Level: developer
8276 
8277 .seealso: DMCreate(), DMPlexGetVTKCellHeight(), DMSetFromOptions()
8278 @*/
8279 PetscErrorCode DMPlexCheckFaces(DM dm, PetscInt cellHeight)
8280 {
8281   PetscInt       dim, depth, vStart, vEnd, cStart, cEnd, c, h;
8282   PetscErrorCode ierr;
8283   DMPlexInterpolatedFlag interpEnum;
8284 
8285   PetscFunctionBegin;
8286   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8287   ierr = DMPlexIsInterpolated(dm, &interpEnum);CHKERRQ(ierr);
8288   if (interpEnum == DMPLEX_INTERPOLATED_NONE) PetscFunctionReturn(0);
8289   if (interpEnum == DMPLEX_INTERPOLATED_PARTIAL) {
8290     PetscMPIInt rank;
8291     MPI_Comm    comm;
8292 
8293     ierr = PetscObjectGetComm((PetscObject) dm, &comm);CHKERRQ(ierr);
8294     ierr = MPI_Comm_rank(comm, &rank);CHKERRMPI(ierr);
8295     SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_SUP, "Mesh is only partially interpolated on rank %d, this is currently not supported", rank);
8296   }
8297 
8298   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
8299   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
8300   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
8301   for (h = cellHeight; h < PetscMin(depth, dim); ++h) {
8302     ierr = DMPlexGetHeightStratum(dm, h, &cStart, &cEnd);CHKERRQ(ierr);
8303     for (c = cStart; c < cEnd; ++c) {
8304       const PetscInt      *cone, *ornt, *faceSizes, *faces;
8305       const DMPolytopeType *faceTypes;
8306       DMPolytopeType        ct;
8307       PetscInt              numFaces, coneSize, f;
8308       PetscInt             *closure = NULL, closureSize, cl, numCorners = 0, fOff = 0, unsplit;
8309 
8310       ierr = DMPlexGetCellType(dm, c, &ct);CHKERRQ(ierr);
8311       ierr = DMPlexCellUnsplitVertices_Private(dm, c, ct, &unsplit);CHKERRQ(ierr);
8312       if (unsplit) continue;
8313       ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
8314       ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
8315       ierr = DMPlexGetConeOrientation(dm, c, &ornt);CHKERRQ(ierr);
8316       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
8317       for (cl = 0; cl < closureSize*2; cl += 2) {
8318         const PetscInt p = closure[cl];
8319         if ((p >= vStart) && (p < vEnd)) closure[numCorners++] = p;
8320       }
8321       ierr = DMPlexGetRawFaces_Internal(dm, ct, closure, &numFaces, &faceTypes, &faceSizes, &faces);CHKERRQ(ierr);
8322       if (coneSize != numFaces) SETERRQ4(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %D of type %s has %D faces but should have %D", c, DMPolytopeTypes[ct], coneSize, numFaces);
8323       for (f = 0; f < numFaces; ++f) {
8324         DMPolytopeType fct;
8325         PetscInt       *fclosure = NULL, fclosureSize, cl, fnumCorners = 0, v;
8326 
8327         ierr = DMPlexGetCellType(dm, cone[f], &fct);CHKERRQ(ierr);
8328         ierr = DMPlexGetTransitiveClosure_Internal(dm, cone[f], ornt[f], PETSC_TRUE, &fclosureSize, &fclosure);CHKERRQ(ierr);
8329         for (cl = 0; cl < fclosureSize*2; cl += 2) {
8330           const PetscInt p = fclosure[cl];
8331           if ((p >= vStart) && (p < vEnd)) fclosure[fnumCorners++] = p;
8332         }
8333         if (fnumCorners != faceSizes[f]) SETERRQ7(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %D of type %s (cone idx %D) of cell %D of type %s has %D vertices but should have %D", cone[f], DMPolytopeTypes[fct], f, c, DMPolytopeTypes[ct], fnumCorners, faceSizes[f]);
8334         for (v = 0; v < fnumCorners; ++v) {
8335           if (fclosure[v] != faces[fOff+v]) SETERRQ8(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %D of type %s (cone idx %d) of cell %D of type %s vertex %D, %D != %D", cone[f], DMPolytopeTypes[fct], f, c, DMPolytopeTypes[ct], v, fclosure[v], faces[fOff+v]);
8336         }
8337         ierr = DMPlexRestoreTransitiveClosure(dm, cone[f], PETSC_TRUE, &fclosureSize, &fclosure);CHKERRQ(ierr);
8338         fOff += faceSizes[f];
8339       }
8340       ierr = DMPlexRestoreRawFaces_Internal(dm, ct, closure, &numFaces, &faceTypes, &faceSizes, &faces);CHKERRQ(ierr);
8341       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
8342     }
8343   }
8344   PetscFunctionReturn(0);
8345 }
8346 
8347 /*@
8348   DMPlexCheckGeometry - Check the geometry of mesh cells
8349 
8350   Input Parameter:
8351 . dm - The DMPlex object
8352 
8353   Notes:
8354   This is a useful diagnostic when creating meshes programmatically.
8355 
8356   For the complete list of DMPlexCheck* functions, see DMSetFromOptions().
8357 
8358   Level: developer
8359 
8360 .seealso: DMCreate(), DMSetFromOptions()
8361 @*/
8362 PetscErrorCode DMPlexCheckGeometry(DM dm)
8363 {
8364   Vec            coordinates;
8365   PetscReal      detJ, J[9], refVol = 1.0;
8366   PetscReal      vol;
8367   PetscBool      periodic;
8368   PetscInt       dim, depth, dE, d, cStart, cEnd, c;
8369   PetscErrorCode ierr;
8370 
8371   PetscFunctionBegin;
8372   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
8373   ierr = DMGetCoordinateDim(dm, &dE);CHKERRQ(ierr);
8374   if (dim != dE) PetscFunctionReturn(0);
8375   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
8376   ierr = DMGetPeriodicity(dm, &periodic, NULL, NULL, NULL);CHKERRQ(ierr);
8377   for (d = 0; d < dim; ++d) refVol *= 2.0;
8378   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
8379   /* Make sure local coordinates are created, because that step is collective */
8380   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
8381   for (c = cStart; c < cEnd; ++c) {
8382     DMPolytopeType ct;
8383     PetscInt       unsplit;
8384     PetscBool      ignoreZeroVol = PETSC_FALSE;
8385 
8386     ierr = DMPlexGetCellType(dm, c, &ct);CHKERRQ(ierr);
8387     switch (ct) {
8388       case DM_POLYTOPE_SEG_PRISM_TENSOR:
8389       case DM_POLYTOPE_TRI_PRISM_TENSOR:
8390       case DM_POLYTOPE_QUAD_PRISM_TENSOR:
8391         ignoreZeroVol = PETSC_TRUE; break;
8392       default: break;
8393     }
8394     switch (ct) {
8395       case DM_POLYTOPE_TRI_PRISM:
8396       case DM_POLYTOPE_TRI_PRISM_TENSOR:
8397       case DM_POLYTOPE_QUAD_PRISM_TENSOR:
8398       case DM_POLYTOPE_PYRAMID:
8399         continue;
8400       default: break;
8401     }
8402     ierr = DMPlexCellUnsplitVertices_Private(dm, c, ct, &unsplit);CHKERRQ(ierr);
8403     if (unsplit) continue;
8404     ierr = DMPlexComputeCellGeometryFEM(dm, c, NULL, NULL, J, NULL, &detJ);CHKERRQ(ierr);
8405     if (detJ < -PETSC_SMALL || (detJ <= 0.0 && !ignoreZeroVol)) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Mesh cell %D of type %s is inverted, |J| = %g", c, DMPolytopeTypes[ct], (double) detJ);
8406     ierr = PetscInfo2(dm, "Cell %D FEM Volume %g\n", c, (double) detJ*refVol);CHKERRQ(ierr);
8407     if (depth > 1 && !periodic) {
8408       ierr = DMPlexComputeCellGeometryFVM(dm, c, &vol, NULL, NULL);CHKERRQ(ierr);
8409       if (vol < -PETSC_SMALL || (vol <= 0.0 && !ignoreZeroVol)) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Mesh cell %D of type %s is inverted, vol = %g", c, DMPolytopeTypes[ct], (double) vol);
8410       ierr = PetscInfo2(dm, "Cell %D FVM Volume %g\n", c, (double) vol);CHKERRQ(ierr);
8411     }
8412   }
8413   PetscFunctionReturn(0);
8414 }
8415 
8416 /*@
8417   DMPlexCheckPointSF - Check that several necessary conditions are met for the point SF of this plex.
8418 
8419   Input Parameters:
8420 . dm - The DMPlex object
8421 
8422   Notes:
8423   This is mainly intended for debugging/testing purposes.
8424   It currently checks only meshes with no partition overlapping.
8425 
8426   For the complete list of DMPlexCheck* functions, see DMSetFromOptions().
8427 
8428   Level: developer
8429 
8430 .seealso: DMGetPointSF(), DMSetFromOptions()
8431 @*/
8432 PetscErrorCode DMPlexCheckPointSF(DM dm)
8433 {
8434   PetscSF         pointSF;
8435   PetscInt        cellHeight, cStart, cEnd, l, nleaves, nroots, overlap;
8436   const PetscInt *locals, *rootdegree;
8437   PetscBool       distributed;
8438   PetscErrorCode  ierr;
8439 
8440   PetscFunctionBegin;
8441   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8442   ierr = DMGetPointSF(dm, &pointSF);CHKERRQ(ierr);
8443   ierr = DMPlexIsDistributed(dm, &distributed);CHKERRQ(ierr);
8444   if (!distributed) PetscFunctionReturn(0);
8445   ierr = DMPlexGetOverlap(dm, &overlap);CHKERRQ(ierr);
8446   if (overlap) {
8447     ierr = PetscPrintf(PetscObjectComm((PetscObject)dm), "Warning: DMPlexCheckPointSF() is currently not implemented for meshes with partition overlapping");CHKERRQ(ierr);
8448     PetscFunctionReturn(0);
8449   }
8450   if (!pointSF) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONGSTATE, "This DMPlex is distributed but does not have PointSF attached");
8451   ierr = PetscSFGetGraph(pointSF, &nroots, &nleaves, &locals, NULL);CHKERRQ(ierr);
8452   if (nroots < 0) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONGSTATE, "This DMPlex is distributed but its PointSF has no graph set");
8453   ierr = PetscSFComputeDegreeBegin(pointSF, &rootdegree);CHKERRQ(ierr);
8454   ierr = PetscSFComputeDegreeEnd(pointSF, &rootdegree);CHKERRQ(ierr);
8455 
8456   /* 1) check there are no faces in 2D, cells in 3D, in interface */
8457   ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
8458   ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
8459   for (l = 0; l < nleaves; ++l) {
8460     const PetscInt point = locals[l];
8461 
8462     if (point >= cStart && point < cEnd) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point SF contains %D which is a cell", point);
8463   }
8464 
8465   /* 2) if some point is in interface, then all its cone points must be also in interface (either as leaves or roots) */
8466   for (l = 0; l < nleaves; ++l) {
8467     const PetscInt  point = locals[l];
8468     const PetscInt *cone;
8469     PetscInt        coneSize, c, idx;
8470 
8471     ierr = DMPlexGetConeSize(dm, point, &coneSize);CHKERRQ(ierr);
8472     ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
8473     for (c = 0; c < coneSize; ++c) {
8474       if (!rootdegree[cone[c]]) {
8475         ierr = PetscFindInt(cone[c], nleaves, locals, &idx);CHKERRQ(ierr);
8476         if (idx < 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point SF contains %D but not %D from its cone", point, cone[c]);
8477       }
8478     }
8479   }
8480   PetscFunctionReturn(0);
8481 }
8482 
8483 typedef struct cell_stats
8484 {
8485   PetscReal min, max, sum, squaresum;
8486   PetscInt  count;
8487 } cell_stats_t;
8488 
8489 static void MPIAPI cell_stats_reduce(void *a, void *b, int * len, MPI_Datatype *datatype)
8490 {
8491   PetscInt i, N = *len;
8492 
8493   for (i = 0; i < N; i++) {
8494     cell_stats_t *A = (cell_stats_t *) a;
8495     cell_stats_t *B = (cell_stats_t *) b;
8496 
8497     B->min = PetscMin(A->min,B->min);
8498     B->max = PetscMax(A->max,B->max);
8499     B->sum += A->sum;
8500     B->squaresum += A->squaresum;
8501     B->count += A->count;
8502   }
8503 }
8504 
8505 /*@
8506   DMPlexCheckCellShape - Checks the Jacobian of the mapping from reference to real cells and computes some minimal statistics.
8507 
8508   Collective on dm
8509 
8510   Input Parameters:
8511 + dm        - The DMPlex object
8512 . output    - If true, statistics will be displayed on stdout
8513 - condLimit - Display all cells above this condition number, or PETSC_DETERMINE for no cell output
8514 
8515   Notes:
8516   This is mainly intended for debugging/testing purposes.
8517 
8518   For the complete list of DMPlexCheck* functions, see DMSetFromOptions().
8519 
8520   Level: developer
8521 
8522 .seealso: DMSetFromOptions(), DMPlexComputeOrthogonalQuality()
8523 @*/
8524 PetscErrorCode DMPlexCheckCellShape(DM dm, PetscBool output, PetscReal condLimit)
8525 {
8526   DM             dmCoarse;
8527   cell_stats_t   stats, globalStats;
8528   MPI_Comm       comm = PetscObjectComm((PetscObject)dm);
8529   PetscReal      *J, *invJ, min = 0, max = 0, mean = 0, stdev = 0;
8530   PetscReal      limit = condLimit > 0 ? condLimit : PETSC_MAX_REAL;
8531   PetscInt       cdim, cStart, cEnd, c, eStart, eEnd, count = 0;
8532   PetscMPIInt    rank,size;
8533   PetscErrorCode ierr;
8534 
8535   PetscFunctionBegin;
8536   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8537   stats.min   = PETSC_MAX_REAL;
8538   stats.max   = PETSC_MIN_REAL;
8539   stats.sum   = stats.squaresum = 0.;
8540   stats.count = 0;
8541 
8542   ierr = MPI_Comm_size(comm, &size);CHKERRMPI(ierr);
8543   ierr = MPI_Comm_rank(comm, &rank);CHKERRMPI(ierr);
8544   ierr = DMGetCoordinateDim(dm,&cdim);CHKERRQ(ierr);
8545   ierr = PetscMalloc2(PetscSqr(cdim), &J, PetscSqr(cdim), &invJ);CHKERRQ(ierr);
8546   ierr = DMPlexGetSimplexOrBoxCells(dm,0,&cStart,&cEnd);CHKERRQ(ierr);
8547   ierr = DMPlexGetDepthStratum(dm,1,&eStart,&eEnd);CHKERRQ(ierr);
8548   for (c = cStart; c < cEnd; c++) {
8549     PetscInt  i;
8550     PetscReal frobJ = 0., frobInvJ = 0., cond2, cond, detJ;
8551 
8552     ierr = DMPlexComputeCellGeometryAffineFEM(dm,c,NULL,J,invJ,&detJ);CHKERRQ(ierr);
8553     if (detJ < 0.0) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Mesh cell %D is inverted", c);
8554     for (i = 0; i < PetscSqr(cdim); ++i) {
8555       frobJ    += J[i] * J[i];
8556       frobInvJ += invJ[i] * invJ[i];
8557     }
8558     cond2 = frobJ * frobInvJ;
8559     cond  = PetscSqrtReal(cond2);
8560 
8561     stats.min        = PetscMin(stats.min,cond);
8562     stats.max        = PetscMax(stats.max,cond);
8563     stats.sum       += cond;
8564     stats.squaresum += cond2;
8565     stats.count++;
8566     if (output && cond > limit) {
8567       PetscSection coordSection;
8568       Vec          coordsLocal;
8569       PetscScalar *coords = NULL;
8570       PetscInt     Nv, d, clSize, cl, *closure = NULL;
8571 
8572       ierr = DMGetCoordinatesLocal(dm, &coordsLocal);CHKERRQ(ierr);
8573       ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
8574       ierr = DMPlexVecGetClosure(dm, coordSection, coordsLocal, c, &Nv, &coords);CHKERRQ(ierr);
8575       ierr = PetscSynchronizedPrintf(comm, "[%d] Cell %D cond %g\n", rank, c, (double) cond);CHKERRQ(ierr);
8576       for (i = 0; i < Nv/cdim; ++i) {
8577         ierr = PetscSynchronizedPrintf(comm, "  Vertex %D: (", i);CHKERRQ(ierr);
8578         for (d = 0; d < cdim; ++d) {
8579           if (d > 0) {ierr = PetscSynchronizedPrintf(comm, ", ");CHKERRQ(ierr);}
8580           ierr = PetscSynchronizedPrintf(comm, "%g", (double) PetscRealPart(coords[i*cdim+d]));CHKERRQ(ierr);
8581         }
8582         ierr = PetscSynchronizedPrintf(comm, ")\n");CHKERRQ(ierr);
8583       }
8584       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &clSize, &closure);CHKERRQ(ierr);
8585       for (cl = 0; cl < clSize*2; cl += 2) {
8586         const PetscInt edge = closure[cl];
8587 
8588         if ((edge >= eStart) && (edge < eEnd)) {
8589           PetscReal len;
8590 
8591           ierr = DMPlexComputeCellGeometryFVM(dm, edge, &len, NULL, NULL);CHKERRQ(ierr);
8592           ierr = PetscSynchronizedPrintf(comm, "  Edge %D: length %g\n", edge, (double) len);CHKERRQ(ierr);
8593         }
8594       }
8595       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &clSize, &closure);CHKERRQ(ierr);
8596       ierr = DMPlexVecRestoreClosure(dm, coordSection, coordsLocal, c, &Nv, &coords);CHKERRQ(ierr);
8597     }
8598   }
8599   if (output) {ierr = PetscSynchronizedFlush(comm, NULL);CHKERRQ(ierr);}
8600 
8601   if (size > 1) {
8602     PetscMPIInt   blockLengths[2] = {4,1};
8603     MPI_Aint      blockOffsets[2] = {offsetof(cell_stats_t,min),offsetof(cell_stats_t,count)};
8604     MPI_Datatype  blockTypes[2]   = {MPIU_REAL,MPIU_INT}, statType;
8605     MPI_Op        statReduce;
8606 
8607     ierr = MPI_Type_create_struct(2,blockLengths,blockOffsets,blockTypes,&statType);CHKERRMPI(ierr);
8608     ierr = MPI_Type_commit(&statType);CHKERRMPI(ierr);
8609     ierr = MPI_Op_create(cell_stats_reduce, PETSC_TRUE, &statReduce);CHKERRMPI(ierr);
8610     ierr = MPI_Reduce(&stats,&globalStats,1,statType,statReduce,0,comm);CHKERRMPI(ierr);
8611     ierr = MPI_Op_free(&statReduce);CHKERRMPI(ierr);
8612     ierr = MPI_Type_free(&statType);CHKERRMPI(ierr);
8613   } else {
8614     ierr = PetscArraycpy(&globalStats,&stats,1);CHKERRQ(ierr);
8615   }
8616   if (!rank) {
8617     count = globalStats.count;
8618     min   = globalStats.min;
8619     max   = globalStats.max;
8620     mean  = globalStats.sum / globalStats.count;
8621     stdev = globalStats.count > 1 ? PetscSqrtReal(PetscMax((globalStats.squaresum - globalStats.count * mean * mean) / (globalStats.count - 1),0)) : 0.0;
8622   }
8623 
8624   if (output) {
8625     ierr = PetscPrintf(comm,"Mesh with %D cells, shape condition numbers: min = %g, max = %g, mean = %g, stddev = %g\n", count, (double) min, (double) max, (double) mean, (double) stdev);CHKERRQ(ierr);
8626   }
8627   ierr = PetscFree2(J,invJ);CHKERRQ(ierr);
8628 
8629   ierr = DMGetCoarseDM(dm,&dmCoarse);CHKERRQ(ierr);
8630   if (dmCoarse) {
8631     PetscBool isplex;
8632 
8633     ierr = PetscObjectTypeCompare((PetscObject)dmCoarse,DMPLEX,&isplex);CHKERRQ(ierr);
8634     if (isplex) {
8635       ierr = DMPlexCheckCellShape(dmCoarse,output,condLimit);CHKERRQ(ierr);
8636     }
8637   }
8638   PetscFunctionReturn(0);
8639 }
8640 
8641 /*@
8642   DMPlexComputeOrthogonalQuality - Compute cell-wise orthogonal quality mesh statistic. Optionally tags all cells with
8643   orthogonal quality below given tolerance.
8644 
8645   Collective on dm
8646 
8647   Input Parameters:
8648 + dm   - The DMPlex object
8649 . fv   - Optional PetscFV object for pre-computed cell/face centroid information
8650 - atol - [0, 1] Absolute tolerance for tagging cells.
8651 
8652   Output Parameters:
8653 + OrthQual      - Vec containing orthogonal quality per cell
8654 - OrthQualLabel - DMLabel tagging cells below atol with DM_ADAPT_REFINE
8655 
8656   Options Database Keys:
8657 + -dm_plex_orthogonal_quality_label_view - view OrthQualLabel if label is requested. Currently only PETSCVIEWERASCII is
8658 supported.
8659 - -dm_plex_orthogonal_quality_vec_view - view OrthQual vector.
8660 
8661   Notes:
8662   Orthogonal quality is given by the following formula:
8663 
8664   \min \left[ \frac{A_i \cdot f_i}{\|A_i\| \|f_i\|} , \frac{A_i \cdot c_i}{\|A_i\| \|c_i\|} \right]
8665 
8666   Where A_i is the i'th face-normal vector, f_i is the vector from the cell centroid to the i'th face centroid, and c_i
8667   is the vector from the current cells centroid to the centroid of its i'th neighbor (which shares a face with the
8668   current cell). This computes the vector similarity between each cell face and its corresponding neighbor centroid by
8669   calculating the cosine of the angle between these vectors.
8670 
8671   Orthogonal quality ranges from 1 (best) to 0 (worst).
8672 
8673   This routine is mainly useful for FVM, however is not restricted to only FVM. The PetscFV object is optionally used to check for
8674   pre-computed FVM cell data, but if it is not passed in then this data will be computed.
8675 
8676   Cells are tagged if they have an orthogonal quality less than or equal to the absolute tolerance.
8677 
8678   Level: intermediate
8679 
8680 .seealso: DMPlexCheckCellShape(), DMCreateLabel()
8681 @*/
8682 PetscErrorCode DMPlexComputeOrthogonalQuality(DM dm, PetscFV fv, PetscReal atol, Vec *OrthQual, DMLabel *OrthQualLabel)
8683 {
8684   PetscInt                nc, cellHeight, cStart, cEnd, cell, cellIter = 0;
8685   PetscInt                *idx;
8686   PetscScalar             *oqVals;
8687   const PetscScalar       *cellGeomArr, *faceGeomArr;
8688   PetscReal               *ci, *fi, *Ai;
8689   MPI_Comm                comm;
8690   Vec                     cellgeom, facegeom;
8691   DM                      dmFace, dmCell;
8692   IS                      glob;
8693   ISLocalToGlobalMapping  ltog;
8694   PetscViewer             vwr;
8695   PetscErrorCode          ierr;
8696 
8697   PetscFunctionBegin;
8698   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8699   if (fv) {PetscValidHeaderSpecific(fv, PETSCFV_CLASSID, 2);}
8700   PetscValidPointer(OrthQual, 4);
8701   if (PetscUnlikelyDebug(atol < 0.0 || atol > 1.0)) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"Absolute tolerance %g not in [0,1]",(double)atol);
8702   ierr = PetscObjectGetComm((PetscObject) dm, &comm);CHKERRQ(ierr);
8703   ierr = DMGetDimension(dm, &nc);CHKERRQ(ierr);
8704   if (nc < 2) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONGSTATE, "DM must have dimension >= 2 (current %D)", nc);
8705   {
8706     DMPlexInterpolatedFlag interpFlag;
8707 
8708     ierr = DMPlexIsInterpolated(dm, &interpFlag);CHKERRQ(ierr);
8709     if (interpFlag != DMPLEX_INTERPOLATED_FULL) {
8710       PetscMPIInt rank;
8711 
8712       ierr = MPI_Comm_rank(comm, &rank);CHKERRMPI(ierr);
8713       SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONGSTATE, "DM must be fully interpolated, DM on rank %d is not fully interpolated", rank);
8714     }
8715   }
8716   if (OrthQualLabel) {
8717     PetscValidPointer(OrthQualLabel, 5);
8718     ierr = DMCreateLabel(dm, "Orthogonal_Quality");CHKERRQ(ierr);
8719     ierr = DMGetLabel(dm, "Orthogonal_Quality", OrthQualLabel);CHKERRQ(ierr);
8720   } else {*OrthQualLabel = NULL;}
8721   ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
8722   ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
8723   ierr = DMPlexCreateCellNumbering_Internal(dm, PETSC_TRUE, &glob);CHKERRQ(ierr);
8724   ierr = ISLocalToGlobalMappingCreateIS(glob, &ltog);CHKERRQ(ierr);
8725   ierr = ISLocalToGlobalMappingSetType(ltog, ISLOCALTOGLOBALMAPPINGHASH);CHKERRQ(ierr);
8726   ierr = VecCreate(comm, OrthQual);CHKERRQ(ierr);
8727   ierr = VecSetType(*OrthQual, VECSTANDARD);CHKERRQ(ierr);
8728   ierr = VecSetSizes(*OrthQual, cEnd-cStart, PETSC_DETERMINE);CHKERRQ(ierr);
8729   ierr = VecSetLocalToGlobalMapping(*OrthQual, ltog);CHKERRQ(ierr);
8730   ierr = VecSetUp(*OrthQual);CHKERRQ(ierr);
8731   ierr = ISDestroy(&glob);CHKERRQ(ierr);
8732   ierr = ISLocalToGlobalMappingDestroy(&ltog);CHKERRQ(ierr);
8733   ierr = DMPlexGetDataFVM(dm, fv, &cellgeom, &facegeom, NULL);CHKERRQ(ierr);
8734   ierr = VecGetArrayRead(cellgeom, &cellGeomArr);CHKERRQ(ierr);
8735   ierr = VecGetArrayRead(facegeom, &faceGeomArr);CHKERRQ(ierr);
8736   ierr = VecGetDM(cellgeom, &dmCell);CHKERRQ(ierr);
8737   ierr = VecGetDM(facegeom, &dmFace);CHKERRQ(ierr);
8738   ierr = PetscMalloc5(cEnd-cStart, &idx, cEnd-cStart, &oqVals, nc, &ci, nc, &fi, nc, &Ai);CHKERRQ(ierr);
8739   for (cell = cStart; cell < cEnd; cellIter++,cell++) {
8740     PetscInt           cellneigh, cellneighiter = 0, adjSize = PETSC_DETERMINE;
8741     PetscInt           cellarr[2], *adj = NULL;
8742     PetscScalar        *cArr, *fArr;
8743     PetscReal          minvalc = 1.0, minvalf = 1.0;
8744     PetscFVCellGeom    *cg;
8745 
8746     idx[cellIter] = cell-cStart;
8747     cellarr[0] = cell;
8748     /* Make indexing into cellGeom easier */
8749     ierr = DMPlexPointLocalRead(dmCell, cell, cellGeomArr, &cg);CHKERRQ(ierr);
8750     ierr = DMPlexGetAdjacency_Internal(dm, cell, PETSC_TRUE, PETSC_FALSE, PETSC_FALSE, &adjSize, &adj);CHKERRQ(ierr);
8751     /* Technically 1 too big, but easier than fiddling with empty adjacency array */
8752     ierr = PetscCalloc2(adjSize, &cArr, adjSize, &fArr);CHKERRQ(ierr);
8753     for (cellneigh = 0; cellneigh < adjSize; cellneighiter++,cellneigh++) {
8754       PetscInt         i;
8755       const PetscInt   neigh = adj[cellneigh];
8756       PetscReal        normci = 0, normfi = 0, normai = 0;
8757       PetscFVCellGeom  *cgneigh;
8758       PetscFVFaceGeom  *fg;
8759 
8760       /* Don't count ourselves in the neighbor list */
8761       if (neigh == cell) continue;
8762       ierr = DMPlexPointLocalRead(dmCell, neigh, cellGeomArr, &cgneigh);CHKERRQ(ierr);
8763       cellarr[1] = neigh;
8764       {
8765         PetscInt       numcovpts;
8766         const PetscInt *covpts;
8767 
8768         ierr = DMPlexGetMeet(dm, 2, cellarr, &numcovpts, &covpts);CHKERRQ(ierr);
8769         ierr = DMPlexPointLocalRead(dmFace, covpts[0], faceGeomArr, &fg);CHKERRQ(ierr);
8770         ierr = DMPlexRestoreMeet(dm, 2, cellarr, &numcovpts, &covpts);CHKERRQ(ierr);
8771       }
8772 
8773       /* Compute c_i, f_i and their norms */
8774       for (i = 0; i < nc; i++) {
8775         ci[i] = cgneigh->centroid[i] - cg->centroid[i];
8776         fi[i] = fg->centroid[i] - cg->centroid[i];
8777         Ai[i] = fg->normal[i];
8778         normci += PetscPowReal(ci[i], 2);
8779         normfi += PetscPowReal(fi[i], 2);
8780         normai += PetscPowReal(Ai[i], 2);
8781       }
8782       normci = PetscSqrtReal(normci);
8783       normfi = PetscSqrtReal(normfi);
8784       normai = PetscSqrtReal(normai);
8785 
8786       /* Normalize and compute for each face-cell-normal pair */
8787       for (i = 0; i < nc; i++) {
8788         ci[i] = ci[i]/normci;
8789         fi[i] = fi[i]/normfi;
8790         Ai[i] = Ai[i]/normai;
8791         /* PetscAbs because I don't know if normals are guaranteed to point out */
8792         cArr[cellneighiter] += PetscAbs(Ai[i]*ci[i]);
8793         fArr[cellneighiter] += PetscAbs(Ai[i]*fi[i]);
8794       }
8795       if (PetscRealPart(cArr[cellneighiter]) < minvalc) {
8796         minvalc = PetscRealPart(cArr[cellneighiter]);
8797       }
8798       if (PetscRealPart(fArr[cellneighiter]) < minvalf) {
8799         minvalf = PetscRealPart(fArr[cellneighiter]);
8800       }
8801     }
8802     ierr = PetscFree(adj);CHKERRQ(ierr);
8803     ierr = PetscFree2(cArr, fArr);CHKERRQ(ierr);
8804     /* Defer to cell if they're equal */
8805     oqVals[cellIter] = PetscMin(minvalf, minvalc);
8806     if (OrthQualLabel) {
8807       if (PetscRealPart(oqVals[cellIter]) <= atol) {ierr = DMLabelSetValue(*OrthQualLabel, cell, DM_ADAPT_REFINE);CHKERRQ(ierr);}
8808     }
8809   }
8810   ierr = VecSetValuesLocal(*OrthQual, cEnd-cStart, idx, oqVals, INSERT_VALUES);CHKERRQ(ierr);
8811   ierr = VecAssemblyBegin(*OrthQual);CHKERRQ(ierr);
8812   ierr = VecAssemblyEnd(*OrthQual);CHKERRQ(ierr);
8813   ierr = VecRestoreArrayRead(cellgeom, &cellGeomArr);CHKERRQ(ierr);
8814   ierr = VecRestoreArrayRead(facegeom, &faceGeomArr);CHKERRQ(ierr);
8815   ierr = PetscOptionsGetViewer(comm, NULL, NULL, "-dm_plex_orthogonal_quality_label_view", &vwr, NULL, NULL);CHKERRQ(ierr);
8816   if (OrthQualLabel) {
8817     if (vwr) {ierr = DMLabelView(*OrthQualLabel, vwr);CHKERRQ(ierr);}
8818   }
8819   ierr = PetscFree5(idx, oqVals, ci, fi, Ai);CHKERRQ(ierr);
8820   ierr = PetscViewerDestroy(&vwr);CHKERRQ(ierr);
8821   ierr = VecViewFromOptions(*OrthQual, NULL, "-dm_plex_orthogonal_quality_vec_view");CHKERRQ(ierr);
8822   PetscFunctionReturn(0);
8823 }
8824 
8825 /* this is here insead of DMGetOutputDM because output DM still has constraints in the local indices that affect
8826  * interpolator construction */
8827 static PetscErrorCode DMGetFullDM(DM dm, DM *odm)
8828 {
8829   PetscSection   section, newSection, gsection;
8830   PetscSF        sf;
8831   PetscBool      hasConstraints, ghasConstraints;
8832   PetscErrorCode ierr;
8833 
8834   PetscFunctionBegin;
8835   PetscValidHeaderSpecific(dm,DM_CLASSID,1);
8836   PetscValidPointer(odm,2);
8837   ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);
8838   ierr = PetscSectionHasConstraints(section, &hasConstraints);CHKERRQ(ierr);
8839   ierr = MPI_Allreduce(&hasConstraints, &ghasConstraints, 1, MPIU_BOOL, MPI_LOR, PetscObjectComm((PetscObject) dm));CHKERRMPI(ierr);
8840   if (!ghasConstraints) {
8841     ierr = PetscObjectReference((PetscObject)dm);CHKERRQ(ierr);
8842     *odm = dm;
8843     PetscFunctionReturn(0);
8844   }
8845   ierr = DMClone(dm, odm);CHKERRQ(ierr);
8846   ierr = DMCopyFields(dm, *odm);CHKERRQ(ierr);
8847   ierr = DMGetLocalSection(*odm, &newSection);CHKERRQ(ierr);
8848   ierr = DMGetPointSF(*odm, &sf);CHKERRQ(ierr);
8849   ierr = PetscSectionCreateGlobalSection(newSection, sf, PETSC_TRUE, PETSC_FALSE, &gsection);CHKERRQ(ierr);
8850   ierr = DMSetGlobalSection(*odm, gsection);CHKERRQ(ierr);
8851   ierr = PetscSectionDestroy(&gsection);CHKERRQ(ierr);
8852   PetscFunctionReturn(0);
8853 }
8854 
8855 static PetscErrorCode DMCreateAffineInterpolationCorrection_Plex(DM dmc, DM dmf, Vec *shift)
8856 {
8857   DM             dmco, dmfo;
8858   Mat            interpo;
8859   Vec            rscale;
8860   Vec            cglobalo, clocal;
8861   Vec            fglobal, fglobalo, flocal;
8862   PetscBool      regular;
8863   PetscErrorCode ierr;
8864 
8865   PetscFunctionBegin;
8866   ierr = DMGetFullDM(dmc, &dmco);CHKERRQ(ierr);
8867   ierr = DMGetFullDM(dmf, &dmfo);CHKERRQ(ierr);
8868   ierr = DMSetCoarseDM(dmfo, dmco);CHKERRQ(ierr);
8869   ierr = DMPlexGetRegularRefinement(dmf, &regular);CHKERRQ(ierr);
8870   ierr = DMPlexSetRegularRefinement(dmfo, regular);CHKERRQ(ierr);
8871   ierr = DMCreateInterpolation(dmco, dmfo, &interpo, &rscale);CHKERRQ(ierr);
8872   ierr = DMCreateGlobalVector(dmco, &cglobalo);CHKERRQ(ierr);
8873   ierr = DMCreateLocalVector(dmc, &clocal);CHKERRQ(ierr);
8874   ierr = VecSet(cglobalo, 0.);CHKERRQ(ierr);
8875   ierr = VecSet(clocal, 0.);CHKERRQ(ierr);
8876   ierr = DMCreateGlobalVector(dmf, &fglobal);CHKERRQ(ierr);
8877   ierr = DMCreateGlobalVector(dmfo, &fglobalo);CHKERRQ(ierr);
8878   ierr = DMCreateLocalVector(dmf, &flocal);CHKERRQ(ierr);
8879   ierr = VecSet(fglobal, 0.);CHKERRQ(ierr);
8880   ierr = VecSet(fglobalo, 0.);CHKERRQ(ierr);
8881   ierr = VecSet(flocal, 0.);CHKERRQ(ierr);
8882   ierr = DMPlexInsertBoundaryValues(dmc, PETSC_TRUE, clocal, 0., NULL, NULL, NULL);CHKERRQ(ierr);
8883   ierr = DMLocalToGlobalBegin(dmco, clocal, INSERT_VALUES, cglobalo);CHKERRQ(ierr);
8884   ierr = DMLocalToGlobalEnd(dmco, clocal, INSERT_VALUES, cglobalo);CHKERRQ(ierr);
8885   ierr = MatMult(interpo, cglobalo, fglobalo);CHKERRQ(ierr);
8886   ierr = DMGlobalToLocalBegin(dmfo, fglobalo, INSERT_VALUES, flocal);CHKERRQ(ierr);
8887   ierr = DMGlobalToLocalEnd(dmfo, fglobalo, INSERT_VALUES, flocal);CHKERRQ(ierr);
8888   ierr = DMLocalToGlobalBegin(dmf, flocal, INSERT_VALUES, fglobal);CHKERRQ(ierr);
8889   ierr = DMLocalToGlobalEnd(dmf, flocal, INSERT_VALUES, fglobal);CHKERRQ(ierr);
8890   *shift = fglobal;
8891   ierr = VecDestroy(&flocal);CHKERRQ(ierr);
8892   ierr = VecDestroy(&fglobalo);CHKERRQ(ierr);
8893   ierr = VecDestroy(&clocal);CHKERRQ(ierr);
8894   ierr = VecDestroy(&cglobalo);CHKERRQ(ierr);
8895   ierr = VecDestroy(&rscale);CHKERRQ(ierr);
8896   ierr = MatDestroy(&interpo);CHKERRQ(ierr);
8897   ierr = DMDestroy(&dmfo);CHKERRQ(ierr);
8898   ierr = DMDestroy(&dmco);CHKERRQ(ierr);
8899   PetscFunctionReturn(0);
8900 }
8901 
8902 PETSC_INTERN PetscErrorCode DMInterpolateSolution_Plex(DM coarse, DM fine, Mat interp, Vec coarseSol, Vec fineSol)
8903 {
8904   PetscObject    shifto;
8905   Vec            shift;
8906 
8907   PetscErrorCode ierr;
8908 
8909   PetscFunctionBegin;
8910   if (!interp) {
8911     Vec rscale;
8912 
8913     ierr = DMCreateInterpolation(coarse, fine, &interp, &rscale);CHKERRQ(ierr);
8914     ierr = VecDestroy(&rscale);CHKERRQ(ierr);
8915   } else {
8916     ierr = PetscObjectReference((PetscObject)interp);CHKERRQ(ierr);
8917   }
8918   ierr = PetscObjectQuery((PetscObject)interp, "_DMInterpolateSolution_Plex_Vec", &shifto);CHKERRQ(ierr);
8919   if (!shifto) {
8920     ierr = DMCreateAffineInterpolationCorrection_Plex(coarse, fine, &shift);CHKERRQ(ierr);
8921     ierr = PetscObjectCompose((PetscObject)interp, "_DMInterpolateSolution_Plex_Vec", (PetscObject) shift);CHKERRQ(ierr);
8922     shifto = (PetscObject) shift;
8923     ierr = VecDestroy(&shift);CHKERRQ(ierr);
8924   }
8925   shift = (Vec) shifto;
8926   ierr = MatInterpolate(interp, coarseSol, fineSol);CHKERRQ(ierr);
8927   ierr = VecAXPY(fineSol, 1.0, shift);CHKERRQ(ierr);
8928   ierr = MatDestroy(&interp);CHKERRQ(ierr);
8929   PetscFunctionReturn(0);
8930 }
8931 
8932 /* Pointwise interpolation
8933      Just code FEM for now
8934      u^f = I u^c
8935      sum_k u^f_k phi^f_k = I sum_j u^c_j phi^c_j
8936      u^f_i = sum_j psi^f_i I phi^c_j u^c_j
8937      I_{ij} = psi^f_i phi^c_j
8938 */
8939 PetscErrorCode DMCreateInterpolation_Plex(DM dmCoarse, DM dmFine, Mat *interpolation, Vec *scaling)
8940 {
8941   PetscSection   gsc, gsf;
8942   PetscInt       m, n;
8943   void          *ctx;
8944   DM             cdm;
8945   PetscBool      regular, ismatis, isRefined = dmCoarse->data == dmFine->data ? PETSC_FALSE : PETSC_TRUE;
8946   PetscErrorCode ierr;
8947 
8948   PetscFunctionBegin;
8949   ierr = DMGetGlobalSection(dmFine, &gsf);CHKERRQ(ierr);
8950   ierr = PetscSectionGetConstrainedStorageSize(gsf, &m);CHKERRQ(ierr);
8951   ierr = DMGetGlobalSection(dmCoarse, &gsc);CHKERRQ(ierr);
8952   ierr = PetscSectionGetConstrainedStorageSize(gsc, &n);CHKERRQ(ierr);
8953 
8954   ierr = PetscStrcmp(dmCoarse->mattype, MATIS, &ismatis);CHKERRQ(ierr);
8955   ierr = MatCreate(PetscObjectComm((PetscObject) dmCoarse), interpolation);CHKERRQ(ierr);
8956   ierr = MatSetSizes(*interpolation, m, n, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
8957   ierr = MatSetType(*interpolation, ismatis ? MATAIJ : dmCoarse->mattype);CHKERRQ(ierr);
8958   ierr = DMGetApplicationContext(dmFine, &ctx);CHKERRQ(ierr);
8959 
8960   ierr = DMGetCoarseDM(dmFine, &cdm);CHKERRQ(ierr);
8961   ierr = DMPlexGetRegularRefinement(dmFine, &regular);CHKERRQ(ierr);
8962   if (!isRefined || (regular && cdm == dmCoarse)) {ierr = DMPlexComputeInterpolatorNested(dmCoarse, dmFine, isRefined, *interpolation, ctx);CHKERRQ(ierr);}
8963   else                                            {ierr = DMPlexComputeInterpolatorGeneral(dmCoarse, dmFine, *interpolation, ctx);CHKERRQ(ierr);}
8964   ierr = MatViewFromOptions(*interpolation, NULL, "-interp_mat_view");CHKERRQ(ierr);
8965   if (scaling) {
8966     /* Use naive scaling */
8967     ierr = DMCreateInterpolationScale(dmCoarse, dmFine, *interpolation, scaling);CHKERRQ(ierr);
8968   }
8969   PetscFunctionReturn(0);
8970 }
8971 
8972 PetscErrorCode DMCreateInjection_Plex(DM dmCoarse, DM dmFine, Mat *mat)
8973 {
8974   PetscErrorCode ierr;
8975   VecScatter     ctx;
8976 
8977   PetscFunctionBegin;
8978   ierr = DMPlexComputeInjectorFEM(dmCoarse, dmFine, &ctx, NULL);CHKERRQ(ierr);
8979   ierr = MatCreateScatter(PetscObjectComm((PetscObject)ctx), ctx, mat);CHKERRQ(ierr);
8980   ierr = VecScatterDestroy(&ctx);CHKERRQ(ierr);
8981   PetscFunctionReturn(0);
8982 }
8983 
8984 static void g0_identity_private(PetscInt dim, PetscInt Nf, PetscInt NfAux,
8985                                 const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[],
8986                                 const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[],
8987                                 PetscReal t, PetscReal u_tShift, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar g0[])
8988 {
8989   g0[0] = 1.0;
8990 }
8991 
8992 PetscErrorCode DMCreateMassMatrix_Plex(DM dmCoarse, DM dmFine, Mat *mass)
8993 {
8994   PetscSection   gsc, gsf;
8995   PetscInt       m, n;
8996   void          *ctx;
8997   DM             cdm;
8998   PetscBool      regular;
8999   PetscErrorCode ierr;
9000 
9001   PetscFunctionBegin;
9002   if (dmFine == dmCoarse) {
9003     DM       dmc;
9004     PetscDS  ds;
9005     Vec      u;
9006     IS       cellIS;
9007     PetscFormKey key;
9008     PetscInt depth;
9009 
9010     ierr = DMClone(dmFine, &dmc);CHKERRQ(ierr);
9011     ierr = DMCopyDisc(dmFine, dmc);CHKERRQ(ierr);
9012     ierr = DMGetDS(dmc, &ds);CHKERRQ(ierr);
9013     ierr = PetscDSSetJacobian(ds, 0, 0, g0_identity_private, NULL, NULL, NULL);CHKERRQ(ierr);
9014     ierr = DMCreateMatrix(dmc, mass);CHKERRQ(ierr);
9015     ierr = DMGetGlobalVector(dmc, &u);CHKERRQ(ierr);
9016     ierr = DMPlexGetDepth(dmc, &depth);CHKERRQ(ierr);
9017     ierr = DMGetStratumIS(dmc, "depth", depth, &cellIS);CHKERRQ(ierr);
9018     ierr = MatZeroEntries(*mass);CHKERRQ(ierr);
9019     key.label = NULL;
9020     key.value = 0;
9021     key.field = 0;
9022     key.part  = 0;
9023     ierr = DMPlexComputeJacobian_Internal(dmc, key, cellIS, 0.0, 0.0, u, NULL, *mass, *mass, NULL);CHKERRQ(ierr);
9024     ierr = ISDestroy(&cellIS);CHKERRQ(ierr);
9025     ierr = DMRestoreGlobalVector(dmc, &u);CHKERRQ(ierr);
9026     ierr = DMDestroy(&dmc);CHKERRQ(ierr);
9027   } else {
9028     ierr = DMGetGlobalSection(dmFine, &gsf);CHKERRQ(ierr);
9029     ierr = PetscSectionGetConstrainedStorageSize(gsf, &m);CHKERRQ(ierr);
9030     ierr = DMGetGlobalSection(dmCoarse, &gsc);CHKERRQ(ierr);
9031     ierr = PetscSectionGetConstrainedStorageSize(gsc, &n);CHKERRQ(ierr);
9032 
9033     ierr = MatCreate(PetscObjectComm((PetscObject) dmCoarse), mass);CHKERRQ(ierr);
9034     ierr = MatSetSizes(*mass, m, n, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
9035     ierr = MatSetType(*mass, dmCoarse->mattype);CHKERRQ(ierr);
9036     ierr = DMGetApplicationContext(dmFine, &ctx);CHKERRQ(ierr);
9037 
9038     ierr = DMGetCoarseDM(dmFine, &cdm);CHKERRQ(ierr);
9039     ierr = DMPlexGetRegularRefinement(dmFine, &regular);CHKERRQ(ierr);
9040     if (regular && cdm == dmCoarse) {ierr = DMPlexComputeMassMatrixNested(dmCoarse, dmFine, *mass, ctx);CHKERRQ(ierr);}
9041     else                            {ierr = DMPlexComputeMassMatrixGeneral(dmCoarse, dmFine, *mass, ctx);CHKERRQ(ierr);}
9042   }
9043   ierr = MatViewFromOptions(*mass, NULL, "-mass_mat_view");CHKERRQ(ierr);
9044   PetscFunctionReturn(0);
9045 }
9046 
9047 /*@
9048   DMPlexGetRegularRefinement - Get the flag indicating that this mesh was obtained by regular refinement from its coarse mesh
9049 
9050   Input Parameter:
9051 . dm - The DMPlex object
9052 
9053   Output Parameter:
9054 . regular - The flag
9055 
9056   Level: intermediate
9057 
9058 .seealso: DMPlexSetRegularRefinement()
9059 @*/
9060 PetscErrorCode DMPlexGetRegularRefinement(DM dm, PetscBool *regular)
9061 {
9062   PetscFunctionBegin;
9063   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9064   PetscValidPointer(regular, 2);
9065   *regular = ((DM_Plex *) dm->data)->regularRefinement;
9066   PetscFunctionReturn(0);
9067 }
9068 
9069 /*@
9070   DMPlexSetRegularRefinement - Set the flag indicating that this mesh was obtained by regular refinement from its coarse mesh
9071 
9072   Input Parameters:
9073 + dm - The DMPlex object
9074 - regular - The flag
9075 
9076   Level: intermediate
9077 
9078 .seealso: DMPlexGetRegularRefinement()
9079 @*/
9080 PetscErrorCode DMPlexSetRegularRefinement(DM dm, PetscBool regular)
9081 {
9082   PetscFunctionBegin;
9083   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9084   ((DM_Plex *) dm->data)->regularRefinement = regular;
9085   PetscFunctionReturn(0);
9086 }
9087 
9088 /*@
9089   DMPlexGetCellRefinerType - Get the strategy for refining a cell
9090 
9091   Input Parameter:
9092 . dm - The DMPlex object
9093 
9094   Output Parameter:
9095 . cr - The strategy number
9096 
9097   Level: intermediate
9098 
9099 .seealso: DMPlexSetCellRefinerType(), DMPlexSetRegularRefinement()
9100 @*/
9101 PetscErrorCode DMPlexGetCellRefinerType(DM dm, DMPlexCellRefinerType *cr)
9102 {
9103   PetscFunctionBegin;
9104   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9105   PetscValidPointer(cr, 2);
9106   *cr = ((DM_Plex *) dm->data)->cellRefiner;
9107   PetscFunctionReturn(0);
9108 }
9109 
9110 /*@
9111   DMPlexSetCellRefinerType - Set the strategy for refining a cell
9112 
9113   Input Parameters:
9114 + dm - The DMPlex object
9115 - cr - The strategy number
9116 
9117   Level: intermediate
9118 
9119 .seealso: DMPlexGetCellRefinerType(), DMPlexGetRegularRefinement()
9120 @*/
9121 PetscErrorCode DMPlexSetCellRefinerType(DM dm, DMPlexCellRefinerType cr)
9122 {
9123   PetscFunctionBegin;
9124   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9125   ((DM_Plex *) dm->data)->cellRefiner = cr;
9126   PetscFunctionReturn(0);
9127 }
9128 
9129 /* anchors */
9130 /*@
9131   DMPlexGetAnchors - Get the layout of the anchor (point-to-point) constraints.  Typically, the user will not have to
9132   call DMPlexGetAnchors() directly: if there are anchors, then DMPlexGetAnchors() is called during DMGetConstraints().
9133 
9134   not collective
9135 
9136   Input Parameters:
9137 . dm - The DMPlex object
9138 
9139   Output Parameters:
9140 + anchorSection - If not NULL, set to the section describing which points anchor the constrained points.
9141 - anchorIS - If not NULL, set to the list of anchors indexed by anchorSection
9142 
9143   Level: intermediate
9144 
9145 .seealso: DMPlexSetAnchors(), DMGetConstraints(), DMSetConstraints()
9146 @*/
9147 PetscErrorCode DMPlexGetAnchors(DM dm, PetscSection *anchorSection, IS *anchorIS)
9148 {
9149   DM_Plex *plex = (DM_Plex *)dm->data;
9150   PetscErrorCode ierr;
9151 
9152   PetscFunctionBegin;
9153   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9154   if (!plex->anchorSection && !plex->anchorIS && plex->createanchors) {ierr = (*plex->createanchors)(dm);CHKERRQ(ierr);}
9155   if (anchorSection) *anchorSection = plex->anchorSection;
9156   if (anchorIS) *anchorIS = plex->anchorIS;
9157   PetscFunctionReturn(0);
9158 }
9159 
9160 /*@
9161   DMPlexSetAnchors - Set the layout of the local anchor (point-to-point) constraints.  Unlike boundary conditions,
9162   when a point's degrees of freedom in a section are constrained to an outside value, the anchor constraints set a
9163   point's degrees of freedom to be a linear combination of other points' degrees of freedom.
9164 
9165   After specifying the layout of constraints with DMPlexSetAnchors(), one specifies the constraints by calling
9166   DMGetConstraints() and filling in the entries in the constraint matrix.
9167 
9168   collective on dm
9169 
9170   Input Parameters:
9171 + dm - The DMPlex object
9172 . anchorSection - The section that describes the mapping from constrained points to the anchor points listed in anchorIS.  Must have a local communicator (PETSC_COMM_SELF or derivative).
9173 - anchorIS - The list of all anchor points.  Must have a local communicator (PETSC_COMM_SELF or derivative).
9174 
9175   The reference counts of anchorSection and anchorIS are incremented.
9176 
9177   Level: intermediate
9178 
9179 .seealso: DMPlexGetAnchors(), DMGetConstraints(), DMSetConstraints()
9180 @*/
9181 PetscErrorCode DMPlexSetAnchors(DM dm, PetscSection anchorSection, IS anchorIS)
9182 {
9183   DM_Plex        *plex = (DM_Plex *)dm->data;
9184   PetscMPIInt    result;
9185   PetscErrorCode ierr;
9186 
9187   PetscFunctionBegin;
9188   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9189   if (anchorSection) {
9190     PetscValidHeaderSpecific(anchorSection,PETSC_SECTION_CLASSID,2);
9191     ierr = MPI_Comm_compare(PETSC_COMM_SELF,PetscObjectComm((PetscObject)anchorSection),&result);CHKERRMPI(ierr);
9192     if (result != MPI_CONGRUENT && result != MPI_IDENT) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_NOTSAMECOMM,"anchor section must have local communicator");
9193   }
9194   if (anchorIS) {
9195     PetscValidHeaderSpecific(anchorIS,IS_CLASSID,3);
9196     ierr = MPI_Comm_compare(PETSC_COMM_SELF,PetscObjectComm((PetscObject)anchorIS),&result);CHKERRMPI(ierr);
9197     if (result != MPI_CONGRUENT && result != MPI_IDENT) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_NOTSAMECOMM,"anchor IS must have local communicator");
9198   }
9199 
9200   ierr = PetscObjectReference((PetscObject)anchorSection);CHKERRQ(ierr);
9201   ierr = PetscSectionDestroy(&plex->anchorSection);CHKERRQ(ierr);
9202   plex->anchorSection = anchorSection;
9203 
9204   ierr = PetscObjectReference((PetscObject)anchorIS);CHKERRQ(ierr);
9205   ierr = ISDestroy(&plex->anchorIS);CHKERRQ(ierr);
9206   plex->anchorIS = anchorIS;
9207 
9208   if (PetscUnlikelyDebug(anchorIS && anchorSection)) {
9209     PetscInt size, a, pStart, pEnd;
9210     const PetscInt *anchors;
9211 
9212     ierr = PetscSectionGetChart(anchorSection,&pStart,&pEnd);CHKERRQ(ierr);
9213     ierr = ISGetLocalSize(anchorIS,&size);CHKERRQ(ierr);
9214     ierr = ISGetIndices(anchorIS,&anchors);CHKERRQ(ierr);
9215     for (a = 0; a < size; a++) {
9216       PetscInt p;
9217 
9218       p = anchors[a];
9219       if (p >= pStart && p < pEnd) {
9220         PetscInt dof;
9221 
9222         ierr = PetscSectionGetDof(anchorSection,p,&dof);CHKERRQ(ierr);
9223         if (dof) {
9224           PetscErrorCode ierr2;
9225 
9226           ierr2 = ISRestoreIndices(anchorIS,&anchors);CHKERRQ(ierr2);
9227           SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_INCOMP,"Point %D cannot be constrained and an anchor",p);
9228         }
9229       }
9230     }
9231     ierr = ISRestoreIndices(anchorIS,&anchors);CHKERRQ(ierr);
9232   }
9233   /* reset the generic constraints */
9234   ierr = DMSetDefaultConstraints(dm,NULL,NULL);CHKERRQ(ierr);
9235   PetscFunctionReturn(0);
9236 }
9237 
9238 static PetscErrorCode DMPlexCreateConstraintSection_Anchors(DM dm, PetscSection section, PetscSection *cSec)
9239 {
9240   PetscSection anchorSection;
9241   PetscInt pStart, pEnd, sStart, sEnd, p, dof, numFields, f;
9242   PetscErrorCode ierr;
9243 
9244   PetscFunctionBegin;
9245   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9246   ierr = DMPlexGetAnchors(dm,&anchorSection,NULL);CHKERRQ(ierr);
9247   ierr = PetscSectionCreate(PETSC_COMM_SELF,cSec);CHKERRQ(ierr);
9248   ierr = PetscSectionGetNumFields(section,&numFields);CHKERRQ(ierr);
9249   if (numFields) {
9250     PetscInt f;
9251     ierr = PetscSectionSetNumFields(*cSec,numFields);CHKERRQ(ierr);
9252 
9253     for (f = 0; f < numFields; f++) {
9254       PetscInt numComp;
9255 
9256       ierr = PetscSectionGetFieldComponents(section,f,&numComp);CHKERRQ(ierr);
9257       ierr = PetscSectionSetFieldComponents(*cSec,f,numComp);CHKERRQ(ierr);
9258     }
9259   }
9260   ierr = PetscSectionGetChart(anchorSection,&pStart,&pEnd);CHKERRQ(ierr);
9261   ierr = PetscSectionGetChart(section,&sStart,&sEnd);CHKERRQ(ierr);
9262   pStart = PetscMax(pStart,sStart);
9263   pEnd   = PetscMin(pEnd,sEnd);
9264   pEnd   = PetscMax(pStart,pEnd);
9265   ierr = PetscSectionSetChart(*cSec,pStart,pEnd);CHKERRQ(ierr);
9266   for (p = pStart; p < pEnd; p++) {
9267     ierr = PetscSectionGetDof(anchorSection,p,&dof);CHKERRQ(ierr);
9268     if (dof) {
9269       ierr = PetscSectionGetDof(section,p,&dof);CHKERRQ(ierr);
9270       ierr = PetscSectionSetDof(*cSec,p,dof);CHKERRQ(ierr);
9271       for (f = 0; f < numFields; f++) {
9272         ierr = PetscSectionGetFieldDof(section,p,f,&dof);CHKERRQ(ierr);
9273         ierr = PetscSectionSetFieldDof(*cSec,p,f,dof);CHKERRQ(ierr);
9274       }
9275     }
9276   }
9277   ierr = PetscSectionSetUp(*cSec);CHKERRQ(ierr);
9278   ierr = PetscObjectSetName((PetscObject) *cSec, "Constraint Section");CHKERRQ(ierr);
9279   PetscFunctionReturn(0);
9280 }
9281 
9282 static PetscErrorCode DMPlexCreateConstraintMatrix_Anchors(DM dm, PetscSection section, PetscSection cSec, Mat *cMat)
9283 {
9284   PetscSection   aSec;
9285   PetscInt       pStart, pEnd, p, sStart, sEnd, dof, aDof, aOff, off, nnz, annz, m, n, q, a, offset, *i, *j;
9286   const PetscInt *anchors;
9287   PetscInt       numFields, f;
9288   IS             aIS;
9289   PetscErrorCode ierr;
9290   MatType        mtype;
9291   PetscBool      iscuda,iskokkos;
9292 
9293   PetscFunctionBegin;
9294   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9295   ierr = PetscSectionGetStorageSize(cSec, &m);CHKERRQ(ierr);
9296   ierr = PetscSectionGetStorageSize(section, &n);CHKERRQ(ierr);
9297   ierr = MatCreate(PETSC_COMM_SELF,cMat);CHKERRQ(ierr);
9298   ierr = MatSetSizes(*cMat,m,n,m,n);CHKERRQ(ierr);
9299   ierr = PetscStrcmp(dm->mattype,MATSEQAIJCUSPARSE,&iscuda);CHKERRQ(ierr);
9300   if (!iscuda) { ierr = PetscStrcmp(dm->mattype,MATMPIAIJCUSPARSE,&iscuda);CHKERRQ(ierr); }
9301   ierr = PetscStrcmp(dm->mattype,MATSEQAIJKOKKOS,&iskokkos);CHKERRQ(ierr);
9302   if (!iskokkos) { ierr = PetscStrcmp(dm->mattype,MATMPIAIJKOKKOS,&iskokkos);CHKERRQ(ierr); }
9303   if (iscuda) mtype = MATSEQAIJCUSPARSE;
9304   else if (iskokkos) mtype = MATSEQAIJKOKKOS;
9305   else mtype = MATSEQAIJ;
9306   ierr = MatSetType(*cMat,mtype);CHKERRQ(ierr);
9307   ierr = DMPlexGetAnchors(dm,&aSec,&aIS);CHKERRQ(ierr);
9308   ierr = ISGetIndices(aIS,&anchors);CHKERRQ(ierr);
9309   /* cSec will be a subset of aSec and section */
9310   ierr = PetscSectionGetChart(cSec,&pStart,&pEnd);CHKERRQ(ierr);
9311   ierr = PetscSectionGetChart(section,&sStart,&sEnd);CHKERRQ(ierr);
9312   ierr = PetscMalloc1(m+1,&i);CHKERRQ(ierr);
9313   i[0] = 0;
9314   ierr = PetscSectionGetNumFields(section,&numFields);CHKERRQ(ierr);
9315   for (p = pStart; p < pEnd; p++) {
9316     PetscInt rDof, rOff, r;
9317 
9318     ierr = PetscSectionGetDof(aSec,p,&rDof);CHKERRQ(ierr);
9319     if (!rDof) continue;
9320     ierr = PetscSectionGetOffset(aSec,p,&rOff);CHKERRQ(ierr);
9321     if (numFields) {
9322       for (f = 0; f < numFields; f++) {
9323         annz = 0;
9324         for (r = 0; r < rDof; r++) {
9325           a = anchors[rOff + r];
9326           if (a < sStart || a >= sEnd) continue;
9327           ierr = PetscSectionGetFieldDof(section,a,f,&aDof);CHKERRQ(ierr);
9328           annz += aDof;
9329         }
9330         ierr = PetscSectionGetFieldDof(cSec,p,f,&dof);CHKERRQ(ierr);
9331         ierr = PetscSectionGetFieldOffset(cSec,p,f,&off);CHKERRQ(ierr);
9332         for (q = 0; q < dof; q++) {
9333           i[off + q + 1] = i[off + q] + annz;
9334         }
9335       }
9336     }
9337     else {
9338       annz = 0;
9339       ierr = PetscSectionGetDof(cSec,p,&dof);CHKERRQ(ierr);
9340       for (q = 0; q < dof; q++) {
9341         a = anchors[rOff + q];
9342         if (a < sStart || a >= sEnd) continue;
9343         ierr = PetscSectionGetDof(section,a,&aDof);CHKERRQ(ierr);
9344         annz += aDof;
9345       }
9346       ierr = PetscSectionGetDof(cSec,p,&dof);CHKERRQ(ierr);
9347       ierr = PetscSectionGetOffset(cSec,p,&off);CHKERRQ(ierr);
9348       for (q = 0; q < dof; q++) {
9349         i[off + q + 1] = i[off + q] + annz;
9350       }
9351     }
9352   }
9353   nnz = i[m];
9354   ierr = PetscMalloc1(nnz,&j);CHKERRQ(ierr);
9355   offset = 0;
9356   for (p = pStart; p < pEnd; p++) {
9357     if (numFields) {
9358       for (f = 0; f < numFields; f++) {
9359         ierr = PetscSectionGetFieldDof(cSec,p,f,&dof);CHKERRQ(ierr);
9360         for (q = 0; q < dof; q++) {
9361           PetscInt rDof, rOff, r;
9362           ierr = PetscSectionGetDof(aSec,p,&rDof);CHKERRQ(ierr);
9363           ierr = PetscSectionGetOffset(aSec,p,&rOff);CHKERRQ(ierr);
9364           for (r = 0; r < rDof; r++) {
9365             PetscInt s;
9366 
9367             a = anchors[rOff + r];
9368             if (a < sStart || a >= sEnd) continue;
9369             ierr = PetscSectionGetFieldDof(section,a,f,&aDof);CHKERRQ(ierr);
9370             ierr = PetscSectionGetFieldOffset(section,a,f,&aOff);CHKERRQ(ierr);
9371             for (s = 0; s < aDof; s++) {
9372               j[offset++] = aOff + s;
9373             }
9374           }
9375         }
9376       }
9377     }
9378     else {
9379       ierr = PetscSectionGetDof(cSec,p,&dof);CHKERRQ(ierr);
9380       for (q = 0; q < dof; q++) {
9381         PetscInt rDof, rOff, r;
9382         ierr = PetscSectionGetDof(aSec,p,&rDof);CHKERRQ(ierr);
9383         ierr = PetscSectionGetOffset(aSec,p,&rOff);CHKERRQ(ierr);
9384         for (r = 0; r < rDof; r++) {
9385           PetscInt s;
9386 
9387           a = anchors[rOff + r];
9388           if (a < sStart || a >= sEnd) continue;
9389           ierr = PetscSectionGetDof(section,a,&aDof);CHKERRQ(ierr);
9390           ierr = PetscSectionGetOffset(section,a,&aOff);CHKERRQ(ierr);
9391           for (s = 0; s < aDof; s++) {
9392             j[offset++] = aOff + s;
9393           }
9394         }
9395       }
9396     }
9397   }
9398   ierr = MatSeqAIJSetPreallocationCSR(*cMat,i,j,NULL);CHKERRQ(ierr);
9399   ierr = PetscFree(i);CHKERRQ(ierr);
9400   ierr = PetscFree(j);CHKERRQ(ierr);
9401   ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);
9402   PetscFunctionReturn(0);
9403 }
9404 
9405 PetscErrorCode DMCreateDefaultConstraints_Plex(DM dm)
9406 {
9407   DM_Plex        *plex = (DM_Plex *)dm->data;
9408   PetscSection   anchorSection, section, cSec;
9409   Mat            cMat;
9410   PetscErrorCode ierr;
9411 
9412   PetscFunctionBegin;
9413   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9414   ierr = DMPlexGetAnchors(dm,&anchorSection,NULL);CHKERRQ(ierr);
9415   if (anchorSection) {
9416     PetscInt Nf;
9417 
9418     ierr = DMGetLocalSection(dm,&section);CHKERRQ(ierr);
9419     ierr = DMPlexCreateConstraintSection_Anchors(dm,section,&cSec);CHKERRQ(ierr);
9420     ierr = DMPlexCreateConstraintMatrix_Anchors(dm,section,cSec,&cMat);CHKERRQ(ierr);
9421     ierr = DMGetNumFields(dm,&Nf);CHKERRQ(ierr);
9422     if (Nf && plex->computeanchormatrix) {ierr = (*plex->computeanchormatrix)(dm,section,cSec,cMat);CHKERRQ(ierr);}
9423     ierr = DMSetDefaultConstraints(dm,cSec,cMat);CHKERRQ(ierr);
9424     ierr = PetscSectionDestroy(&cSec);CHKERRQ(ierr);
9425     ierr = MatDestroy(&cMat);CHKERRQ(ierr);
9426   }
9427   PetscFunctionReturn(0);
9428 }
9429 
9430 PetscErrorCode DMCreateSubDomainDM_Plex(DM dm, DMLabel label, PetscInt value, IS *is, DM *subdm)
9431 {
9432   IS             subis;
9433   PetscSection   section, subsection;
9434   PetscErrorCode ierr;
9435 
9436   PetscFunctionBegin;
9437   ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);
9438   if (!section) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Must set default section for DM before splitting subdomain");
9439   if (!subdm)   SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Must set output subDM for splitting subdomain");
9440   /* Create subdomain */
9441   ierr = DMPlexFilter(dm, label, value, subdm);CHKERRQ(ierr);
9442   /* Create submodel */
9443   ierr = DMPlexGetSubpointIS(*subdm, &subis);CHKERRQ(ierr);
9444   ierr = PetscSectionCreateSubmeshSection(section, subis, &subsection);CHKERRQ(ierr);
9445   ierr = DMSetLocalSection(*subdm, subsection);CHKERRQ(ierr);
9446   ierr = PetscSectionDestroy(&subsection);CHKERRQ(ierr);
9447   ierr = DMCopyDisc(dm, *subdm);CHKERRQ(ierr);
9448   /* Create map from submodel to global model */
9449   if (is) {
9450     PetscSection    sectionGlobal, subsectionGlobal;
9451     IS              spIS;
9452     const PetscInt *spmap;
9453     PetscInt       *subIndices;
9454     PetscInt        subSize = 0, subOff = 0, pStart, pEnd, p;
9455     PetscInt        Nf, f, bs = -1, bsLocal[2], bsMinMax[2];
9456 
9457     ierr = DMPlexGetSubpointIS(*subdm, &spIS);CHKERRQ(ierr);
9458     ierr = ISGetIndices(spIS, &spmap);CHKERRQ(ierr);
9459     ierr = PetscSectionGetNumFields(section, &Nf);CHKERRQ(ierr);
9460     ierr = DMGetGlobalSection(dm, &sectionGlobal);CHKERRQ(ierr);
9461     ierr = DMGetGlobalSection(*subdm, &subsectionGlobal);CHKERRQ(ierr);
9462     ierr = PetscSectionGetChart(subsection, &pStart, &pEnd);CHKERRQ(ierr);
9463     for (p = pStart; p < pEnd; ++p) {
9464       PetscInt gdof, pSubSize  = 0;
9465 
9466       ierr = PetscSectionGetDof(sectionGlobal, p, &gdof);CHKERRQ(ierr);
9467       if (gdof > 0) {
9468         for (f = 0; f < Nf; ++f) {
9469           PetscInt fdof, fcdof;
9470 
9471           ierr     = PetscSectionGetFieldDof(subsection, p, f, &fdof);CHKERRQ(ierr);
9472           ierr     = PetscSectionGetFieldConstraintDof(subsection, p, f, &fcdof);CHKERRQ(ierr);
9473           pSubSize += fdof-fcdof;
9474         }
9475         subSize += pSubSize;
9476         if (pSubSize) {
9477           if (bs < 0) {
9478             bs = pSubSize;
9479           } else if (bs != pSubSize) {
9480             /* Layout does not admit a pointwise block size */
9481             bs = 1;
9482           }
9483         }
9484       }
9485     }
9486     /* Must have same blocksize on all procs (some might have no points) */
9487     bsLocal[0] = bs < 0 ? PETSC_MAX_INT : bs; bsLocal[1] = bs;
9488     ierr = PetscGlobalMinMaxInt(PetscObjectComm((PetscObject) dm), bsLocal, bsMinMax);CHKERRQ(ierr);
9489     if (bsMinMax[0] != bsMinMax[1]) {bs = 1;}
9490     else                            {bs = bsMinMax[0];}
9491     ierr = PetscMalloc1(subSize, &subIndices);CHKERRQ(ierr);
9492     for (p = pStart; p < pEnd; ++p) {
9493       PetscInt gdof, goff;
9494 
9495       ierr = PetscSectionGetDof(subsectionGlobal, p, &gdof);CHKERRQ(ierr);
9496       if (gdof > 0) {
9497         const PetscInt point = spmap[p];
9498 
9499         ierr = PetscSectionGetOffset(sectionGlobal, point, &goff);CHKERRQ(ierr);
9500         for (f = 0; f < Nf; ++f) {
9501           PetscInt fdof, fcdof, fc, f2, poff = 0;
9502 
9503           /* Can get rid of this loop by storing field information in the global section */
9504           for (f2 = 0; f2 < f; ++f2) {
9505             ierr  = PetscSectionGetFieldDof(section, p, f2, &fdof);CHKERRQ(ierr);
9506             ierr  = PetscSectionGetFieldConstraintDof(section, p, f2, &fcdof);CHKERRQ(ierr);
9507             poff += fdof-fcdof;
9508           }
9509           ierr = PetscSectionGetFieldDof(section, p, f, &fdof);CHKERRQ(ierr);
9510           ierr = PetscSectionGetFieldConstraintDof(section, p, f, &fcdof);CHKERRQ(ierr);
9511           for (fc = 0; fc < fdof-fcdof; ++fc, ++subOff) {
9512             subIndices[subOff] = goff+poff+fc;
9513           }
9514         }
9515       }
9516     }
9517     ierr = ISRestoreIndices(spIS, &spmap);CHKERRQ(ierr);
9518     ierr = ISCreateGeneral(PetscObjectComm((PetscObject)dm), subSize, subIndices, PETSC_OWN_POINTER, is);CHKERRQ(ierr);
9519     if (bs > 1) {
9520       /* We need to check that the block size does not come from non-contiguous fields */
9521       PetscInt i, j, set = 1;
9522       for (i = 0; i < subSize; i += bs) {
9523         for (j = 0; j < bs; ++j) {
9524           if (subIndices[i+j] != subIndices[i]+j) {set = 0; break;}
9525         }
9526       }
9527       if (set) {ierr = ISSetBlockSize(*is, bs);CHKERRQ(ierr);}
9528     }
9529     /* Attach nullspace */
9530     for (f = 0; f < Nf; ++f) {
9531       (*subdm)->nullspaceConstructors[f] = dm->nullspaceConstructors[f];
9532       if ((*subdm)->nullspaceConstructors[f]) break;
9533     }
9534     if (f < Nf) {
9535       MatNullSpace nullSpace;
9536       ierr = (*(*subdm)->nullspaceConstructors[f])(*subdm, f, f, &nullSpace);CHKERRQ(ierr);
9537 
9538       ierr = PetscObjectCompose((PetscObject) *is, "nullspace", (PetscObject) nullSpace);CHKERRQ(ierr);
9539       ierr = MatNullSpaceDestroy(&nullSpace);CHKERRQ(ierr);
9540     }
9541   }
9542   PetscFunctionReturn(0);
9543 }
9544 
9545 /*@
9546   DMPlexMonitorThroughput - Report the cell throughput of FE integration
9547 
9548   Input Parameter:
9549 - dm - The DM
9550 
9551   Level: developer
9552 
9553   Options Database Keys:
9554 . -dm_plex_monitor_throughput - Activate the monitor
9555 
9556 .seealso: DMSetFromOptions(), DMPlexCreate()
9557 @*/
9558 PetscErrorCode DMPlexMonitorThroughput(DM dm, void *dummy)
9559 {
9560 #if defined(PETSC_USE_LOG)
9561   PetscStageLog      stageLog;
9562   PetscLogEvent      event;
9563   PetscLogStage      stage;
9564   PetscEventPerfInfo eventInfo;
9565   PetscReal          cellRate, flopRate;
9566   PetscInt           cStart, cEnd, Nf, N;
9567   const char        *name;
9568   PetscErrorCode     ierr;
9569 #endif
9570 
9571   PetscFunctionBegin;
9572   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9573 #if defined(PETSC_USE_LOG)
9574   ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
9575   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
9576   ierr = DMGetNumFields(dm, &Nf);CHKERRQ(ierr);
9577   ierr = PetscLogGetStageLog(&stageLog);CHKERRQ(ierr);
9578   ierr = PetscStageLogGetCurrent(stageLog, &stage);CHKERRQ(ierr);
9579   ierr = PetscLogEventGetId("DMPlexResidualFE", &event);CHKERRQ(ierr);
9580   ierr = PetscLogEventGetPerfInfo(stage, event, &eventInfo);CHKERRQ(ierr);
9581   N        = (cEnd - cStart)*Nf*eventInfo.count;
9582   flopRate = eventInfo.flops/eventInfo.time;
9583   cellRate = N/eventInfo.time;
9584   ierr = PetscPrintf(PetscObjectComm((PetscObject) dm), "DM (%s) FE Residual Integration: %D integrals %D reps\n  Cell rate: %.2g/s flop rate: %.2g MF/s\n", name ? name : "unknown", N, eventInfo.count, (double) cellRate, (double) (flopRate/1.e6));CHKERRQ(ierr);
9585 #else
9586   SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Plex Throughput Monitor is not supported if logging is turned off. Reconfigure using --with-log.");
9587 #endif
9588   PetscFunctionReturn(0);
9589 }
9590