xref: /petsc/src/dm/impls/plex/plex.c (revision cb9b7bb0b9f68fc121f797778e409e2e056048fa) 
1 #include <petsc/private/dmpleximpl.h>   /*I      "petscdmplex.h"   I*/
2 #include <petsc/private/isimpl.h>
3 #include <petsc/private/vecimpl.h>
4 #include <petsc/private/glvisvecimpl.h>
5 #include <petscsf.h>
6 #include <petscds.h>
7 #include <petscdraw.h>
8 #include <petscdmfield.h>
9 #include <petscdmplextransform.h>
10 
11 /* Logging support */
12 PetscLogEvent DMPLEX_Interpolate, DMPLEX_Partition, DMPLEX_Distribute, DMPLEX_DistributeCones, DMPLEX_DistributeLabels, DMPLEX_DistributeSF, DMPLEX_DistributeOverlap, DMPLEX_DistributeField, DMPLEX_DistributeData, DMPLEX_Migrate, DMPLEX_InterpolateSF, DMPLEX_GlobalToNaturalBegin, DMPLEX_GlobalToNaturalEnd, DMPLEX_NaturalToGlobalBegin, DMPLEX_NaturalToGlobalEnd, DMPLEX_Stratify, DMPLEX_Symmetrize, DMPLEX_Preallocate, DMPLEX_ResidualFEM, DMPLEX_JacobianFEM, DMPLEX_InterpolatorFEM, DMPLEX_InjectorFEM, DMPLEX_IntegralFEM, DMPLEX_CreateGmsh, DMPLEX_RebalanceSharedPoints, DMPLEX_PartSelf, DMPLEX_PartLabelInvert, DMPLEX_PartLabelCreateSF, DMPLEX_PartStratSF, DMPLEX_CreatePointSF,DMPLEX_LocatePoints,DMPLEX_TopologyView,DMPLEX_LabelsView,DMPLEX_CoordinatesView,DMPLEX_SectionView,DMPLEX_GlobalVectorView,DMPLEX_LocalVectorView,DMPLEX_TopologyLoad,DMPLEX_LabelsLoad,DMPLEX_CoordinatesLoad,DMPLEX_SectionLoad,DMPLEX_GlobalVectorLoad,DMPLEX_LocalVectorLoad;
13 
14 PETSC_EXTERN PetscErrorCode VecView_MPI(Vec, PetscViewer);
15 
16 /*@
17   DMPlexIsSimplex - Is the first cell in this mesh a simplex?
18 
19   Input Parameter:
20 . dm      - The DMPlex object
21 
22   Output Parameter:
23 . simplex - Flag checking for a simplex
24 
25   Note: This just gives the first range of cells found. If the mesh has several cell types, it will only give the first.
26   If the mesh has no cells, this returns PETSC_FALSE.
27 
28   Level: intermediate
29 
30 .seealso DMPlexGetSimplexOrBoxCells(), DMPlexGetCellType(), DMPlexGetHeightStratum(), DMPolytopeTypeGetNumVertices()
31 @*/
32 PetscErrorCode DMPlexIsSimplex(DM dm, PetscBool *simplex)
33 {
34   DMPolytopeType ct;
35   PetscInt       cStart, cEnd;
36   PetscErrorCode ierr;
37 
38   PetscFunctionBegin;
39   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
40   if (cEnd <= cStart) {*simplex = PETSC_FALSE; PetscFunctionReturn(0);}
41   ierr = DMPlexGetCellType(dm, cStart, &ct);CHKERRQ(ierr);
42   *simplex = DMPolytopeTypeGetNumVertices(ct) == DMPolytopeTypeGetDim(ct)+1 ? PETSC_TRUE : PETSC_FALSE;
43   PetscFunctionReturn(0);
44 }
45 
46 /*@
47   DMPlexGetSimplexOrBoxCells - Get the range of cells which are neither prisms nor ghost FV cells
48 
49   Input Parameters:
50 + dm     - The DMPlex object
51 - height - The cell height in the Plex, 0 is the default
52 
53   Output Parameters:
54 + cStart - The first "normal" cell
55 - cEnd   - The upper bound on "normal"" cells
56 
57   Note: This just gives the first range of cells found. If the mesh has several cell types, it will only give the first.
58 
59   Level: developer
60 
61 .seealso DMPlexConstructGhostCells(), DMPlexGetGhostCellStratum()
62 @*/
63 PetscErrorCode DMPlexGetSimplexOrBoxCells(DM dm, PetscInt height, PetscInt *cStart, PetscInt *cEnd)
64 {
65   DMPolytopeType ct = DM_POLYTOPE_UNKNOWN;
66   PetscInt       cS, cE, c;
67   PetscErrorCode ierr;
68 
69   PetscFunctionBegin;
70   ierr = DMPlexGetHeightStratum(dm, PetscMax(height, 0), &cS, &cE);CHKERRQ(ierr);
71   for (c = cS; c < cE; ++c) {
72     DMPolytopeType cct;
73 
74     ierr = DMPlexGetCellType(dm, c, &cct);CHKERRQ(ierr);
75     if ((PetscInt) cct < 0) break;
76     switch (cct) {
77       case DM_POLYTOPE_POINT:
78       case DM_POLYTOPE_SEGMENT:
79       case DM_POLYTOPE_TRIANGLE:
80       case DM_POLYTOPE_QUADRILATERAL:
81       case DM_POLYTOPE_TETRAHEDRON:
82       case DM_POLYTOPE_HEXAHEDRON:
83         ct = cct;
84         break;
85       default: break;
86     }
87     if (ct != DM_POLYTOPE_UNKNOWN) break;
88   }
89   if (ct != DM_POLYTOPE_UNKNOWN) {
90     DMLabel ctLabel;
91 
92     ierr = DMPlexGetCellTypeLabel(dm, &ctLabel);CHKERRQ(ierr);
93     ierr = DMLabelGetStratumBounds(ctLabel, ct, &cS, &cE);CHKERRQ(ierr);
94   }
95   if (cStart) *cStart = cS;
96   if (cEnd)   *cEnd   = cE;
97   PetscFunctionReturn(0);
98 }
99 
100 PetscErrorCode DMPlexGetFieldType_Internal(DM dm, PetscSection section, PetscInt field, PetscInt *sStart, PetscInt *sEnd, PetscViewerVTKFieldType *ft)
101 {
102   PetscInt       cdim, pStart, pEnd, vStart, vEnd, cStart, cEnd;
103   PetscInt       vcdof[2] = {0,0}, globalvcdof[2];
104   PetscErrorCode ierr;
105 
106   PetscFunctionBegin;
107   *ft  = PETSC_VTK_INVALID;
108   ierr = DMGetCoordinateDim(dm, &cdim);CHKERRQ(ierr);
109   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
110   ierr = DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
111   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
112   if (field >= 0) {
113     if ((vStart >= pStart) && (vStart < pEnd)) {ierr = PetscSectionGetFieldDof(section, vStart, field, &vcdof[0]);CHKERRQ(ierr);}
114     if ((cStart >= pStart) && (cStart < pEnd)) {ierr = PetscSectionGetFieldDof(section, cStart, field, &vcdof[1]);CHKERRQ(ierr);}
115   } else {
116     if ((vStart >= pStart) && (vStart < pEnd)) {ierr = PetscSectionGetDof(section, vStart, &vcdof[0]);CHKERRQ(ierr);}
117     if ((cStart >= pStart) && (cStart < pEnd)) {ierr = PetscSectionGetDof(section, cStart, &vcdof[1]);CHKERRQ(ierr);}
118   }
119   ierr = MPI_Allreduce(vcdof, globalvcdof, 2, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject)dm));CHKERRMPI(ierr);
120   if (globalvcdof[0]) {
121     *sStart = vStart;
122     *sEnd   = vEnd;
123     if (globalvcdof[0] == cdim) *ft = PETSC_VTK_POINT_VECTOR_FIELD;
124     else                        *ft = PETSC_VTK_POINT_FIELD;
125   } else if (globalvcdof[1]) {
126     *sStart = cStart;
127     *sEnd   = cEnd;
128     if (globalvcdof[1] == cdim) *ft = PETSC_VTK_CELL_VECTOR_FIELD;
129     else                        *ft = PETSC_VTK_CELL_FIELD;
130   } else {
131     if (field >= 0) {
132       const char *fieldname;
133 
134       ierr = PetscSectionGetFieldName(section, field, &fieldname);CHKERRQ(ierr);
135       ierr = PetscInfo((PetscObject) dm, "Could not classify VTK output type of section field %D \"%s\"\n", field, fieldname);CHKERRQ(ierr);
136     } else {
137       ierr = PetscInfo((PetscObject) dm, "Could not classify VTK output type of section\"%s\"\n");CHKERRQ(ierr);
138     }
139   }
140   PetscFunctionReturn(0);
141 }
142 
143 static PetscErrorCode VecView_Plex_Local_Draw(Vec v, PetscViewer viewer)
144 {
145   DM                 dm;
146   PetscSection       s;
147   PetscDraw          draw, popup;
148   DM                 cdm;
149   PetscSection       coordSection;
150   Vec                coordinates;
151   const PetscScalar *coords, *array;
152   PetscReal          bound[4] = {PETSC_MAX_REAL, PETSC_MAX_REAL, PETSC_MIN_REAL, PETSC_MIN_REAL};
153   PetscReal          vbound[2], time;
154   PetscBool          isnull, flg;
155   PetscInt           dim, Nf, f, Nc, comp, vStart, vEnd, cStart, cEnd, c, N, level, step, w = 0;
156   const char        *name;
157   char               title[PETSC_MAX_PATH_LEN];
158   PetscErrorCode     ierr;
159 
160   PetscFunctionBegin;
161   ierr = PetscViewerDrawGetDraw(viewer, 0, &draw);CHKERRQ(ierr);
162   ierr = PetscDrawIsNull(draw, &isnull);CHKERRQ(ierr);
163   if (isnull) PetscFunctionReturn(0);
164 
165   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
166   ierr = DMGetCoordinateDim(dm, &dim);CHKERRQ(ierr);
167   PetscAssertFalse(dim != 2,PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Cannot draw meshes of dimension %D. Use PETSCVIEWERGLVIS", dim);
168   ierr = DMGetLocalSection(dm, &s);CHKERRQ(ierr);
169   ierr = PetscSectionGetNumFields(s, &Nf);CHKERRQ(ierr);
170   ierr = DMGetCoarsenLevel(dm, &level);CHKERRQ(ierr);
171   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
172   ierr = DMGetLocalSection(cdm, &coordSection);CHKERRQ(ierr);
173   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
174   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
175   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
176 
177   ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
178   ierr = DMGetOutputSequenceNumber(dm, &step, &time);CHKERRQ(ierr);
179 
180   ierr = VecGetLocalSize(coordinates, &N);CHKERRQ(ierr);
181   ierr = VecGetArrayRead(coordinates, &coords);CHKERRQ(ierr);
182   for (c = 0; c < N; c += dim) {
183     bound[0] = PetscMin(bound[0], PetscRealPart(coords[c]));   bound[2] = PetscMax(bound[2], PetscRealPart(coords[c]));
184     bound[1] = PetscMin(bound[1], PetscRealPart(coords[c+1])); bound[3] = PetscMax(bound[3], PetscRealPart(coords[c+1]));
185   }
186   ierr = VecRestoreArrayRead(coordinates, &coords);CHKERRQ(ierr);
187   ierr = PetscDrawClear(draw);CHKERRQ(ierr);
188 
189   /* Could implement something like DMDASelectFields() */
190   for (f = 0; f < Nf; ++f) {
191     DM   fdm = dm;
192     Vec  fv  = v;
193     IS   fis;
194     char prefix[PETSC_MAX_PATH_LEN];
195     const char *fname;
196 
197     ierr = PetscSectionGetFieldComponents(s, f, &Nc);CHKERRQ(ierr);
198     ierr = PetscSectionGetFieldName(s, f, &fname);CHKERRQ(ierr);
199 
200     if (v->hdr.prefix) {ierr = PetscStrncpy(prefix, v->hdr.prefix,sizeof(prefix));CHKERRQ(ierr);}
201     else               {prefix[0] = '\0';}
202     if (Nf > 1) {
203       ierr = DMCreateSubDM(dm, 1, &f, &fis, &fdm);CHKERRQ(ierr);
204       ierr = VecGetSubVector(v, fis, &fv);CHKERRQ(ierr);
205       ierr = PetscStrlcat(prefix, fname,sizeof(prefix));CHKERRQ(ierr);
206       ierr = PetscStrlcat(prefix, "_",sizeof(prefix));CHKERRQ(ierr);
207     }
208     for (comp = 0; comp < Nc; ++comp, ++w) {
209       PetscInt nmax = 2;
210 
211       ierr = PetscViewerDrawGetDraw(viewer, w, &draw);CHKERRQ(ierr);
212       if (Nc > 1) {ierr = PetscSNPrintf(title, sizeof(title), "%s:%s_%D Step: %D Time: %.4g", name, fname, comp, step, time);CHKERRQ(ierr);}
213       else        {ierr = PetscSNPrintf(title, sizeof(title), "%s:%s Step: %D Time: %.4g", name, fname, step, time);CHKERRQ(ierr);}
214       ierr = PetscDrawSetTitle(draw, title);CHKERRQ(ierr);
215 
216       /* TODO Get max and min only for this component */
217       ierr = PetscOptionsGetRealArray(NULL, prefix, "-vec_view_bounds", vbound, &nmax, &flg);CHKERRQ(ierr);
218       if (!flg) {
219         ierr = VecMin(fv, NULL, &vbound[0]);CHKERRQ(ierr);
220         ierr = VecMax(fv, NULL, &vbound[1]);CHKERRQ(ierr);
221         if (vbound[1] <= vbound[0]) vbound[1] = vbound[0] + 1.0;
222       }
223       ierr = PetscDrawGetPopup(draw, &popup);CHKERRQ(ierr);
224       ierr = PetscDrawScalePopup(popup, vbound[0], vbound[1]);CHKERRQ(ierr);
225       ierr = PetscDrawSetCoordinates(draw, bound[0], bound[1], bound[2], bound[3]);CHKERRQ(ierr);
226 
227       ierr = VecGetArrayRead(fv, &array);CHKERRQ(ierr);
228       for (c = cStart; c < cEnd; ++c) {
229         PetscScalar *coords = NULL, *a = NULL;
230         PetscInt     numCoords, color[4] = {-1,-1,-1,-1};
231 
232         ierr = DMPlexPointLocalRead(fdm, c, array, &a);CHKERRQ(ierr);
233         if (a) {
234           color[0] = PetscDrawRealToColor(PetscRealPart(a[comp]), vbound[0], vbound[1]);
235           color[1] = color[2] = color[3] = color[0];
236         } else {
237           PetscScalar *vals = NULL;
238           PetscInt     numVals, va;
239 
240           ierr = DMPlexVecGetClosure(fdm, NULL, fv, c, &numVals, &vals);CHKERRQ(ierr);
241           PetscAssertFalse(numVals % Nc,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "The number of components %D does not divide the number of values in the closure %D", Nc, numVals);
242           switch (numVals/Nc) {
243           case 3: /* P1 Triangle */
244           case 4: /* P1 Quadrangle */
245             for (va = 0; va < numVals/Nc; ++va) color[va] = PetscDrawRealToColor(PetscRealPart(vals[va*Nc+comp]), vbound[0], vbound[1]);
246             break;
247           case 6: /* P2 Triangle */
248           case 8: /* P2 Quadrangle */
249             for (va = 0; va < numVals/(Nc*2); ++va) color[va] = PetscDrawRealToColor(PetscRealPart(vals[va*Nc+comp + numVals/(Nc*2)]), vbound[0], vbound[1]);
250             break;
251           default: SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of values for cell closure %D cannot be handled", numVals/Nc);
252           }
253           ierr = DMPlexVecRestoreClosure(fdm, NULL, fv, c, &numVals, &vals);CHKERRQ(ierr);
254         }
255         ierr = DMPlexVecGetClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
256         switch (numCoords) {
257         case 6:
258         case 12: /* Localized triangle */
259           ierr = PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), color[0], color[1], color[2]);CHKERRQ(ierr);
260           break;
261         case 8:
262         case 16: /* Localized quadrilateral */
263           ierr = PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), color[0], color[1], color[2]);CHKERRQ(ierr);
264           ierr = PetscDrawTriangle(draw, PetscRealPart(coords[4]), PetscRealPart(coords[5]), PetscRealPart(coords[6]), PetscRealPart(coords[7]), PetscRealPart(coords[0]), PetscRealPart(coords[1]), color[2], color[3], color[0]);CHKERRQ(ierr);
265           break;
266         default: SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Cannot draw cells with %D coordinates", numCoords);
267         }
268         ierr = DMPlexVecRestoreClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
269       }
270       ierr = VecRestoreArrayRead(fv, &array);CHKERRQ(ierr);
271       ierr = PetscDrawFlush(draw);CHKERRQ(ierr);
272       ierr = PetscDrawPause(draw);CHKERRQ(ierr);
273       ierr = PetscDrawSave(draw);CHKERRQ(ierr);
274     }
275     if (Nf > 1) {
276       ierr = VecRestoreSubVector(v, fis, &fv);CHKERRQ(ierr);
277       ierr = ISDestroy(&fis);CHKERRQ(ierr);
278       ierr = DMDestroy(&fdm);CHKERRQ(ierr);
279     }
280   }
281   PetscFunctionReturn(0);
282 }
283 
284 static PetscErrorCode VecView_Plex_Local_VTK(Vec v, PetscViewer viewer)
285 {
286   DM                      dm;
287   Vec                     locv;
288   const char              *name;
289   PetscSection            section;
290   PetscInt                pStart, pEnd;
291   PetscInt                numFields;
292   PetscViewerVTKFieldType ft;
293   PetscErrorCode          ierr;
294 
295   PetscFunctionBegin;
296   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
297   ierr = DMCreateLocalVector(dm, &locv);CHKERRQ(ierr); /* VTK viewer requires exclusive ownership of the vector */
298   ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
299   ierr = PetscObjectSetName((PetscObject) locv, name);CHKERRQ(ierr);
300   ierr = VecCopy(v, locv);CHKERRQ(ierr);
301   ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);
302   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
303   if (!numFields) {
304     ierr = DMPlexGetFieldType_Internal(dm, section, PETSC_DETERMINE, &pStart, &pEnd, &ft);CHKERRQ(ierr);
305     ierr = PetscViewerVTKAddField(viewer, (PetscObject) dm, DMPlexVTKWriteAll, PETSC_DEFAULT, ft, PETSC_TRUE,(PetscObject) locv);CHKERRQ(ierr);
306   } else {
307     PetscInt f;
308 
309     for (f = 0; f < numFields; f++) {
310       ierr = DMPlexGetFieldType_Internal(dm, section, f, &pStart, &pEnd, &ft);CHKERRQ(ierr);
311       if (ft == PETSC_VTK_INVALID) continue;
312       ierr = PetscObjectReference((PetscObject)locv);CHKERRQ(ierr);
313       ierr = PetscViewerVTKAddField(viewer, (PetscObject) dm, DMPlexVTKWriteAll, f, ft, PETSC_TRUE,(PetscObject) locv);CHKERRQ(ierr);
314     }
315     ierr = VecDestroy(&locv);CHKERRQ(ierr);
316   }
317   PetscFunctionReturn(0);
318 }
319 
320 PetscErrorCode VecView_Plex_Local(Vec v, PetscViewer viewer)
321 {
322   DM             dm;
323   PetscBool      isvtk, ishdf5, isdraw, isglvis;
324   PetscErrorCode ierr;
325 
326   PetscFunctionBegin;
327   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
328   PetscAssertFalse(!dm,PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
329   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,   &isvtk);CHKERRQ(ierr);
330   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,  &ishdf5);CHKERRQ(ierr);
331   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERDRAW,  &isdraw);CHKERRQ(ierr);
332   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERGLVIS, &isglvis);CHKERRQ(ierr);
333   if (isvtk || ishdf5 || isdraw || isglvis) {
334     PetscInt    i,numFields;
335     PetscObject fe;
336     PetscBool   fem = PETSC_FALSE;
337     Vec         locv = v;
338     const char  *name;
339     PetscInt    step;
340     PetscReal   time;
341 
342     ierr = DMGetNumFields(dm, &numFields);CHKERRQ(ierr);
343     for (i=0; i<numFields; i++) {
344       ierr = DMGetField(dm, i, NULL, &fe);CHKERRQ(ierr);
345       if (fe->classid == PETSCFE_CLASSID) { fem = PETSC_TRUE; break; }
346     }
347     if (fem) {
348       PetscObject isZero;
349 
350       ierr = DMGetLocalVector(dm, &locv);CHKERRQ(ierr);
351       ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
352       ierr = PetscObjectSetName((PetscObject) locv, name);CHKERRQ(ierr);
353       ierr = PetscObjectQuery((PetscObject) v, "__Vec_bc_zero__", &isZero);CHKERRQ(ierr);
354       ierr = PetscObjectCompose((PetscObject) locv, "__Vec_bc_zero__", isZero);CHKERRQ(ierr);
355       ierr = VecCopy(v, locv);CHKERRQ(ierr);
356       ierr = DMGetOutputSequenceNumber(dm, NULL, &time);CHKERRQ(ierr);
357       ierr = DMPlexInsertBoundaryValues(dm, PETSC_TRUE, locv, time, NULL, NULL, NULL);CHKERRQ(ierr);
358     }
359     if (isvtk) {
360       ierr = VecView_Plex_Local_VTK(locv, viewer);CHKERRQ(ierr);
361     } else if (ishdf5) {
362 #if defined(PETSC_HAVE_HDF5)
363       ierr = VecView_Plex_Local_HDF5_Internal(locv, viewer);CHKERRQ(ierr);
364 #else
365       SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
366 #endif
367     } else if (isdraw) {
368       ierr = VecView_Plex_Local_Draw(locv, viewer);CHKERRQ(ierr);
369     } else if (isglvis) {
370       ierr = DMGetOutputSequenceNumber(dm, &step, NULL);CHKERRQ(ierr);
371       ierr = PetscViewerGLVisSetSnapId(viewer, step);CHKERRQ(ierr);
372       ierr = VecView_GLVis(locv, viewer);CHKERRQ(ierr);
373     }
374     if (fem) {
375       ierr = PetscObjectCompose((PetscObject) locv, "__Vec_bc_zero__", NULL);CHKERRQ(ierr);
376       ierr = DMRestoreLocalVector(dm, &locv);CHKERRQ(ierr);
377     }
378   } else {
379     PetscBool isseq;
380 
381     ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
382     if (isseq) {ierr = VecView_Seq(v, viewer);CHKERRQ(ierr);}
383     else       {ierr = VecView_MPI(v, viewer);CHKERRQ(ierr);}
384   }
385   PetscFunctionReturn(0);
386 }
387 
388 PetscErrorCode VecView_Plex(Vec v, PetscViewer viewer)
389 {
390   DM             dm;
391   PetscBool      isvtk, ishdf5, isdraw, isglvis, isexodusii;
392   PetscErrorCode ierr;
393 
394   PetscFunctionBegin;
395   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
396   PetscAssertFalse(!dm,PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
397   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,      &isvtk);CHKERRQ(ierr);
398   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,     &ishdf5);CHKERRQ(ierr);
399   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERDRAW,     &isdraw);CHKERRQ(ierr);
400   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERGLVIS,    &isglvis);CHKERRQ(ierr);
401   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWEREXODUSII, &isexodusii);CHKERRQ(ierr);
402   if (isvtk || isdraw || isglvis) {
403     Vec         locv;
404     PetscObject isZero;
405     const char *name;
406 
407     ierr = DMGetLocalVector(dm, &locv);CHKERRQ(ierr);
408     ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
409     ierr = PetscObjectSetName((PetscObject) locv, name);CHKERRQ(ierr);
410     ierr = DMGlobalToLocalBegin(dm, v, INSERT_VALUES, locv);CHKERRQ(ierr);
411     ierr = DMGlobalToLocalEnd(dm, v, INSERT_VALUES, locv);CHKERRQ(ierr);
412     ierr = PetscObjectQuery((PetscObject) v, "__Vec_bc_zero__", &isZero);CHKERRQ(ierr);
413     ierr = PetscObjectCompose((PetscObject) locv, "__Vec_bc_zero__", isZero);CHKERRQ(ierr);
414     ierr = VecView_Plex_Local(locv, viewer);CHKERRQ(ierr);
415     ierr = PetscObjectCompose((PetscObject) locv, "__Vec_bc_zero__", NULL);CHKERRQ(ierr);
416     ierr = DMRestoreLocalVector(dm, &locv);CHKERRQ(ierr);
417   } else if (ishdf5) {
418 #if defined(PETSC_HAVE_HDF5)
419     ierr = VecView_Plex_HDF5_Internal(v, viewer);CHKERRQ(ierr);
420 #else
421     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
422 #endif
423   } else if (isexodusii) {
424 #if defined(PETSC_HAVE_EXODUSII)
425     ierr = VecView_PlexExodusII_Internal(v, viewer);CHKERRQ(ierr);
426 #else
427     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "ExodusII not supported in this build.\nPlease reconfigure using --download-exodusii");
428 #endif
429   } else {
430     PetscBool isseq;
431 
432     ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
433     if (isseq) {ierr = VecView_Seq(v, viewer);CHKERRQ(ierr);}
434     else       {ierr = VecView_MPI(v, viewer);CHKERRQ(ierr);}
435   }
436   PetscFunctionReturn(0);
437 }
438 
439 PetscErrorCode VecView_Plex_Native(Vec originalv, PetscViewer viewer)
440 {
441   DM                dm;
442   MPI_Comm          comm;
443   PetscViewerFormat format;
444   Vec               v;
445   PetscBool         isvtk, ishdf5;
446   PetscErrorCode    ierr;
447 
448   PetscFunctionBegin;
449   ierr = VecGetDM(originalv, &dm);CHKERRQ(ierr);
450   ierr = PetscObjectGetComm((PetscObject) originalv, &comm);CHKERRQ(ierr);
451   PetscAssertFalse(!dm,comm, PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
452   ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
453   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
454   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,  &isvtk);CHKERRQ(ierr);
455   if (format == PETSC_VIEWER_NATIVE) {
456     /* Natural ordering is the common case for DMDA, NATIVE means plain vector, for PLEX is the opposite */
457     /* this need a better fix */
458     if (dm->useNatural) {
459       if (dm->sfNatural) {
460         const char *vecname;
461         PetscInt    n, nroots;
462 
463         ierr = VecGetLocalSize(originalv, &n);CHKERRQ(ierr);
464         ierr = PetscSFGetGraph(dm->sfNatural, &nroots, NULL, NULL, NULL);CHKERRQ(ierr);
465         if (n == nroots) {
466           ierr = DMGetGlobalVector(dm, &v);CHKERRQ(ierr);
467           ierr = DMPlexGlobalToNaturalBegin(dm, originalv, v);CHKERRQ(ierr);
468           ierr = DMPlexGlobalToNaturalEnd(dm, originalv, v);CHKERRQ(ierr);
469           ierr = PetscObjectGetName((PetscObject) originalv, &vecname);CHKERRQ(ierr);
470           ierr = PetscObjectSetName((PetscObject) v, vecname);CHKERRQ(ierr);
471         } else SETERRQ(comm, PETSC_ERR_ARG_WRONG, "DM global to natural SF only handles global vectors");
472       } else SETERRQ(comm, PETSC_ERR_ARG_WRONGSTATE, "DM global to natural SF was not created");
473     } else v = originalv;
474   } else v = originalv;
475 
476   if (ishdf5) {
477 #if defined(PETSC_HAVE_HDF5)
478     ierr = VecView_Plex_HDF5_Native_Internal(v, viewer);CHKERRQ(ierr);
479 #else
480     SETERRQ(comm, PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
481 #endif
482   } else if (isvtk) {
483     SETERRQ(comm, PETSC_ERR_SUP, "VTK format does not support viewing in natural order. Please switch to HDF5.");
484   } else {
485     PetscBool isseq;
486 
487     ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
488     if (isseq) {ierr = VecView_Seq(v, viewer);CHKERRQ(ierr);}
489     else       {ierr = VecView_MPI(v, viewer);CHKERRQ(ierr);}
490   }
491   if (v != originalv) {ierr = DMRestoreGlobalVector(dm, &v);CHKERRQ(ierr);}
492   PetscFunctionReturn(0);
493 }
494 
495 PetscErrorCode VecLoad_Plex_Local(Vec v, PetscViewer viewer)
496 {
497   DM             dm;
498   PetscBool      ishdf5;
499   PetscErrorCode ierr;
500 
501   PetscFunctionBegin;
502   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
503   PetscAssertFalse(!dm,PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
504   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
505   if (ishdf5) {
506     DM          dmBC;
507     Vec         gv;
508     const char *name;
509 
510     ierr = DMGetOutputDM(dm, &dmBC);CHKERRQ(ierr);
511     ierr = DMGetGlobalVector(dmBC, &gv);CHKERRQ(ierr);
512     ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
513     ierr = PetscObjectSetName((PetscObject) gv, name);CHKERRQ(ierr);
514     ierr = VecLoad_Default(gv, viewer);CHKERRQ(ierr);
515     ierr = DMGlobalToLocalBegin(dmBC, gv, INSERT_VALUES, v);CHKERRQ(ierr);
516     ierr = DMGlobalToLocalEnd(dmBC, gv, INSERT_VALUES, v);CHKERRQ(ierr);
517     ierr = DMRestoreGlobalVector(dmBC, &gv);CHKERRQ(ierr);
518   } else {
519     ierr = VecLoad_Default(v, viewer);CHKERRQ(ierr);
520   }
521   PetscFunctionReturn(0);
522 }
523 
524 PetscErrorCode VecLoad_Plex(Vec v, PetscViewer viewer)
525 {
526   DM             dm;
527   PetscBool      ishdf5,isexodusii;
528   PetscErrorCode ierr;
529 
530   PetscFunctionBegin;
531   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
532   PetscAssertFalse(!dm,PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
533   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,     &ishdf5);CHKERRQ(ierr);
534   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWEREXODUSII, &isexodusii);CHKERRQ(ierr);
535   if (ishdf5) {
536 #if defined(PETSC_HAVE_HDF5)
537     ierr = VecLoad_Plex_HDF5_Internal(v, viewer);CHKERRQ(ierr);
538 #else
539     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
540 #endif
541   } else if (isexodusii) {
542 #if defined(PETSC_HAVE_EXODUSII)
543     ierr = VecLoad_PlexExodusII_Internal(v, viewer);CHKERRQ(ierr);
544 #else
545     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "ExodusII not supported in this build.\nPlease reconfigure using --download-exodusii");
546 #endif
547   } else {
548     ierr = VecLoad_Default(v, viewer);CHKERRQ(ierr);
549   }
550   PetscFunctionReturn(0);
551 }
552 
553 PetscErrorCode VecLoad_Plex_Native(Vec originalv, PetscViewer viewer)
554 {
555   DM                dm;
556   PetscViewerFormat format;
557   PetscBool         ishdf5;
558   PetscErrorCode    ierr;
559 
560   PetscFunctionBegin;
561   ierr = VecGetDM(originalv, &dm);CHKERRQ(ierr);
562   PetscAssertFalse(!dm,PetscObjectComm((PetscObject) originalv), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
563   ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
564   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
565   if (format == PETSC_VIEWER_NATIVE) {
566     if (dm->useNatural) {
567       if (dm->sfNatural) {
568         if (ishdf5) {
569 #if defined(PETSC_HAVE_HDF5)
570           Vec         v;
571           const char *vecname;
572 
573           ierr = DMGetGlobalVector(dm, &v);CHKERRQ(ierr);
574           ierr = PetscObjectGetName((PetscObject) originalv, &vecname);CHKERRQ(ierr);
575           ierr = PetscObjectSetName((PetscObject) v, vecname);CHKERRQ(ierr);
576           ierr = VecLoad_Plex_HDF5_Native_Internal(v, viewer);CHKERRQ(ierr);
577           ierr = DMPlexNaturalToGlobalBegin(dm, v, originalv);CHKERRQ(ierr);
578           ierr = DMPlexNaturalToGlobalEnd(dm, v, originalv);CHKERRQ(ierr);
579           ierr = DMRestoreGlobalVector(dm, &v);CHKERRQ(ierr);
580 #else
581           SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
582 #endif
583         } else SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Reading in natural order is not supported for anything but HDF5.");
584       }
585     } else {
586       ierr = VecLoad_Default(originalv, viewer);CHKERRQ(ierr);
587     }
588   }
589   PetscFunctionReturn(0);
590 }
591 
592 PETSC_UNUSED static PetscErrorCode DMPlexView_Ascii_Geometry(DM dm, PetscViewer viewer)
593 {
594   PetscSection       coordSection;
595   Vec                coordinates;
596   DMLabel            depthLabel, celltypeLabel;
597   const char        *name[4];
598   const PetscScalar *a;
599   PetscInt           dim, pStart, pEnd, cStart, cEnd, c;
600   PetscErrorCode     ierr;
601 
602   PetscFunctionBegin;
603   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
604   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
605   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
606   ierr = DMPlexGetDepthLabel(dm, &depthLabel);CHKERRQ(ierr);
607   ierr = DMPlexGetCellTypeLabel(dm, &celltypeLabel);CHKERRQ(ierr);
608   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
609   ierr = PetscSectionGetChart(coordSection, &pStart, &pEnd);CHKERRQ(ierr);
610   ierr = VecGetArrayRead(coordinates, &a);CHKERRQ(ierr);
611   name[0]     = "vertex";
612   name[1]     = "edge";
613   name[dim-1] = "face";
614   name[dim]   = "cell";
615   for (c = cStart; c < cEnd; ++c) {
616     PetscInt *closure = NULL;
617     PetscInt  closureSize, cl, ct;
618 
619     ierr = DMLabelGetValue(celltypeLabel, c, &ct);CHKERRQ(ierr);
620     ierr = PetscViewerASCIIPrintf(viewer, "Geometry for cell %D polytope type %s:\n", c, DMPolytopeTypes[ct]);CHKERRQ(ierr);
621     ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
622     ierr = PetscViewerASCIIPushTab(viewer);CHKERRQ(ierr);
623     for (cl = 0; cl < closureSize*2; cl += 2) {
624       PetscInt point = closure[cl], depth, dof, off, d, p;
625 
626       if ((point < pStart) || (point >= pEnd)) continue;
627       ierr = PetscSectionGetDof(coordSection, point, &dof);CHKERRQ(ierr);
628       if (!dof) continue;
629       ierr = DMLabelGetValue(depthLabel, point, &depth);CHKERRQ(ierr);
630       ierr = PetscSectionGetOffset(coordSection, point, &off);CHKERRQ(ierr);
631       ierr = PetscViewerASCIIPrintf(viewer, "%s %D coords:", name[depth], point);CHKERRQ(ierr);
632       for (p = 0; p < dof/dim; ++p) {
633         ierr = PetscViewerASCIIPrintf(viewer, " (");CHKERRQ(ierr);
634         for (d = 0; d < dim; ++d) {
635           if (d > 0) {ierr = PetscViewerASCIIPrintf(viewer, ", ");CHKERRQ(ierr);}
636           ierr = PetscViewerASCIIPrintf(viewer, "%g", (double) PetscRealPart(a[off+p*dim+d]));CHKERRQ(ierr);
637         }
638         ierr = PetscViewerASCIIPrintf(viewer, ")");CHKERRQ(ierr);
639       }
640       ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
641     }
642     ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
643     ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr);
644   }
645   ierr = VecRestoreArrayRead(coordinates, &a);CHKERRQ(ierr);
646   PetscFunctionReturn(0);
647 }
648 
649 typedef enum {CS_CARTESIAN, CS_POLAR, CS_CYLINDRICAL, CS_SPHERICAL} CoordSystem;
650 const char *CoordSystems[] = {"cartesian", "polar", "cylindrical", "spherical", "CoordSystem", "CS_", NULL};
651 
652 static PetscErrorCode DMPlexView_Ascii_Coordinates(PetscViewer viewer, CoordSystem cs, PetscInt dim, const PetscScalar x[])
653 {
654   PetscInt       i;
655   PetscErrorCode ierr;
656 
657   PetscFunctionBegin;
658   if (dim > 3) {
659     for (i = 0; i < dim; ++i) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, " %g", (double) PetscRealPart(x[i]));CHKERRQ(ierr);}
660   } else {
661     PetscReal coords[3], trcoords[3];
662 
663     for (i = 0; i < dim; ++i) coords[i] = PetscRealPart(x[i]);
664     switch (cs) {
665       case CS_CARTESIAN: for (i = 0; i < dim; ++i) trcoords[i] = coords[i];break;
666       case CS_POLAR:
667         PetscAssertFalse(dim != 2,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Polar coordinates are for 2 dimension, not %D", dim);
668         trcoords[0] = PetscSqrtReal(PetscSqr(coords[0]) + PetscSqr(coords[1]));
669         trcoords[1] = PetscAtan2Real(coords[1], coords[0]);
670         break;
671       case CS_CYLINDRICAL:
672         PetscAssertFalse(dim != 3,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cylindrical coordinates are for 3 dimension, not %D", dim);
673         trcoords[0] = PetscSqrtReal(PetscSqr(coords[0]) + PetscSqr(coords[1]));
674         trcoords[1] = PetscAtan2Real(coords[1], coords[0]);
675         trcoords[2] = coords[2];
676         break;
677       case CS_SPHERICAL:
678         PetscAssertFalse(dim != 3,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Spherical coordinates are for 3 dimension, not %D", dim);
679         trcoords[0] = PetscSqrtReal(PetscSqr(coords[0]) + PetscSqr(coords[1]) + PetscSqr(coords[2]));
680         trcoords[1] = PetscAtan2Real(PetscSqrtReal(PetscSqr(coords[0]) + PetscSqr(coords[1])), coords[2]);
681         trcoords[2] = PetscAtan2Real(coords[1], coords[0]);
682         break;
683     }
684     for (i = 0; i < dim; ++i) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, " %g", (double) trcoords[i]);CHKERRQ(ierr);}
685   }
686   PetscFunctionReturn(0);
687 }
688 
689 static PetscErrorCode DMPlexView_Ascii(DM dm, PetscViewer viewer)
690 {
691   DM_Plex          *mesh = (DM_Plex*) dm->data;
692   DM                cdm;
693   PetscSection      coordSection;
694   Vec               coordinates;
695   PetscViewerFormat format;
696   PetscErrorCode    ierr;
697 
698   PetscFunctionBegin;
699   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
700   ierr = DMGetLocalSection(cdm, &coordSection);CHKERRQ(ierr);
701   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
702   ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
703   if (format == PETSC_VIEWER_ASCII_INFO_DETAIL) {
704     const char *name;
705     PetscInt    dim, cellHeight, maxConeSize, maxSupportSize;
706     PetscInt    pStart, pEnd, p, numLabels, l;
707     PetscMPIInt rank, size;
708 
709     ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRMPI(ierr);
710     ierr = MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size);CHKERRMPI(ierr);
711     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
712     ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
713     ierr = DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize);CHKERRQ(ierr);
714     ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
715     ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
716     if (name) {ierr = PetscViewerASCIIPrintf(viewer, "%s in %D dimension%s:\n", name, dim, dim == 1 ? "" : "s");CHKERRQ(ierr);}
717     else      {ierr = PetscViewerASCIIPrintf(viewer, "Mesh in %D dimension%s:\n", dim, dim == 1 ? "" : "s");CHKERRQ(ierr);}
718     if (cellHeight) {ierr = PetscViewerASCIIPrintf(viewer, "  Cells are at height %D\n", cellHeight);CHKERRQ(ierr);}
719     ierr = PetscViewerASCIIPrintf(viewer, "Supports:\n", name);CHKERRQ(ierr);
720     ierr = PetscViewerASCIIPushSynchronized(viewer);CHKERRQ(ierr);
721     ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%d] Max support size: %D\n", rank, maxSupportSize);CHKERRQ(ierr);
722     for (p = pStart; p < pEnd; ++p) {
723       PetscInt dof, off, s;
724 
725       ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
726       ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
727       for (s = off; s < off+dof; ++s) {
728         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%d]: %D ----> %D\n", rank, p, mesh->supports[s]);CHKERRQ(ierr);
729       }
730     }
731     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
732     ierr = PetscViewerASCIIPrintf(viewer, "Cones:\n", name);CHKERRQ(ierr);
733     ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%d] Max cone size: %D\n", rank, maxConeSize);CHKERRQ(ierr);
734     for (p = pStart; p < pEnd; ++p) {
735       PetscInt dof, off, c;
736 
737       ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
738       ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
739       for (c = off; c < off+dof; ++c) {
740         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%d]: %D <---- %D (%D)\n", rank, p, mesh->cones[c], mesh->coneOrientations[c]);CHKERRQ(ierr);
741       }
742     }
743     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
744     ierr = PetscViewerASCIIPopSynchronized(viewer);CHKERRQ(ierr);
745     if (coordSection && coordinates) {
746       CoordSystem        cs = CS_CARTESIAN;
747       const PetscScalar *array;
748       PetscInt           Nf, Nc, pStart, pEnd, p;
749       PetscMPIInt        rank;
750       const char        *name;
751 
752       ierr = PetscOptionsGetEnum(((PetscObject) viewer)->options, ((PetscObject) viewer)->prefix, "-dm_plex_view_coord_system", CoordSystems, (PetscEnum *) &cs, NULL);CHKERRQ(ierr);
753       ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)viewer), &rank);CHKERRMPI(ierr);
754       ierr = PetscSectionGetNumFields(coordSection, &Nf);CHKERRQ(ierr);
755       PetscAssertFalse(Nf != 1,PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Coordinate section should have 1 field, not %D", Nf);
756       ierr = PetscSectionGetFieldComponents(coordSection, 0, &Nc);CHKERRQ(ierr);
757       ierr = PetscSectionGetChart(coordSection, &pStart, &pEnd);CHKERRQ(ierr);
758       ierr = PetscObjectGetName((PetscObject) coordinates, &name);CHKERRQ(ierr);
759       ierr = PetscViewerASCIIPrintf(viewer, "%s with %D fields\n", name, Nf);CHKERRQ(ierr);
760       ierr = PetscViewerASCIIPrintf(viewer, "  field 0 with %D components\n", Nc);CHKERRQ(ierr);
761       if (cs != CS_CARTESIAN) {ierr = PetscViewerASCIIPrintf(viewer, "  output coordinate system: %s\n", CoordSystems[cs]);CHKERRQ(ierr);}
762 
763       ierr = VecGetArrayRead(coordinates, &array);CHKERRQ(ierr);
764       ierr = PetscViewerASCIIPushSynchronized(viewer);CHKERRQ(ierr);
765       ierr = PetscViewerASCIISynchronizedPrintf(viewer, "Process %d:\n", rank);CHKERRQ(ierr);
766       for (p = pStart; p < pEnd; ++p) {
767         PetscInt dof, off;
768 
769         ierr = PetscSectionGetDof(coordSection, p, &dof);CHKERRQ(ierr);
770         ierr = PetscSectionGetOffset(coordSection, p, &off);CHKERRQ(ierr);
771         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "  (%4D) dim %2D offset %3D", p, dof, off);CHKERRQ(ierr);
772         ierr = DMPlexView_Ascii_Coordinates(viewer, cs, dof, &array[off]);CHKERRQ(ierr);
773         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\n");CHKERRQ(ierr);
774       }
775       ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
776       ierr = PetscViewerASCIIPopSynchronized(viewer);CHKERRQ(ierr);
777       ierr = VecRestoreArrayRead(coordinates, &array);CHKERRQ(ierr);
778     }
779     ierr = DMGetNumLabels(dm, &numLabels);CHKERRQ(ierr);
780     if (numLabels) {ierr = PetscViewerASCIIPrintf(viewer, "Labels:\n");CHKERRQ(ierr);}
781     for (l = 0; l < numLabels; ++l) {
782       DMLabel     label;
783       PetscBool   isdepth;
784       const char *name;
785 
786       ierr = DMGetLabelName(dm, l, &name);CHKERRQ(ierr);
787       ierr = PetscStrcmp(name, "depth", &isdepth);CHKERRQ(ierr);
788       if (isdepth) continue;
789       ierr = DMGetLabel(dm, name, &label);CHKERRQ(ierr);
790       ierr = DMLabelView(label, viewer);CHKERRQ(ierr);
791     }
792     if (size > 1) {
793       PetscSF sf;
794 
795       ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr);
796       ierr = PetscSFView(sf, viewer);CHKERRQ(ierr);
797     }
798     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
799   } else if (format == PETSC_VIEWER_ASCII_LATEX) {
800     const char  *name, *color;
801     const char  *defcolors[3]  = {"gray", "orange", "green"};
802     const char  *deflcolors[4] = {"blue", "cyan", "red", "magenta"};
803     char         lname[PETSC_MAX_PATH_LEN];
804     PetscReal    scale         = 2.0;
805     PetscReal    tikzscale     = 1.0;
806     PetscBool    useNumbers    = PETSC_TRUE, drawNumbers[4], drawColors[4], useLabels, useColors, plotEdges, drawHasse = PETSC_FALSE;
807     double       tcoords[3];
808     PetscScalar *coords;
809     PetscInt     numLabels, l, numColors, numLColors, dim, d, depth, cStart, cEnd, c, vStart, vEnd, v, eStart = 0, eEnd = 0, e, p, n;
810     PetscMPIInt  rank, size;
811     char         **names, **colors, **lcolors;
812     PetscBool    flg, lflg;
813     PetscBT      wp = NULL;
814     PetscInt     pEnd, pStart;
815 
816     ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
817     ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
818     ierr = DMGetNumLabels(dm, &numLabels);CHKERRQ(ierr);
819     numLabels  = PetscMax(numLabels, 10);
820     numColors  = 10;
821     numLColors = 10;
822     ierr = PetscCalloc3(numLabels, &names, numColors, &colors, numLColors, &lcolors);CHKERRQ(ierr);
823     ierr = PetscOptionsGetReal(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_scale", &scale, NULL);CHKERRQ(ierr);
824     ierr = PetscOptionsGetReal(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_tikzscale", &tikzscale, NULL);CHKERRQ(ierr);
825     ierr = PetscOptionsGetBool(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_numbers", &useNumbers, NULL);CHKERRQ(ierr);
826     for (d = 0; d < 4; ++d) drawNumbers[d] = useNumbers;
827     for (d = 0; d < 4; ++d) drawColors[d]  = PETSC_TRUE;
828     n = 4;
829     ierr = PetscOptionsGetBoolArray(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_numbers_depth", drawNumbers, &n, &flg);CHKERRQ(ierr);
830     PetscAssertFalse(flg && n != dim+1,PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_SIZ, "Number of flags %D != %D dim+1", n, dim+1);
831     ierr = PetscOptionsGetBoolArray(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_colors_depth", drawColors, &n, &flg);CHKERRQ(ierr);
832     PetscAssertFalse(flg && n != dim+1,PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_SIZ, "Number of flags %D != %D dim+1", n, dim+1);
833     ierr = PetscOptionsGetStringArray(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_labels", names, &numLabels, &useLabels);CHKERRQ(ierr);
834     if (!useLabels) numLabels = 0;
835     ierr = PetscOptionsGetStringArray(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_colors", colors, &numColors, &useColors);CHKERRQ(ierr);
836     if (!useColors) {
837       numColors = 3;
838       for (c = 0; c < numColors; ++c) {ierr = PetscStrallocpy(defcolors[c], &colors[c]);CHKERRQ(ierr);}
839     }
840     ierr = PetscOptionsGetStringArray(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_lcolors", lcolors, &numLColors, &useColors);CHKERRQ(ierr);
841     if (!useColors) {
842       numLColors = 4;
843       for (c = 0; c < numLColors; ++c) {ierr = PetscStrallocpy(deflcolors[c], &lcolors[c]);CHKERRQ(ierr);}
844     }
845     ierr = PetscOptionsGetString(((PetscObject) viewer)->options, ((PetscObject) viewer)->prefix, "-dm_plex_view_label_filter", lname, sizeof(lname), &lflg);CHKERRQ(ierr);
846     plotEdges = (PetscBool)(depth > 1 && drawNumbers[1] && dim < 3);
847     ierr = PetscOptionsGetBool(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_edges", &plotEdges, &flg);CHKERRQ(ierr);
848     PetscAssertFalse(flg && plotEdges && depth < dim,PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Mesh must be interpolated");
849     if (depth < dim) plotEdges = PETSC_FALSE;
850     ierr = PetscOptionsGetBool(((PetscObject) viewer)->options, ((PetscObject) viewer)->prefix, "-dm_plex_view_hasse", &drawHasse, NULL);CHKERRQ(ierr);
851 
852     /* filter points with labelvalue != labeldefaultvalue */
853     ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
854     ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
855     ierr = DMPlexGetDepthStratum(dm, 1, &eStart, &eEnd);CHKERRQ(ierr);
856     ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
857     if (lflg) {
858       DMLabel lbl;
859 
860       ierr = DMGetLabel(dm, lname, &lbl);CHKERRQ(ierr);
861       if (lbl) {
862         PetscInt val, defval;
863 
864         ierr = DMLabelGetDefaultValue(lbl, &defval);CHKERRQ(ierr);
865         ierr = PetscBTCreate(pEnd-pStart, &wp);CHKERRQ(ierr);
866         for (c = pStart;  c < pEnd; c++) {
867           PetscInt *closure = NULL;
868           PetscInt  closureSize;
869 
870           ierr = DMLabelGetValue(lbl, c, &val);CHKERRQ(ierr);
871           if (val == defval) continue;
872 
873           ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
874           for (p = 0; p < closureSize*2; p += 2) {
875             ierr = PetscBTSet(wp, closure[p] - pStart);CHKERRQ(ierr);
876           }
877           ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
878         }
879       }
880     }
881 
882     ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRMPI(ierr);
883     ierr = MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size);CHKERRMPI(ierr);
884     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
885     ierr = PetscViewerASCIIPrintf(viewer, "\
886 \\documentclass[tikz]{standalone}\n\n\
887 \\usepackage{pgflibraryshapes}\n\
888 \\usetikzlibrary{backgrounds}\n\
889 \\usetikzlibrary{arrows}\n\
890 \\begin{document}\n");CHKERRQ(ierr);
891     if (size > 1) {
892       ierr = PetscViewerASCIIPrintf(viewer, "%s for process ", name);CHKERRQ(ierr);
893       for (p = 0; p < size; ++p) {
894         if (p > 0 && p == size-1) {
895           ierr = PetscViewerASCIIPrintf(viewer, ", and ", colors[p%numColors], p);CHKERRQ(ierr);
896         } else if (p > 0) {
897           ierr = PetscViewerASCIIPrintf(viewer, ", ", colors[p%numColors], p);CHKERRQ(ierr);
898         }
899         ierr = PetscViewerASCIIPrintf(viewer, "{\\textcolor{%s}%D}", colors[p%numColors], p);CHKERRQ(ierr);
900       }
901       ierr = PetscViewerASCIIPrintf(viewer, ".\n\n\n");CHKERRQ(ierr);
902     }
903     if (drawHasse) {
904       PetscInt maxStratum = PetscMax(vEnd-vStart, PetscMax(eEnd-eStart, cEnd-cStart));
905 
906       ierr = PetscViewerASCIIPrintf(viewer, "\\newcommand{\\vStart}{%D}\n", vStart);CHKERRQ(ierr);
907       ierr = PetscViewerASCIIPrintf(viewer, "\\newcommand{\\vEnd}{%D}\n", vEnd-1);CHKERRQ(ierr);
908       ierr = PetscViewerASCIIPrintf(viewer, "\\newcommand{\\numVertices}{%D}\n", vEnd-vStart);CHKERRQ(ierr);
909       ierr = PetscViewerASCIIPrintf(viewer, "\\newcommand{\\vShift}{%.2f}\n", 3 + (maxStratum-(vEnd-vStart))/2.);CHKERRQ(ierr);
910       ierr = PetscViewerASCIIPrintf(viewer, "\\newcommand{\\eStart}{%D}\n", eStart);CHKERRQ(ierr);
911       ierr = PetscViewerASCIIPrintf(viewer, "\\newcommand{\\eEnd}{%D}\n", eEnd-1);CHKERRQ(ierr);
912       ierr = PetscViewerASCIIPrintf(viewer, "\\newcommand{\\eShift}{%.2f}\n", 3 + (maxStratum-(eEnd-eStart))/2.);CHKERRQ(ierr);
913       ierr = PetscViewerASCIIPrintf(viewer, "\\newcommand{\\numEdges}{%D}\n", eEnd-eStart);CHKERRQ(ierr);
914       ierr = PetscViewerASCIIPrintf(viewer, "\\newcommand{\\cStart}{%D}\n", cStart);CHKERRQ(ierr);
915       ierr = PetscViewerASCIIPrintf(viewer, "\\newcommand{\\cEnd}{%D}\n", cEnd-1);CHKERRQ(ierr);
916       ierr = PetscViewerASCIIPrintf(viewer, "\\newcommand{\\numCells}{%D}\n", cEnd-cStart);CHKERRQ(ierr);
917       ierr = PetscViewerASCIIPrintf(viewer, "\\newcommand{\\cShift}{%.2f}\n", 3 + (maxStratum-(cEnd-cStart))/2.);CHKERRQ(ierr);
918     }
919     ierr = PetscViewerASCIIPrintf(viewer, "\\begin{tikzpicture}[scale = %g,font=\\fontsize{8}{8}\\selectfont]\n", (double) tikzscale);CHKERRQ(ierr);
920 
921     /* Plot vertices */
922     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
923     ierr = PetscViewerASCIIPushSynchronized(viewer);CHKERRQ(ierr);
924     for (v = vStart; v < vEnd; ++v) {
925       PetscInt  off, dof, d;
926       PetscBool isLabeled = PETSC_FALSE;
927 
928       if (wp && !PetscBTLookup(wp,v - pStart)) continue;
929       ierr = PetscSectionGetDof(coordSection, v, &dof);CHKERRQ(ierr);
930       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
931       ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\path (");CHKERRQ(ierr);
932       PetscAssertFalse(dof > 3,PETSC_COMM_SELF,PETSC_ERR_PLIB,"coordSection vertex %D has dof %D > 3",v,dof);
933       for (d = 0; d < dof; ++d) {
934         tcoords[d] = (double) (scale*PetscRealPart(coords[off+d]));
935         tcoords[d] = PetscAbs(tcoords[d]) < 1e-10 ? 0.0 : tcoords[d];
936       }
937       /* Rotate coordinates since PGF makes z point out of the page instead of up */
938       if (dim == 3) {PetscReal tmp = tcoords[1]; tcoords[1] = tcoords[2]; tcoords[2] = -tmp;}
939       for (d = 0; d < dof; ++d) {
940         if (d > 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, ",");CHKERRQ(ierr);}
941         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "%g", (double) tcoords[d]);CHKERRQ(ierr);
942       }
943       if (drawHasse) color = colors[0%numColors];
944       else           color = colors[rank%numColors];
945       for (l = 0; l < numLabels; ++l) {
946         PetscInt val;
947         ierr = DMGetLabelValue(dm, names[l], v, &val);CHKERRQ(ierr);
948         if (val >= 0) {color = lcolors[l%numLColors]; isLabeled = PETSC_TRUE; break;}
949       }
950       if (drawNumbers[0]) {
951         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [draw,shape=circle,color=%s] {%D};\n", v, rank, color, v);CHKERRQ(ierr);
952       } else if (drawColors[0]) {
953         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [fill,inner sep=%dpt,shape=circle,color=%s] {};\n", v, rank, !isLabeled ? 1 : 2, color);CHKERRQ(ierr);
954       } else {
955         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [] {};\n", v, rank);CHKERRQ(ierr);
956       }
957     }
958     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
959     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
960     /* Plot edges */
961     if (plotEdges) {
962       ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
963       ierr = PetscViewerASCIIPrintf(viewer, "\\path\n");CHKERRQ(ierr);
964       for (e = eStart; e < eEnd; ++e) {
965         const PetscInt *cone;
966         PetscInt        coneSize, offA, offB, dof, d;
967 
968         if (wp && !PetscBTLookup(wp,e - pStart)) continue;
969         ierr = DMPlexGetConeSize(dm, e, &coneSize);CHKERRQ(ierr);
970         PetscAssertFalse(coneSize != 2,PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Edge %D cone should have two vertices, not %D", e, coneSize);
971         ierr = DMPlexGetCone(dm, e, &cone);CHKERRQ(ierr);
972         ierr = PetscSectionGetDof(coordSection, cone[0], &dof);CHKERRQ(ierr);
973         ierr = PetscSectionGetOffset(coordSection, cone[0], &offA);CHKERRQ(ierr);
974         ierr = PetscSectionGetOffset(coordSection, cone[1], &offB);CHKERRQ(ierr);
975         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "(");CHKERRQ(ierr);
976         for (d = 0; d < dof; ++d) {
977           tcoords[d] = (double) (0.5*scale*PetscRealPart(coords[offA+d]+coords[offB+d]));
978           tcoords[d] = PetscAbs(tcoords[d]) < 1e-10 ? 0.0 : tcoords[d];
979         }
980         /* Rotate coordinates since PGF makes z point out of the page instead of up */
981         if (dim == 3) {PetscReal tmp = tcoords[1]; tcoords[1] = tcoords[2]; tcoords[2] = -tmp;}
982         for (d = 0; d < dof; ++d) {
983           if (d > 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, ",");CHKERRQ(ierr);}
984           ierr = PetscViewerASCIISynchronizedPrintf(viewer, "%g", (double)tcoords[d]);CHKERRQ(ierr);
985         }
986         if (drawHasse) color = colors[1%numColors];
987         else           color = colors[rank%numColors];
988         for (l = 0; l < numLabels; ++l) {
989           PetscInt val;
990           ierr = DMGetLabelValue(dm, names[l], v, &val);CHKERRQ(ierr);
991           if (val >= 0) {color = lcolors[l%numLColors]; break;}
992         }
993         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [draw,shape=circle,color=%s] {%D} --\n", e, rank, color, e);CHKERRQ(ierr);
994       }
995       ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
996       ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
997       ierr = PetscViewerASCIIPrintf(viewer, "(0,0);\n");CHKERRQ(ierr);
998     }
999     /* Plot cells */
1000     if (dim == 3 || !drawNumbers[1]) {
1001       for (e = eStart; e < eEnd; ++e) {
1002         const PetscInt *cone;
1003 
1004         if (wp && !PetscBTLookup(wp,e - pStart)) continue;
1005         color = colors[rank%numColors];
1006         for (l = 0; l < numLabels; ++l) {
1007           PetscInt val;
1008           ierr = DMGetLabelValue(dm, names[l], e, &val);CHKERRQ(ierr);
1009           if (val >= 0) {color = lcolors[l%numLColors]; break;}
1010         }
1011         ierr = DMPlexGetCone(dm, e, &cone);CHKERRQ(ierr);
1012         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\draw[color=%s] (%D_%d) -- (%D_%d);\n", color, cone[0], rank, cone[1], rank);CHKERRQ(ierr);
1013       }
1014     } else {
1015        DMPolytopeType ct;
1016 
1017       /* Drawing a 2D polygon */
1018       for (c = cStart; c < cEnd; ++c) {
1019         if (wp && !PetscBTLookup(wp, c - pStart)) continue;
1020         ierr = DMPlexGetCellType(dm, c, &ct);CHKERRQ(ierr);
1021         if (ct == DM_POLYTOPE_SEG_PRISM_TENSOR ||
1022             ct == DM_POLYTOPE_TRI_PRISM_TENSOR ||
1023             ct == DM_POLYTOPE_QUAD_PRISM_TENSOR) {
1024           const PetscInt *cone;
1025           PetscInt        coneSize, e;
1026 
1027           ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
1028           ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
1029           for (e = 0; e < coneSize; ++e) {
1030             const PetscInt *econe;
1031 
1032             ierr = DMPlexGetCone(dm, cone[e], &econe);CHKERRQ(ierr);
1033             ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\draw[color=%s] (%D_%d) -- (%D_%d) -- (%D_%d);\n", colors[rank%numColors], econe[0], rank, cone[e], rank, econe[1], rank);CHKERRQ(ierr);
1034           }
1035         } else {
1036           PetscInt *closure = NULL;
1037           PetscInt  closureSize, Nv = 0, v;
1038 
1039           ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1040           for (p = 0; p < closureSize*2; p += 2) {
1041             const PetscInt point = closure[p];
1042 
1043             if ((point >= vStart) && (point < vEnd)) closure[Nv++] = point;
1044           }
1045           ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\draw[color=%s] ", colors[rank%numColors]);CHKERRQ(ierr);
1046           for (v = 0; v <= Nv; ++v) {
1047             const PetscInt vertex = closure[v%Nv];
1048 
1049             if (v > 0) {
1050               if (plotEdges) {
1051                 const PetscInt *edge;
1052                 PetscInt        endpoints[2], ne;
1053 
1054                 endpoints[0] = closure[v-1]; endpoints[1] = vertex;
1055                 ierr = DMPlexGetJoin(dm, 2, endpoints, &ne, &edge);CHKERRQ(ierr);
1056                 PetscAssertFalse(ne != 1,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Could not find edge for vertices %D, %D", endpoints[0], endpoints[1]);
1057                 ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- (%D_%d) -- ", edge[0], rank);CHKERRQ(ierr);
1058                 ierr = DMPlexRestoreJoin(dm, 2, endpoints, &ne, &edge);CHKERRQ(ierr);
1059               } else {
1060                 ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- ");CHKERRQ(ierr);
1061               }
1062             }
1063             ierr = PetscViewerASCIISynchronizedPrintf(viewer, "(%D_%d)", vertex, rank);CHKERRQ(ierr);
1064           }
1065           ierr = PetscViewerASCIISynchronizedPrintf(viewer, ";\n");CHKERRQ(ierr);
1066           ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1067         }
1068       }
1069     }
1070     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
1071     for (c = cStart; c < cEnd; ++c) {
1072       double    ccoords[3] = {0.0, 0.0, 0.0};
1073       PetscBool isLabeled  = PETSC_FALSE;
1074       PetscInt *closure    = NULL;
1075       PetscInt  closureSize, dof, d, n = 0;
1076 
1077       if (wp && !PetscBTLookup(wp,c - pStart)) continue;
1078       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1079       ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\path (");CHKERRQ(ierr);
1080       for (p = 0; p < closureSize*2; p += 2) {
1081         const PetscInt point = closure[p];
1082         PetscInt       off;
1083 
1084         if ((point < vStart) || (point >= vEnd)) continue;
1085         ierr = PetscSectionGetDof(coordSection, point, &dof);CHKERRQ(ierr);
1086         ierr = PetscSectionGetOffset(coordSection, point, &off);CHKERRQ(ierr);
1087         for (d = 0; d < dof; ++d) {
1088           tcoords[d] = (double) (scale*PetscRealPart(coords[off+d]));
1089           tcoords[d] = PetscAbs(tcoords[d]) < 1e-10 ? 0.0 : tcoords[d];
1090         }
1091         /* Rotate coordinates since PGF makes z point out of the page instead of up */
1092         if (dof == 3) {PetscReal tmp = tcoords[1]; tcoords[1] = tcoords[2]; tcoords[2] = -tmp;}
1093         for (d = 0; d < dof; ++d) {ccoords[d] += tcoords[d];}
1094         ++n;
1095       }
1096       for (d = 0; d < dof; ++d) {ccoords[d] /= n;}
1097       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1098       for (d = 0; d < dof; ++d) {
1099         if (d > 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, ",");CHKERRQ(ierr);}
1100         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "%g", (double) ccoords[d]);CHKERRQ(ierr);
1101       }
1102       if (drawHasse) color = colors[depth%numColors];
1103       else           color = colors[rank%numColors];
1104       for (l = 0; l < numLabels; ++l) {
1105         PetscInt val;
1106         ierr = DMGetLabelValue(dm, names[l], c, &val);CHKERRQ(ierr);
1107         if (val >= 0) {color = lcolors[l%numLColors]; isLabeled = PETSC_TRUE; break;}
1108       }
1109       if (drawNumbers[dim]) {
1110         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [draw,shape=circle,color=%s] {%D};\n", c, rank, color, c);CHKERRQ(ierr);
1111       } else if (drawColors[dim]) {
1112         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [fill,inner sep=%dpt,shape=circle,color=%s] {};\n", c, rank, !isLabeled ? 1 : 2, color);CHKERRQ(ierr);
1113       } else {
1114         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [] {};\n", c, rank);CHKERRQ(ierr);
1115       }
1116     }
1117     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
1118     if (drawHasse) {
1119       color = colors[depth%numColors];
1120       ierr = PetscViewerASCIIPrintf(viewer, "%% Cells\n");CHKERRQ(ierr);
1121       ierr = PetscViewerASCIIPrintf(viewer, "\\foreach \\c in {\\cStart,...,\\cEnd}\n");CHKERRQ(ierr);
1122       ierr = PetscViewerASCIIPrintf(viewer, "{\n");CHKERRQ(ierr);
1123       ierr = PetscViewerASCIIPrintf(viewer, "  \\node(\\c_%d) [draw,shape=circle,color=%s,minimum size = 6mm] at (\\cShift+\\c-\\cStart,0) {\\c};\n", rank, color);CHKERRQ(ierr);
1124       ierr = PetscViewerASCIIPrintf(viewer, "}\n");CHKERRQ(ierr);
1125 
1126       color = colors[1%numColors];
1127       ierr = PetscViewerASCIIPrintf(viewer, "%% Edges\n");CHKERRQ(ierr);
1128       ierr = PetscViewerASCIIPrintf(viewer, "\\foreach \\e in {\\eStart,...,\\eEnd}\n");CHKERRQ(ierr);
1129       ierr = PetscViewerASCIIPrintf(viewer, "{\n");CHKERRQ(ierr);
1130       ierr = PetscViewerASCIIPrintf(viewer, "  \\node(\\e_%d) [draw,shape=circle,color=%s,minimum size = 6mm] at (\\eShift+\\e-\\eStart,1) {\\e};\n", rank, color);CHKERRQ(ierr);
1131       ierr = PetscViewerASCIIPrintf(viewer, "}\n");CHKERRQ(ierr);
1132 
1133       color = colors[0%numColors];
1134       ierr = PetscViewerASCIIPrintf(viewer, "%% Vertices\n");CHKERRQ(ierr);
1135       ierr = PetscViewerASCIIPrintf(viewer, "\\foreach \\v in {\\vStart,...,\\vEnd}\n");CHKERRQ(ierr);
1136       ierr = PetscViewerASCIIPrintf(viewer, "{\n");CHKERRQ(ierr);
1137       ierr = PetscViewerASCIIPrintf(viewer, "  \\node(\\v_%d) [draw,shape=circle,color=%s,minimum size = 6mm] at (\\vShift+\\v-\\vStart,2) {\\v};\n", rank, color);CHKERRQ(ierr);
1138       ierr = PetscViewerASCIIPrintf(viewer, "}\n");CHKERRQ(ierr);
1139 
1140       for (p = pStart; p < pEnd; ++p) {
1141         const PetscInt *cone;
1142         PetscInt        coneSize, cp;
1143 
1144         ierr = DMPlexGetCone(dm, p, &cone);CHKERRQ(ierr);
1145         ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
1146         for (cp = 0; cp < coneSize; ++cp) {
1147           ierr = PetscViewerASCIIPrintf(viewer, "\\draw[->, shorten >=1pt] (%D_%d) -- (%D_%d);\n", cone[cp], rank, p, rank);CHKERRQ(ierr);
1148         }
1149       }
1150     }
1151     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
1152     ierr = PetscViewerASCIIPopSynchronized(viewer);CHKERRQ(ierr);
1153     ierr = PetscViewerASCIIPrintf(viewer, "\\end{tikzpicture}\n");CHKERRQ(ierr);
1154     ierr = PetscViewerASCIIPrintf(viewer, "\\end{document}\n", name);CHKERRQ(ierr);
1155     for (l = 0; l < numLabels;  ++l) {ierr = PetscFree(names[l]);CHKERRQ(ierr);}
1156     for (c = 0; c < numColors;  ++c) {ierr = PetscFree(colors[c]);CHKERRQ(ierr);}
1157     for (c = 0; c < numLColors; ++c) {ierr = PetscFree(lcolors[c]);CHKERRQ(ierr);}
1158     ierr = PetscFree3(names, colors, lcolors);CHKERRQ(ierr);
1159     ierr = PetscBTDestroy(&wp);CHKERRQ(ierr);
1160   } else if (format == PETSC_VIEWER_LOAD_BALANCE) {
1161     Vec                    cown,acown;
1162     VecScatter             sct;
1163     ISLocalToGlobalMapping g2l;
1164     IS                     gid,acis;
1165     MPI_Comm               comm,ncomm = MPI_COMM_NULL;
1166     MPI_Group              ggroup,ngroup;
1167     PetscScalar            *array,nid;
1168     const PetscInt         *idxs;
1169     PetscInt               *idxs2,*start,*adjacency,*work;
1170     PetscInt64             lm[3],gm[3];
1171     PetscInt               i,c,cStart,cEnd,cum,numVertices,ect,ectn,cellHeight;
1172     PetscMPIInt            d1,d2,rank;
1173 
1174     ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
1175     ierr = MPI_Comm_rank(comm,&rank);CHKERRMPI(ierr);
1176 #if defined(PETSC_HAVE_MPI_PROCESS_SHARED_MEMORY)
1177     ierr = MPI_Comm_split_type(comm,MPI_COMM_TYPE_SHARED,rank,MPI_INFO_NULL,&ncomm);CHKERRMPI(ierr);
1178 #endif
1179     if (ncomm != MPI_COMM_NULL) {
1180       ierr = MPI_Comm_group(comm,&ggroup);CHKERRMPI(ierr);
1181       ierr = MPI_Comm_group(ncomm,&ngroup);CHKERRMPI(ierr);
1182       d1   = 0;
1183       ierr = MPI_Group_translate_ranks(ngroup,1,&d1,ggroup,&d2);CHKERRMPI(ierr);
1184       nid  = d2;
1185       ierr = MPI_Group_free(&ggroup);CHKERRMPI(ierr);
1186       ierr = MPI_Group_free(&ngroup);CHKERRMPI(ierr);
1187       ierr = MPI_Comm_free(&ncomm);CHKERRMPI(ierr);
1188     } else nid = 0.0;
1189 
1190     /* Get connectivity */
1191     ierr = DMPlexGetVTKCellHeight(dm,&cellHeight);CHKERRQ(ierr);
1192     ierr = DMPlexCreatePartitionerGraph(dm,cellHeight,&numVertices,&start,&adjacency,&gid);CHKERRQ(ierr);
1193 
1194     /* filter overlapped local cells */
1195     ierr = DMPlexGetHeightStratum(dm,cellHeight,&cStart,&cEnd);CHKERRQ(ierr);
1196     ierr = ISGetIndices(gid,&idxs);CHKERRQ(ierr);
1197     ierr = ISGetLocalSize(gid,&cum);CHKERRQ(ierr);
1198     ierr = PetscMalloc1(cum,&idxs2);CHKERRQ(ierr);
1199     for (c = cStart, cum = 0; c < cEnd; c++) {
1200       if (idxs[c-cStart] < 0) continue;
1201       idxs2[cum++] = idxs[c-cStart];
1202     }
1203     ierr = ISRestoreIndices(gid,&idxs);CHKERRQ(ierr);
1204     PetscAssertFalse(numVertices != cum,PETSC_COMM_SELF,PETSC_ERR_PLIB,"Unexpected %D != %D",numVertices,cum);
1205     ierr = ISDestroy(&gid);CHKERRQ(ierr);
1206     ierr = ISCreateGeneral(comm,numVertices,idxs2,PETSC_OWN_POINTER,&gid);CHKERRQ(ierr);
1207 
1208     /* support for node-aware cell locality */
1209     ierr = ISCreateGeneral(comm,start[numVertices],adjacency,PETSC_USE_POINTER,&acis);CHKERRQ(ierr);
1210     ierr = VecCreateSeq(PETSC_COMM_SELF,start[numVertices],&acown);CHKERRQ(ierr);
1211     ierr = VecCreateMPI(comm,numVertices,PETSC_DECIDE,&cown);CHKERRQ(ierr);
1212     ierr = VecGetArray(cown,&array);CHKERRQ(ierr);
1213     for (c = 0; c < numVertices; c++) array[c] = nid;
1214     ierr = VecRestoreArray(cown,&array);CHKERRQ(ierr);
1215     ierr = VecScatterCreate(cown,acis,acown,NULL,&sct);CHKERRQ(ierr);
1216     ierr = VecScatterBegin(sct,cown,acown,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
1217     ierr = VecScatterEnd(sct,cown,acown,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
1218     ierr = ISDestroy(&acis);CHKERRQ(ierr);
1219     ierr = VecScatterDestroy(&sct);CHKERRQ(ierr);
1220     ierr = VecDestroy(&cown);CHKERRQ(ierr);
1221 
1222     /* compute edgeCut */
1223     for (c = 0, cum = 0; c < numVertices; c++) cum = PetscMax(cum,start[c+1]-start[c]);
1224     ierr = PetscMalloc1(cum,&work);CHKERRQ(ierr);
1225     ierr = ISLocalToGlobalMappingCreateIS(gid,&g2l);CHKERRQ(ierr);
1226     ierr = ISLocalToGlobalMappingSetType(g2l,ISLOCALTOGLOBALMAPPINGHASH);CHKERRQ(ierr);
1227     ierr = ISDestroy(&gid);CHKERRQ(ierr);
1228     ierr = VecGetArray(acown,&array);CHKERRQ(ierr);
1229     for (c = 0, ect = 0, ectn = 0; c < numVertices; c++) {
1230       PetscInt totl;
1231 
1232       totl = start[c+1]-start[c];
1233       ierr = ISGlobalToLocalMappingApply(g2l,IS_GTOLM_MASK,totl,adjacency+start[c],NULL,work);CHKERRQ(ierr);
1234       for (i = 0; i < totl; i++) {
1235         if (work[i] < 0) {
1236           ect  += 1;
1237           ectn += (array[i + start[c]] != nid) ? 0 : 1;
1238         }
1239       }
1240     }
1241     ierr  = PetscFree(work);CHKERRQ(ierr);
1242     ierr  = VecRestoreArray(acown,&array);CHKERRQ(ierr);
1243     lm[0] = numVertices > 0 ?  numVertices : PETSC_MAX_INT;
1244     lm[1] = -numVertices;
1245     ierr  = MPIU_Allreduce(lm,gm,2,MPIU_INT64,MPI_MIN,comm);CHKERRMPI(ierr);
1246     ierr  = PetscViewerASCIIPrintf(viewer,"  Cell balance: %.2f (max %D, min %D",-((double)gm[1])/((double)gm[0]),-(PetscInt)gm[1],(PetscInt)gm[0]);CHKERRQ(ierr);
1247     lm[0] = ect; /* edgeCut */
1248     lm[1] = ectn; /* node-aware edgeCut */
1249     lm[2] = numVertices > 0 ? 0 : 1; /* empty processes */
1250     ierr  = MPIU_Allreduce(lm,gm,3,MPIU_INT64,MPI_SUM,comm);CHKERRMPI(ierr);
1251     ierr  = PetscViewerASCIIPrintf(viewer,", empty %D)\n",(PetscInt)gm[2]);CHKERRQ(ierr);
1252 #if defined(PETSC_HAVE_MPI_PROCESS_SHARED_MEMORY)
1253     ierr  = PetscViewerASCIIPrintf(viewer,"  Edge Cut: %D (on node %.3f)\n",(PetscInt)(gm[0]/2),gm[0] ? ((double)(gm[1]))/((double)gm[0]) : 1.);CHKERRQ(ierr);
1254 #else
1255     ierr  = PetscViewerASCIIPrintf(viewer,"  Edge Cut: %D (on node %.3f)\n",(PetscInt)(gm[0]/2),0.0);CHKERRQ(ierr);
1256 #endif
1257     ierr  = ISLocalToGlobalMappingDestroy(&g2l);CHKERRQ(ierr);
1258     ierr  = PetscFree(start);CHKERRQ(ierr);
1259     ierr  = PetscFree(adjacency);CHKERRQ(ierr);
1260     ierr  = VecDestroy(&acown);CHKERRQ(ierr);
1261   } else {
1262     const char    *name;
1263     PetscInt      *sizes, *hybsizes, *ghostsizes;
1264     PetscInt       locDepth, depth, cellHeight, dim, d;
1265     PetscInt       pStart, pEnd, p, gcStart, gcEnd, gcNum;
1266     PetscInt       numLabels, l;
1267     DMPolytopeType ct0 = DM_POLYTOPE_UNKNOWN;
1268     MPI_Comm       comm;
1269     PetscMPIInt    size, rank;
1270 
1271     ierr = PetscObjectGetComm((PetscObject) dm, &comm);CHKERRQ(ierr);
1272     ierr = MPI_Comm_size(comm, &size);CHKERRMPI(ierr);
1273     ierr = MPI_Comm_rank(comm, &rank);CHKERRMPI(ierr);
1274     ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
1275     ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
1276     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
1277     if (name) {ierr = PetscViewerASCIIPrintf(viewer, "%s in %D dimension%s:\n", name, dim, dim == 1 ? "" : "s");CHKERRQ(ierr);}
1278     else      {ierr = PetscViewerASCIIPrintf(viewer, "Mesh in %D dimension%s:\n", dim, dim == 1 ? "" : "s");CHKERRQ(ierr);}
1279     if (cellHeight) {ierr = PetscViewerASCIIPrintf(viewer, "  Cells are at height %D\n", cellHeight);CHKERRQ(ierr);}
1280     ierr = DMPlexGetDepth(dm, &locDepth);CHKERRQ(ierr);
1281     ierr = MPIU_Allreduce(&locDepth, &depth, 1, MPIU_INT, MPI_MAX, comm);CHKERRMPI(ierr);
1282     ierr = DMPlexGetGhostCellStratum(dm, &gcStart, &gcEnd);CHKERRQ(ierr);
1283     gcNum = gcEnd - gcStart;
1284     ierr = PetscCalloc3(size,&sizes,size,&hybsizes,size,&ghostsizes);CHKERRQ(ierr);
1285     for (d = 0; d <= depth; d++) {
1286       PetscInt Nc[2] = {0, 0}, ict;
1287 
1288       ierr = DMPlexGetDepthStratum(dm, d, &pStart, &pEnd);CHKERRQ(ierr);
1289       if (pStart < pEnd) {ierr = DMPlexGetCellType(dm, pStart, &ct0);CHKERRQ(ierr);}
1290       ict  = ct0;
1291       ierr = MPI_Bcast(&ict, 1, MPIU_INT, 0, comm);CHKERRMPI(ierr);
1292       ct0  = (DMPolytopeType) ict;
1293       for (p = pStart; p < pEnd; ++p) {
1294         DMPolytopeType ct;
1295 
1296         ierr = DMPlexGetCellType(dm, p, &ct);CHKERRQ(ierr);
1297         if (ct == ct0) ++Nc[0];
1298         else           ++Nc[1];
1299       }
1300       ierr = MPI_Gather(&Nc[0], 1, MPIU_INT, sizes,    1, MPIU_INT, 0, comm);CHKERRMPI(ierr);
1301       ierr = MPI_Gather(&Nc[1], 1, MPIU_INT, hybsizes, 1, MPIU_INT, 0, comm);CHKERRMPI(ierr);
1302       if (d == depth) {ierr = MPI_Gather(&gcNum, 1, MPIU_INT, ghostsizes, 1, MPIU_INT, 0, comm);CHKERRMPI(ierr);}
1303       ierr = PetscViewerASCIIPrintf(viewer, "  Number of %D-cells per rank:", (depth == 1) && d ? dim : d);CHKERRQ(ierr);
1304       for (p = 0; p < size; ++p) {
1305         if (rank == 0) {
1306           ierr = PetscViewerASCIIPrintf(viewer, " %D", sizes[p]+hybsizes[p]);CHKERRQ(ierr);
1307           if (hybsizes[p]   > 0) {ierr = PetscViewerASCIIPrintf(viewer, " (%D)", hybsizes[p]);CHKERRQ(ierr);}
1308           if (ghostsizes[p] > 0) {ierr = PetscViewerASCIIPrintf(viewer, " [%D]", ghostsizes[p]);CHKERRQ(ierr);}
1309         }
1310       }
1311       ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
1312     }
1313     ierr = PetscFree3(sizes,hybsizes,ghostsizes);CHKERRQ(ierr);
1314     {
1315       const PetscReal      *maxCell;
1316       const PetscReal      *L;
1317       const DMBoundaryType *bd;
1318       PetscBool             per, localized;
1319 
1320       ierr = DMGetPeriodicity(dm, &per, &maxCell, &L, &bd);CHKERRQ(ierr);
1321       ierr = DMGetCoordinatesLocalized(dm, &localized);CHKERRQ(ierr);
1322       if (per) {
1323         ierr = PetscViewerASCIIPrintf(viewer, "Periodic mesh (");CHKERRQ(ierr);
1324         ierr = PetscViewerASCIIUseTabs(viewer, PETSC_FALSE);CHKERRQ(ierr);
1325         for (d = 0; d < dim; ++d) {
1326           if (bd && d > 0) {ierr = PetscViewerASCIIPrintf(viewer, ", ");CHKERRQ(ierr);}
1327           if (bd)    {ierr = PetscViewerASCIIPrintf(viewer, "%s", DMBoundaryTypes[bd[d]]);CHKERRQ(ierr);}
1328         }
1329         ierr = PetscViewerASCIIPrintf(viewer, ") coordinates %s\n", localized ? "localized" : "not localized");CHKERRQ(ierr);
1330         ierr = PetscViewerASCIIUseTabs(viewer, PETSC_TRUE);CHKERRQ(ierr);
1331       }
1332     }
1333     ierr = DMGetNumLabels(dm, &numLabels);CHKERRQ(ierr);
1334     if (numLabels) {ierr = PetscViewerASCIIPrintf(viewer, "Labels:\n");CHKERRQ(ierr);}
1335     for (l = 0; l < numLabels; ++l) {
1336       DMLabel         label;
1337       const char     *name;
1338       IS              valueIS;
1339       const PetscInt *values;
1340       PetscInt        numValues, v;
1341 
1342       ierr = DMGetLabelName(dm, l, &name);CHKERRQ(ierr);
1343       ierr = DMGetLabel(dm, name, &label);CHKERRQ(ierr);
1344       ierr = DMLabelGetNumValues(label, &numValues);CHKERRQ(ierr);
1345       ierr = PetscViewerASCIIPrintf(viewer, "  %s: %D strata with value/size (", name, numValues);CHKERRQ(ierr);
1346       ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
1347       ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
1348       ierr = PetscViewerASCIIUseTabs(viewer, PETSC_FALSE);CHKERRQ(ierr);
1349       for (v = 0; v < numValues; ++v) {
1350         PetscInt size;
1351 
1352         ierr = DMLabelGetStratumSize(label, values[v], &size);CHKERRQ(ierr);
1353         if (v > 0) {ierr = PetscViewerASCIIPrintf(viewer, ", ");CHKERRQ(ierr);}
1354         ierr = PetscViewerASCIIPrintf(viewer, "%D (%D)", values[v], size);CHKERRQ(ierr);
1355       }
1356       ierr = PetscViewerASCIIPrintf(viewer, ")\n");CHKERRQ(ierr);
1357       ierr = PetscViewerASCIIUseTabs(viewer, PETSC_TRUE);CHKERRQ(ierr);
1358       ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
1359       ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
1360     }
1361     {
1362       char    **labelNames;
1363       PetscInt  Nl = numLabels;
1364       PetscBool flg;
1365 
1366       ierr = PetscMalloc1(Nl, &labelNames);CHKERRQ(ierr);
1367       ierr = PetscOptionsGetStringArray(((PetscObject) dm)->options, ((PetscObject) dm)->prefix, "-dm_plex_view_labels", labelNames, &Nl, &flg);CHKERRQ(ierr);
1368       for (l = 0; l < Nl; ++l) {
1369         DMLabel label;
1370 
1371         ierr = DMHasLabel(dm, labelNames[l], &flg);CHKERRQ(ierr);
1372         if (flg) {
1373           ierr = DMGetLabel(dm, labelNames[l], &label);CHKERRQ(ierr);
1374           ierr = DMLabelView(label, viewer);CHKERRQ(ierr);
1375         }
1376         ierr = PetscFree(labelNames[l]);CHKERRQ(ierr);
1377       }
1378       ierr = PetscFree(labelNames);CHKERRQ(ierr);
1379     }
1380     /* If no fields are specified, people do not want to see adjacency */
1381     if (dm->Nf) {
1382       PetscInt f;
1383 
1384       for (f = 0; f < dm->Nf; ++f) {
1385         const char *name;
1386 
1387         ierr = PetscObjectGetName(dm->fields[f].disc, &name);CHKERRQ(ierr);
1388         if (numLabels) {ierr = PetscViewerASCIIPrintf(viewer, "Field %s:\n", name);CHKERRQ(ierr);}
1389         ierr = PetscViewerASCIIPushTab(viewer);CHKERRQ(ierr);
1390         if (dm->fields[f].label) {ierr = DMLabelView(dm->fields[f].label, viewer);CHKERRQ(ierr);}
1391         if (dm->fields[f].adjacency[0]) {
1392           if (dm->fields[f].adjacency[1]) {ierr = PetscViewerASCIIPrintf(viewer, "adjacency FVM++\n");CHKERRQ(ierr);}
1393           else                            {ierr = PetscViewerASCIIPrintf(viewer, "adjacency FVM\n");CHKERRQ(ierr);}
1394         } else {
1395           if (dm->fields[f].adjacency[1]) {ierr = PetscViewerASCIIPrintf(viewer, "adjacency FEM\n");CHKERRQ(ierr);}
1396           else                            {ierr = PetscViewerASCIIPrintf(viewer, "adjacency FUNKY\n");CHKERRQ(ierr);}
1397         }
1398         ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr);
1399       }
1400     }
1401     ierr = DMGetCoarseDM(dm, &cdm);CHKERRQ(ierr);
1402     if (cdm) {
1403       ierr = PetscViewerASCIIPushTab(viewer);CHKERRQ(ierr);
1404       ierr = DMPlexView_Ascii(cdm, viewer);CHKERRQ(ierr);
1405       ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr);
1406     }
1407   }
1408   PetscFunctionReturn(0);
1409 }
1410 
1411 static PetscErrorCode DMPlexDrawCell(DM dm, PetscDraw draw, PetscInt cell, const PetscScalar coords[])
1412 {
1413   DMPolytopeType ct;
1414   PetscMPIInt    rank;
1415   PetscInt       cdim;
1416   PetscErrorCode ierr;
1417 
1418   PetscFunctionBegin;
1419   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRMPI(ierr);
1420   ierr = DMPlexGetCellType(dm, cell, &ct);CHKERRQ(ierr);
1421   ierr = DMGetCoordinateDim(dm, &cdim);CHKERRQ(ierr);
1422   switch (ct) {
1423   case DM_POLYTOPE_SEGMENT:
1424   case DM_POLYTOPE_POINT_PRISM_TENSOR:
1425     switch (cdim) {
1426     case 1:
1427     {
1428       const PetscReal y  = 0.5;  /* TODO Put it in the middle of the viewport */
1429       const PetscReal dy = 0.05; /* TODO Make it a fraction of the total length */
1430 
1431       ierr = PetscDrawLine(draw, PetscRealPart(coords[0]), y,    PetscRealPart(coords[1]), y,    PETSC_DRAW_BLACK);CHKERRQ(ierr);
1432       ierr = PetscDrawLine(draw, PetscRealPart(coords[0]), y+dy, PetscRealPart(coords[0]), y-dy, PETSC_DRAW_BLACK);CHKERRQ(ierr);
1433       ierr = PetscDrawLine(draw, PetscRealPart(coords[1]), y+dy, PetscRealPart(coords[1]), y-dy, PETSC_DRAW_BLACK);CHKERRQ(ierr);
1434     }
1435     break;
1436     case 2:
1437     {
1438       const PetscReal dx = (PetscRealPart(coords[3]) - PetscRealPart(coords[1]));
1439       const PetscReal dy = (PetscRealPart(coords[2]) - PetscRealPart(coords[0]));
1440       const PetscReal l  = 0.1/PetscSqrtReal(dx*dx + dy*dy);
1441 
1442       ierr = PetscDrawLine(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1443       ierr = PetscDrawLine(draw, PetscRealPart(coords[0])+l*dx, PetscRealPart(coords[1])+l*dy, PetscRealPart(coords[0])-l*dx, PetscRealPart(coords[1])-l*dy, PETSC_DRAW_BLACK);CHKERRQ(ierr);
1444       ierr = PetscDrawLine(draw, PetscRealPart(coords[2])+l*dx, PetscRealPart(coords[3])+l*dy, PetscRealPart(coords[2])-l*dx, PetscRealPart(coords[3])-l*dy, PETSC_DRAW_BLACK);CHKERRQ(ierr);
1445     }
1446     break;
1447     default: SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Cannot draw cells of dimension %D", cdim);
1448     }
1449     break;
1450   case DM_POLYTOPE_TRIANGLE:
1451     ierr = PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]),
1452                              PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1453                              PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1454                              PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2);CHKERRQ(ierr);
1455     ierr = PetscDrawLine(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1456     ierr = PetscDrawLine(draw, PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1457     ierr = PetscDrawLine(draw, PetscRealPart(coords[4]), PetscRealPart(coords[5]), PetscRealPart(coords[0]), PetscRealPart(coords[1]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1458     break;
1459   case DM_POLYTOPE_QUADRILATERAL:
1460     ierr = PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]),
1461                               PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1462                               PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1463                               PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2);CHKERRQ(ierr);
1464     ierr = PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), PetscRealPart(coords[6]), PetscRealPart(coords[7]),
1465                               PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1466                               PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1467                               PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2);CHKERRQ(ierr);
1468     ierr = PetscDrawLine(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1469     ierr = PetscDrawLine(draw, PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1470     ierr = PetscDrawLine(draw, PetscRealPart(coords[4]), PetscRealPart(coords[5]), PetscRealPart(coords[6]), PetscRealPart(coords[7]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1471     ierr = PetscDrawLine(draw, PetscRealPart(coords[6]), PetscRealPart(coords[7]), PetscRealPart(coords[0]), PetscRealPart(coords[1]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1472     break;
1473   default: SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Cannot draw cells of type %s", DMPolytopeTypes[ct]);
1474   }
1475   PetscFunctionReturn(0);
1476 }
1477 
1478 static PetscErrorCode DMPlexDrawCellHighOrder(DM dm, PetscDraw draw, PetscInt cell, const PetscScalar coords[], PetscInt edgeDiv, PetscReal refCoords[], PetscReal edgeCoords[])
1479 {
1480   DMPolytopeType ct;
1481   PetscReal      centroid[2] = {0., 0.};
1482   PetscMPIInt    rank;
1483   PetscInt       fillColor, v, e, d;
1484   PetscErrorCode ierr;
1485 
1486   PetscFunctionBegin;
1487   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRMPI(ierr);
1488   ierr = DMPlexGetCellType(dm, cell, &ct);CHKERRQ(ierr);
1489   fillColor = PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2;
1490   switch (ct) {
1491   case DM_POLYTOPE_TRIANGLE:
1492     {
1493       PetscReal refVertices[6] = {-1., -1., 1., -1., -1., 1.};
1494 
1495       for (v = 0; v < 3; ++v) {centroid[0] += PetscRealPart(coords[v*2+0])/3.;centroid[1] += PetscRealPart(coords[v*2+1])/3.;}
1496       for (e = 0; e < 3; ++e) {
1497         refCoords[0] = refVertices[e*2+0];
1498         refCoords[1] = refVertices[e*2+1];
1499         for (d = 1; d <= edgeDiv; ++d) {
1500           refCoords[d*2+0] = refCoords[0] + (refVertices[(e+1)%3 * 2 + 0] - refCoords[0])*d/edgeDiv;
1501           refCoords[d*2+1] = refCoords[1] + (refVertices[(e+1)%3 * 2 + 1] - refCoords[1])*d/edgeDiv;
1502         }
1503         ierr = DMPlexReferenceToCoordinates(dm, cell, edgeDiv+1, refCoords, edgeCoords);CHKERRQ(ierr);
1504         for (d = 0; d < edgeDiv; ++d) {
1505           ierr = PetscDrawTriangle(draw, centroid[0], centroid[1], edgeCoords[d*2+0], edgeCoords[d*2+1], edgeCoords[(d+1)*2+0], edgeCoords[(d+1)*2+1], fillColor, fillColor, fillColor);CHKERRQ(ierr);
1506           ierr = PetscDrawLine(draw, edgeCoords[d*2+0], edgeCoords[d*2+1], edgeCoords[(d+1)*2+0], edgeCoords[(d+1)*2+1], PETSC_DRAW_BLACK);CHKERRQ(ierr);
1507         }
1508       }
1509     }
1510     break;
1511   default: SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Cannot draw cells of type %s", DMPolytopeTypes[ct]);
1512   }
1513   PetscFunctionReturn(0);
1514 }
1515 
1516 static PetscErrorCode DMPlexView_Draw(DM dm, PetscViewer viewer)
1517 {
1518   PetscDraw          draw;
1519   DM                 cdm;
1520   PetscSection       coordSection;
1521   Vec                coordinates;
1522   const PetscScalar *coords;
1523   PetscReal          xyl[2],xyr[2],bound[4] = {PETSC_MAX_REAL, PETSC_MAX_REAL, PETSC_MIN_REAL, PETSC_MIN_REAL};
1524   PetscReal         *refCoords, *edgeCoords;
1525   PetscBool          isnull, drawAffine = PETSC_TRUE;
1526   PetscInt           dim, vStart, vEnd, cStart, cEnd, c, N, edgeDiv = 4;
1527   PetscErrorCode     ierr;
1528 
1529   PetscFunctionBegin;
1530   ierr = DMGetCoordinateDim(dm, &dim);CHKERRQ(ierr);
1531   PetscAssertFalse(dim > 2,PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Cannot draw meshes of dimension %D", dim);
1532   ierr = PetscOptionsGetBool(((PetscObject) dm)->options, ((PetscObject) dm)->prefix, "-dm_view_draw_affine", &drawAffine, NULL);CHKERRQ(ierr);
1533   if (!drawAffine) {ierr = PetscMalloc2((edgeDiv+1)*dim, &refCoords, (edgeDiv+1)*dim, &edgeCoords);CHKERRQ(ierr);}
1534   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
1535   ierr = DMGetLocalSection(cdm, &coordSection);CHKERRQ(ierr);
1536   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
1537   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
1538   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
1539 
1540   ierr = PetscViewerDrawGetDraw(viewer, 0, &draw);CHKERRQ(ierr);
1541   ierr = PetscDrawIsNull(draw, &isnull);CHKERRQ(ierr);
1542   if (isnull) PetscFunctionReturn(0);
1543   ierr = PetscDrawSetTitle(draw, "Mesh");CHKERRQ(ierr);
1544 
1545   ierr = VecGetLocalSize(coordinates, &N);CHKERRQ(ierr);
1546   ierr = VecGetArrayRead(coordinates, &coords);CHKERRQ(ierr);
1547   for (c = 0; c < N; c += dim) {
1548     bound[0] = PetscMin(bound[0], PetscRealPart(coords[c]));   bound[2] = PetscMax(bound[2], PetscRealPart(coords[c]));
1549     bound[1] = PetscMin(bound[1], PetscRealPart(coords[c+1])); bound[3] = PetscMax(bound[3], PetscRealPart(coords[c+1]));
1550   }
1551   ierr = VecRestoreArrayRead(coordinates, &coords);CHKERRQ(ierr);
1552   ierr = MPIU_Allreduce(&bound[0],xyl,2,MPIU_REAL,MPIU_MIN,PetscObjectComm((PetscObject)dm));CHKERRMPI(ierr);
1553   ierr = MPIU_Allreduce(&bound[2],xyr,2,MPIU_REAL,MPIU_MAX,PetscObjectComm((PetscObject)dm));CHKERRMPI(ierr);
1554   ierr = PetscDrawSetCoordinates(draw, xyl[0], xyl[1], xyr[0], xyr[1]);CHKERRQ(ierr);
1555   ierr = PetscDrawClear(draw);CHKERRQ(ierr);
1556 
1557   for (c = cStart; c < cEnd; ++c) {
1558     PetscScalar *coords = NULL;
1559     PetscInt     numCoords;
1560 
1561     ierr = DMPlexVecGetClosureAtDepth_Internal(dm, coordSection, coordinates, c, 0, &numCoords, &coords);CHKERRQ(ierr);
1562     if (drawAffine) {
1563       ierr = DMPlexDrawCell(dm, draw, c, coords);CHKERRQ(ierr);
1564     } else {
1565       ierr = DMPlexDrawCellHighOrder(dm, draw, c, coords, edgeDiv, refCoords, edgeCoords);CHKERRQ(ierr);
1566     }
1567     ierr = DMPlexVecRestoreClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
1568   }
1569   if (!drawAffine) {ierr = PetscFree2(refCoords, edgeCoords);CHKERRQ(ierr);}
1570   ierr = PetscDrawFlush(draw);CHKERRQ(ierr);
1571   ierr = PetscDrawPause(draw);CHKERRQ(ierr);
1572   ierr = PetscDrawSave(draw);CHKERRQ(ierr);
1573   PetscFunctionReturn(0);
1574 }
1575 
1576 #if defined(PETSC_HAVE_EXODUSII)
1577 #include <exodusII.h>
1578 #include <petscviewerexodusii.h>
1579 #endif
1580 
1581 PetscErrorCode DMView_Plex(DM dm, PetscViewer viewer)
1582 {
1583   PetscBool      iascii, ishdf5, isvtk, isdraw, flg, isglvis, isexodus;
1584   char           name[PETSC_MAX_PATH_LEN];
1585   PetscErrorCode ierr;
1586 
1587   PetscFunctionBegin;
1588   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1589   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1590   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERASCII,    &iascii);CHKERRQ(ierr);
1591   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,      &isvtk);CHKERRQ(ierr);
1592   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,     &ishdf5);CHKERRQ(ierr);
1593   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERDRAW,     &isdraw);CHKERRQ(ierr);
1594   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERGLVIS,    &isglvis);CHKERRQ(ierr);
1595   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWEREXODUSII, &isexodus);CHKERRQ(ierr);
1596   if (iascii) {
1597     PetscViewerFormat format;
1598     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
1599     if (format == PETSC_VIEWER_ASCII_GLVIS) {
1600       ierr = DMPlexView_GLVis(dm, viewer);CHKERRQ(ierr);
1601     } else {
1602       ierr = DMPlexView_Ascii(dm, viewer);CHKERRQ(ierr);
1603     }
1604   } else if (ishdf5) {
1605 #if defined(PETSC_HAVE_HDF5)
1606     ierr = DMPlexView_HDF5_Internal(dm, viewer);CHKERRQ(ierr);
1607 #else
1608     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1609 #endif
1610   } else if (isvtk) {
1611     ierr = DMPlexVTKWriteAll((PetscObject) dm,viewer);CHKERRQ(ierr);
1612   } else if (isdraw) {
1613     ierr = DMPlexView_Draw(dm, viewer);CHKERRQ(ierr);
1614   } else if (isglvis) {
1615     ierr = DMPlexView_GLVis(dm, viewer);CHKERRQ(ierr);
1616 #if defined(PETSC_HAVE_EXODUSII)
1617   } else if (isexodus) {
1618 /*
1619       exodusII requires that all sets be part of exactly one cell set.
1620       If the dm does not have a "Cell Sets" label defined, we create one
1621       with ID 1, containig all cells.
1622       Note that if the Cell Sets label is defined but does not cover all cells,
1623       we may still have a problem. This should probably be checked here or in the viewer;
1624     */
1625     PetscInt numCS;
1626     ierr = DMGetLabelSize(dm,"Cell Sets",&numCS);CHKERRQ(ierr);
1627     if (!numCS) {
1628       PetscInt cStart, cEnd, c;
1629       ierr = DMCreateLabel(dm, "Cell Sets");CHKERRQ(ierr);
1630       ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
1631       for (c = cStart; c < cEnd; ++c) {ierr = DMSetLabelValue(dm, "Cell Sets", c, 1);CHKERRQ(ierr);}
1632     }
1633     ierr = DMView_PlexExodusII(dm, viewer);CHKERRQ(ierr);
1634 #endif
1635   } else {
1636     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Viewer type %s not yet supported for DMPlex writing", ((PetscObject)viewer)->type_name);
1637   }
1638   /* Optionally view the partition */
1639   ierr = PetscOptionsHasName(((PetscObject) dm)->options, ((PetscObject) dm)->prefix, "-dm_partition_view", &flg);CHKERRQ(ierr);
1640   if (flg) {
1641     Vec ranks;
1642     ierr = DMPlexCreateRankField(dm, &ranks);CHKERRQ(ierr);
1643     ierr = VecView(ranks, viewer);CHKERRQ(ierr);
1644     ierr = VecDestroy(&ranks);CHKERRQ(ierr);
1645   }
1646   /* Optionally view a label */
1647   ierr = PetscOptionsGetString(((PetscObject) dm)->options, ((PetscObject) dm)->prefix, "-dm_label_view", name, sizeof(name), &flg);CHKERRQ(ierr);
1648   if (flg) {
1649     DMLabel label;
1650     Vec     val;
1651 
1652     ierr = DMGetLabel(dm, name, &label);CHKERRQ(ierr);
1653     PetscAssertFalse(!label,PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Label %s provided to -dm_label_view does not exist in this DM", name);
1654     ierr = DMPlexCreateLabelField(dm, label, &val);CHKERRQ(ierr);
1655     ierr = VecView(val, viewer);CHKERRQ(ierr);
1656     ierr = VecDestroy(&val);CHKERRQ(ierr);
1657   }
1658   PetscFunctionReturn(0);
1659 }
1660 
1661 /*@
1662   DMPlexTopologyView - Saves a DMPlex topology into a file
1663 
1664   Collective on DM
1665 
1666   Input Parameters:
1667 + dm     - The DM whose topology is to be saved
1668 - viewer - The PetscViewer for saving
1669 
1670   Level: advanced
1671 
1672 .seealso: DMView(), DMPlexCoordinatesView(), DMPlexLabelsView(), DMPlexTopologyLoad()
1673 @*/
1674 PetscErrorCode DMPlexTopologyView(DM dm, PetscViewer viewer)
1675 {
1676   PetscBool      ishdf5;
1677   PetscErrorCode ierr;
1678 
1679   PetscFunctionBegin;
1680   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1681   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1682   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
1683   ierr = PetscLogEventBegin(DMPLEX_TopologyView,viewer,0,0,0);CHKERRQ(ierr);
1684   if (ishdf5) {
1685 #if defined(PETSC_HAVE_HDF5)
1686     PetscViewerFormat format;
1687     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
1688     if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
1689       IS globalPointNumbering;
1690 
1691       ierr = DMPlexCreatePointNumbering(dm, &globalPointNumbering);CHKERRQ(ierr);
1692       ierr = DMPlexTopologyView_HDF5_Internal(dm, globalPointNumbering, viewer);CHKERRQ(ierr);
1693       ierr = ISDestroy(&globalPointNumbering);CHKERRQ(ierr);
1694     } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 output.", PetscViewerFormats[format]);
1695 #else
1696     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1697 #endif
1698   }
1699   ierr = PetscLogEventEnd(DMPLEX_TopologyView,viewer,0,0,0);CHKERRQ(ierr);
1700   PetscFunctionReturn(0);
1701 }
1702 
1703 /*@
1704   DMPlexCoordinatesView - Saves DMPlex coordinates into a file
1705 
1706   Collective on DM
1707 
1708   Input Parameters:
1709 + dm     - The DM whose coordinates are to be saved
1710 - viewer - The PetscViewer for saving
1711 
1712   Level: advanced
1713 
1714 .seealso: DMView(), DMPlexTopologyView(), DMPlexLabelsView(), DMPlexCoordinatesLoad()
1715 @*/
1716 PetscErrorCode DMPlexCoordinatesView(DM dm, PetscViewer viewer)
1717 {
1718   PetscBool      ishdf5;
1719   PetscErrorCode ierr;
1720 
1721   PetscFunctionBegin;
1722   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1723   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1724   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
1725   ierr = PetscLogEventBegin(DMPLEX_CoordinatesView,viewer,0,0,0);CHKERRQ(ierr);
1726   if (ishdf5) {
1727 #if defined(PETSC_HAVE_HDF5)
1728     PetscViewerFormat format;
1729     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
1730     if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
1731       ierr = DMPlexCoordinatesView_HDF5_Internal(dm, viewer);CHKERRQ(ierr);
1732     } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 input.", PetscViewerFormats[format]);
1733 #else
1734     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1735 #endif
1736   }
1737   ierr = PetscLogEventEnd(DMPLEX_CoordinatesView,viewer,0,0,0);CHKERRQ(ierr);
1738   PetscFunctionReturn(0);
1739 }
1740 
1741 /*@
1742   DMPlexLabelsView - Saves DMPlex labels into a file
1743 
1744   Collective on DM
1745 
1746   Input Parameters:
1747 + dm     - The DM whose labels are to be saved
1748 - viewer - The PetscViewer for saving
1749 
1750   Level: advanced
1751 
1752 .seealso: DMView(), DMPlexTopologyView(), DMPlexCoordinatesView(), DMPlexLabelsLoad()
1753 @*/
1754 PetscErrorCode DMPlexLabelsView(DM dm, PetscViewer viewer)
1755 {
1756   PetscBool      ishdf5;
1757   PetscErrorCode ierr;
1758 
1759   PetscFunctionBegin;
1760   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1761   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1762   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
1763   ierr = PetscLogEventBegin(DMPLEX_LabelsView,viewer,0,0,0);CHKERRQ(ierr);
1764   if (ishdf5) {
1765 #if defined(PETSC_HAVE_HDF5)
1766     IS                globalPointNumbering;
1767     PetscViewerFormat format;
1768 
1769     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
1770     if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
1771       ierr = DMPlexCreatePointNumbering(dm, &globalPointNumbering);CHKERRQ(ierr);
1772       ierr = DMPlexLabelsView_HDF5_Internal(dm, globalPointNumbering, viewer);CHKERRQ(ierr);
1773       ierr = ISDestroy(&globalPointNumbering);CHKERRQ(ierr);
1774     } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 input.", PetscViewerFormats[format]);
1775 #else
1776     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1777 #endif
1778   }
1779   ierr = PetscLogEventEnd(DMPLEX_LabelsView,viewer,0,0,0);CHKERRQ(ierr);
1780   PetscFunctionReturn(0);
1781 }
1782 
1783 /*@
1784   DMPlexSectionView - Saves a section associated with a DMPlex
1785 
1786   Collective on DM
1787 
1788   Input Parameters:
1789 + dm         - The DM that contains the topology on which the section to be saved is defined
1790 . viewer     - The PetscViewer for saving
1791 - sectiondm  - The DM that contains the section to be saved
1792 
1793   Level: advanced
1794 
1795   Notes:
1796   This function is a wrapper around PetscSectionView(); in addition to the raw section, it saves information that associates the section points to the topology (dm) points. When the topology (dm) and the section are later loaded with DMPlexTopologyLoad() and DMPlexSectionLoad(), respectively, this information is used to match section points with topology points.
1797 
1798   In general dm and sectiondm are two different objects, the former carrying the topology and the latter carrying the section, and have been given a topology name and a section name, respectively, with PetscObjectSetName(). In practice, however, they can be the same object if it carries both topology and section; in that case the name of the object is used as both the topology name and the section name.
1799 
1800 .seealso: DMView(), DMPlexTopologyView(), DMPlexCoordinatesView(), DMPlexLabelsView(), DMPlexGlobalVectorView(), DMPlexLocalVectorView(), PetscSectionView(), DMPlexSectionLoad()
1801 @*/
1802 PetscErrorCode DMPlexSectionView(DM dm, PetscViewer viewer, DM sectiondm)
1803 {
1804   PetscBool      ishdf5;
1805   PetscErrorCode ierr;
1806 
1807   PetscFunctionBegin;
1808   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1809   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1810   PetscValidHeaderSpecific(sectiondm, DM_CLASSID, 3);
1811   ierr = PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERHDF5,&ishdf5);CHKERRQ(ierr);
1812   ierr = PetscLogEventBegin(DMPLEX_SectionView,viewer,0,0,0);CHKERRQ(ierr);
1813   if (ishdf5) {
1814 #if defined(PETSC_HAVE_HDF5)
1815     ierr = DMPlexSectionView_HDF5_Internal(dm, viewer, sectiondm);CHKERRQ(ierr);
1816 #else
1817     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1818 #endif
1819   }
1820   ierr = PetscLogEventEnd(DMPLEX_SectionView,viewer,0,0,0);CHKERRQ(ierr);
1821   PetscFunctionReturn(0);
1822 }
1823 
1824 /*@
1825   DMPlexGlobalVectorView - Saves a global vector
1826 
1827   Collective on DM
1828 
1829   Input Parameters:
1830 + dm        - The DM that represents the topology
1831 . viewer    - The PetscViewer to save data with
1832 . sectiondm - The DM that contains the global section on which vec is defined
1833 - vec       - The global vector to be saved
1834 
1835   Level: advanced
1836 
1837   Notes:
1838   In general dm and sectiondm are two different objects, the former carrying the topology and the latter carrying the section, and have been given a topology name and a section name, respectively, with PetscObjectSetName(). In practice, however, they can be the same object if it carries both topology and section; in that case the name of the object is used as both the topology name and the section name.
1839 
1840   Typical calling sequence
1841 $       DMCreate(PETSC_COMM_WORLD, &dm);
1842 $       DMSetType(dm, DMPLEX);
1843 $       PetscObjectSetName((PetscObject)dm, "topologydm_name");
1844 $       DMClone(dm, &sectiondm);
1845 $       PetscObjectSetName((PetscObject)sectiondm, "sectiondm_name");
1846 $       PetscSectionCreate(PETSC_COMM_WORLD, &section);
1847 $       DMPlexGetChart(sectiondm, &pStart, &pEnd);
1848 $       PetscSectionSetChart(section, pStart, pEnd);
1849 $       PetscSectionSetUp(section);
1850 $       DMSetLocalSection(sectiondm, section);
1851 $       PetscSectionDestroy(&section);
1852 $       DMGetGlobalVector(sectiondm, &vec);
1853 $       PetscObjectSetName((PetscObject)vec, "vec_name");
1854 $       DMPlexTopologyView(dm, viewer);
1855 $       DMPlexSectionView(dm, viewer, sectiondm);
1856 $       DMPlexGlobalVectorView(dm, viewer, sectiondm, vec);
1857 $       DMRestoreGlobalVector(sectiondm, &vec);
1858 $       DMDestroy(&sectiondm);
1859 $       DMDestroy(&dm);
1860 
1861 .seealso: DMPlexTopologyView(), DMPlexSectionView(), DMPlexLocalVectorView(), DMPlexGlobalVectorLoad(), DMPlexLocalVectorLoad()
1862 @*/
1863 PetscErrorCode DMPlexGlobalVectorView(DM dm, PetscViewer viewer, DM sectiondm, Vec vec)
1864 {
1865   PetscBool       ishdf5;
1866   PetscErrorCode  ierr;
1867 
1868   PetscFunctionBegin;
1869   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1870   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1871   PetscValidHeaderSpecific(sectiondm, DM_CLASSID, 3);
1872   PetscValidHeaderSpecific(vec, VEC_CLASSID, 4);
1873   /* Check consistency */
1874   {
1875     PetscSection  section;
1876     PetscBool     includesConstraints;
1877     PetscInt      m, m1;
1878 
1879     ierr = VecGetLocalSize(vec, &m1);CHKERRQ(ierr);
1880     ierr = DMGetGlobalSection(sectiondm, &section);CHKERRQ(ierr);
1881     ierr = PetscSectionGetIncludesConstraints(section, &includesConstraints);CHKERRQ(ierr);
1882     if (includesConstraints) {ierr = PetscSectionGetStorageSize(section, &m);CHKERRQ(ierr);}
1883     else {ierr = PetscSectionGetConstrainedStorageSize(section, &m);CHKERRQ(ierr);}
1884     PetscAssertFalse(m1 != m,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Global vector size (%D) != global section storage size (%D)", m1, m);
1885   }
1886   ierr = PetscObjectTypeCompare((PetscObject)viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
1887   ierr = PetscLogEventBegin(DMPLEX_GlobalVectorView,viewer,0,0,0);CHKERRQ(ierr);
1888   if (ishdf5) {
1889 #if defined(PETSC_HAVE_HDF5)
1890     ierr = DMPlexGlobalVectorView_HDF5_Internal(dm, viewer, sectiondm, vec);CHKERRQ(ierr);
1891 #else
1892     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1893 #endif
1894   }
1895   ierr = PetscLogEventEnd(DMPLEX_GlobalVectorView,viewer,0,0,0);CHKERRQ(ierr);
1896   PetscFunctionReturn(0);
1897 }
1898 
1899 /*@
1900   DMPlexLocalVectorView - Saves a local vector
1901 
1902   Collective on DM
1903 
1904   Input Parameters:
1905 + dm        - The DM that represents the topology
1906 . viewer    - The PetscViewer to save data with
1907 . sectiondm - The DM that contains the local section on which vec is defined; may be the same as dm
1908 - vec       - The local vector to be saved
1909 
1910   Level: advanced
1911 
1912   Notes:
1913   In general dm and sectiondm are two different objects, the former carrying the topology and the latter carrying the section, and have been given a topology name and a section name, respectively, with PetscObjectSetName(). In practice, however, they can be the same object if it carries both topology and section; in that case the name of the object is used as both the topology name and the section name.
1914 
1915   Typical calling sequence
1916 $       DMCreate(PETSC_COMM_WORLD, &dm);
1917 $       DMSetType(dm, DMPLEX);
1918 $       PetscObjectSetName((PetscObject)dm, "topologydm_name");
1919 $       DMClone(dm, &sectiondm);
1920 $       PetscObjectSetName((PetscObject)sectiondm, "sectiondm_name");
1921 $       PetscSectionCreate(PETSC_COMM_WORLD, &section);
1922 $       DMPlexGetChart(sectiondm, &pStart, &pEnd);
1923 $       PetscSectionSetChart(section, pStart, pEnd);
1924 $       PetscSectionSetUp(section);
1925 $       DMSetLocalSection(sectiondm, section);
1926 $       DMGetLocalVector(sectiondm, &vec);
1927 $       PetscObjectSetName((PetscObject)vec, "vec_name");
1928 $       DMPlexTopologyView(dm, viewer);
1929 $       DMPlexSectionView(dm, viewer, sectiondm);
1930 $       DMPlexLocalVectorView(dm, viewer, sectiondm, vec);
1931 $       DMRestoreLocalVector(sectiondm, &vec);
1932 $       DMDestroy(&sectiondm);
1933 $       DMDestroy(&dm);
1934 
1935 .seealso: DMPlexTopologyView(), DMPlexSectionView(), DMPlexGlobalVectorView(), DMPlexGlobalVectorLoad(), DMPlexLocalVectorLoad()
1936 @*/
1937 PetscErrorCode DMPlexLocalVectorView(DM dm, PetscViewer viewer, DM sectiondm, Vec vec)
1938 {
1939   PetscBool       ishdf5;
1940   PetscErrorCode  ierr;
1941 
1942   PetscFunctionBegin;
1943   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1944   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1945   PetscValidHeaderSpecific(sectiondm, DM_CLASSID, 3);
1946   PetscValidHeaderSpecific(vec, VEC_CLASSID, 4);
1947   /* Check consistency */
1948   {
1949     PetscSection  section;
1950     PetscBool     includesConstraints;
1951     PetscInt      m, m1;
1952 
1953     ierr = VecGetLocalSize(vec, &m1);CHKERRQ(ierr);
1954     ierr = DMGetLocalSection(sectiondm, &section);CHKERRQ(ierr);
1955     ierr = PetscSectionGetIncludesConstraints(section, &includesConstraints);CHKERRQ(ierr);
1956     if (includesConstraints) {ierr = PetscSectionGetStorageSize(section, &m);CHKERRQ(ierr);}
1957     else {ierr = PetscSectionGetConstrainedStorageSize(section, &m);CHKERRQ(ierr);}
1958     PetscAssertFalse(m1 != m,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Local vector size (%D) != local section storage size (%D)", m1, m);
1959   }
1960   ierr = PetscObjectTypeCompare((PetscObject)viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
1961   ierr = PetscLogEventBegin(DMPLEX_LocalVectorView,viewer,0,0,0);CHKERRQ(ierr);
1962   if (ishdf5) {
1963 #if defined(PETSC_HAVE_HDF5)
1964     ierr = DMPlexLocalVectorView_HDF5_Internal(dm, viewer, sectiondm, vec);CHKERRQ(ierr);
1965 #else
1966     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1967 #endif
1968   }
1969   ierr = PetscLogEventEnd(DMPLEX_LocalVectorView,viewer,0,0,0);CHKERRQ(ierr);
1970   PetscFunctionReturn(0);
1971 }
1972 
1973 PetscErrorCode DMLoad_Plex(DM dm, PetscViewer viewer)
1974 {
1975   PetscBool      ishdf5;
1976   PetscErrorCode ierr;
1977 
1978   PetscFunctionBegin;
1979   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1980   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1981   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,   &ishdf5);CHKERRQ(ierr);
1982   if (ishdf5) {
1983 #if defined(PETSC_HAVE_HDF5)
1984     PetscViewerFormat format;
1985     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
1986     if (format == PETSC_VIEWER_HDF5_XDMF || format == PETSC_VIEWER_HDF5_VIZ) {
1987       ierr = DMPlexLoad_HDF5_Xdmf_Internal(dm, viewer);CHKERRQ(ierr);
1988     } else if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
1989       ierr = DMPlexLoad_HDF5_Internal(dm, viewer);CHKERRQ(ierr);
1990     } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 input.", PetscViewerFormats[format]);
1991     PetscFunctionReturn(0);
1992 #else
1993     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1994 #endif
1995   } else SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Viewer type %s not yet supported for DMPlex loading", ((PetscObject)viewer)->type_name);
1996 }
1997 
1998 /*@
1999   DMPlexTopologyLoad - Loads a topology into a DMPlex
2000 
2001   Collective on DM
2002 
2003   Input Parameters:
2004 + dm     - The DM into which the topology is loaded
2005 - viewer - The PetscViewer for the saved topology
2006 
2007   Output Parameters:
2008 . globalToLocalPointSF - The PetscSF that pushes points in [0, N) to the associated points in the loaded plex, where N is the global number of points; NULL if unneeded
2009 
2010   Level: advanced
2011 
2012 .seealso: DMLoad(), DMPlexCoordinatesLoad(), DMPlexLabelsLoad(), DMView(), PetscViewerHDF5Open(), PetscViewerPushFormat()
2013 @*/
2014 PetscErrorCode DMPlexTopologyLoad(DM dm, PetscViewer viewer, PetscSF *globalToLocalPointSF)
2015 {
2016   PetscBool      ishdf5;
2017   PetscErrorCode ierr;
2018 
2019   PetscFunctionBegin;
2020   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2021   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
2022   if (globalToLocalPointSF) PetscValidPointer(globalToLocalPointSF, 3);
2023   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
2024   ierr = PetscLogEventBegin(DMPLEX_TopologyLoad,viewer,0,0,0);CHKERRQ(ierr);
2025   if (ishdf5) {
2026 #if defined(PETSC_HAVE_HDF5)
2027     PetscViewerFormat format;
2028     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
2029     if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
2030       ierr = DMPlexTopologyLoad_HDF5_Internal(dm, viewer, globalToLocalPointSF);CHKERRQ(ierr);
2031     } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 input.", PetscViewerFormats[format]);
2032 #else
2033     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
2034 #endif
2035   }
2036   ierr = PetscLogEventEnd(DMPLEX_TopologyLoad,viewer,0,0,0);CHKERRQ(ierr);
2037   PetscFunctionReturn(0);
2038 }
2039 
2040 /*@
2041   DMPlexCoordinatesLoad - Loads coordinates into a DMPlex
2042 
2043   Collective on DM
2044 
2045   Input Parameters:
2046 + dm     - The DM into which the coordinates are loaded
2047 . viewer - The PetscViewer for the saved coordinates
2048 - globalToLocalPointSF - The SF returned by DMPlexTopologyLoad() when loading dm from viewer
2049 
2050   Level: advanced
2051 
2052 .seealso: DMLoad(), DMPlexTopologyLoad(), DMPlexLabelsLoad(), DMView(), PetscViewerHDF5Open(), PetscViewerPushFormat()
2053 @*/
2054 PetscErrorCode DMPlexCoordinatesLoad(DM dm, PetscViewer viewer, PetscSF globalToLocalPointSF)
2055 {
2056   PetscBool      ishdf5;
2057   PetscErrorCode ierr;
2058 
2059   PetscFunctionBegin;
2060   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2061   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
2062   PetscValidHeaderSpecific(globalToLocalPointSF, PETSCSF_CLASSID, 3);
2063   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
2064   ierr = PetscLogEventBegin(DMPLEX_CoordinatesLoad,viewer,0,0,0);CHKERRQ(ierr);
2065   if (ishdf5) {
2066 #if defined(PETSC_HAVE_HDF5)
2067     PetscViewerFormat format;
2068     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
2069     if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
2070       ierr = DMPlexCoordinatesLoad_HDF5_Internal(dm, viewer, globalToLocalPointSF);CHKERRQ(ierr);
2071     } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 input.", PetscViewerFormats[format]);
2072 #else
2073     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
2074 #endif
2075   }
2076   ierr = PetscLogEventEnd(DMPLEX_CoordinatesLoad,viewer,0,0,0);CHKERRQ(ierr);
2077   PetscFunctionReturn(0);
2078 }
2079 
2080 /*@
2081   DMPlexLabelsLoad - Loads labels into a DMPlex
2082 
2083   Collective on DM
2084 
2085   Input Parameters:
2086 + dm     - The DM into which the labels are loaded
2087 . viewer - The PetscViewer for the saved labels
2088 - globalToLocalPointSF - The SF returned by DMPlexTopologyLoad() when loading dm from viewer
2089 
2090   Level: advanced
2091 
2092   Notes:
2093   The PetscSF argument must not be NULL if the DM is distributed, otherwise an error occurs.
2094 
2095 .seealso: DMLoad(), DMPlexTopologyLoad(), DMPlexCoordinatesLoad(), DMView(), PetscViewerHDF5Open(), PetscViewerPushFormat()
2096 @*/
2097 PetscErrorCode DMPlexLabelsLoad(DM dm, PetscViewer viewer, PetscSF globalToLocalPointSF)
2098 {
2099   PetscBool      ishdf5;
2100   PetscErrorCode ierr;
2101 
2102   PetscFunctionBegin;
2103   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2104   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
2105   if (globalToLocalPointSF) PetscValidHeaderSpecific(globalToLocalPointSF, PETSCSF_CLASSID, 3);
2106   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
2107   ierr = PetscLogEventBegin(DMPLEX_LabelsLoad,viewer,0,0,0);CHKERRQ(ierr);
2108   if (ishdf5) {
2109 #if defined(PETSC_HAVE_HDF5)
2110     PetscViewerFormat format;
2111 
2112     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
2113     if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
2114       ierr = DMPlexLabelsLoad_HDF5_Internal(dm, viewer, globalToLocalPointSF);CHKERRQ(ierr);
2115     } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 input.", PetscViewerFormats[format]);
2116 #else
2117     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
2118 #endif
2119   }
2120   ierr = PetscLogEventEnd(DMPLEX_LabelsLoad,viewer,0,0,0);CHKERRQ(ierr);
2121   PetscFunctionReturn(0);
2122 }
2123 
2124 /*@
2125   DMPlexSectionLoad - Loads section into a DMPlex
2126 
2127   Collective on DM
2128 
2129   Input Parameters:
2130 + dm          - The DM that represents the topology
2131 . viewer      - The PetscViewer that represents the on-disk section (sectionA)
2132 . sectiondm   - The DM into which the on-disk section (sectionA) is migrated
2133 - globalToLocalPointSF - The SF returned by DMPlexTopologyLoad() when loading dm from viewer
2134 
2135   Output Parameters
2136 + globalDofSF - The SF that migrates any on-disk Vec data associated with sectionA into a global Vec associated with the sectiondm's global section (NULL if not needed)
2137 - localDofSF  - The SF that migrates any on-disk Vec data associated with sectionA into a local Vec associated with the sectiondm's local section (NULL if not needed)
2138 
2139   Level: advanced
2140 
2141   Notes:
2142   This function is a wrapper around PetscSectionLoad(); it loads, in addition to the raw section, a list of global point numbers that associates each on-disk section point with a global point number in [0, NX), where NX is the number of topology points in dm. Noting that globalToLocalPointSF associates each topology point in dm with a global number in [0, NX), one can readily establish an association of the on-disk section points with the topology points.
2143 
2144   In general dm and sectiondm are two different objects, the former carrying the topology and the latter carrying the section, and have been given a topology name and a section name, respectively, with PetscObjectSetName(). In practice, however, they can be the same object if it carries both topology and section; in that case the name of the object is used as both the topology name and the section name.
2145 
2146   The output parameter, globalDofSF (localDofSF), can later be used with DMPlexGlobalVectorLoad() (DMPlexLocalVectorLoad()) to load on-disk vectors into global (local) vectors associated with sectiondm's global (local) section.
2147 
2148   Example using 2 processes:
2149 $  NX (number of points on dm): 4
2150 $  sectionA                   : the on-disk section
2151 $  vecA                       : a vector associated with sectionA
2152 $  sectionB                   : sectiondm's local section constructed in this function
2153 $  vecB (local)               : a vector associated with sectiondm's local section
2154 $  vecB (global)              : a vector associated with sectiondm's global section
2155 $
2156 $                                     rank 0    rank 1
2157 $  vecA (global)                  : [.0 .4 .1 | .2 .3]        <- to be loaded in DMPlexGlobalVectorLoad() or DMPlexLocalVectorLoad()
2158 $  sectionA->atlasOff             :       0 2 | 1             <- loaded in PetscSectionLoad()
2159 $  sectionA->atlasDof             :       1 3 | 1             <- loaded in PetscSectionLoad()
2160 $  sectionA's global point numbers:       0 2 | 3             <- loaded in DMPlexSectionLoad()
2161 $  [0, NX)                        :       0 1 | 2 3           <- conceptual partition used in globalToLocalPointSF
2162 $  sectionB's global point numbers:     0 1 3 | 3 2           <- associated with [0, NX) by globalToLocalPointSF
2163 $  sectionB->atlasDof             :     1 0 1 | 1 3
2164 $  sectionB->atlasOff (no perm)   :     0 1 1 | 0 1
2165 $  vecB (local)                   :   [.0 .4] | [.4 .1 .2 .3] <- to be constructed by calling DMPlexLocalVectorLoad() with localDofSF
2166 $  vecB (global)                  :    [.0 .4 | .1 .2 .3]     <- to be constructed by calling DMPlexGlobalVectorLoad() with globalDofSF
2167 $
2168 $  where "|" represents a partition of loaded data, and global point 3 is assumed to be owned by rank 0.
2169 
2170 .seealso: DMLoad(), DMPlexTopologyLoad(), DMPlexCoordinatesLoad(), DMPlexLabelsLoad(), DMPlexGlobalVectorLoad(), DMPlexLocalVectorLoad(), PetscSectionLoad(), DMPlexSectionView()
2171 @*/
2172 PetscErrorCode DMPlexSectionLoad(DM dm, PetscViewer viewer, DM sectiondm, PetscSF globalToLocalPointSF, PetscSF *globalDofSF, PetscSF *localDofSF)
2173 {
2174   PetscBool      ishdf5;
2175   PetscErrorCode ierr;
2176 
2177   PetscFunctionBegin;
2178   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2179   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
2180   PetscValidHeaderSpecific(sectiondm, DM_CLASSID, 3);
2181   PetscValidHeaderSpecific(globalToLocalPointSF, PETSCSF_CLASSID, 4);
2182   if (globalDofSF) PetscValidPointer(globalDofSF, 5);
2183   if (localDofSF) PetscValidPointer(localDofSF, 6);
2184   ierr = PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERHDF5,&ishdf5);CHKERRQ(ierr);
2185   ierr = PetscLogEventBegin(DMPLEX_SectionLoad,viewer,0,0,0);CHKERRQ(ierr);
2186   if (ishdf5) {
2187 #if defined(PETSC_HAVE_HDF5)
2188     ierr = DMPlexSectionLoad_HDF5_Internal(dm, viewer, sectiondm, globalToLocalPointSF, globalDofSF, localDofSF);CHKERRQ(ierr);
2189 #else
2190     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
2191 #endif
2192   }
2193   ierr = PetscLogEventEnd(DMPLEX_SectionLoad,viewer,0,0,0);CHKERRQ(ierr);
2194   PetscFunctionReturn(0);
2195 }
2196 
2197 /*@
2198   DMPlexGlobalVectorLoad - Loads on-disk vector data into a global vector
2199 
2200   Collective on DM
2201 
2202   Input Parameters:
2203 + dm        - The DM that represents the topology
2204 . viewer    - The PetscViewer that represents the on-disk vector data
2205 . sectiondm - The DM that contains the global section on which vec is defined
2206 . sf        - The SF that migrates the on-disk vector data into vec
2207 - vec       - The global vector to set values of
2208 
2209   Level: advanced
2210 
2211   Notes:
2212   In general dm and sectiondm are two different objects, the former carrying the topology and the latter carrying the section, and have been given a topology name and a section name, respectively, with PetscObjectSetName(). In practice, however, they can be the same object if it carries both topology and section; in that case the name of the object is used as both the topology name and the section name.
2213 
2214   Typical calling sequence
2215 $       DMCreate(PETSC_COMM_WORLD, &dm);
2216 $       DMSetType(dm, DMPLEX);
2217 $       PetscObjectSetName((PetscObject)dm, "topologydm_name");
2218 $       DMPlexTopologyLoad(dm, viewer, &sfX);
2219 $       DMClone(dm, &sectiondm);
2220 $       PetscObjectSetName((PetscObject)sectiondm, "sectiondm_name");
2221 $       DMPlexSectionLoad(dm, viewer, sectiondm, sfX, &gsf, NULL);
2222 $       DMGetGlobalVector(sectiondm, &vec);
2223 $       PetscObjectSetName((PetscObject)vec, "vec_name");
2224 $       DMPlexGlobalVectorLoad(dm, viewer, sectiondm, gsf, vec);
2225 $       DMRestoreGlobalVector(sectiondm, &vec);
2226 $       PetscSFDestroy(&gsf);
2227 $       PetscSFDestroy(&sfX);
2228 $       DMDestroy(&sectiondm);
2229 $       DMDestroy(&dm);
2230 
2231 .seealso: DMPlexTopologyLoad(), DMPlexSectionLoad(), DMPlexLocalVectorLoad(), DMPlexGlobalVectorView(), DMPlexLocalVectorView()
2232 @*/
2233 PetscErrorCode DMPlexGlobalVectorLoad(DM dm, PetscViewer viewer, DM sectiondm, PetscSF sf, Vec vec)
2234 {
2235   PetscBool       ishdf5;
2236   PetscErrorCode  ierr;
2237 
2238   PetscFunctionBegin;
2239   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2240   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
2241   PetscValidHeaderSpecific(sectiondm, DM_CLASSID, 3);
2242   PetscValidHeaderSpecific(sf, PETSCSF_CLASSID, 4);
2243   PetscValidHeaderSpecific(vec, VEC_CLASSID, 5);
2244   /* Check consistency */
2245   {
2246     PetscSection  section;
2247     PetscBool     includesConstraints;
2248     PetscInt      m, m1;
2249 
2250     ierr = VecGetLocalSize(vec, &m1);CHKERRQ(ierr);
2251     ierr = DMGetGlobalSection(sectiondm, &section);CHKERRQ(ierr);
2252     ierr = PetscSectionGetIncludesConstraints(section, &includesConstraints);CHKERRQ(ierr);
2253     if (includesConstraints) {ierr = PetscSectionGetStorageSize(section, &m);CHKERRQ(ierr);}
2254     else {ierr = PetscSectionGetConstrainedStorageSize(section, &m);CHKERRQ(ierr);}
2255     PetscAssertFalse(m1 != m,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Global vector size (%D) != global section storage size (%D)", m1, m);
2256   }
2257   ierr = PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERHDF5,&ishdf5);CHKERRQ(ierr);
2258   ierr = PetscLogEventBegin(DMPLEX_GlobalVectorLoad,viewer,0,0,0);CHKERRQ(ierr);
2259   if (ishdf5) {
2260 #if defined(PETSC_HAVE_HDF5)
2261     ierr = DMPlexVecLoad_HDF5_Internal(dm, viewer, sectiondm, sf, vec);CHKERRQ(ierr);
2262 #else
2263     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
2264 #endif
2265   }
2266   ierr = PetscLogEventEnd(DMPLEX_GlobalVectorLoad,viewer,0,0,0);CHKERRQ(ierr);
2267   PetscFunctionReturn(0);
2268 }
2269 
2270 /*@
2271   DMPlexLocalVectorLoad - Loads on-disk vector data into a local vector
2272 
2273   Collective on DM
2274 
2275   Input Parameters:
2276 + dm        - The DM that represents the topology
2277 . viewer    - The PetscViewer that represents the on-disk vector data
2278 . sectiondm - The DM that contains the local section on which vec is defined
2279 . sf        - The SF that migrates the on-disk vector data into vec
2280 - vec       - The local vector to set values of
2281 
2282   Level: advanced
2283 
2284   Notes:
2285   In general dm and sectiondm are two different objects, the former carrying the topology and the latter carrying the section, and have been given a topology name and a section name, respectively, with PetscObjectSetName(). In practice, however, they can be the same object if it carries both topology and section; in that case the name of the object is used as both the topology name and the section name.
2286 
2287   Typical calling sequence
2288 $       DMCreate(PETSC_COMM_WORLD, &dm);
2289 $       DMSetType(dm, DMPLEX);
2290 $       PetscObjectSetName((PetscObject)dm, "topologydm_name");
2291 $       DMPlexTopologyLoad(dm, viewer, &sfX);
2292 $       DMClone(dm, &sectiondm);
2293 $       PetscObjectSetName((PetscObject)sectiondm, "sectiondm_name");
2294 $       DMPlexSectionLoad(dm, viewer, sectiondm, sfX, NULL, &lsf);
2295 $       DMGetLocalVector(sectiondm, &vec);
2296 $       PetscObjectSetName((PetscObject)vec, "vec_name");
2297 $       DMPlexLocalVectorLoad(dm, viewer, sectiondm, lsf, vec);
2298 $       DMRestoreLocalVector(sectiondm, &vec);
2299 $       PetscSFDestroy(&lsf);
2300 $       PetscSFDestroy(&sfX);
2301 $       DMDestroy(&sectiondm);
2302 $       DMDestroy(&dm);
2303 
2304 .seealso: DMPlexTopologyLoad(), DMPlexSectionLoad(), DMPlexGlobalVectorLoad(), DMPlexGlobalVectorView(), DMPlexLocalVectorView()
2305 @*/
2306 PetscErrorCode DMPlexLocalVectorLoad(DM dm, PetscViewer viewer, DM sectiondm, PetscSF sf, Vec vec)
2307 {
2308   PetscBool       ishdf5;
2309   PetscErrorCode  ierr;
2310 
2311   PetscFunctionBegin;
2312   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2313   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
2314   PetscValidHeaderSpecific(sectiondm, DM_CLASSID, 3);
2315   PetscValidHeaderSpecific(sf, PETSCSF_CLASSID, 4);
2316   PetscValidHeaderSpecific(vec, VEC_CLASSID, 5);
2317   /* Check consistency */
2318   {
2319     PetscSection  section;
2320     PetscBool     includesConstraints;
2321     PetscInt      m, m1;
2322 
2323     ierr = VecGetLocalSize(vec, &m1);CHKERRQ(ierr);
2324     ierr = DMGetLocalSection(sectiondm, &section);CHKERRQ(ierr);
2325     ierr = PetscSectionGetIncludesConstraints(section, &includesConstraints);CHKERRQ(ierr);
2326     if (includesConstraints) {ierr = PetscSectionGetStorageSize(section, &m);CHKERRQ(ierr);}
2327     else {ierr = PetscSectionGetConstrainedStorageSize(section, &m);CHKERRQ(ierr);}
2328     PetscAssertFalse(m1 != m,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Local vector size (%D) != local section storage size (%D)", m1, m);
2329   }
2330   ierr = PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERHDF5,&ishdf5);CHKERRQ(ierr);
2331   ierr = PetscLogEventBegin(DMPLEX_LocalVectorLoad,viewer,0,0,0);CHKERRQ(ierr);
2332   if (ishdf5) {
2333 #if defined(PETSC_HAVE_HDF5)
2334     ierr = DMPlexVecLoad_HDF5_Internal(dm, viewer, sectiondm, sf, vec);CHKERRQ(ierr);
2335 #else
2336     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
2337 #endif
2338   }
2339   ierr = PetscLogEventEnd(DMPLEX_LocalVectorLoad,viewer,0,0,0);CHKERRQ(ierr);
2340   PetscFunctionReturn(0);
2341 }
2342 
2343 PetscErrorCode DMDestroy_Plex(DM dm)
2344 {
2345   DM_Plex       *mesh = (DM_Plex*) dm->data;
2346   PetscErrorCode ierr;
2347 
2348   PetscFunctionBegin;
2349   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMSetUpGLVisViewer_C",NULL);CHKERRQ(ierr);
2350   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMPlexInsertBoundaryValues_C", NULL);CHKERRQ(ierr);
2351   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMCreateNeumannOverlap_C", NULL);CHKERRQ(ierr);
2352   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMInterpolateSolution_C", NULL);CHKERRQ(ierr);
2353   if (--mesh->refct > 0) PetscFunctionReturn(0);
2354   ierr = PetscSectionDestroy(&mesh->coneSection);CHKERRQ(ierr);
2355   ierr = PetscFree(mesh->cones);CHKERRQ(ierr);
2356   ierr = PetscFree(mesh->coneOrientations);CHKERRQ(ierr);
2357   ierr = PetscSectionDestroy(&mesh->supportSection);CHKERRQ(ierr);
2358   ierr = PetscSectionDestroy(&mesh->subdomainSection);CHKERRQ(ierr);
2359   ierr = PetscFree(mesh->supports);CHKERRQ(ierr);
2360   ierr = PetscFree(mesh->facesTmp);CHKERRQ(ierr);
2361   ierr = PetscFree(mesh->tetgenOpts);CHKERRQ(ierr);
2362   ierr = PetscFree(mesh->triangleOpts);CHKERRQ(ierr);
2363   ierr = PetscFree(mesh->transformType);CHKERRQ(ierr);
2364   ierr = PetscPartitionerDestroy(&mesh->partitioner);CHKERRQ(ierr);
2365   ierr = DMLabelDestroy(&mesh->subpointMap);CHKERRQ(ierr);
2366   ierr = ISDestroy(&mesh->subpointIS);CHKERRQ(ierr);
2367   ierr = ISDestroy(&mesh->globalVertexNumbers);CHKERRQ(ierr);
2368   ierr = ISDestroy(&mesh->globalCellNumbers);CHKERRQ(ierr);
2369   ierr = PetscSectionDestroy(&mesh->anchorSection);CHKERRQ(ierr);
2370   ierr = ISDestroy(&mesh->anchorIS);CHKERRQ(ierr);
2371   ierr = PetscSectionDestroy(&mesh->parentSection);CHKERRQ(ierr);
2372   ierr = PetscFree(mesh->parents);CHKERRQ(ierr);
2373   ierr = PetscFree(mesh->childIDs);CHKERRQ(ierr);
2374   ierr = PetscSectionDestroy(&mesh->childSection);CHKERRQ(ierr);
2375   ierr = PetscFree(mesh->children);CHKERRQ(ierr);
2376   ierr = DMDestroy(&mesh->referenceTree);CHKERRQ(ierr);
2377   ierr = PetscGridHashDestroy(&mesh->lbox);CHKERRQ(ierr);
2378   ierr = PetscFree(mesh->neighbors);CHKERRQ(ierr);
2379   if (mesh->metricCtx) { ierr = PetscFree(mesh->metricCtx);CHKERRQ(ierr); }
2380   /* This was originally freed in DMDestroy(), but that prevents reference counting of backend objects */
2381   ierr = PetscFree(mesh);CHKERRQ(ierr);
2382   PetscFunctionReturn(0);
2383 }
2384 
2385 PetscErrorCode DMCreateMatrix_Plex(DM dm, Mat *J)
2386 {
2387   PetscSection           sectionGlobal;
2388   PetscInt               bs = -1, mbs;
2389   PetscInt               localSize;
2390   PetscBool              isShell, isBlock, isSeqBlock, isMPIBlock, isSymBlock, isSymSeqBlock, isSymMPIBlock, isMatIS;
2391   PetscErrorCode         ierr;
2392   MatType                mtype;
2393   ISLocalToGlobalMapping ltog;
2394 
2395   PetscFunctionBegin;
2396   ierr = MatInitializePackage();CHKERRQ(ierr);
2397   mtype = dm->mattype;
2398   ierr = DMGetGlobalSection(dm, &sectionGlobal);CHKERRQ(ierr);
2399   /* ierr = PetscSectionGetStorageSize(sectionGlobal, &localSize);CHKERRQ(ierr); */
2400   ierr = PetscSectionGetConstrainedStorageSize(sectionGlobal, &localSize);CHKERRQ(ierr);
2401   ierr = MatCreate(PetscObjectComm((PetscObject)dm), J);CHKERRQ(ierr);
2402   ierr = MatSetSizes(*J, localSize, localSize, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
2403   ierr = MatSetType(*J, mtype);CHKERRQ(ierr);
2404   ierr = MatSetFromOptions(*J);CHKERRQ(ierr);
2405   ierr = MatGetBlockSize(*J, &mbs);CHKERRQ(ierr);
2406   if (mbs > 1) bs = mbs;
2407   ierr = PetscStrcmp(mtype, MATSHELL, &isShell);CHKERRQ(ierr);
2408   ierr = PetscStrcmp(mtype, MATBAIJ, &isBlock);CHKERRQ(ierr);
2409   ierr = PetscStrcmp(mtype, MATSEQBAIJ, &isSeqBlock);CHKERRQ(ierr);
2410   ierr = PetscStrcmp(mtype, MATMPIBAIJ, &isMPIBlock);CHKERRQ(ierr);
2411   ierr = PetscStrcmp(mtype, MATSBAIJ, &isSymBlock);CHKERRQ(ierr);
2412   ierr = PetscStrcmp(mtype, MATSEQSBAIJ, &isSymSeqBlock);CHKERRQ(ierr);
2413   ierr = PetscStrcmp(mtype, MATMPISBAIJ, &isSymMPIBlock);CHKERRQ(ierr);
2414   ierr = PetscStrcmp(mtype, MATIS, &isMatIS);CHKERRQ(ierr);
2415   if (!isShell) {
2416     PetscSection subSection;
2417     PetscBool    fillMatrix = (PetscBool)(!dm->prealloc_only && !isMatIS);
2418     PetscInt    *dnz, *onz, *dnzu, *onzu, bsLocal[2], bsMinMax[2], *ltogidx, lsize;
2419     PetscInt     pStart, pEnd, p, dof, cdof;
2420 
2421     /* Set localtoglobalmapping on the matrix for MatSetValuesLocal() to work (it also creates the local matrices in case of MATIS) */
2422     if (isMatIS) { /* need a different l2g map than the one computed by DMGetLocalToGlobalMapping */
2423       PetscSection section;
2424       PetscInt     size;
2425 
2426       ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);
2427       ierr = PetscSectionGetStorageSize(section, &size);CHKERRQ(ierr);
2428       ierr = PetscMalloc1(size,&ltogidx);CHKERRQ(ierr);
2429       ierr = DMPlexGetSubdomainSection(dm, &subSection);CHKERRQ(ierr);
2430     } else {
2431       ierr = DMGetLocalToGlobalMapping(dm,&ltog);CHKERRQ(ierr);
2432     }
2433     ierr = PetscSectionGetChart(sectionGlobal, &pStart, &pEnd);CHKERRQ(ierr);
2434     for (p = pStart, lsize = 0; p < pEnd; ++p) {
2435       PetscInt bdof;
2436 
2437       ierr = PetscSectionGetDof(sectionGlobal, p, &dof);CHKERRQ(ierr);
2438       ierr = PetscSectionGetConstraintDof(sectionGlobal, p, &cdof);CHKERRQ(ierr);
2439       dof  = dof < 0 ? -(dof+1) : dof;
2440       bdof = cdof && (dof-cdof) ? 1 : dof;
2441       if (dof) {
2442         if (bs < 0)          {bs = bdof;}
2443         else if (bs != bdof) {bs = 1; if (!isMatIS) break;}
2444       }
2445       if (isMatIS) {
2446         PetscInt loff,c,off;
2447         ierr = PetscSectionGetOffset(subSection, p, &loff);CHKERRQ(ierr);
2448         ierr = PetscSectionGetOffset(sectionGlobal, p, &off);CHKERRQ(ierr);
2449         for (c = 0; c < dof-cdof; ++c, ++lsize) ltogidx[loff+c] = off > -1 ? off+c : -(off+1)+c;
2450       }
2451     }
2452     /* Must have same blocksize on all procs (some might have no points) */
2453     bsLocal[0] = bs < 0 ? PETSC_MAX_INT : bs; bsLocal[1] = bs;
2454     ierr = PetscGlobalMinMaxInt(PetscObjectComm((PetscObject) dm), bsLocal, bsMinMax);CHKERRQ(ierr);
2455     if (bsMinMax[0] != bsMinMax[1]) {bs = 1;}
2456     else                            {bs = bsMinMax[0];}
2457     bs = PetscMax(1,bs);
2458     if (isMatIS) { /* Must reduce indices by blocksize */
2459       PetscInt l;
2460 
2461       lsize = lsize/bs;
2462       if (bs > 1) for (l = 0; l < lsize; ++l) ltogidx[l] = ltogidx[l*bs]/bs;
2463       ierr = ISLocalToGlobalMappingCreate(PetscObjectComm((PetscObject)dm), bs, lsize, ltogidx, PETSC_OWN_POINTER, &ltog);CHKERRQ(ierr);
2464     }
2465     ierr = MatSetLocalToGlobalMapping(*J,ltog,ltog);CHKERRQ(ierr);
2466     if (isMatIS) {
2467       ierr = ISLocalToGlobalMappingDestroy(&ltog);CHKERRQ(ierr);
2468     }
2469     ierr = PetscCalloc4(localSize/bs, &dnz, localSize/bs, &onz, localSize/bs, &dnzu, localSize/bs, &onzu);CHKERRQ(ierr);
2470     ierr = DMPlexPreallocateOperator(dm, bs, dnz, onz, dnzu, onzu, *J, fillMatrix);CHKERRQ(ierr);
2471     ierr = PetscFree4(dnz, onz, dnzu, onzu);CHKERRQ(ierr);
2472   }
2473   ierr = MatSetDM(*J, dm);CHKERRQ(ierr);
2474   PetscFunctionReturn(0);
2475 }
2476 
2477 /*@
2478   DMPlexGetSubdomainSection - Returns the section associated with the subdomain
2479 
2480   Not collective
2481 
2482   Input Parameter:
2483 . mesh - The DMPlex
2484 
2485   Output Parameters:
2486 . subsection - The subdomain section
2487 
2488   Level: developer
2489 
2490 .seealso:
2491 @*/
2492 PetscErrorCode DMPlexGetSubdomainSection(DM dm, PetscSection *subsection)
2493 {
2494   DM_Plex       *mesh = (DM_Plex*) dm->data;
2495   PetscErrorCode ierr;
2496 
2497   PetscFunctionBegin;
2498   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2499   if (!mesh->subdomainSection) {
2500     PetscSection section;
2501     PetscSF      sf;
2502 
2503     ierr = PetscSFCreate(PETSC_COMM_SELF,&sf);CHKERRQ(ierr);
2504     ierr = DMGetLocalSection(dm,&section);CHKERRQ(ierr);
2505     ierr = PetscSectionCreateGlobalSection(section,sf,PETSC_FALSE,PETSC_TRUE,&mesh->subdomainSection);CHKERRQ(ierr);
2506     ierr = PetscSFDestroy(&sf);CHKERRQ(ierr);
2507   }
2508   *subsection = mesh->subdomainSection;
2509   PetscFunctionReturn(0);
2510 }
2511 
2512 /*@
2513   DMPlexGetChart - Return the interval for all mesh points [pStart, pEnd)
2514 
2515   Not collective
2516 
2517   Input Parameter:
2518 . mesh - The DMPlex
2519 
2520   Output Parameters:
2521 + pStart - The first mesh point
2522 - pEnd   - The upper bound for mesh points
2523 
2524   Level: beginner
2525 
2526 .seealso: DMPlexCreate(), DMPlexSetChart()
2527 @*/
2528 PetscErrorCode DMPlexGetChart(DM dm, PetscInt *pStart, PetscInt *pEnd)
2529 {
2530   DM_Plex       *mesh = (DM_Plex*) dm->data;
2531   PetscErrorCode ierr;
2532 
2533   PetscFunctionBegin;
2534   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2535   ierr = PetscSectionGetChart(mesh->coneSection, pStart, pEnd);CHKERRQ(ierr);
2536   PetscFunctionReturn(0);
2537 }
2538 
2539 /*@
2540   DMPlexSetChart - Set the interval for all mesh points [pStart, pEnd)
2541 
2542   Not collective
2543 
2544   Input Parameters:
2545 + mesh - The DMPlex
2546 . pStart - The first mesh point
2547 - pEnd   - The upper bound for mesh points
2548 
2549   Output Parameters:
2550 
2551   Level: beginner
2552 
2553 .seealso: DMPlexCreate(), DMPlexGetChart()
2554 @*/
2555 PetscErrorCode DMPlexSetChart(DM dm, PetscInt pStart, PetscInt pEnd)
2556 {
2557   DM_Plex       *mesh = (DM_Plex*) dm->data;
2558   PetscErrorCode ierr;
2559 
2560   PetscFunctionBegin;
2561   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2562   ierr = PetscSectionSetChart(mesh->coneSection, pStart, pEnd);CHKERRQ(ierr);
2563   ierr = PetscSectionSetChart(mesh->supportSection, pStart, pEnd);CHKERRQ(ierr);
2564   PetscFunctionReturn(0);
2565 }
2566 
2567 /*@
2568   DMPlexGetConeSize - Return the number of in-edges for this point in the DAG
2569 
2570   Not collective
2571 
2572   Input Parameters:
2573 + mesh - The DMPlex
2574 - p - The point, which must lie in the chart set with DMPlexSetChart()
2575 
2576   Output Parameter:
2577 . size - The cone size for point p
2578 
2579   Level: beginner
2580 
2581 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart()
2582 @*/
2583 PetscErrorCode DMPlexGetConeSize(DM dm, PetscInt p, PetscInt *size)
2584 {
2585   DM_Plex       *mesh = (DM_Plex*) dm->data;
2586   PetscErrorCode ierr;
2587 
2588   PetscFunctionBegin;
2589   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2590   PetscValidPointer(size, 3);
2591   ierr = PetscSectionGetDof(mesh->coneSection, p, size);CHKERRQ(ierr);
2592   PetscFunctionReturn(0);
2593 }
2594 
2595 /*@
2596   DMPlexSetConeSize - Set the number of in-edges for this point in the DAG
2597 
2598   Not collective
2599 
2600   Input Parameters:
2601 + mesh - The DMPlex
2602 . p - The point, which must lie in the chart set with DMPlexSetChart()
2603 - size - The cone size for point p
2604 
2605   Output Parameter:
2606 
2607   Note:
2608   This should be called after DMPlexSetChart().
2609 
2610   Level: beginner
2611 
2612 .seealso: DMPlexCreate(), DMPlexGetConeSize(), DMPlexSetChart()
2613 @*/
2614 PetscErrorCode DMPlexSetConeSize(DM dm, PetscInt p, PetscInt size)
2615 {
2616   DM_Plex       *mesh = (DM_Plex*) dm->data;
2617   PetscErrorCode ierr;
2618 
2619   PetscFunctionBegin;
2620   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2621   ierr = PetscSectionSetDof(mesh->coneSection, p, size);CHKERRQ(ierr);
2622 
2623   mesh->maxConeSize = PetscMax(mesh->maxConeSize, size);
2624   PetscFunctionReturn(0);
2625 }
2626 
2627 /*@
2628   DMPlexAddConeSize - Add the given number of in-edges to this point in the DAG
2629 
2630   Not collective
2631 
2632   Input Parameters:
2633 + mesh - The DMPlex
2634 . p - The point, which must lie in the chart set with DMPlexSetChart()
2635 - size - The additional cone size for point p
2636 
2637   Output Parameter:
2638 
2639   Note:
2640   This should be called after DMPlexSetChart().
2641 
2642   Level: beginner
2643 
2644 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexGetConeSize(), DMPlexSetChart()
2645 @*/
2646 PetscErrorCode DMPlexAddConeSize(DM dm, PetscInt p, PetscInt size)
2647 {
2648   DM_Plex       *mesh = (DM_Plex*) dm->data;
2649   PetscInt       csize;
2650   PetscErrorCode ierr;
2651 
2652   PetscFunctionBegin;
2653   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2654   ierr = PetscSectionAddDof(mesh->coneSection, p, size);CHKERRQ(ierr);
2655   ierr = PetscSectionGetDof(mesh->coneSection, p, &csize);CHKERRQ(ierr);
2656 
2657   mesh->maxConeSize = PetscMax(mesh->maxConeSize, csize);
2658   PetscFunctionReturn(0);
2659 }
2660 
2661 /*@C
2662   DMPlexGetCone - Return the points on the in-edges for this point in the DAG
2663 
2664   Not collective
2665 
2666   Input Parameters:
2667 + dm - The DMPlex
2668 - p - The point, which must lie in the chart set with DMPlexSetChart()
2669 
2670   Output Parameter:
2671 . cone - An array of points which are on the in-edges for point p
2672 
2673   Level: beginner
2674 
2675   Fortran Notes:
2676   Since it returns an array, this routine is only available in Fortran 90, and you must
2677   include petsc.h90 in your code.
2678   You must also call DMPlexRestoreCone() after you finish using the returned array.
2679   DMPlexRestoreCone() is not needed/available in C.
2680 
2681 .seealso: DMPlexGetConeSize(), DMPlexSetCone(), DMPlexGetConeTuple(), DMPlexSetChart()
2682 @*/
2683 PetscErrorCode DMPlexGetCone(DM dm, PetscInt p, const PetscInt *cone[])
2684 {
2685   DM_Plex       *mesh = (DM_Plex*) dm->data;
2686   PetscInt       off;
2687   PetscErrorCode ierr;
2688 
2689   PetscFunctionBegin;
2690   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2691   PetscValidPointer(cone, 3);
2692   ierr  = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2693   *cone = &mesh->cones[off];
2694   PetscFunctionReturn(0);
2695 }
2696 
2697 /*@C
2698   DMPlexGetConeTuple - Return the points on the in-edges of several points in the DAG
2699 
2700   Not collective
2701 
2702   Input Parameters:
2703 + dm - The DMPlex
2704 - p - The IS of points, which must lie in the chart set with DMPlexSetChart()
2705 
2706   Output Parameters:
2707 + pConesSection - PetscSection describing the layout of pCones
2708 - pCones - An array of points which are on the in-edges for the point set p
2709 
2710   Level: intermediate
2711 
2712 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexGetConeRecursive(), DMPlexSetChart()
2713 @*/
2714 PetscErrorCode DMPlexGetConeTuple(DM dm, IS p, PetscSection *pConesSection, IS *pCones)
2715 {
2716   PetscSection        cs, newcs;
2717   PetscInt            *cones;
2718   PetscInt            *newarr=NULL;
2719   PetscInt            n;
2720   PetscErrorCode      ierr;
2721 
2722   PetscFunctionBegin;
2723   ierr = DMPlexGetCones(dm, &cones);CHKERRQ(ierr);
2724   ierr = DMPlexGetConeSection(dm, &cs);CHKERRQ(ierr);
2725   ierr = PetscSectionExtractDofsFromArray(cs, MPIU_INT, cones, p, &newcs, pCones ? ((void**)&newarr) : NULL);CHKERRQ(ierr);
2726   if (pConesSection) *pConesSection = newcs;
2727   if (pCones) {
2728     ierr = PetscSectionGetStorageSize(newcs, &n);CHKERRQ(ierr);
2729     ierr = ISCreateGeneral(PetscObjectComm((PetscObject)p), n, newarr, PETSC_OWN_POINTER, pCones);CHKERRQ(ierr);
2730   }
2731   PetscFunctionReturn(0);
2732 }
2733 
2734 /*@
2735   DMPlexGetConeRecursiveVertices - Expand each given point into its cone points and do that recursively until we end up just with vertices.
2736 
2737   Not collective
2738 
2739   Input Parameters:
2740 + dm - The DMPlex
2741 - points - The IS of points, which must lie in the chart set with DMPlexSetChart()
2742 
2743   Output Parameter:
2744 . expandedPoints - An array of vertices recursively expanded from input points
2745 
2746   Level: advanced
2747 
2748   Notes:
2749   Like DMPlexGetConeRecursive but returns only the 0-depth IS (i.e. vertices only) and no sections.
2750   There is no corresponding Restore function, just call ISDestroy() on the returned IS to deallocate.
2751 
2752 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexGetConeTuple(), DMPlexGetConeRecursive(), DMPlexRestoreConeRecursive(), DMPlexGetDepth()
2753 @*/
2754 PetscErrorCode DMPlexGetConeRecursiveVertices(DM dm, IS points, IS *expandedPoints)
2755 {
2756   IS                  *expandedPointsAll;
2757   PetscInt            depth;
2758   PetscErrorCode      ierr;
2759 
2760   PetscFunctionBegin;
2761   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2762   PetscValidHeaderSpecific(points, IS_CLASSID, 2);
2763   PetscValidPointer(expandedPoints, 3);
2764   ierr = DMPlexGetConeRecursive(dm, points, &depth, &expandedPointsAll, NULL);CHKERRQ(ierr);
2765   *expandedPoints = expandedPointsAll[0];
2766   ierr = PetscObjectReference((PetscObject)expandedPointsAll[0]);CHKERRQ(ierr);
2767   ierr = DMPlexRestoreConeRecursive(dm, points, &depth, &expandedPointsAll, NULL);CHKERRQ(ierr);
2768   PetscFunctionReturn(0);
2769 }
2770 
2771 /*@
2772   DMPlexGetConeRecursive - Expand each given point into its cone points and do that recursively until we end up just with vertices (DAG points of depth 0, i.e. without cones).
2773 
2774   Not collective
2775 
2776   Input Parameters:
2777 + dm - The DMPlex
2778 - points - The IS of points, which must lie in the chart set with DMPlexSetChart()
2779 
2780   Output Parameters:
2781 + depth - (optional) Size of the output arrays, equal to DMPlex depth, returned by DMPlexGetDepth()
2782 . expandedPoints - (optional) An array of index sets with recursively expanded cones
2783 - sections - (optional) An array of sections which describe mappings from points to their cone points
2784 
2785   Level: advanced
2786 
2787   Notes:
2788   Like DMPlexGetConeTuple() but recursive.
2789 
2790   Array expandedPoints has size equal to depth. Each expandedPoints[d] contains DAG points with maximum depth d, recursively cone-wise expanded from the input points.
2791   For example, for d=0 it contains only vertices, for d=1 it can contain vertices and edges, etc.
2792 
2793   Array section has size equal to depth.  Each PetscSection sections[d] realizes mapping from expandedPoints[d+1] (section points) to expandedPoints[d] (section dofs) as follows:
2794   (1) DAG points in expandedPoints[d+1] with depth d+1 to their cone points in expandedPoints[d];
2795   (2) DAG points in expandedPoints[d+1] with depth in [0,d] to the same points in expandedPoints[d].
2796 
2797 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexGetConeTuple(), DMPlexRestoreConeRecursive(), DMPlexGetConeRecursiveVertices(), DMPlexGetDepth()
2798 @*/
2799 PetscErrorCode DMPlexGetConeRecursive(DM dm, IS points, PetscInt *depth, IS *expandedPoints[], PetscSection *sections[])
2800 {
2801   const PetscInt      *arr0=NULL, *cone=NULL;
2802   PetscInt            *arr=NULL, *newarr=NULL;
2803   PetscInt            d, depth_, i, n, newn, cn, co, start, end;
2804   IS                  *expandedPoints_;
2805   PetscSection        *sections_;
2806   PetscErrorCode      ierr;
2807 
2808   PetscFunctionBegin;
2809   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2810   PetscValidHeaderSpecific(points, IS_CLASSID, 2);
2811   if (depth) PetscValidIntPointer(depth, 3);
2812   if (expandedPoints) PetscValidPointer(expandedPoints, 4);
2813   if (sections) PetscValidPointer(sections, 5);
2814   ierr = ISGetLocalSize(points, &n);CHKERRQ(ierr);
2815   ierr = ISGetIndices(points, &arr0);CHKERRQ(ierr);
2816   ierr = DMPlexGetDepth(dm, &depth_);CHKERRQ(ierr);
2817   ierr = PetscCalloc1(depth_, &expandedPoints_);CHKERRQ(ierr);
2818   ierr = PetscCalloc1(depth_, &sections_);CHKERRQ(ierr);
2819   arr = (PetscInt*) arr0; /* this is ok because first generation of arr is not modified */
2820   for (d=depth_-1; d>=0; d--) {
2821     ierr = PetscSectionCreate(PETSC_COMM_SELF, &sections_[d]);CHKERRQ(ierr);
2822     ierr = PetscSectionSetChart(sections_[d], 0, n);CHKERRQ(ierr);
2823     for (i=0; i<n; i++) {
2824       ierr = DMPlexGetDepthStratum(dm, d+1, &start, &end);CHKERRQ(ierr);
2825       if (arr[i] >= start && arr[i] < end) {
2826         ierr = DMPlexGetConeSize(dm, arr[i], &cn);CHKERRQ(ierr);
2827         ierr = PetscSectionSetDof(sections_[d], i, cn);CHKERRQ(ierr);
2828       } else {
2829         ierr = PetscSectionSetDof(sections_[d], i, 1);CHKERRQ(ierr);
2830       }
2831     }
2832     ierr = PetscSectionSetUp(sections_[d]);CHKERRQ(ierr);
2833     ierr = PetscSectionGetStorageSize(sections_[d], &newn);CHKERRQ(ierr);
2834     ierr = PetscMalloc1(newn, &newarr);CHKERRQ(ierr);
2835     for (i=0; i<n; i++) {
2836       ierr = PetscSectionGetDof(sections_[d], i, &cn);CHKERRQ(ierr);
2837       ierr = PetscSectionGetOffset(sections_[d], i, &co);CHKERRQ(ierr);
2838       if (cn > 1) {
2839         ierr = DMPlexGetCone(dm, arr[i], &cone);CHKERRQ(ierr);
2840         ierr = PetscMemcpy(&newarr[co], cone, cn*sizeof(PetscInt));CHKERRQ(ierr);
2841       } else {
2842         newarr[co] = arr[i];
2843       }
2844     }
2845     ierr = ISCreateGeneral(PETSC_COMM_SELF, newn, newarr, PETSC_OWN_POINTER, &expandedPoints_[d]);CHKERRQ(ierr);
2846     arr = newarr;
2847     n = newn;
2848   }
2849   ierr = ISRestoreIndices(points, &arr0);CHKERRQ(ierr);
2850   *depth = depth_;
2851   if (expandedPoints) *expandedPoints = expandedPoints_;
2852   else {
2853     for (d=0; d<depth_; d++) {ierr = ISDestroy(&expandedPoints_[d]);CHKERRQ(ierr);}
2854     ierr = PetscFree(expandedPoints_);CHKERRQ(ierr);
2855   }
2856   if (sections) *sections = sections_;
2857   else {
2858     for (d=0; d<depth_; d++) {ierr = PetscSectionDestroy(&sections_[d]);CHKERRQ(ierr);}
2859     ierr = PetscFree(sections_);CHKERRQ(ierr);
2860   }
2861   PetscFunctionReturn(0);
2862 }
2863 
2864 /*@
2865   DMPlexRestoreConeRecursive - Deallocates arrays created by DMPlexGetConeRecursive
2866 
2867   Not collective
2868 
2869   Input Parameters:
2870 + dm - The DMPlex
2871 - points - The IS of points, which must lie in the chart set with DMPlexSetChart()
2872 
2873   Output Parameters:
2874 + depth - (optional) Size of the output arrays, equal to DMPlex depth, returned by DMPlexGetDepth()
2875 . expandedPoints - (optional) An array of recursively expanded cones
2876 - sections - (optional) An array of sections which describe mappings from points to their cone points
2877 
2878   Level: advanced
2879 
2880   Notes:
2881   See DMPlexGetConeRecursive() for details.
2882 
2883 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexGetConeTuple(), DMPlexGetConeRecursive(), DMPlexGetConeRecursiveVertices(), DMPlexGetDepth()
2884 @*/
2885 PetscErrorCode DMPlexRestoreConeRecursive(DM dm, IS points, PetscInt *depth, IS *expandedPoints[], PetscSection *sections[])
2886 {
2887   PetscInt            d, depth_;
2888   PetscErrorCode      ierr;
2889 
2890   PetscFunctionBegin;
2891   ierr = DMPlexGetDepth(dm, &depth_);CHKERRQ(ierr);
2892   PetscAssertFalse(depth && *depth != depth_,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "depth changed since last call to DMPlexGetConeRecursive");
2893   if (depth) *depth = 0;
2894   if (expandedPoints) {
2895     for (d=0; d<depth_; d++) {ierr = ISDestroy(&((*expandedPoints)[d]));CHKERRQ(ierr);}
2896     ierr = PetscFree(*expandedPoints);CHKERRQ(ierr);
2897   }
2898   if (sections)  {
2899     for (d=0; d<depth_; d++) {ierr = PetscSectionDestroy(&((*sections)[d]));CHKERRQ(ierr);}
2900     ierr = PetscFree(*sections);CHKERRQ(ierr);
2901   }
2902   PetscFunctionReturn(0);
2903 }
2904 
2905 /*@
2906   DMPlexSetCone - Set the points on the in-edges for this point in the DAG; that is these are the points that cover the specific point
2907 
2908   Not collective
2909 
2910   Input Parameters:
2911 + mesh - The DMPlex
2912 . p - The point, which must lie in the chart set with DMPlexSetChart()
2913 - cone - An array of points which are on the in-edges for point p
2914 
2915   Output Parameter:
2916 
2917   Note:
2918   This should be called after all calls to DMPlexSetConeSize() and DMSetUp().
2919 
2920   Level: beginner
2921 
2922 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp(), DMPlexSetSupport(), DMPlexSetSupportSize()
2923 @*/
2924 PetscErrorCode DMPlexSetCone(DM dm, PetscInt p, const PetscInt cone[])
2925 {
2926   DM_Plex       *mesh = (DM_Plex*) dm->data;
2927   PetscInt       pStart, pEnd;
2928   PetscInt       dof, off, c;
2929   PetscErrorCode ierr;
2930 
2931   PetscFunctionBegin;
2932   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2933   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
2934   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2935   if (dof) PetscValidPointer(cone, 3);
2936   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2937   PetscAssertFalse((p < pStart) || (p >= pEnd),PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
2938   for (c = 0; c < dof; ++c) {
2939     PetscAssertFalse((cone[c] < pStart) || (cone[c] >= pEnd),PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone point %D is not in the valid range [%D, %D)", cone[c], pStart, pEnd);
2940     mesh->cones[off+c] = cone[c];
2941   }
2942   PetscFunctionReturn(0);
2943 }
2944 
2945 /*@C
2946   DMPlexGetConeOrientation - Return the orientations on the in-edges for this point in the DAG
2947 
2948   Not collective
2949 
2950   Input Parameters:
2951 + mesh - The DMPlex
2952 - p - The point, which must lie in the chart set with DMPlexSetChart()
2953 
2954   Output Parameter:
2955 . coneOrientation - An array of orientations which are on the in-edges for point p. An orientation is an
2956                     integer giving the prescription for cone traversal.
2957 
2958   Level: beginner
2959 
2960   Notes:
2961   The number indexes the symmetry transformations for the cell type (see manual). Orientation 0 is always
2962   the identity transformation. Negative orientation indicates reflection so that -(o+1) is the reflection
2963   of o, however it is not necessarily the inverse. To get the inverse, use DMPolytopeTypeComposeOrientationInv()
2964   with the identity.
2965 
2966   Fortran Notes:
2967   Since it returns an array, this routine is only available in Fortran 90, and you must
2968   include petsc.h90 in your code.
2969   You must also call DMPlexRestoreConeOrientation() after you finish using the returned array.
2970   DMPlexRestoreConeOrientation() is not needed/available in C.
2971 
2972 .seealso: DMPolytopeTypeComposeOrientation(), DMPolytopeTypeComposeOrientationInv(), DMPlexCreate(), DMPlexGetCone(), DMPlexSetCone(), DMPlexSetChart()
2973 @*/
2974 PetscErrorCode DMPlexGetConeOrientation(DM dm, PetscInt p, const PetscInt *coneOrientation[])
2975 {
2976   DM_Plex       *mesh = (DM_Plex*) dm->data;
2977   PetscInt       off;
2978   PetscErrorCode ierr;
2979 
2980   PetscFunctionBegin;
2981   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2982   if (PetscDefined(USE_DEBUG)) {
2983     PetscInt dof;
2984     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2985     if (dof) PetscValidPointer(coneOrientation, 3);
2986   }
2987   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2988 
2989   *coneOrientation = &mesh->coneOrientations[off];
2990   PetscFunctionReturn(0);
2991 }
2992 
2993 /*@
2994   DMPlexSetConeOrientation - Set the orientations on the in-edges for this point in the DAG
2995 
2996   Not collective
2997 
2998   Input Parameters:
2999 + mesh - The DMPlex
3000 . p - The point, which must lie in the chart set with DMPlexSetChart()
3001 - coneOrientation - An array of orientations
3002   Output Parameter:
3003 
3004   Notes:
3005   This should be called after all calls to DMPlexSetConeSize() and DMSetUp().
3006 
3007   The meaning of coneOrientation is detailed in DMPlexGetConeOrientation().
3008 
3009   Level: beginner
3010 
3011 .seealso: DMPlexCreate(), DMPlexGetConeOrientation(), DMPlexSetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
3012 @*/
3013 PetscErrorCode DMPlexSetConeOrientation(DM dm, PetscInt p, const PetscInt coneOrientation[])
3014 {
3015   DM_Plex       *mesh = (DM_Plex*) dm->data;
3016   PetscInt       pStart, pEnd;
3017   PetscInt       dof, off, c;
3018   PetscErrorCode ierr;
3019 
3020   PetscFunctionBegin;
3021   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3022   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
3023   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
3024   if (dof) PetscValidPointer(coneOrientation, 3);
3025   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
3026   PetscAssertFalse((p < pStart) || (p >= pEnd),PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
3027   for (c = 0; c < dof; ++c) {
3028     PetscInt cdof, o = coneOrientation[c];
3029 
3030     ierr = PetscSectionGetDof(mesh->coneSection, mesh->cones[off+c], &cdof);CHKERRQ(ierr);
3031     PetscAssertFalse(o && ((o < -(cdof+1)) || (o >= cdof)),PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone orientation %D is not in the valid range [%D. %D)", o, -(cdof+1), cdof);
3032     mesh->coneOrientations[off+c] = o;
3033   }
3034   PetscFunctionReturn(0);
3035 }
3036 
3037 /*@
3038   DMPlexInsertCone - Insert a point into the in-edges for the point p in the DAG
3039 
3040   Not collective
3041 
3042   Input Parameters:
3043 + mesh - The DMPlex
3044 . p - The point, which must lie in the chart set with DMPlexSetChart()
3045 . conePos - The local index in the cone where the point should be put
3046 - conePoint - The mesh point to insert
3047 
3048   Level: beginner
3049 
3050 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
3051 @*/
3052 PetscErrorCode DMPlexInsertCone(DM dm, PetscInt p, PetscInt conePos, PetscInt conePoint)
3053 {
3054   DM_Plex       *mesh = (DM_Plex*) dm->data;
3055   PetscInt       pStart, pEnd;
3056   PetscInt       dof, off;
3057   PetscErrorCode ierr;
3058 
3059   PetscFunctionBegin;
3060   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3061   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
3062   PetscAssertFalse((p < pStart) || (p >= pEnd),PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
3063   PetscAssertFalse((conePoint < pStart) || (conePoint >= pEnd),PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone point %D is not in the valid range [%D, %D)", conePoint, pStart, pEnd);
3064   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
3065   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
3066   PetscAssertFalse((conePos < 0) || (conePos >= dof),PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone position %D of point %D is not in the valid range [0, %D)", conePos, p, dof);
3067   mesh->cones[off+conePos] = conePoint;
3068   PetscFunctionReturn(0);
3069 }
3070 
3071 /*@
3072   DMPlexInsertConeOrientation - Insert a point orientation for the in-edge for the point p in the DAG
3073 
3074   Not collective
3075 
3076   Input Parameters:
3077 + mesh - The DMPlex
3078 . p - The point, which must lie in the chart set with DMPlexSetChart()
3079 . conePos - The local index in the cone where the point should be put
3080 - coneOrientation - The point orientation to insert
3081 
3082   Level: beginner
3083 
3084   Notes:
3085   The meaning of coneOrientation values is detailed in DMPlexGetConeOrientation().
3086 
3087 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
3088 @*/
3089 PetscErrorCode DMPlexInsertConeOrientation(DM dm, PetscInt p, PetscInt conePos, PetscInt coneOrientation)
3090 {
3091   DM_Plex       *mesh = (DM_Plex*) dm->data;
3092   PetscInt       pStart, pEnd;
3093   PetscInt       dof, off;
3094   PetscErrorCode ierr;
3095 
3096   PetscFunctionBegin;
3097   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3098   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
3099   PetscAssertFalse((p < pStart) || (p >= pEnd),PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
3100   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
3101   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
3102   PetscAssertFalse((conePos < 0) || (conePos >= dof),PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone position %D of point %D is not in the valid range [0, %D)", conePos, p, dof);
3103   mesh->coneOrientations[off+conePos] = coneOrientation;
3104   PetscFunctionReturn(0);
3105 }
3106 
3107 /*@
3108   DMPlexGetSupportSize - Return the number of out-edges for this point in the DAG
3109 
3110   Not collective
3111 
3112   Input Parameters:
3113 + mesh - The DMPlex
3114 - p - The point, which must lie in the chart set with DMPlexSetChart()
3115 
3116   Output Parameter:
3117 . size - The support size for point p
3118 
3119   Level: beginner
3120 
3121 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart(), DMPlexGetConeSize()
3122 @*/
3123 PetscErrorCode DMPlexGetSupportSize(DM dm, PetscInt p, PetscInt *size)
3124 {
3125   DM_Plex       *mesh = (DM_Plex*) dm->data;
3126   PetscErrorCode ierr;
3127 
3128   PetscFunctionBegin;
3129   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3130   PetscValidPointer(size, 3);
3131   ierr = PetscSectionGetDof(mesh->supportSection, p, size);CHKERRQ(ierr);
3132   PetscFunctionReturn(0);
3133 }
3134 
3135 /*@
3136   DMPlexSetSupportSize - Set the number of out-edges for this point in the DAG
3137 
3138   Not collective
3139 
3140   Input Parameters:
3141 + mesh - The DMPlex
3142 . p - The point, which must lie in the chart set with DMPlexSetChart()
3143 - size - The support size for point p
3144 
3145   Output Parameter:
3146 
3147   Note:
3148   This should be called after DMPlexSetChart().
3149 
3150   Level: beginner
3151 
3152 .seealso: DMPlexCreate(), DMPlexGetSupportSize(), DMPlexSetChart()
3153 @*/
3154 PetscErrorCode DMPlexSetSupportSize(DM dm, PetscInt p, PetscInt size)
3155 {
3156   DM_Plex       *mesh = (DM_Plex*) dm->data;
3157   PetscErrorCode ierr;
3158 
3159   PetscFunctionBegin;
3160   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3161   ierr = PetscSectionSetDof(mesh->supportSection, p, size);CHKERRQ(ierr);
3162 
3163   mesh->maxSupportSize = PetscMax(mesh->maxSupportSize, size);
3164   PetscFunctionReturn(0);
3165 }
3166 
3167 /*@C
3168   DMPlexGetSupport - Return the points on the out-edges for this point in the DAG
3169 
3170   Not collective
3171 
3172   Input Parameters:
3173 + mesh - The DMPlex
3174 - p - The point, which must lie in the chart set with DMPlexSetChart()
3175 
3176   Output Parameter:
3177 . support - An array of points which are on the out-edges for point p
3178 
3179   Level: beginner
3180 
3181   Fortran Notes:
3182   Since it returns an array, this routine is only available in Fortran 90, and you must
3183   include petsc.h90 in your code.
3184   You must also call DMPlexRestoreSupport() after you finish using the returned array.
3185   DMPlexRestoreSupport() is not needed/available in C.
3186 
3187 .seealso: DMPlexGetSupportSize(), DMPlexSetSupport(), DMPlexGetCone(), DMPlexSetChart()
3188 @*/
3189 PetscErrorCode DMPlexGetSupport(DM dm, PetscInt p, const PetscInt *support[])
3190 {
3191   DM_Plex       *mesh = (DM_Plex*) dm->data;
3192   PetscInt       off;
3193   PetscErrorCode ierr;
3194 
3195   PetscFunctionBegin;
3196   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3197   PetscValidPointer(support, 3);
3198   ierr     = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
3199   *support = &mesh->supports[off];
3200   PetscFunctionReturn(0);
3201 }
3202 
3203 /*@
3204   DMPlexSetSupport - Set the points on the out-edges for this point in the DAG, that is the list of points that this point covers
3205 
3206   Not collective
3207 
3208   Input Parameters:
3209 + mesh - The DMPlex
3210 . p - The point, which must lie in the chart set with DMPlexSetChart()
3211 - support - An array of points which are on the out-edges for point p
3212 
3213   Output Parameter:
3214 
3215   Note:
3216   This should be called after all calls to DMPlexSetSupportSize() and DMSetUp().
3217 
3218   Level: beginner
3219 
3220 .seealso: DMPlexSetCone(), DMPlexSetConeSize(), DMPlexCreate(), DMPlexGetSupport(), DMPlexSetChart(), DMPlexSetSupportSize(), DMSetUp()
3221 @*/
3222 PetscErrorCode DMPlexSetSupport(DM dm, PetscInt p, const PetscInt support[])
3223 {
3224   DM_Plex       *mesh = (DM_Plex*) dm->data;
3225   PetscInt       pStart, pEnd;
3226   PetscInt       dof, off, c;
3227   PetscErrorCode ierr;
3228 
3229   PetscFunctionBegin;
3230   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3231   ierr = PetscSectionGetChart(mesh->supportSection, &pStart, &pEnd);CHKERRQ(ierr);
3232   ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
3233   if (dof) PetscValidPointer(support, 3);
3234   ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
3235   PetscAssertFalse((p < pStart) || (p >= pEnd),PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
3236   for (c = 0; c < dof; ++c) {
3237     PetscAssertFalse((support[c] < pStart) || (support[c] >= pEnd),PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support point %D is not in the valid range [%D, %D)", support[c], pStart, pEnd);
3238     mesh->supports[off+c] = support[c];
3239   }
3240   PetscFunctionReturn(0);
3241 }
3242 
3243 /*@
3244   DMPlexInsertSupport - Insert a point into the out-edges for the point p in the DAG
3245 
3246   Not collective
3247 
3248   Input Parameters:
3249 + mesh - The DMPlex
3250 . p - The point, which must lie in the chart set with DMPlexSetChart()
3251 . supportPos - The local index in the cone where the point should be put
3252 - supportPoint - The mesh point to insert
3253 
3254   Level: beginner
3255 
3256 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
3257 @*/
3258 PetscErrorCode DMPlexInsertSupport(DM dm, PetscInt p, PetscInt supportPos, PetscInt supportPoint)
3259 {
3260   DM_Plex       *mesh = (DM_Plex*) dm->data;
3261   PetscInt       pStart, pEnd;
3262   PetscInt       dof, off;
3263   PetscErrorCode ierr;
3264 
3265   PetscFunctionBegin;
3266   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3267   ierr = PetscSectionGetChart(mesh->supportSection, &pStart, &pEnd);CHKERRQ(ierr);
3268   ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
3269   ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
3270   PetscAssertFalse((p < pStart) || (p >= pEnd),PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
3271   PetscAssertFalse((supportPoint < pStart) || (supportPoint >= pEnd),PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support point %D is not in the valid range [%D, %D)", supportPoint, pStart, pEnd);
3272   PetscAssertFalse(supportPos >= dof,PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support position %D of point %D is not in the valid range [0, %D)", supportPos, p, dof);
3273   mesh->supports[off+supportPos] = supportPoint;
3274   PetscFunctionReturn(0);
3275 }
3276 
3277 /* Converts an orientation o in the current numbering to the previous scheme used in Plex */
3278 PetscInt DMPolytopeConvertNewOrientation_Internal(DMPolytopeType ct, PetscInt o)
3279 {
3280   switch (ct) {
3281     case DM_POLYTOPE_SEGMENT:
3282       if (o == -1) return -2;
3283       break;
3284     case DM_POLYTOPE_TRIANGLE:
3285       if (o == -3) return -1;
3286       if (o == -2) return -3;
3287       if (o == -1) return -2;
3288       break;
3289     case DM_POLYTOPE_QUADRILATERAL:
3290       if (o == -4) return -2;
3291       if (o == -3) return -1;
3292       if (o == -2) return -4;
3293       if (o == -1) return -3;
3294       break;
3295     default: return o;
3296   }
3297   return o;
3298 }
3299 
3300 /* Converts an orientation o in the previous scheme used in Plex to the current numbering */
3301 PetscInt DMPolytopeConvertOldOrientation_Internal(DMPolytopeType ct, PetscInt o)
3302 {
3303   switch (ct) {
3304     case DM_POLYTOPE_SEGMENT:
3305       if ((o == -2) || (o == 1)) return -1;
3306       if (o == -1) return 0;
3307       break;
3308     case DM_POLYTOPE_TRIANGLE:
3309       if (o == -3) return -2;
3310       if (o == -2) return -1;
3311       if (o == -1) return -3;
3312       break;
3313     case DM_POLYTOPE_QUADRILATERAL:
3314       if (o == -4) return -2;
3315       if (o == -3) return -1;
3316       if (o == -2) return -4;
3317       if (o == -1) return -3;
3318       break;
3319     default: return o;
3320   }
3321   return o;
3322 }
3323 
3324 /* Takes in a mesh whose orientations are in the previous scheme and converts them all to the current numbering */
3325 PetscErrorCode DMPlexConvertOldOrientations_Internal(DM dm)
3326 {
3327   PetscInt       pStart, pEnd, p;
3328   PetscErrorCode ierr;
3329 
3330   PetscFunctionBegin;
3331   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
3332   for (p = pStart; p < pEnd; ++p) {
3333     const PetscInt *cone, *ornt;
3334     PetscInt        coneSize, c;
3335 
3336     ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
3337     ierr = DMPlexGetCone(dm, p, &cone);CHKERRQ(ierr);
3338     ierr = DMPlexGetConeOrientation(dm, p, &ornt);CHKERRQ(ierr);
3339     for (c = 0; c < coneSize; ++c) {
3340       DMPolytopeType ct;
3341       const PetscInt o = ornt[c];
3342 
3343       ierr = DMPlexGetCellType(dm, cone[c], &ct);CHKERRQ(ierr);
3344       switch (ct) {
3345         case DM_POLYTOPE_SEGMENT:
3346           if ((o == -2) || (o == 1)) {ierr = DMPlexInsertConeOrientation(dm, p, c, -1);CHKERRQ(ierr);}
3347           if (o == -1) {ierr = DMPlexInsertConeOrientation(dm, p, c, 0);CHKERRQ(ierr);}
3348           break;
3349         case DM_POLYTOPE_TRIANGLE:
3350           if (o == -3) {ierr = DMPlexInsertConeOrientation(dm, p, c, -2);CHKERRQ(ierr);}
3351           if (o == -2) {ierr = DMPlexInsertConeOrientation(dm, p, c, -1);CHKERRQ(ierr);}
3352           if (o == -1) {ierr = DMPlexInsertConeOrientation(dm, p, c, -3);CHKERRQ(ierr);}
3353           break;
3354         case DM_POLYTOPE_QUADRILATERAL:
3355           if (o == -4) {ierr = DMPlexInsertConeOrientation(dm, p, c, -2);CHKERRQ(ierr);}
3356           if (o == -3) {ierr = DMPlexInsertConeOrientation(dm, p, c, -1);CHKERRQ(ierr);}
3357           if (o == -2) {ierr = DMPlexInsertConeOrientation(dm, p, c, -4);CHKERRQ(ierr);}
3358           if (o == -1) {ierr = DMPlexInsertConeOrientation(dm, p, c, -3);CHKERRQ(ierr);}
3359           break;
3360         default: break;
3361       }
3362     }
3363   }
3364   PetscFunctionReturn(0);
3365 }
3366 
3367 static PetscErrorCode DMPlexGetTransitiveClosure_Depth1_Private(DM dm, PetscInt p, PetscInt ornt, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
3368 {
3369   DMPolytopeType  ct = DM_POLYTOPE_UNKNOWN;
3370   PetscInt       *closure;
3371   const PetscInt *tmp = NULL, *tmpO = NULL;
3372   PetscInt        off = 0, tmpSize, t;
3373   PetscErrorCode  ierr;
3374 
3375   PetscFunctionBeginHot;
3376   if (ornt) {
3377     ierr = DMPlexGetCellType(dm, p, &ct);CHKERRQ(ierr);
3378     if (ct == DM_POLYTOPE_FV_GHOST || ct == DM_POLYTOPE_INTERIOR_GHOST || ct == DM_POLYTOPE_UNKNOWN) ct = DM_POLYTOPE_UNKNOWN;
3379   }
3380   if (*points) {
3381     closure = *points;
3382   } else {
3383     PetscInt maxConeSize, maxSupportSize;
3384     ierr = DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize);CHKERRQ(ierr);
3385     ierr = DMGetWorkArray(dm, 2*(PetscMax(maxConeSize, maxSupportSize)+1), MPIU_INT, &closure);CHKERRQ(ierr);
3386   }
3387   if (useCone) {
3388     ierr = DMPlexGetConeSize(dm, p, &tmpSize);CHKERRQ(ierr);
3389     ierr = DMPlexGetCone(dm, p, &tmp);CHKERRQ(ierr);
3390     ierr = DMPlexGetConeOrientation(dm, p, &tmpO);CHKERRQ(ierr);
3391   } else {
3392     ierr = DMPlexGetSupportSize(dm, p, &tmpSize);CHKERRQ(ierr);
3393     ierr = DMPlexGetSupport(dm, p, &tmp);CHKERRQ(ierr);
3394   }
3395   if (ct == DM_POLYTOPE_UNKNOWN) {
3396     closure[off++] = p;
3397     closure[off++] = 0;
3398     for (t = 0; t < tmpSize; ++t) {
3399       closure[off++] = tmp[t];
3400       closure[off++] = tmpO ? tmpO[t] : 0;
3401     }
3402   } else {
3403     const PetscInt *arr = DMPolytopeTypeGetArrangment(ct, ornt);CHKERRQ(ierr);
3404 
3405     /* We assume that cells with a valid type have faces with a valid type */
3406     closure[off++] = p;
3407     closure[off++] = ornt;
3408     for (t = 0; t < tmpSize; ++t) {
3409       DMPolytopeType ft;
3410 
3411       ierr = DMPlexGetCellType(dm, tmp[t], &ft);CHKERRQ(ierr);
3412       closure[off++] = tmp[arr[t]];
3413       closure[off++] = tmpO ? DMPolytopeTypeComposeOrientation(ft, ornt, tmpO[t]) : 0;
3414     }
3415   }
3416   if (numPoints) *numPoints = tmpSize+1;
3417   if (points)    *points    = closure;
3418   PetscFunctionReturn(0);
3419 }
3420 
3421 /* We need a special tensor verison becasue we want to allow duplicate points in the endcaps for hybrid cells */
3422 static PetscErrorCode DMPlexTransitiveClosure_Tensor_Internal(DM dm, PetscInt point, DMPolytopeType ct, PetscInt o, PetscBool useCone, PetscInt *numPoints, PetscInt **points)
3423 {
3424   const PetscInt *arr = DMPolytopeTypeGetArrangment(ct, o);
3425   const PetscInt *cone, *ornt;
3426   PetscInt       *pts,  *closure = NULL;
3427   DMPolytopeType  ft;
3428   PetscInt        maxConeSize, maxSupportSize, coneSeries, supportSeries, maxSize;
3429   PetscInt        dim, coneSize, c, d, clSize, cl;
3430   PetscErrorCode  ierr;
3431 
3432   PetscFunctionBeginHot;
3433   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
3434   ierr = DMPlexGetConeSize(dm, point, &coneSize);CHKERRQ(ierr);
3435   ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
3436   ierr = DMPlexGetConeOrientation(dm, point, &ornt);CHKERRQ(ierr);
3437   ierr = DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize);CHKERRQ(ierr);
3438   coneSeries    = (maxConeSize    > 1) ? ((PetscPowInt(maxConeSize,    dim+1)-1)/(maxConeSize-1))    : dim+1;
3439   supportSeries = (maxSupportSize > 1) ? ((PetscPowInt(maxSupportSize, dim+1)-1)/(maxSupportSize-1)) : dim+1;
3440   maxSize       = PetscMax(coneSeries, supportSeries);
3441   if (*points) {pts  = *points;}
3442   else         {ierr = DMGetWorkArray(dm, 2*maxSize, MPIU_INT, &pts);CHKERRQ(ierr);}
3443   c    = 0;
3444   pts[c++] = point;
3445   pts[c++] = o;
3446   ierr = DMPlexGetCellType(dm, cone[arr[0*2+0]], &ft);CHKERRQ(ierr);
3447   ierr = DMPlexGetTransitiveClosure_Internal(dm, cone[arr[0*2+0]], DMPolytopeTypeComposeOrientation(ft, arr[0*2+1], ornt[0]), useCone, &clSize, &closure);CHKERRQ(ierr);
3448   for (cl = 0; cl < clSize*2; cl += 2) {pts[c++] = closure[cl]; pts[c++] = closure[cl+1];}
3449   ierr = DMPlexGetTransitiveClosure_Internal(dm, cone[arr[1*2+0]], DMPolytopeTypeComposeOrientation(ft, arr[1*2+1], ornt[1]), useCone, &clSize, &closure);CHKERRQ(ierr);
3450   for (cl = 0; cl < clSize*2; cl += 2) {pts[c++] = closure[cl]; pts[c++] = closure[cl+1];}
3451   ierr = DMPlexRestoreTransitiveClosure(dm, cone[0], useCone, &clSize, &closure);CHKERRQ(ierr);
3452   for (d = 2; d < coneSize; ++d) {
3453     ierr = DMPlexGetCellType(dm, cone[arr[d*2+0]], &ft);CHKERRQ(ierr);
3454     pts[c++] = cone[arr[d*2+0]];
3455     pts[c++] = DMPolytopeTypeComposeOrientation(ft, arr[d*2+1], ornt[d]);
3456   }
3457   if (dim >= 3) {
3458     for (d = 2; d < coneSize; ++d) {
3459       const PetscInt  fpoint = cone[arr[d*2+0]];
3460       const PetscInt *fcone, *fornt;
3461       PetscInt        fconeSize, fc, i;
3462 
3463       ierr = DMPlexGetCellType(dm, fpoint, &ft);CHKERRQ(ierr);
3464       const PetscInt *farr = DMPolytopeTypeGetArrangment(ft, DMPolytopeTypeComposeOrientation(ft, arr[d*2+1], ornt[d]));
3465       ierr = DMPlexGetConeSize(dm, fpoint, &fconeSize);CHKERRQ(ierr);
3466       ierr = DMPlexGetCone(dm, fpoint, &fcone);CHKERRQ(ierr);
3467       ierr = DMPlexGetConeOrientation(dm, fpoint, &fornt);CHKERRQ(ierr);
3468       for (fc = 0; fc < fconeSize; ++fc) {
3469         const PetscInt cp = fcone[farr[fc*2+0]];
3470         const PetscInt co = farr[fc*2+1];
3471 
3472         for (i = 0; i < c; i += 2) if (pts[i] == cp) break;
3473         if (i == c) {
3474           ierr = DMPlexGetCellType(dm, cp, &ft);CHKERRQ(ierr);
3475           pts[c++] = cp;
3476           pts[c++] = DMPolytopeTypeComposeOrientation(ft, co, fornt[farr[fc*2+0]]);
3477         }
3478       }
3479     }
3480   }
3481   *numPoints = c/2;
3482   *points    = pts;
3483   PetscFunctionReturn(0);
3484 }
3485 
3486 PetscErrorCode DMPlexGetTransitiveClosure_Internal(DM dm, PetscInt p, PetscInt ornt, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
3487 {
3488   DMPolytopeType ct;
3489   PetscInt      *closure, *fifo;
3490   PetscInt       closureSize = 0, fifoStart = 0, fifoSize = 0;
3491   PetscInt       maxConeSize, maxSupportSize, coneSeries, supportSeries;
3492   PetscInt       depth, maxSize;
3493   PetscErrorCode ierr;
3494 
3495   PetscFunctionBeginHot;
3496   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3497   if (depth == 1) {
3498     ierr = DMPlexGetTransitiveClosure_Depth1_Private(dm, p, ornt, useCone, numPoints, points);CHKERRQ(ierr);
3499     PetscFunctionReturn(0);
3500   }
3501   ierr = DMPlexGetCellType(dm, p, &ct);CHKERRQ(ierr);
3502   if (ct == DM_POLYTOPE_FV_GHOST || ct == DM_POLYTOPE_INTERIOR_GHOST || ct == DM_POLYTOPE_UNKNOWN) ct = DM_POLYTOPE_UNKNOWN;
3503   if (ct == DM_POLYTOPE_SEG_PRISM_TENSOR || ct == DM_POLYTOPE_TRI_PRISM_TENSOR || ct == DM_POLYTOPE_QUAD_PRISM_TENSOR) {
3504     ierr = DMPlexTransitiveClosure_Tensor_Internal(dm, p, ct, ornt, useCone, numPoints, points);CHKERRQ(ierr);
3505     PetscFunctionReturn(0);
3506   }
3507   ierr = DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize);CHKERRQ(ierr);
3508   coneSeries    = (maxConeSize    > 1) ? ((PetscPowInt(maxConeSize,    depth+1)-1)/(maxConeSize-1))    : depth+1;
3509   supportSeries = (maxSupportSize > 1) ? ((PetscPowInt(maxSupportSize, depth+1)-1)/(maxSupportSize-1)) : depth+1;
3510   maxSize       = PetscMax(coneSeries, supportSeries);
3511   ierr = DMGetWorkArray(dm, 3*maxSize, MPIU_INT, &fifo);CHKERRQ(ierr);
3512   if (*points) {closure = *points;}
3513   else         {ierr = DMGetWorkArray(dm, 2*maxSize, MPIU_INT, &closure);CHKERRQ(ierr);}
3514   closure[closureSize++] = p;
3515   closure[closureSize++] = ornt;
3516   fifo[fifoSize++]       = p;
3517   fifo[fifoSize++]       = ornt;
3518   fifo[fifoSize++]       = ct;
3519   /* Should kick out early when depth is reached, rather than checking all vertices for empty cones */
3520   while (fifoSize - fifoStart) {
3521     const PetscInt       q    = fifo[fifoStart++];
3522     const PetscInt       o    = fifo[fifoStart++];
3523     const DMPolytopeType qt   = (DMPolytopeType) fifo[fifoStart++];
3524     const PetscInt      *qarr = DMPolytopeTypeGetArrangment(qt, o);
3525     const PetscInt      *tmp, *tmpO;
3526     PetscInt             tmpSize, t;
3527 
3528     if (PetscDefined(USE_DEBUG)) {
3529       PetscInt nO = DMPolytopeTypeGetNumArrangments(qt)/2;
3530       PetscAssertFalse(o && (o >= nO || o < -nO),PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid orientation %D not in [%D,%D) for %s %D", o, -nO, nO, DMPolytopeTypes[qt], q);
3531     }
3532     if (useCone) {
3533       ierr = DMPlexGetConeSize(dm, q, &tmpSize);CHKERRQ(ierr);
3534       ierr = DMPlexGetCone(dm, q, &tmp);CHKERRQ(ierr);
3535       ierr = DMPlexGetConeOrientation(dm, q, &tmpO);CHKERRQ(ierr);
3536     } else {
3537       ierr = DMPlexGetSupportSize(dm, q, &tmpSize);CHKERRQ(ierr);
3538       ierr = DMPlexGetSupport(dm, q, &tmp);CHKERRQ(ierr);
3539       tmpO = NULL;
3540     }
3541     for (t = 0; t < tmpSize; ++t) {
3542       const PetscInt ip = useCone && qarr ? qarr[t*2]   : t;
3543       const PetscInt io = useCone && qarr ? qarr[t*2+1] : 0;
3544       const PetscInt cp = tmp[ip];
3545       ierr = DMPlexGetCellType(dm, cp, &ct);CHKERRQ(ierr);
3546       const PetscInt co = tmpO ? DMPolytopeTypeComposeOrientation(ct, io, tmpO[ip]) : 0;
3547       PetscInt       c;
3548 
3549       /* Check for duplicate */
3550       for (c = 0; c < closureSize; c += 2) {
3551         if (closure[c] == cp) break;
3552       }
3553       if (c == closureSize) {
3554         closure[closureSize++] = cp;
3555         closure[closureSize++] = co;
3556         fifo[fifoSize++]       = cp;
3557         fifo[fifoSize++]       = co;
3558         fifo[fifoSize++]       = ct;
3559       }
3560     }
3561   }
3562   ierr = DMRestoreWorkArray(dm, 3*maxSize, MPIU_INT, &fifo);CHKERRQ(ierr);
3563   if (numPoints) *numPoints = closureSize/2;
3564   if (points)    *points    = closure;
3565   PetscFunctionReturn(0);
3566 }
3567 
3568 /*@C
3569   DMPlexGetTransitiveClosure - Return the points on the transitive closure of the in-edges or out-edges for this point in the DAG
3570 
3571   Not collective
3572 
3573   Input Parameters:
3574 + dm      - The DMPlex
3575 . p       - The mesh point
3576 - useCone - PETSC_TRUE for the closure, otherwise return the star
3577 
3578   Input/Output Parameter:
3579 . points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...];
3580            if NULL on input, internal storage will be returned, otherwise the provided array is used
3581 
3582   Output Parameter:
3583 . numPoints - The number of points in the closure, so points[] is of size 2*numPoints
3584 
3585   Note:
3586   If using internal storage (points is NULL on input), each call overwrites the last output.
3587 
3588   Fortran Notes:
3589   Since it returns an array, this routine is only available in Fortran 90, and you must include petsc.h90 in your code.
3590 
3591   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3592 
3593   Level: beginner
3594 
3595 .seealso: DMPlexRestoreTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
3596 @*/
3597 PetscErrorCode DMPlexGetTransitiveClosure(DM dm, PetscInt p, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
3598 {
3599   PetscErrorCode ierr;
3600 
3601   PetscFunctionBeginHot;
3602   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3603   if (numPoints) PetscValidIntPointer(numPoints, 4);
3604   if (points)    PetscValidPointer(points, 5);
3605   ierr = DMPlexGetTransitiveClosure_Internal(dm, p, 0, useCone, numPoints, points);CHKERRQ(ierr);
3606   PetscFunctionReturn(0);
3607 }
3608 
3609 /*@C
3610   DMPlexRestoreTransitiveClosure - Restore the array of points on the transitive closure of the in-edges or out-edges for this point in the DAG
3611 
3612   Not collective
3613 
3614   Input Parameters:
3615 + dm        - The DMPlex
3616 . p         - The mesh point
3617 . useCone   - PETSC_TRUE for the closure, otherwise return the star
3618 . numPoints - The number of points in the closure, so points[] is of size 2*numPoints
3619 - points    - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...]
3620 
3621   Note:
3622   If not using internal storage (points is not NULL on input), this call is unnecessary
3623 
3624   Fortran Notes:
3625   Since it returns an array, this routine is only available in Fortran 90, and you must include petsc.h90 in your code.
3626 
3627   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3628 
3629   Level: beginner
3630 
3631 .seealso: DMPlexGetTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
3632 @*/
3633 PetscErrorCode DMPlexRestoreTransitiveClosure(DM dm, PetscInt p, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
3634 {
3635   PetscErrorCode ierr;
3636 
3637   PetscFunctionBeginHot;
3638   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3639   if (numPoints) *numPoints = 0;
3640   ierr = DMRestoreWorkArray(dm, 0, MPIU_INT, points);CHKERRQ(ierr);
3641   PetscFunctionReturn(0);
3642 }
3643 
3644 /*@
3645   DMPlexGetMaxSizes - Return the maximum number of in-edges (cone) and out-edges (support) for any point in the DAG
3646 
3647   Not collective
3648 
3649   Input Parameter:
3650 . mesh - The DMPlex
3651 
3652   Output Parameters:
3653 + maxConeSize - The maximum number of in-edges
3654 - maxSupportSize - The maximum number of out-edges
3655 
3656   Level: beginner
3657 
3658 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart()
3659 @*/
3660 PetscErrorCode DMPlexGetMaxSizes(DM dm, PetscInt *maxConeSize, PetscInt *maxSupportSize)
3661 {
3662   DM_Plex *mesh = (DM_Plex*) dm->data;
3663 
3664   PetscFunctionBegin;
3665   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3666   if (maxConeSize)    *maxConeSize    = mesh->maxConeSize;
3667   if (maxSupportSize) *maxSupportSize = mesh->maxSupportSize;
3668   PetscFunctionReturn(0);
3669 }
3670 
3671 PetscErrorCode DMSetUp_Plex(DM dm)
3672 {
3673   DM_Plex       *mesh = (DM_Plex*) dm->data;
3674   PetscInt       size;
3675   PetscErrorCode ierr;
3676 
3677   PetscFunctionBegin;
3678   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3679   ierr = PetscSectionSetUp(mesh->coneSection);CHKERRQ(ierr);
3680   ierr = PetscSectionGetStorageSize(mesh->coneSection, &size);CHKERRQ(ierr);
3681   ierr = PetscMalloc1(size, &mesh->cones);CHKERRQ(ierr);
3682   ierr = PetscCalloc1(size, &mesh->coneOrientations);CHKERRQ(ierr);
3683   ierr = PetscLogObjectMemory((PetscObject) dm, size*2*sizeof(PetscInt));CHKERRQ(ierr);
3684   if (mesh->maxSupportSize) {
3685     ierr = PetscSectionSetUp(mesh->supportSection);CHKERRQ(ierr);
3686     ierr = PetscSectionGetStorageSize(mesh->supportSection, &size);CHKERRQ(ierr);
3687     ierr = PetscMalloc1(size, &mesh->supports);CHKERRQ(ierr);
3688     ierr = PetscLogObjectMemory((PetscObject) dm, size*sizeof(PetscInt));CHKERRQ(ierr);
3689   }
3690   PetscFunctionReturn(0);
3691 }
3692 
3693 PetscErrorCode DMCreateSubDM_Plex(DM dm, PetscInt numFields, const PetscInt fields[], IS *is, DM *subdm)
3694 {
3695   PetscErrorCode ierr;
3696 
3697   PetscFunctionBegin;
3698   if (subdm) {ierr = DMClone(dm, subdm);CHKERRQ(ierr);}
3699   ierr = DMCreateSectionSubDM(dm, numFields, fields, is, subdm);CHKERRQ(ierr);
3700   if (subdm) {(*subdm)->useNatural = dm->useNatural;}
3701   if (dm->useNatural && dm->sfMigration) {
3702     PetscSF        sfMigrationInv,sfNatural;
3703     PetscSection   section, sectionSeq;
3704 
3705     (*subdm)->sfMigration = dm->sfMigration;
3706     ierr = PetscObjectReference((PetscObject) dm->sfMigration);CHKERRQ(ierr);
3707     ierr = DMGetLocalSection((*subdm), &section);CHKERRQ(ierr);
3708     ierr = PetscSFCreateInverseSF((*subdm)->sfMigration, &sfMigrationInv);CHKERRQ(ierr);
3709     ierr = PetscSectionCreate(PetscObjectComm((PetscObject) (*subdm)), &sectionSeq);CHKERRQ(ierr);
3710     ierr = PetscSFDistributeSection(sfMigrationInv, section, NULL, sectionSeq);CHKERRQ(ierr);
3711 
3712     ierr = DMPlexCreateGlobalToNaturalSF(*subdm, sectionSeq, (*subdm)->sfMigration, &sfNatural);CHKERRQ(ierr);
3713     (*subdm)->sfNatural = sfNatural;
3714     ierr = PetscSectionDestroy(&sectionSeq);CHKERRQ(ierr);
3715     ierr = PetscSFDestroy(&sfMigrationInv);CHKERRQ(ierr);
3716   }
3717   PetscFunctionReturn(0);
3718 }
3719 
3720 PetscErrorCode DMCreateSuperDM_Plex(DM dms[], PetscInt len, IS **is, DM *superdm)
3721 {
3722   PetscErrorCode ierr;
3723   PetscInt       i = 0;
3724 
3725   PetscFunctionBegin;
3726   ierr = DMClone(dms[0], superdm);CHKERRQ(ierr);
3727   ierr = DMCreateSectionSuperDM(dms, len, is, superdm);CHKERRQ(ierr);
3728   (*superdm)->useNatural = PETSC_FALSE;
3729   for (i = 0; i < len; i++) {
3730     if (dms[i]->useNatural && dms[i]->sfMigration) {
3731       PetscSF        sfMigrationInv,sfNatural;
3732       PetscSection   section, sectionSeq;
3733 
3734       (*superdm)->sfMigration = dms[i]->sfMigration;
3735       ierr = PetscObjectReference((PetscObject) dms[i]->sfMigration);CHKERRQ(ierr);
3736       (*superdm)->useNatural = PETSC_TRUE;
3737       ierr = DMGetLocalSection((*superdm), &section);CHKERRQ(ierr);
3738       ierr = PetscSFCreateInverseSF((*superdm)->sfMigration, &sfMigrationInv);CHKERRQ(ierr);
3739       ierr = PetscSectionCreate(PetscObjectComm((PetscObject) (*superdm)), &sectionSeq);CHKERRQ(ierr);
3740       ierr = PetscSFDistributeSection(sfMigrationInv, section, NULL, sectionSeq);CHKERRQ(ierr);
3741 
3742       ierr = DMPlexCreateGlobalToNaturalSF(*superdm, sectionSeq, (*superdm)->sfMigration, &sfNatural);CHKERRQ(ierr);
3743       (*superdm)->sfNatural = sfNatural;
3744       ierr = PetscSectionDestroy(&sectionSeq);CHKERRQ(ierr);
3745       ierr = PetscSFDestroy(&sfMigrationInv);CHKERRQ(ierr);
3746       break;
3747     }
3748   }
3749   PetscFunctionReturn(0);
3750 }
3751 
3752 /*@
3753   DMPlexSymmetrize - Create support (out-edge) information from cone (in-edge) information
3754 
3755   Not collective
3756 
3757   Input Parameter:
3758 . mesh - The DMPlex
3759 
3760   Output Parameter:
3761 
3762   Note:
3763   This should be called after all calls to DMPlexSetCone()
3764 
3765   Level: beginner
3766 
3767 .seealso: DMPlexCreate(), DMPlexSetChart(), DMPlexSetConeSize(), DMPlexSetCone()
3768 @*/
3769 PetscErrorCode DMPlexSymmetrize(DM dm)
3770 {
3771   DM_Plex       *mesh = (DM_Plex*) dm->data;
3772   PetscInt      *offsets;
3773   PetscInt       supportSize;
3774   PetscInt       pStart, pEnd, p;
3775   PetscErrorCode ierr;
3776 
3777   PetscFunctionBegin;
3778   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3779   PetscAssertFalse(mesh->supports,PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONGSTATE, "Supports were already setup in this DMPlex");
3780   ierr = PetscLogEventBegin(DMPLEX_Symmetrize,dm,0,0,0);CHKERRQ(ierr);
3781   /* Calculate support sizes */
3782   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
3783   for (p = pStart; p < pEnd; ++p) {
3784     PetscInt dof, off, c;
3785 
3786     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
3787     ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
3788     for (c = off; c < off+dof; ++c) {
3789       ierr = PetscSectionAddDof(mesh->supportSection, mesh->cones[c], 1);CHKERRQ(ierr);
3790     }
3791   }
3792   for (p = pStart; p < pEnd; ++p) {
3793     PetscInt dof;
3794 
3795     ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
3796 
3797     mesh->maxSupportSize = PetscMax(mesh->maxSupportSize, dof);
3798   }
3799   ierr = PetscSectionSetUp(mesh->supportSection);CHKERRQ(ierr);
3800   /* Calculate supports */
3801   ierr = PetscSectionGetStorageSize(mesh->supportSection, &supportSize);CHKERRQ(ierr);
3802   ierr = PetscMalloc1(supportSize, &mesh->supports);CHKERRQ(ierr);
3803   ierr = PetscCalloc1(pEnd - pStart, &offsets);CHKERRQ(ierr);
3804   for (p = pStart; p < pEnd; ++p) {
3805     PetscInt dof, off, c;
3806 
3807     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
3808     ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
3809     for (c = off; c < off+dof; ++c) {
3810       const PetscInt q = mesh->cones[c];
3811       PetscInt       offS;
3812 
3813       ierr = PetscSectionGetOffset(mesh->supportSection, q, &offS);CHKERRQ(ierr);
3814 
3815       mesh->supports[offS+offsets[q]] = p;
3816       ++offsets[q];
3817     }
3818   }
3819   ierr = PetscFree(offsets);CHKERRQ(ierr);
3820   ierr = PetscLogEventEnd(DMPLEX_Symmetrize,dm,0,0,0);CHKERRQ(ierr);
3821   PetscFunctionReturn(0);
3822 }
3823 
3824 static PetscErrorCode DMPlexCreateDepthStratum(DM dm, DMLabel label, PetscInt depth, PetscInt pStart, PetscInt pEnd)
3825 {
3826   IS             stratumIS;
3827   PetscErrorCode ierr;
3828 
3829   PetscFunctionBegin;
3830   if (pStart >= pEnd) PetscFunctionReturn(0);
3831   if (PetscDefined(USE_DEBUG)) {
3832     PetscInt  qStart, qEnd, numLevels, level;
3833     PetscBool overlap = PETSC_FALSE;
3834     ierr = DMLabelGetNumValues(label, &numLevels);CHKERRQ(ierr);
3835     for (level = 0; level < numLevels; level++) {
3836       ierr = DMLabelGetStratumBounds(label, level, &qStart, &qEnd);CHKERRQ(ierr);
3837       if ((pStart >= qStart && pStart < qEnd) || (pEnd > qStart && pEnd <= qEnd)) {overlap = PETSC_TRUE; break;}
3838     }
3839     PetscAssertFalse(overlap,PETSC_COMM_SELF, PETSC_ERR_PLIB, "New depth %D range [%D,%D) overlaps with depth %D range [%D,%D)", depth, pStart, pEnd, level, qStart, qEnd);
3840   }
3841   ierr = ISCreateStride(PETSC_COMM_SELF, pEnd-pStart, pStart, 1, &stratumIS);CHKERRQ(ierr);
3842   ierr = DMLabelSetStratumIS(label, depth, stratumIS);CHKERRQ(ierr);
3843   ierr = ISDestroy(&stratumIS);CHKERRQ(ierr);
3844   PetscFunctionReturn(0);
3845 }
3846 
3847 /*@
3848   DMPlexStratify - The DAG for most topologies is a graded poset (https://en.wikipedia.org/wiki/Graded_poset), and
3849   can be illustrated by a Hasse Diagram (https://en.wikipedia.org/wiki/Hasse_diagram). The strata group all points of the
3850   same grade, and this function calculates the strata. This grade can be seen as the height (or depth) of the point in
3851   the DAG.
3852 
3853   Collective on dm
3854 
3855   Input Parameter:
3856 . mesh - The DMPlex
3857 
3858   Output Parameter:
3859 
3860   Notes:
3861   Concretely, DMPlexStratify() creates a new label named "depth" containing the depth in the DAG of each point. For cell-vertex
3862   meshes, vertices are depth 0 and cells are depth 1. For fully interpolated meshes, depth 0 for vertices, 1 for edges, and so on
3863   until cells have depth equal to the dimension of the mesh. The depth label can be accessed through DMPlexGetDepthLabel() or DMPlexGetDepthStratum(), or
3864   manually via DMGetLabel().  The height is defined implicitly by height = maxDimension - depth, and can be accessed
3865   via DMPlexGetHeightStratum().  For example, cells have height 0 and faces have height 1.
3866 
3867   The depth of a point is calculated by executing a breadth-first search (BFS) on the DAG. This could produce surprising results
3868   if run on a partially interpolated mesh, meaning one that had some edges and faces, but not others. For example, suppose that
3869   we had a mesh consisting of one triangle (c0) and three vertices (v0, v1, v2), and only one edge is on the boundary so we choose
3870   to interpolate only that one (e0), so that
3871 $  cone(c0) = {e0, v2}
3872 $  cone(e0) = {v0, v1}
3873   If DMPlexStratify() is run on this mesh, it will give depths
3874 $  depth 0 = {v0, v1, v2}
3875 $  depth 1 = {e0, c0}
3876   where the triangle has been given depth 1, instead of 2, because it is reachable from vertex v2.
3877 
3878   DMPlexStratify() should be called after all calls to DMPlexSymmetrize()
3879 
3880   Level: beginner
3881 
3882 .seealso: DMPlexCreate(), DMPlexSymmetrize(), DMPlexComputeCellTypes()
3883 @*/
3884 PetscErrorCode DMPlexStratify(DM dm)
3885 {
3886   DM_Plex       *mesh = (DM_Plex*) dm->data;
3887   DMLabel        label;
3888   PetscInt       pStart, pEnd, p;
3889   PetscInt       numRoots = 0, numLeaves = 0;
3890   PetscErrorCode ierr;
3891 
3892   PetscFunctionBegin;
3893   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3894   ierr = PetscLogEventBegin(DMPLEX_Stratify,dm,0,0,0);CHKERRQ(ierr);
3895 
3896   /* Create depth label */
3897   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
3898   ierr = DMCreateLabel(dm, "depth");CHKERRQ(ierr);
3899   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
3900 
3901   {
3902     /* Initialize roots and count leaves */
3903     PetscInt sMin = PETSC_MAX_INT;
3904     PetscInt sMax = PETSC_MIN_INT;
3905     PetscInt coneSize, supportSize;
3906 
3907     for (p = pStart; p < pEnd; ++p) {
3908       ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
3909       ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
3910       if (!coneSize && supportSize) {
3911         sMin = PetscMin(p, sMin);
3912         sMax = PetscMax(p, sMax);
3913         ++numRoots;
3914       } else if (!supportSize && coneSize) {
3915         ++numLeaves;
3916       } else if (!supportSize && !coneSize) {
3917         /* Isolated points */
3918         sMin = PetscMin(p, sMin);
3919         sMax = PetscMax(p, sMax);
3920       }
3921     }
3922     ierr = DMPlexCreateDepthStratum(dm, label, 0, sMin, sMax+1);CHKERRQ(ierr);
3923   }
3924 
3925   if (numRoots + numLeaves == (pEnd - pStart)) {
3926     PetscInt sMin = PETSC_MAX_INT;
3927     PetscInt sMax = PETSC_MIN_INT;
3928     PetscInt coneSize, supportSize;
3929 
3930     for (p = pStart; p < pEnd; ++p) {
3931       ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
3932       ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
3933       if (!supportSize && coneSize) {
3934         sMin = PetscMin(p, sMin);
3935         sMax = PetscMax(p, sMax);
3936       }
3937     }
3938     ierr = DMPlexCreateDepthStratum(dm, label, 1, sMin, sMax+1);CHKERRQ(ierr);
3939   } else {
3940     PetscInt level = 0;
3941     PetscInt qStart, qEnd, q;
3942 
3943     ierr = DMLabelGetStratumBounds(label, level, &qStart, &qEnd);CHKERRQ(ierr);
3944     while (qEnd > qStart) {
3945       PetscInt sMin = PETSC_MAX_INT;
3946       PetscInt sMax = PETSC_MIN_INT;
3947 
3948       for (q = qStart; q < qEnd; ++q) {
3949         const PetscInt *support;
3950         PetscInt        supportSize, s;
3951 
3952         ierr = DMPlexGetSupportSize(dm, q, &supportSize);CHKERRQ(ierr);
3953         ierr = DMPlexGetSupport(dm, q, &support);CHKERRQ(ierr);
3954         for (s = 0; s < supportSize; ++s) {
3955           sMin = PetscMin(support[s], sMin);
3956           sMax = PetscMax(support[s], sMax);
3957         }
3958       }
3959       ierr = DMLabelGetNumValues(label, &level);CHKERRQ(ierr);
3960       ierr = DMPlexCreateDepthStratum(dm, label, level, sMin, sMax+1);CHKERRQ(ierr);
3961       ierr = DMLabelGetStratumBounds(label, level, &qStart, &qEnd);CHKERRQ(ierr);
3962     }
3963   }
3964   { /* just in case there is an empty process */
3965     PetscInt numValues, maxValues = 0, v;
3966 
3967     ierr = DMLabelGetNumValues(label, &numValues);CHKERRQ(ierr);
3968     ierr = MPI_Allreduce(&numValues,&maxValues,1,MPIU_INT,MPI_MAX,PetscObjectComm((PetscObject)dm));CHKERRMPI(ierr);
3969     for (v = numValues; v < maxValues; v++) {
3970       ierr = DMLabelAddStratum(label, v);CHKERRQ(ierr);
3971     }
3972   }
3973   ierr = PetscObjectStateGet((PetscObject) label, &mesh->depthState);CHKERRQ(ierr);
3974   ierr = PetscLogEventEnd(DMPLEX_Stratify,dm,0,0,0);CHKERRQ(ierr);
3975   PetscFunctionReturn(0);
3976 }
3977 
3978 PetscErrorCode DMPlexComputeCellType_Internal(DM dm, PetscInt p, PetscInt pdepth, DMPolytopeType *pt)
3979 {
3980   DMPolytopeType ct = DM_POLYTOPE_UNKNOWN;
3981   PetscInt       dim, depth, pheight, coneSize;
3982   PetscErrorCode ierr;
3983 
3984   PetscFunctionBeginHot;
3985   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
3986   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3987   ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
3988   pheight = depth - pdepth;
3989   if (depth <= 1) {
3990     switch (pdepth) {
3991       case 0: ct = DM_POLYTOPE_POINT;break;
3992       case 1:
3993         switch (coneSize) {
3994           case 2: ct = DM_POLYTOPE_SEGMENT;break;
3995           case 3: ct = DM_POLYTOPE_TRIANGLE;break;
3996           case 4:
3997           switch (dim) {
3998             case 2: ct = DM_POLYTOPE_QUADRILATERAL;break;
3999             case 3: ct = DM_POLYTOPE_TETRAHEDRON;break;
4000             default: break;
4001           }
4002           break;
4003         case 5: ct = DM_POLYTOPE_PYRAMID;break;
4004         case 6: ct = DM_POLYTOPE_TRI_PRISM_TENSOR;break;
4005         case 8: ct = DM_POLYTOPE_HEXAHEDRON;break;
4006         default: break;
4007       }
4008     }
4009   } else {
4010     if (pdepth == 0) {
4011       ct = DM_POLYTOPE_POINT;
4012     } else if (pheight == 0) {
4013       switch (dim) {
4014         case 1:
4015           switch (coneSize) {
4016             case 2: ct = DM_POLYTOPE_SEGMENT;break;
4017             default: break;
4018           }
4019           break;
4020         case 2:
4021           switch (coneSize) {
4022             case 3: ct = DM_POLYTOPE_TRIANGLE;break;
4023             case 4: ct = DM_POLYTOPE_QUADRILATERAL;break;
4024             default: break;
4025           }
4026           break;
4027         case 3:
4028           switch (coneSize) {
4029             case 4: ct = DM_POLYTOPE_TETRAHEDRON;break;
4030             case 5:
4031             {
4032               const PetscInt *cone;
4033               PetscInt        faceConeSize;
4034 
4035               ierr = DMPlexGetCone(dm, p, &cone);CHKERRQ(ierr);
4036               ierr = DMPlexGetConeSize(dm, cone[0], &faceConeSize);CHKERRQ(ierr);
4037               switch (faceConeSize) {
4038                 case 3: ct = DM_POLYTOPE_TRI_PRISM_TENSOR;break;
4039                 case 4: ct = DM_POLYTOPE_PYRAMID;break;
4040               }
4041             }
4042             break;
4043             case 6: ct = DM_POLYTOPE_HEXAHEDRON;break;
4044             default: break;
4045           }
4046           break;
4047         default: break;
4048       }
4049     } else if (pheight > 0) {
4050       switch (coneSize) {
4051         case 2: ct = DM_POLYTOPE_SEGMENT;break;
4052         case 3: ct = DM_POLYTOPE_TRIANGLE;break;
4053         case 4: ct = DM_POLYTOPE_QUADRILATERAL;break;
4054         default: break;
4055       }
4056     }
4057   }
4058   *pt = ct;
4059   PetscFunctionReturn(0);
4060 }
4061 
4062 /*@
4063   DMPlexComputeCellTypes - Infer the polytope type of every cell using its dimension and cone size.
4064 
4065   Collective on dm
4066 
4067   Input Parameter:
4068 . mesh - The DMPlex
4069 
4070   DMPlexComputeCellTypes() should be called after all calls to DMPlexSymmetrize() and DMPlexStratify()
4071 
4072   Level: developer
4073 
4074   Note: This function is normally called automatically by Plex when a cell type is requested. It creates an
4075   internal DMLabel named "celltype" which can be directly accessed using DMGetLabel(). A user may disable
4076   automatic creation by creating the label manually, using DMCreateLabel(dm, "celltype").
4077 
4078 .seealso: DMPlexCreate(), DMPlexSymmetrize(), DMPlexStratify(), DMGetLabel(), DMCreateLabel()
4079 @*/
4080 PetscErrorCode DMPlexComputeCellTypes(DM dm)
4081 {
4082   DM_Plex       *mesh;
4083   DMLabel        ctLabel;
4084   PetscInt       pStart, pEnd, p;
4085   PetscErrorCode ierr;
4086 
4087   PetscFunctionBegin;
4088   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4089   mesh = (DM_Plex *) dm->data;
4090   ierr = DMCreateLabel(dm, "celltype");CHKERRQ(ierr);
4091   ierr = DMPlexGetCellTypeLabel(dm, &ctLabel);CHKERRQ(ierr);
4092   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
4093   for (p = pStart; p < pEnd; ++p) {
4094     DMPolytopeType ct = DM_POLYTOPE_UNKNOWN;
4095     PetscInt       pdepth;
4096 
4097     ierr = DMPlexGetPointDepth(dm, p, &pdepth);CHKERRQ(ierr);
4098     ierr = DMPlexComputeCellType_Internal(dm, p, pdepth, &ct);CHKERRQ(ierr);
4099     PetscAssertFalse(ct == DM_POLYTOPE_UNKNOWN,PETSC_COMM_SELF, PETSC_ERR_SUP, "Point %D is screwed up", p);
4100     ierr = DMLabelSetValue(ctLabel, p, ct);CHKERRQ(ierr);
4101   }
4102   ierr = PetscObjectStateGet((PetscObject) ctLabel, &mesh->celltypeState);CHKERRQ(ierr);
4103   ierr = PetscObjectViewFromOptions((PetscObject) ctLabel, NULL, "-dm_plex_celltypes_view");CHKERRQ(ierr);
4104   PetscFunctionReturn(0);
4105 }
4106 
4107 /*@C
4108   DMPlexGetJoin - Get an array for the join of the set of points
4109 
4110   Not Collective
4111 
4112   Input Parameters:
4113 + dm - The DMPlex object
4114 . numPoints - The number of input points for the join
4115 - points - The input points
4116 
4117   Output Parameters:
4118 + numCoveredPoints - The number of points in the join
4119 - coveredPoints - The points in the join
4120 
4121   Level: intermediate
4122 
4123   Note: Currently, this is restricted to a single level join
4124 
4125   Fortran Notes:
4126   Since it returns an array, this routine is only available in Fortran 90, and you must
4127   include petsc.h90 in your code.
4128 
4129   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
4130 
4131 .seealso: DMPlexRestoreJoin(), DMPlexGetMeet()
4132 @*/
4133 PetscErrorCode DMPlexGetJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
4134 {
4135   DM_Plex       *mesh = (DM_Plex*) dm->data;
4136   PetscInt      *join[2];
4137   PetscInt       joinSize, i = 0;
4138   PetscInt       dof, off, p, c, m;
4139   PetscErrorCode ierr;
4140 
4141   PetscFunctionBegin;
4142   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4143   PetscValidIntPointer(points, 3);
4144   PetscValidIntPointer(numCoveredPoints, 4);
4145   PetscValidPointer(coveredPoints, 5);
4146   ierr = DMGetWorkArray(dm, mesh->maxSupportSize, MPIU_INT, &join[0]);CHKERRQ(ierr);
4147   ierr = DMGetWorkArray(dm, mesh->maxSupportSize, MPIU_INT, &join[1]);CHKERRQ(ierr);
4148   /* Copy in support of first point */
4149   ierr = PetscSectionGetDof(mesh->supportSection, points[0], &dof);CHKERRQ(ierr);
4150   ierr = PetscSectionGetOffset(mesh->supportSection, points[0], &off);CHKERRQ(ierr);
4151   for (joinSize = 0; joinSize < dof; ++joinSize) {
4152     join[i][joinSize] = mesh->supports[off+joinSize];
4153   }
4154   /* Check each successive support */
4155   for (p = 1; p < numPoints; ++p) {
4156     PetscInt newJoinSize = 0;
4157 
4158     ierr = PetscSectionGetDof(mesh->supportSection, points[p], &dof);CHKERRQ(ierr);
4159     ierr = PetscSectionGetOffset(mesh->supportSection, points[p], &off);CHKERRQ(ierr);
4160     for (c = 0; c < dof; ++c) {
4161       const PetscInt point = mesh->supports[off+c];
4162 
4163       for (m = 0; m < joinSize; ++m) {
4164         if (point == join[i][m]) {
4165           join[1-i][newJoinSize++] = point;
4166           break;
4167         }
4168       }
4169     }
4170     joinSize = newJoinSize;
4171     i        = 1-i;
4172   }
4173   *numCoveredPoints = joinSize;
4174   *coveredPoints    = join[i];
4175   ierr              = DMRestoreWorkArray(dm, mesh->maxSupportSize, MPIU_INT, &join[1-i]);CHKERRQ(ierr);
4176   PetscFunctionReturn(0);
4177 }
4178 
4179 /*@C
4180   DMPlexRestoreJoin - Restore an array for the join of the set of points
4181 
4182   Not Collective
4183 
4184   Input Parameters:
4185 + dm - The DMPlex object
4186 . numPoints - The number of input points for the join
4187 - points - The input points
4188 
4189   Output Parameters:
4190 + numCoveredPoints - The number of points in the join
4191 - coveredPoints - The points in the join
4192 
4193   Fortran Notes:
4194   Since it returns an array, this routine is only available in Fortran 90, and you must
4195   include petsc.h90 in your code.
4196 
4197   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
4198 
4199   Level: intermediate
4200 
4201 .seealso: DMPlexGetJoin(), DMPlexGetFullJoin(), DMPlexGetMeet()
4202 @*/
4203 PetscErrorCode DMPlexRestoreJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
4204 {
4205   PetscErrorCode ierr;
4206 
4207   PetscFunctionBegin;
4208   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4209   if (points) PetscValidIntPointer(points,3);
4210   if (numCoveredPoints) PetscValidIntPointer(numCoveredPoints,4);
4211   PetscValidPointer(coveredPoints, 5);
4212   ierr = DMRestoreWorkArray(dm, 0, MPIU_INT, (void*) coveredPoints);CHKERRQ(ierr);
4213   if (numCoveredPoints) *numCoveredPoints = 0;
4214   PetscFunctionReturn(0);
4215 }
4216 
4217 /*@C
4218   DMPlexGetFullJoin - Get an array for the join of the set of points
4219 
4220   Not Collective
4221 
4222   Input Parameters:
4223 + dm - The DMPlex object
4224 . numPoints - The number of input points for the join
4225 - points - The input points
4226 
4227   Output Parameters:
4228 + numCoveredPoints - The number of points in the join
4229 - coveredPoints - The points in the join
4230 
4231   Fortran Notes:
4232   Since it returns an array, this routine is only available in Fortran 90, and you must
4233   include petsc.h90 in your code.
4234 
4235   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
4236 
4237   Level: intermediate
4238 
4239 .seealso: DMPlexGetJoin(), DMPlexRestoreJoin(), DMPlexGetMeet()
4240 @*/
4241 PetscErrorCode DMPlexGetFullJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
4242 {
4243   DM_Plex       *mesh = (DM_Plex*) dm->data;
4244   PetscInt      *offsets, **closures;
4245   PetscInt      *join[2];
4246   PetscInt       depth = 0, maxSize, joinSize = 0, i = 0;
4247   PetscInt       p, d, c, m, ms;
4248   PetscErrorCode ierr;
4249 
4250   PetscFunctionBegin;
4251   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4252   PetscValidIntPointer(points, 3);
4253   PetscValidIntPointer(numCoveredPoints, 4);
4254   PetscValidPointer(coveredPoints, 5);
4255 
4256   ierr    = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
4257   ierr    = PetscCalloc1(numPoints, &closures);CHKERRQ(ierr);
4258   ierr    = DMGetWorkArray(dm, numPoints*(depth+2), MPIU_INT, &offsets);CHKERRQ(ierr);
4259   ms      = mesh->maxSupportSize;
4260   maxSize = (ms > 1) ? ((PetscPowInt(ms,depth+1)-1)/(ms-1)) : depth + 1;
4261   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &join[0]);CHKERRQ(ierr);
4262   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &join[1]);CHKERRQ(ierr);
4263 
4264   for (p = 0; p < numPoints; ++p) {
4265     PetscInt closureSize;
4266 
4267     ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_FALSE, &closureSize, &closures[p]);CHKERRQ(ierr);
4268 
4269     offsets[p*(depth+2)+0] = 0;
4270     for (d = 0; d < depth+1; ++d) {
4271       PetscInt pStart, pEnd, i;
4272 
4273       ierr = DMPlexGetDepthStratum(dm, d, &pStart, &pEnd);CHKERRQ(ierr);
4274       for (i = offsets[p*(depth+2)+d]; i < closureSize; ++i) {
4275         if ((pStart > closures[p][i*2]) || (pEnd <= closures[p][i*2])) {
4276           offsets[p*(depth+2)+d+1] = i;
4277           break;
4278         }
4279       }
4280       if (i == closureSize) offsets[p*(depth+2)+d+1] = i;
4281     }
4282     PetscAssertFalse(offsets[p*(depth+2)+depth+1] != closureSize,PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Total size of closure %D should be %D", offsets[p*(depth+2)+depth+1], closureSize);
4283   }
4284   for (d = 0; d < depth+1; ++d) {
4285     PetscInt dof;
4286 
4287     /* Copy in support of first point */
4288     dof = offsets[d+1] - offsets[d];
4289     for (joinSize = 0; joinSize < dof; ++joinSize) {
4290       join[i][joinSize] = closures[0][(offsets[d]+joinSize)*2];
4291     }
4292     /* Check each successive cone */
4293     for (p = 1; p < numPoints && joinSize; ++p) {
4294       PetscInt newJoinSize = 0;
4295 
4296       dof = offsets[p*(depth+2)+d+1] - offsets[p*(depth+2)+d];
4297       for (c = 0; c < dof; ++c) {
4298         const PetscInt point = closures[p][(offsets[p*(depth+2)+d]+c)*2];
4299 
4300         for (m = 0; m < joinSize; ++m) {
4301           if (point == join[i][m]) {
4302             join[1-i][newJoinSize++] = point;
4303             break;
4304           }
4305         }
4306       }
4307       joinSize = newJoinSize;
4308       i        = 1-i;
4309     }
4310     if (joinSize) break;
4311   }
4312   *numCoveredPoints = joinSize;
4313   *coveredPoints    = join[i];
4314   for (p = 0; p < numPoints; ++p) {
4315     ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_FALSE, NULL, &closures[p]);CHKERRQ(ierr);
4316   }
4317   ierr = PetscFree(closures);CHKERRQ(ierr);
4318   ierr = DMRestoreWorkArray(dm, numPoints*(depth+2), MPIU_INT, &offsets);CHKERRQ(ierr);
4319   ierr = DMRestoreWorkArray(dm, mesh->maxSupportSize, MPIU_INT, &join[1-i]);CHKERRQ(ierr);
4320   PetscFunctionReturn(0);
4321 }
4322 
4323 /*@C
4324   DMPlexGetMeet - Get an array for the meet of the set of points
4325 
4326   Not Collective
4327 
4328   Input Parameters:
4329 + dm - The DMPlex object
4330 . numPoints - The number of input points for the meet
4331 - points - The input points
4332 
4333   Output Parameters:
4334 + numCoveredPoints - The number of points in the meet
4335 - coveredPoints - The points in the meet
4336 
4337   Level: intermediate
4338 
4339   Note: Currently, this is restricted to a single level meet
4340 
4341   Fortran Notes:
4342   Since it returns an array, this routine is only available in Fortran 90, and you must
4343   include petsc.h90 in your code.
4344 
4345   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
4346 
4347 .seealso: DMPlexRestoreMeet(), DMPlexGetJoin()
4348 @*/
4349 PetscErrorCode DMPlexGetMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveringPoints, const PetscInt **coveringPoints)
4350 {
4351   DM_Plex       *mesh = (DM_Plex*) dm->data;
4352   PetscInt      *meet[2];
4353   PetscInt       meetSize, i = 0;
4354   PetscInt       dof, off, p, c, m;
4355   PetscErrorCode ierr;
4356 
4357   PetscFunctionBegin;
4358   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4359   PetscValidPointer(points, 3);
4360   PetscValidPointer(numCoveringPoints, 4);
4361   PetscValidPointer(coveringPoints, 5);
4362   ierr = DMGetWorkArray(dm, mesh->maxConeSize, MPIU_INT, &meet[0]);CHKERRQ(ierr);
4363   ierr = DMGetWorkArray(dm, mesh->maxConeSize, MPIU_INT, &meet[1]);CHKERRQ(ierr);
4364   /* Copy in cone of first point */
4365   ierr = PetscSectionGetDof(mesh->coneSection, points[0], &dof);CHKERRQ(ierr);
4366   ierr = PetscSectionGetOffset(mesh->coneSection, points[0], &off);CHKERRQ(ierr);
4367   for (meetSize = 0; meetSize < dof; ++meetSize) {
4368     meet[i][meetSize] = mesh->cones[off+meetSize];
4369   }
4370   /* Check each successive cone */
4371   for (p = 1; p < numPoints; ++p) {
4372     PetscInt newMeetSize = 0;
4373 
4374     ierr = PetscSectionGetDof(mesh->coneSection, points[p], &dof);CHKERRQ(ierr);
4375     ierr = PetscSectionGetOffset(mesh->coneSection, points[p], &off);CHKERRQ(ierr);
4376     for (c = 0; c < dof; ++c) {
4377       const PetscInt point = mesh->cones[off+c];
4378 
4379       for (m = 0; m < meetSize; ++m) {
4380         if (point == meet[i][m]) {
4381           meet[1-i][newMeetSize++] = point;
4382           break;
4383         }
4384       }
4385     }
4386     meetSize = newMeetSize;
4387     i        = 1-i;
4388   }
4389   *numCoveringPoints = meetSize;
4390   *coveringPoints    = meet[i];
4391   ierr               = DMRestoreWorkArray(dm, mesh->maxConeSize, MPIU_INT, &meet[1-i]);CHKERRQ(ierr);
4392   PetscFunctionReturn(0);
4393 }
4394 
4395 /*@C
4396   DMPlexRestoreMeet - Restore an array for the meet of the set of points
4397 
4398   Not Collective
4399 
4400   Input Parameters:
4401 + dm - The DMPlex object
4402 . numPoints - The number of input points for the meet
4403 - points - The input points
4404 
4405   Output Parameters:
4406 + numCoveredPoints - The number of points in the meet
4407 - coveredPoints - The points in the meet
4408 
4409   Level: intermediate
4410 
4411   Fortran Notes:
4412   Since it returns an array, this routine is only available in Fortran 90, and you must
4413   include petsc.h90 in your code.
4414 
4415   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
4416 
4417 .seealso: DMPlexGetMeet(), DMPlexGetFullMeet(), DMPlexGetJoin()
4418 @*/
4419 PetscErrorCode DMPlexRestoreMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
4420 {
4421   PetscErrorCode ierr;
4422 
4423   PetscFunctionBegin;
4424   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4425   if (points) PetscValidIntPointer(points,3);
4426   if (numCoveredPoints) PetscValidIntPointer(numCoveredPoints,4);
4427   PetscValidPointer(coveredPoints,5);
4428   ierr = DMRestoreWorkArray(dm, 0, MPIU_INT, (void*) coveredPoints);CHKERRQ(ierr);
4429   if (numCoveredPoints) *numCoveredPoints = 0;
4430   PetscFunctionReturn(0);
4431 }
4432 
4433 /*@C
4434   DMPlexGetFullMeet - Get an array for the meet of the set of points
4435 
4436   Not Collective
4437 
4438   Input Parameters:
4439 + dm - The DMPlex object
4440 . numPoints - The number of input points for the meet
4441 - points - The input points
4442 
4443   Output Parameters:
4444 + numCoveredPoints - The number of points in the meet
4445 - coveredPoints - The points in the meet
4446 
4447   Level: intermediate
4448 
4449   Fortran Notes:
4450   Since it returns an array, this routine is only available in Fortran 90, and you must
4451   include petsc.h90 in your code.
4452 
4453   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
4454 
4455 .seealso: DMPlexGetMeet(), DMPlexRestoreMeet(), DMPlexGetJoin()
4456 @*/
4457 PetscErrorCode DMPlexGetFullMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
4458 {
4459   DM_Plex       *mesh = (DM_Plex*) dm->data;
4460   PetscInt      *offsets, **closures;
4461   PetscInt      *meet[2];
4462   PetscInt       height = 0, maxSize, meetSize = 0, i = 0;
4463   PetscInt       p, h, c, m, mc;
4464   PetscErrorCode ierr;
4465 
4466   PetscFunctionBegin;
4467   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4468   PetscValidPointer(points, 3);
4469   PetscValidPointer(numCoveredPoints, 4);
4470   PetscValidPointer(coveredPoints, 5);
4471 
4472   ierr    = DMPlexGetDepth(dm, &height);CHKERRQ(ierr);
4473   ierr    = PetscMalloc1(numPoints, &closures);CHKERRQ(ierr);
4474   ierr    = DMGetWorkArray(dm, numPoints*(height+2), MPIU_INT, &offsets);CHKERRQ(ierr);
4475   mc      = mesh->maxConeSize;
4476   maxSize = (mc > 1) ? ((PetscPowInt(mc,height+1)-1)/(mc-1)) : height + 1;
4477   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &meet[0]);CHKERRQ(ierr);
4478   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &meet[1]);CHKERRQ(ierr);
4479 
4480   for (p = 0; p < numPoints; ++p) {
4481     PetscInt closureSize;
4482 
4483     ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_TRUE, &closureSize, &closures[p]);CHKERRQ(ierr);
4484 
4485     offsets[p*(height+2)+0] = 0;
4486     for (h = 0; h < height+1; ++h) {
4487       PetscInt pStart, pEnd, i;
4488 
4489       ierr = DMPlexGetHeightStratum(dm, h, &pStart, &pEnd);CHKERRQ(ierr);
4490       for (i = offsets[p*(height+2)+h]; i < closureSize; ++i) {
4491         if ((pStart > closures[p][i*2]) || (pEnd <= closures[p][i*2])) {
4492           offsets[p*(height+2)+h+1] = i;
4493           break;
4494         }
4495       }
4496       if (i == closureSize) offsets[p*(height+2)+h+1] = i;
4497     }
4498     PetscAssertFalse(offsets[p*(height+2)+height+1] != closureSize,PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Total size of closure %D should be %D", offsets[p*(height+2)+height+1], closureSize);
4499   }
4500   for (h = 0; h < height+1; ++h) {
4501     PetscInt dof;
4502 
4503     /* Copy in cone of first point */
4504     dof = offsets[h+1] - offsets[h];
4505     for (meetSize = 0; meetSize < dof; ++meetSize) {
4506       meet[i][meetSize] = closures[0][(offsets[h]+meetSize)*2];
4507     }
4508     /* Check each successive cone */
4509     for (p = 1; p < numPoints && meetSize; ++p) {
4510       PetscInt newMeetSize = 0;
4511 
4512       dof = offsets[p*(height+2)+h+1] - offsets[p*(height+2)+h];
4513       for (c = 0; c < dof; ++c) {
4514         const PetscInt point = closures[p][(offsets[p*(height+2)+h]+c)*2];
4515 
4516         for (m = 0; m < meetSize; ++m) {
4517           if (point == meet[i][m]) {
4518             meet[1-i][newMeetSize++] = point;
4519             break;
4520           }
4521         }
4522       }
4523       meetSize = newMeetSize;
4524       i        = 1-i;
4525     }
4526     if (meetSize) break;
4527   }
4528   *numCoveredPoints = meetSize;
4529   *coveredPoints    = meet[i];
4530   for (p = 0; p < numPoints; ++p) {
4531     ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_TRUE, NULL, &closures[p]);CHKERRQ(ierr);
4532   }
4533   ierr = PetscFree(closures);CHKERRQ(ierr);
4534   ierr = DMRestoreWorkArray(dm, numPoints*(height+2), MPIU_INT, &offsets);CHKERRQ(ierr);
4535   ierr = DMRestoreWorkArray(dm, mesh->maxConeSize, MPIU_INT, &meet[1-i]);CHKERRQ(ierr);
4536   PetscFunctionReturn(0);
4537 }
4538 
4539 /*@C
4540   DMPlexEqual - Determine if two DMs have the same topology
4541 
4542   Not Collective
4543 
4544   Input Parameters:
4545 + dmA - A DMPlex object
4546 - dmB - A DMPlex object
4547 
4548   Output Parameters:
4549 . equal - PETSC_TRUE if the topologies are identical
4550 
4551   Level: intermediate
4552 
4553   Notes:
4554   We are not solving graph isomorphism, so we do not permutation.
4555 
4556 .seealso: DMPlexGetCone()
4557 @*/
4558 PetscErrorCode DMPlexEqual(DM dmA, DM dmB, PetscBool *equal)
4559 {
4560   PetscInt       depth, depthB, pStart, pEnd, pStartB, pEndB, p;
4561   PetscErrorCode ierr;
4562 
4563   PetscFunctionBegin;
4564   PetscValidHeaderSpecific(dmA, DM_CLASSID, 1);
4565   PetscValidHeaderSpecific(dmB, DM_CLASSID, 2);
4566   PetscValidPointer(equal, 3);
4567 
4568   *equal = PETSC_FALSE;
4569   ierr = DMPlexGetDepth(dmA, &depth);CHKERRQ(ierr);
4570   ierr = DMPlexGetDepth(dmB, &depthB);CHKERRQ(ierr);
4571   if (depth != depthB) PetscFunctionReturn(0);
4572   ierr = DMPlexGetChart(dmA, &pStart,  &pEnd);CHKERRQ(ierr);
4573   ierr = DMPlexGetChart(dmB, &pStartB, &pEndB);CHKERRQ(ierr);
4574   if ((pStart != pStartB) || (pEnd != pEndB)) PetscFunctionReturn(0);
4575   for (p = pStart; p < pEnd; ++p) {
4576     const PetscInt *cone, *coneB, *ornt, *orntB, *support, *supportB;
4577     PetscInt        coneSize, coneSizeB, c, supportSize, supportSizeB, s;
4578 
4579     ierr = DMPlexGetConeSize(dmA, p, &coneSize);CHKERRQ(ierr);
4580     ierr = DMPlexGetCone(dmA, p, &cone);CHKERRQ(ierr);
4581     ierr = DMPlexGetConeOrientation(dmA, p, &ornt);CHKERRQ(ierr);
4582     ierr = DMPlexGetConeSize(dmB, p, &coneSizeB);CHKERRQ(ierr);
4583     ierr = DMPlexGetCone(dmB, p, &coneB);CHKERRQ(ierr);
4584     ierr = DMPlexGetConeOrientation(dmB, p, &orntB);CHKERRQ(ierr);
4585     if (coneSize != coneSizeB) PetscFunctionReturn(0);
4586     for (c = 0; c < coneSize; ++c) {
4587       if (cone[c] != coneB[c]) PetscFunctionReturn(0);
4588       if (ornt[c] != orntB[c]) PetscFunctionReturn(0);
4589     }
4590     ierr = DMPlexGetSupportSize(dmA, p, &supportSize);CHKERRQ(ierr);
4591     ierr = DMPlexGetSupport(dmA, p, &support);CHKERRQ(ierr);
4592     ierr = DMPlexGetSupportSize(dmB, p, &supportSizeB);CHKERRQ(ierr);
4593     ierr = DMPlexGetSupport(dmB, p, &supportB);CHKERRQ(ierr);
4594     if (supportSize != supportSizeB) PetscFunctionReturn(0);
4595     for (s = 0; s < supportSize; ++s) {
4596       if (support[s] != supportB[s]) PetscFunctionReturn(0);
4597     }
4598   }
4599   *equal = PETSC_TRUE;
4600   PetscFunctionReturn(0);
4601 }
4602 
4603 /*@C
4604   DMPlexGetNumFaceVertices - Returns the number of vertices on a face
4605 
4606   Not Collective
4607 
4608   Input Parameters:
4609 + dm         - The DMPlex
4610 . cellDim    - The cell dimension
4611 - numCorners - The number of vertices on a cell
4612 
4613   Output Parameters:
4614 . numFaceVertices - The number of vertices on a face
4615 
4616   Level: developer
4617 
4618   Notes:
4619   Of course this can only work for a restricted set of symmetric shapes
4620 
4621 .seealso: DMPlexGetCone()
4622 @*/
4623 PetscErrorCode DMPlexGetNumFaceVertices(DM dm, PetscInt cellDim, PetscInt numCorners, PetscInt *numFaceVertices)
4624 {
4625   MPI_Comm       comm;
4626   PetscErrorCode ierr;
4627 
4628   PetscFunctionBegin;
4629   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
4630   PetscValidPointer(numFaceVertices,4);
4631   switch (cellDim) {
4632   case 0:
4633     *numFaceVertices = 0;
4634     break;
4635   case 1:
4636     *numFaceVertices = 1;
4637     break;
4638   case 2:
4639     switch (numCorners) {
4640     case 3: /* triangle */
4641       *numFaceVertices = 2; /* Edge has 2 vertices */
4642       break;
4643     case 4: /* quadrilateral */
4644       *numFaceVertices = 2; /* Edge has 2 vertices */
4645       break;
4646     case 6: /* quadratic triangle, tri and quad cohesive Lagrange cells */
4647       *numFaceVertices = 3; /* Edge has 3 vertices */
4648       break;
4649     case 9: /* quadratic quadrilateral, quadratic quad cohesive Lagrange cells */
4650       *numFaceVertices = 3; /* Edge has 3 vertices */
4651       break;
4652     default:
4653       SETERRQ(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid number of face corners %D for dimension %D", numCorners, cellDim);
4654     }
4655     break;
4656   case 3:
4657     switch (numCorners) {
4658     case 4: /* tetradehdron */
4659       *numFaceVertices = 3; /* Face has 3 vertices */
4660       break;
4661     case 6: /* tet cohesive cells */
4662       *numFaceVertices = 4; /* Face has 4 vertices */
4663       break;
4664     case 8: /* hexahedron */
4665       *numFaceVertices = 4; /* Face has 4 vertices */
4666       break;
4667     case 9: /* tet cohesive Lagrange cells */
4668       *numFaceVertices = 6; /* Face has 6 vertices */
4669       break;
4670     case 10: /* quadratic tetrahedron */
4671       *numFaceVertices = 6; /* Face has 6 vertices */
4672       break;
4673     case 12: /* hex cohesive Lagrange cells */
4674       *numFaceVertices = 6; /* Face has 6 vertices */
4675       break;
4676     case 18: /* quadratic tet cohesive Lagrange cells */
4677       *numFaceVertices = 6; /* Face has 6 vertices */
4678       break;
4679     case 27: /* quadratic hexahedron, quadratic hex cohesive Lagrange cells */
4680       *numFaceVertices = 9; /* Face has 9 vertices */
4681       break;
4682     default:
4683       SETERRQ(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid number of face corners %D for dimension %D", numCorners, cellDim);
4684     }
4685     break;
4686   default:
4687     SETERRQ(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid cell dimension %D", cellDim);
4688   }
4689   PetscFunctionReturn(0);
4690 }
4691 
4692 /*@
4693   DMPlexGetDepthLabel - Get the DMLabel recording the depth of each point
4694 
4695   Not Collective
4696 
4697   Input Parameter:
4698 . dm    - The DMPlex object
4699 
4700   Output Parameter:
4701 . depthLabel - The DMLabel recording point depth
4702 
4703   Level: developer
4704 
4705 .seealso: DMPlexGetDepth(), DMPlexGetHeightStratum(), DMPlexGetDepthStratum(), DMPlexGetPointDepth(),
4706 @*/
4707 PetscErrorCode DMPlexGetDepthLabel(DM dm, DMLabel *depthLabel)
4708 {
4709   PetscFunctionBegin;
4710   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4711   PetscValidPointer(depthLabel, 2);
4712   *depthLabel = dm->depthLabel;
4713   PetscFunctionReturn(0);
4714 }
4715 
4716 /*@
4717   DMPlexGetDepth - Get the depth of the DAG representing this mesh
4718 
4719   Not Collective
4720 
4721   Input Parameter:
4722 . dm    - The DMPlex object
4723 
4724   Output Parameter:
4725 . depth - The number of strata (breadth first levels) in the DAG
4726 
4727   Level: developer
4728 
4729   Notes:
4730   This returns maximum of point depths over all points, i.e. maximum value of the label returned by DMPlexGetDepthLabel().
4731   The point depth is described more in detail in DMPlexGetDepthStratum().
4732   An empty mesh gives -1.
4733 
4734 .seealso: DMPlexGetDepthLabel(), DMPlexGetDepthStratum(), DMPlexGetPointDepth(), DMPlexSymmetrize()
4735 @*/
4736 PetscErrorCode DMPlexGetDepth(DM dm, PetscInt *depth)
4737 {
4738   DMLabel        label;
4739   PetscInt       d = 0;
4740   PetscErrorCode ierr;
4741 
4742   PetscFunctionBegin;
4743   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4744   PetscValidPointer(depth, 2);
4745   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
4746   if (label) {ierr = DMLabelGetNumValues(label, &d);CHKERRQ(ierr);}
4747   *depth = d-1;
4748   PetscFunctionReturn(0);
4749 }
4750 
4751 /*@
4752   DMPlexGetDepthStratum - Get the bounds [start, end) for all points at a certain depth.
4753 
4754   Not Collective
4755 
4756   Input Parameters:
4757 + dm           - The DMPlex object
4758 - stratumValue - The requested depth
4759 
4760   Output Parameters:
4761 + start - The first point at this depth
4762 - end   - One beyond the last point at this depth
4763 
4764   Notes:
4765   Depth indexing is related to topological dimension.  Depth stratum 0 contains the lowest topological dimension points,
4766   often "vertices".  If the mesh is "interpolated" (see DMPlexInterpolate()), then depth stratum 1 contains the next
4767   higher dimension, e.g., "edges".
4768 
4769   Level: developer
4770 
4771 .seealso: DMPlexGetHeightStratum(), DMPlexGetDepth(), DMPlexGetDepthLabel(), DMPlexGetPointDepth(), DMPlexSymmetrize(), DMPlexInterpolate()
4772 @*/
4773 PetscErrorCode DMPlexGetDepthStratum(DM dm, PetscInt stratumValue, PetscInt *start, PetscInt *end)
4774 {
4775   DMLabel        label;
4776   PetscInt       pStart, pEnd;
4777   PetscErrorCode ierr;
4778 
4779   PetscFunctionBegin;
4780   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4781   if (start) {PetscValidPointer(start, 3); *start = 0;}
4782   if (end)   {PetscValidPointer(end,   4); *end   = 0;}
4783   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
4784   if (pStart == pEnd) PetscFunctionReturn(0);
4785   if (stratumValue < 0) {
4786     if (start) *start = pStart;
4787     if (end)   *end   = pEnd;
4788     PetscFunctionReturn(0);
4789   }
4790   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
4791   PetscAssertFalse(!label,PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "No label named depth was found");
4792   ierr = DMLabelGetStratumBounds(label, stratumValue, start, end);CHKERRQ(ierr);
4793   PetscFunctionReturn(0);
4794 }
4795 
4796 /*@
4797   DMPlexGetHeightStratum - Get the bounds [start, end) for all points at a certain height.
4798 
4799   Not Collective
4800 
4801   Input Parameters:
4802 + dm           - The DMPlex object
4803 - stratumValue - The requested height
4804 
4805   Output Parameters:
4806 + start - The first point at this height
4807 - end   - One beyond the last point at this height
4808 
4809   Notes:
4810   Height indexing is related to topological codimension.  Height stratum 0 contains the highest topological dimension
4811   points, often called "cells" or "elements".  If the mesh is "interpolated" (see DMPlexInterpolate()), then height
4812   stratum 1 contains the boundary of these "cells", often called "faces" or "facets".
4813 
4814   Level: developer
4815 
4816 .seealso: DMPlexGetDepthStratum(), DMPlexGetDepth(), DMPlexGetPointHeight()
4817 @*/
4818 PetscErrorCode DMPlexGetHeightStratum(DM dm, PetscInt stratumValue, PetscInt *start, PetscInt *end)
4819 {
4820   DMLabel        label;
4821   PetscInt       depth, pStart, pEnd;
4822   PetscErrorCode ierr;
4823 
4824   PetscFunctionBegin;
4825   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4826   if (start) {PetscValidPointer(start, 3); *start = 0;}
4827   if (end)   {PetscValidPointer(end,   4); *end   = 0;}
4828   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
4829   if (pStart == pEnd) PetscFunctionReturn(0);
4830   if (stratumValue < 0) {
4831     if (start) *start = pStart;
4832     if (end)   *end   = pEnd;
4833     PetscFunctionReturn(0);
4834   }
4835   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
4836   PetscAssertFalse(!label,PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "No label named depth was found");
4837   ierr = DMLabelGetNumValues(label, &depth);CHKERRQ(ierr);
4838   ierr = DMLabelGetStratumBounds(label, depth-1-stratumValue, start, end);CHKERRQ(ierr);
4839   PetscFunctionReturn(0);
4840 }
4841 
4842 /*@
4843   DMPlexGetPointDepth - Get the depth of a given point
4844 
4845   Not Collective
4846 
4847   Input Parameters:
4848 + dm    - The DMPlex object
4849 - point - The point
4850 
4851   Output Parameter:
4852 . depth - The depth of the point
4853 
4854   Level: intermediate
4855 
4856 .seealso: DMPlexGetCellType(), DMPlexGetDepthLabel(), DMPlexGetDepth(), DMPlexGetPointHeight()
4857 @*/
4858 PetscErrorCode DMPlexGetPointDepth(DM dm, PetscInt point, PetscInt *depth)
4859 {
4860   PetscErrorCode ierr;
4861 
4862   PetscFunctionBegin;
4863   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4864   PetscValidIntPointer(depth, 3);
4865   ierr = DMLabelGetValue(dm->depthLabel, point, depth);CHKERRQ(ierr);
4866   PetscFunctionReturn(0);
4867 }
4868 
4869 /*@
4870   DMPlexGetPointHeight - Get the height of a given point
4871 
4872   Not Collective
4873 
4874   Input Parameters:
4875 + dm    - The DMPlex object
4876 - point - The point
4877 
4878   Output Parameter:
4879 . height - The height of the point
4880 
4881   Level: intermediate
4882 
4883 .seealso: DMPlexGetCellType(), DMPlexGetDepthLabel(), DMPlexGetDepth(), DMPlexGetPointDepth()
4884 @*/
4885 PetscErrorCode DMPlexGetPointHeight(DM dm, PetscInt point, PetscInt *height)
4886 {
4887   PetscInt       n, pDepth;
4888   PetscErrorCode ierr;
4889 
4890   PetscFunctionBegin;
4891   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4892   PetscValidIntPointer(height, 3);
4893   ierr = DMLabelGetNumValues(dm->depthLabel, &n);CHKERRQ(ierr);
4894   ierr = DMLabelGetValue(dm->depthLabel, point, &pDepth);CHKERRQ(ierr);
4895   *height = n - 1 - pDepth;  /* DAG depth is n-1 */
4896   PetscFunctionReturn(0);
4897 }
4898 
4899 /*@
4900   DMPlexGetCellTypeLabel - Get the DMLabel recording the polytope type of each cell
4901 
4902   Not Collective
4903 
4904   Input Parameter:
4905 . dm - The DMPlex object
4906 
4907   Output Parameter:
4908 . celltypeLabel - The DMLabel recording cell polytope type
4909 
4910   Note: This function will trigger automatica computation of cell types. This can be disabled by calling
4911   DMCreateLabel(dm, "celltype") beforehand.
4912 
4913   Level: developer
4914 
4915 .seealso: DMPlexGetCellType(), DMPlexGetDepthLabel(), DMCreateLabel()
4916 @*/
4917 PetscErrorCode DMPlexGetCellTypeLabel(DM dm, DMLabel *celltypeLabel)
4918 {
4919   PetscErrorCode ierr;
4920 
4921   PetscFunctionBegin;
4922   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4923   PetscValidPointer(celltypeLabel, 2);
4924   if (!dm->celltypeLabel) {ierr = DMPlexComputeCellTypes(dm);CHKERRQ(ierr);}
4925   *celltypeLabel = dm->celltypeLabel;
4926   PetscFunctionReturn(0);
4927 }
4928 
4929 /*@
4930   DMPlexGetCellType - Get the polytope type of a given cell
4931 
4932   Not Collective
4933 
4934   Input Parameters:
4935 + dm   - The DMPlex object
4936 - cell - The cell
4937 
4938   Output Parameter:
4939 . celltype - The polytope type of the cell
4940 
4941   Level: intermediate
4942 
4943 .seealso: DMPlexGetCellTypeLabel(), DMPlexGetDepthLabel(), DMPlexGetDepth()
4944 @*/
4945 PetscErrorCode DMPlexGetCellType(DM dm, PetscInt cell, DMPolytopeType *celltype)
4946 {
4947   DMLabel        label;
4948   PetscInt       ct;
4949   PetscErrorCode ierr;
4950 
4951   PetscFunctionBegin;
4952   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4953   PetscValidPointer(celltype, 3);
4954   ierr = DMPlexGetCellTypeLabel(dm, &label);CHKERRQ(ierr);
4955   ierr = DMLabelGetValue(label, cell, &ct);CHKERRQ(ierr);
4956   PetscAssertFalse(ct < 0,PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Cell %D has not been assigned a cell type", cell);
4957   *celltype = (DMPolytopeType) ct;
4958   PetscFunctionReturn(0);
4959 }
4960 
4961 /*@
4962   DMPlexSetCellType - Set the polytope type of a given cell
4963 
4964   Not Collective
4965 
4966   Input Parameters:
4967 + dm   - The DMPlex object
4968 . cell - The cell
4969 - celltype - The polytope type of the cell
4970 
4971   Note: By default, cell types will be automatically computed using DMPlexComputeCellTypes() before this function
4972   is executed. This function will override the computed type. However, if automatic classification will not succeed
4973   and a user wants to manually specify all types, the classification must be disabled by calling
4974   DMCreaateLabel(dm, "celltype") before getting or setting any cell types.
4975 
4976   Level: advanced
4977 
4978 .seealso: DMPlexGetCellTypeLabel(), DMPlexGetDepthLabel(), DMPlexGetDepth(), DMPlexComputeCellTypes(), DMCreateLabel()
4979 @*/
4980 PetscErrorCode DMPlexSetCellType(DM dm, PetscInt cell, DMPolytopeType celltype)
4981 {
4982   DMLabel        label;
4983   PetscErrorCode ierr;
4984 
4985   PetscFunctionBegin;
4986   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4987   ierr = DMPlexGetCellTypeLabel(dm, &label);CHKERRQ(ierr);
4988   ierr = DMLabelSetValue(label, cell, celltype);CHKERRQ(ierr);
4989   PetscFunctionReturn(0);
4990 }
4991 
4992 PetscErrorCode DMCreateCoordinateDM_Plex(DM dm, DM *cdm)
4993 {
4994   PetscSection   section, s;
4995   Mat            m;
4996   PetscInt       maxHeight;
4997   PetscErrorCode ierr;
4998 
4999   PetscFunctionBegin;
5000   ierr = DMClone(dm, cdm);CHKERRQ(ierr);
5001   ierr = DMPlexGetMaxProjectionHeight(dm, &maxHeight);CHKERRQ(ierr);
5002   ierr = DMPlexSetMaxProjectionHeight(*cdm, maxHeight);CHKERRQ(ierr);
5003   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &section);CHKERRQ(ierr);
5004   ierr = DMSetLocalSection(*cdm, section);CHKERRQ(ierr);
5005   ierr = PetscSectionDestroy(&section);CHKERRQ(ierr);
5006   ierr = PetscSectionCreate(PETSC_COMM_SELF, &s);CHKERRQ(ierr);
5007   ierr = MatCreate(PETSC_COMM_SELF, &m);CHKERRQ(ierr);
5008   ierr = DMSetDefaultConstraints(*cdm, s, m);CHKERRQ(ierr);
5009   ierr = PetscSectionDestroy(&s);CHKERRQ(ierr);
5010   ierr = MatDestroy(&m);CHKERRQ(ierr);
5011 
5012   ierr = DMSetNumFields(*cdm, 1);CHKERRQ(ierr);
5013   ierr = DMCreateDS(*cdm);CHKERRQ(ierr);
5014   PetscFunctionReturn(0);
5015 }
5016 
5017 PetscErrorCode DMCreateCoordinateField_Plex(DM dm, DMField *field)
5018 {
5019   Vec            coordsLocal;
5020   DM             coordsDM;
5021   PetscErrorCode ierr;
5022 
5023   PetscFunctionBegin;
5024   *field = NULL;
5025   ierr = DMGetCoordinatesLocal(dm,&coordsLocal);CHKERRQ(ierr);
5026   ierr = DMGetCoordinateDM(dm,&coordsDM);CHKERRQ(ierr);
5027   if (coordsLocal && coordsDM) {
5028     ierr = DMFieldCreateDS(coordsDM, 0, coordsLocal, field);CHKERRQ(ierr);
5029   }
5030   PetscFunctionReturn(0);
5031 }
5032 
5033 /*@C
5034   DMPlexGetConeSection - Return a section which describes the layout of cone data
5035 
5036   Not Collective
5037 
5038   Input Parameters:
5039 . dm        - The DMPlex object
5040 
5041   Output Parameter:
5042 . section - The PetscSection object
5043 
5044   Level: developer
5045 
5046 .seealso: DMPlexGetSupportSection(), DMPlexGetCones(), DMPlexGetConeOrientations()
5047 @*/
5048 PetscErrorCode DMPlexGetConeSection(DM dm, PetscSection *section)
5049 {
5050   DM_Plex *mesh = (DM_Plex*) dm->data;
5051 
5052   PetscFunctionBegin;
5053   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5054   if (section) *section = mesh->coneSection;
5055   PetscFunctionReturn(0);
5056 }
5057 
5058 /*@C
5059   DMPlexGetSupportSection - Return a section which describes the layout of support data
5060 
5061   Not Collective
5062 
5063   Input Parameters:
5064 . dm        - The DMPlex object
5065 
5066   Output Parameter:
5067 . section - The PetscSection object
5068 
5069   Level: developer
5070 
5071 .seealso: DMPlexGetConeSection()
5072 @*/
5073 PetscErrorCode DMPlexGetSupportSection(DM dm, PetscSection *section)
5074 {
5075   DM_Plex *mesh = (DM_Plex*) dm->data;
5076 
5077   PetscFunctionBegin;
5078   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5079   if (section) *section = mesh->supportSection;
5080   PetscFunctionReturn(0);
5081 }
5082 
5083 /*@C
5084   DMPlexGetCones - Return cone data
5085 
5086   Not Collective
5087 
5088   Input Parameters:
5089 . dm        - The DMPlex object
5090 
5091   Output Parameter:
5092 . cones - The cone for each point
5093 
5094   Level: developer
5095 
5096 .seealso: DMPlexGetConeSection()
5097 @*/
5098 PetscErrorCode DMPlexGetCones(DM dm, PetscInt *cones[])
5099 {
5100   DM_Plex *mesh = (DM_Plex*) dm->data;
5101 
5102   PetscFunctionBegin;
5103   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5104   if (cones) *cones = mesh->cones;
5105   PetscFunctionReturn(0);
5106 }
5107 
5108 /*@C
5109   DMPlexGetConeOrientations - Return cone orientation data
5110 
5111   Not Collective
5112 
5113   Input Parameters:
5114 . dm        - The DMPlex object
5115 
5116   Output Parameter:
5117 . coneOrientations - The array of cone orientations for all points
5118 
5119   Level: developer
5120 
5121   Notes:
5122   The PetscSection returned by DMPlexGetConeSection() partitions coneOrientations into cone orientations of particular points as returned by DMPlexGetConeOrientation().
5123 
5124   The meaning of coneOrientations values is detailed in DMPlexGetConeOrientation().
5125 
5126 .seealso: DMPlexGetConeSection(), DMPlexGetConeOrientation()
5127 @*/
5128 PetscErrorCode DMPlexGetConeOrientations(DM dm, PetscInt *coneOrientations[])
5129 {
5130   DM_Plex *mesh = (DM_Plex*) dm->data;
5131 
5132   PetscFunctionBegin;
5133   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5134   if (coneOrientations) *coneOrientations = mesh->coneOrientations;
5135   PetscFunctionReturn(0);
5136 }
5137 
5138 /******************************** FEM Support **********************************/
5139 
5140 /*
5141  Returns number of components and tensor degree for the field.  For interpolated meshes, line should be a point
5142  representing a line in the section.
5143 */
5144 static PetscErrorCode PetscSectionFieldGetTensorDegree_Private(PetscSection section,PetscInt field,PetscInt line,PetscBool vertexchart,PetscInt *Nc,PetscInt *k)
5145 {
5146   PetscErrorCode ierr;
5147 
5148   PetscFunctionBeginHot;
5149   ierr = PetscSectionGetFieldComponents(section, field, Nc);CHKERRQ(ierr);
5150   if (line < 0) {
5151     *k = 0;
5152     *Nc = 0;
5153   } else if (vertexchart) {            /* If we only have a vertex chart, we must have degree k=1 */
5154     *k = 1;
5155   } else {                      /* Assume the full interpolated mesh is in the chart; lines in particular */
5156     /* An order k SEM disc has k-1 dofs on an edge */
5157     ierr = PetscSectionGetFieldDof(section, line, field, k);CHKERRQ(ierr);
5158     *k = *k / *Nc + 1;
5159   }
5160   PetscFunctionReturn(0);
5161 }
5162 
5163 /*@
5164 
5165   DMPlexSetClosurePermutationTensor - Create a permutation from the default (BFS) point ordering in the closure, to a
5166   lexicographic ordering over the tensor product cell (i.e., line, quad, hex, etc.), and set this permutation in the
5167   section provided (or the section of the DM).
5168 
5169   Input Parameters:
5170 + dm      - The DM
5171 . point   - Either a cell (highest dim point) or an edge (dim 1 point), or PETSC_DETERMINE
5172 - section - The PetscSection to reorder, or NULL for the default section
5173 
5174   Note: The point is used to determine the number of dofs/field on an edge. For SEM, this is related to the polynomial
5175   degree of the basis.
5176 
5177   Example:
5178   A typical interpolated single-quad mesh might order points as
5179 .vb
5180   [c0, v1, v2, v3, v4, e5, e6, e7, e8]
5181 
5182   v4 -- e6 -- v3
5183   |           |
5184   e7    c0    e8
5185   |           |
5186   v1 -- e5 -- v2
5187 .ve
5188 
5189   (There is no significance to the ordering described here.)  The default section for a Q3 quad might typically assign
5190   dofs in the order of points, e.g.,
5191 .vb
5192     c0 -> [0,1,2,3]
5193     v1 -> [4]
5194     ...
5195     e5 -> [8, 9]
5196 .ve
5197 
5198   which corresponds to the dofs
5199 .vb
5200     6   10  11  7
5201     13  2   3   15
5202     12  0   1   14
5203     4   8   9   5
5204 .ve
5205 
5206   The closure in BFS ordering works through height strata (cells, edges, vertices) to produce the ordering
5207 .vb
5208   0 1 2 3 8 9 14 15 11 10 13 12 4 5 7 6
5209 .ve
5210 
5211   After calling DMPlexSetClosurePermutationTensor(), the closure will be ordered lexicographically,
5212 .vb
5213    4 8 9 5 12 0 1 14 13 2 3 15 6 10 11 7
5214 .ve
5215 
5216   Level: developer
5217 
5218 .seealso: DMGetLocalSection(), PetscSectionSetClosurePermutation(), DMSetGlobalSection()
5219 @*/
5220 PetscErrorCode DMPlexSetClosurePermutationTensor(DM dm, PetscInt point, PetscSection section)
5221 {
5222   DMLabel        label;
5223   PetscInt       dim, depth = -1, eStart = -1, Nf;
5224   PetscBool      vertexchart;
5225   PetscErrorCode ierr;
5226 
5227   PetscFunctionBegin;
5228   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
5229   if (dim < 1) PetscFunctionReturn(0);
5230   if (point < 0) {
5231     PetscInt sStart,sEnd;
5232 
5233     ierr = DMPlexGetDepthStratum(dm, 1, &sStart, &sEnd);CHKERRQ(ierr);
5234     point = sEnd-sStart ? sStart : point;
5235   }
5236   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
5237   if (point >= 0) { ierr = DMLabelGetValue(label, point, &depth);CHKERRQ(ierr); }
5238   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
5239   if (depth == 1) {eStart = point;}
5240   else if  (depth == dim) {
5241     const PetscInt *cone;
5242 
5243     ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
5244     if (dim == 2) eStart = cone[0];
5245     else if (dim == 3) {
5246       const PetscInt *cone2;
5247       ierr = DMPlexGetCone(dm, cone[0], &cone2);CHKERRQ(ierr);
5248       eStart = cone2[0];
5249     } else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Point %D of depth %D cannot be used to bootstrap spectral ordering for dim %D", point, depth, dim);
5250   } else PetscAssertFalse(depth >= 0,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Point %D of depth %D cannot be used to bootstrap spectral ordering for dim %D", point, depth, dim);
5251   {                             /* Determine whether the chart covers all points or just vertices. */
5252     PetscInt pStart,pEnd,cStart,cEnd;
5253     ierr = DMPlexGetDepthStratum(dm,0,&pStart,&pEnd);CHKERRQ(ierr);
5254     ierr = PetscSectionGetChart(section,&cStart,&cEnd);CHKERRQ(ierr);
5255     if (pStart == cStart && pEnd == cEnd) vertexchart = PETSC_TRUE; /* Just vertices */
5256     else vertexchart = PETSC_FALSE;                                 /* Assume all interpolated points are in chart */
5257   }
5258   ierr = PetscSectionGetNumFields(section, &Nf);CHKERRQ(ierr);
5259   for (PetscInt d=1; d<=dim; d++) {
5260     PetscInt k, f, Nc, c, i, j, size = 0, offset = 0, foffset = 0;
5261     PetscInt *perm;
5262 
5263     for (f = 0; f < Nf; ++f) {
5264       ierr = PetscSectionFieldGetTensorDegree_Private(section,f,eStart,vertexchart,&Nc,&k);CHKERRQ(ierr);
5265       size += PetscPowInt(k+1, d)*Nc;
5266     }
5267     ierr = PetscMalloc1(size, &perm);CHKERRQ(ierr);
5268     for (f = 0; f < Nf; ++f) {
5269       switch (d) {
5270       case 1:
5271         ierr = PetscSectionFieldGetTensorDegree_Private(section,f,eStart,vertexchart,&Nc,&k);CHKERRQ(ierr);
5272         /*
5273          Original ordering is [ edge of length k-1; vtx0; vtx1 ]
5274          We want              [ vtx0; edge of length k-1; vtx1 ]
5275          */
5276         for (c=0; c<Nc; c++,offset++) perm[offset] = (k-1)*Nc + c + foffset;
5277         for (i=0; i<k-1; i++) for (c=0; c<Nc; c++,offset++) perm[offset] = i*Nc + c + foffset;
5278         for (c=0; c<Nc; c++,offset++) perm[offset] = k*Nc + c + foffset;
5279         foffset = offset;
5280         break;
5281       case 2:
5282         /* The original quad closure is oriented clockwise, {f, e_b, e_r, e_t, e_l, v_lb, v_rb, v_tr, v_tl} */
5283         ierr = PetscSectionFieldGetTensorDegree_Private(section,f,eStart,vertexchart,&Nc,&k);CHKERRQ(ierr);
5284         /* The SEM order is
5285 
5286          v_lb, {e_b}, v_rb,
5287          e^{(k-1)-i}_l, {f^{i*(k-1)}}, e^i_r,
5288          v_lt, reverse {e_t}, v_rt
5289          */
5290         {
5291           const PetscInt of   = 0;
5292           const PetscInt oeb  = of   + PetscSqr(k-1);
5293           const PetscInt oer  = oeb  + (k-1);
5294           const PetscInt oet  = oer  + (k-1);
5295           const PetscInt oel  = oet  + (k-1);
5296           const PetscInt ovlb = oel  + (k-1);
5297           const PetscInt ovrb = ovlb + 1;
5298           const PetscInt ovrt = ovrb + 1;
5299           const PetscInt ovlt = ovrt + 1;
5300           PetscInt       o;
5301 
5302           /* bottom */
5303           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovlb*Nc + c + foffset;
5304           for (o = oeb; o < oer; ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5305           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovrb*Nc + c + foffset;
5306           /* middle */
5307           for (i = 0; i < k-1; ++i) {
5308             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oel+(k-2)-i)*Nc + c + foffset;
5309             for (o = of+(k-1)*i; o < of+(k-1)*(i+1); ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5310             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oer+i)*Nc + c + foffset;
5311           }
5312           /* top */
5313           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovlt*Nc + c + foffset;
5314           for (o = oel-1; o >= oet; --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5315           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovrt*Nc + c + foffset;
5316           foffset = offset;
5317         }
5318         break;
5319       case 3:
5320         /* The original hex closure is
5321 
5322          {c,
5323          f_b, f_t, f_f, f_b, f_r, f_l,
5324          e_bl, e_bb, e_br, e_bf,  e_tf, e_tr, e_tb, e_tl,  e_rf, e_lf, e_lb, e_rb,
5325          v_blf, v_blb, v_brb, v_brf, v_tlf, v_trf, v_trb, v_tlb}
5326          */
5327         ierr = PetscSectionFieldGetTensorDegree_Private(section,f,eStart,vertexchart,&Nc,&k);CHKERRQ(ierr);
5328         /* The SEM order is
5329          Bottom Slice
5330          v_blf, {e^{(k-1)-n}_bf}, v_brf,
5331          e^{i}_bl, f^{n*(k-1)+(k-1)-i}_b, e^{(k-1)-i}_br,
5332          v_blb, {e_bb}, v_brb,
5333 
5334          Middle Slice (j)
5335          {e^{(k-1)-j}_lf}, {f^{j*(k-1)+n}_f}, e^j_rf,
5336          f^{i*(k-1)+j}_l, {c^{(j*(k-1) + i)*(k-1)+n}_t}, f^{j*(k-1)+i}_r,
5337          e^j_lb, {f^{j*(k-1)+(k-1)-n}_b}, e^{(k-1)-j}_rb,
5338 
5339          Top Slice
5340          v_tlf, {e_tf}, v_trf,
5341          e^{(k-1)-i}_tl, {f^{i*(k-1)}_t}, e^{i}_tr,
5342          v_tlb, {e^{(k-1)-n}_tb}, v_trb,
5343          */
5344         {
5345           const PetscInt oc    = 0;
5346           const PetscInt ofb   = oc    + PetscSqr(k-1)*(k-1);
5347           const PetscInt oft   = ofb   + PetscSqr(k-1);
5348           const PetscInt off   = oft   + PetscSqr(k-1);
5349           const PetscInt ofk   = off   + PetscSqr(k-1);
5350           const PetscInt ofr   = ofk   + PetscSqr(k-1);
5351           const PetscInt ofl   = ofr   + PetscSqr(k-1);
5352           const PetscInt oebl  = ofl   + PetscSqr(k-1);
5353           const PetscInt oebb  = oebl  + (k-1);
5354           const PetscInt oebr  = oebb  + (k-1);
5355           const PetscInt oebf  = oebr  + (k-1);
5356           const PetscInt oetf  = oebf  + (k-1);
5357           const PetscInt oetr  = oetf  + (k-1);
5358           const PetscInt oetb  = oetr  + (k-1);
5359           const PetscInt oetl  = oetb  + (k-1);
5360           const PetscInt oerf  = oetl  + (k-1);
5361           const PetscInt oelf  = oerf  + (k-1);
5362           const PetscInt oelb  = oelf  + (k-1);
5363           const PetscInt oerb  = oelb  + (k-1);
5364           const PetscInt ovblf = oerb  + (k-1);
5365           const PetscInt ovblb = ovblf + 1;
5366           const PetscInt ovbrb = ovblb + 1;
5367           const PetscInt ovbrf = ovbrb + 1;
5368           const PetscInt ovtlf = ovbrf + 1;
5369           const PetscInt ovtrf = ovtlf + 1;
5370           const PetscInt ovtrb = ovtrf + 1;
5371           const PetscInt ovtlb = ovtrb + 1;
5372           PetscInt       o, n;
5373 
5374           /* Bottom Slice */
5375           /*   bottom */
5376           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovblf*Nc + c + foffset;
5377           for (o = oetf-1; o >= oebf; --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5378           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovbrf*Nc + c + foffset;
5379           /*   middle */
5380           for (i = 0; i < k-1; ++i) {
5381             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oebl+i)*Nc + c + foffset;
5382             for (n = 0; n < k-1; ++n) {o = ofb+n*(k-1)+i; for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;}
5383             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oebr+(k-2)-i)*Nc + c + foffset;
5384           }
5385           /*   top */
5386           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovblb*Nc + c + foffset;
5387           for (o = oebb; o < oebr; ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5388           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovbrb*Nc + c + foffset;
5389 
5390           /* Middle Slice */
5391           for (j = 0; j < k-1; ++j) {
5392             /*   bottom */
5393             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oelf+(k-2)-j)*Nc + c + foffset;
5394             for (o = off+j*(k-1); o < off+(j+1)*(k-1); ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5395             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oerf+j)*Nc + c + foffset;
5396             /*   middle */
5397             for (i = 0; i < k-1; ++i) {
5398               for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (ofl+i*(k-1)+j)*Nc + c + foffset;
5399               for (n = 0; n < k-1; ++n) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oc+(j*(k-1)+i)*(k-1)+n)*Nc + c + foffset;
5400               for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (ofr+j*(k-1)+i)*Nc + c + foffset;
5401             }
5402             /*   top */
5403             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oelb+j)*Nc + c + foffset;
5404             for (o = ofk+j*(k-1)+(k-2); o >= ofk+j*(k-1); --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5405             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oerb+(k-2)-j)*Nc + c + foffset;
5406           }
5407 
5408           /* Top Slice */
5409           /*   bottom */
5410           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtlf*Nc + c + foffset;
5411           for (o = oetf; o < oetr; ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5412           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtrf*Nc + c + foffset;
5413           /*   middle */
5414           for (i = 0; i < k-1; ++i) {
5415             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oetl+(k-2)-i)*Nc + c + foffset;
5416             for (n = 0; n < k-1; ++n) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oft+i*(k-1)+n)*Nc + c + foffset;
5417             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oetr+i)*Nc + c + foffset;
5418           }
5419           /*   top */
5420           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtlb*Nc + c + foffset;
5421           for (o = oetl-1; o >= oetb; --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5422           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtrb*Nc + c + foffset;
5423 
5424           foffset = offset;
5425         }
5426         break;
5427       default: SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_OUTOFRANGE, "No spectral ordering for dimension %D", d);
5428       }
5429     }
5430     PetscAssertFalse(offset != size,PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Number of permutation entries %D != %D", offset, size);
5431     /* Check permutation */
5432     {
5433       PetscInt *check;
5434 
5435       ierr = PetscMalloc1(size, &check);CHKERRQ(ierr);
5436       for (i = 0; i < size; ++i) {check[i] = -1; PetscAssertFalse(perm[i] < 0 || perm[i] >= size,PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Invalid permutation index p[%D] = %D", i, perm[i]);}
5437       for (i = 0; i < size; ++i) check[perm[i]] = i;
5438       for (i = 0; i < size; ++i) {PetscAssertFalse(check[i] < 0,PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Missing permutation index %D", i);}
5439       ierr = PetscFree(check);CHKERRQ(ierr);
5440     }
5441     ierr = PetscSectionSetClosurePermutation_Internal(section, (PetscObject) dm, d, size, PETSC_OWN_POINTER, perm);CHKERRQ(ierr);
5442   }
5443   PetscFunctionReturn(0);
5444 }
5445 
5446 PetscErrorCode DMPlexGetPointDualSpaceFEM(DM dm, PetscInt point, PetscInt field, PetscDualSpace *dspace)
5447 {
5448   PetscDS        prob;
5449   PetscInt       depth, Nf, h;
5450   DMLabel        label;
5451   PetscErrorCode ierr;
5452 
5453   PetscFunctionBeginHot;
5454   ierr = DMGetDS(dm, &prob);CHKERRQ(ierr);
5455   Nf      = prob->Nf;
5456   label   = dm->depthLabel;
5457   *dspace = NULL;
5458   if (field < Nf) {
5459     PetscObject disc = prob->disc[field];
5460 
5461     if (disc->classid == PETSCFE_CLASSID) {
5462       PetscDualSpace dsp;
5463 
5464       ierr = PetscFEGetDualSpace((PetscFE)disc,&dsp);CHKERRQ(ierr);
5465       ierr = DMLabelGetNumValues(label,&depth);CHKERRQ(ierr);
5466       ierr = DMLabelGetValue(label,point,&h);CHKERRQ(ierr);
5467       h    = depth - 1 - h;
5468       if (h) {
5469         ierr = PetscDualSpaceGetHeightSubspace(dsp,h,dspace);CHKERRQ(ierr);
5470       } else {
5471         *dspace = dsp;
5472       }
5473     }
5474   }
5475   PetscFunctionReturn(0);
5476 }
5477 
5478 static inline PetscErrorCode DMPlexVecGetClosure_Depth1_Static(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
5479 {
5480   PetscScalar    *array, *vArray;
5481   const PetscInt *cone, *coneO;
5482   PetscInt        pStart, pEnd, p, numPoints, size = 0, offset = 0;
5483   PetscErrorCode  ierr;
5484 
5485   PetscFunctionBeginHot;
5486   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
5487   ierr = DMPlexGetConeSize(dm, point, &numPoints);CHKERRQ(ierr);
5488   ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
5489   ierr = DMPlexGetConeOrientation(dm, point, &coneO);CHKERRQ(ierr);
5490   if (!values || !*values) {
5491     if ((point >= pStart) && (point < pEnd)) {
5492       PetscInt dof;
5493 
5494       ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
5495       size += dof;
5496     }
5497     for (p = 0; p < numPoints; ++p) {
5498       const PetscInt cp = cone[p];
5499       PetscInt       dof;
5500 
5501       if ((cp < pStart) || (cp >= pEnd)) continue;
5502       ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
5503       size += dof;
5504     }
5505     if (!values) {
5506       if (csize) *csize = size;
5507       PetscFunctionReturn(0);
5508     }
5509     ierr = DMGetWorkArray(dm, size, MPIU_SCALAR, &array);CHKERRQ(ierr);
5510   } else {
5511     array = *values;
5512   }
5513   size = 0;
5514   ierr = VecGetArray(v, &vArray);CHKERRQ(ierr);
5515   if ((point >= pStart) && (point < pEnd)) {
5516     PetscInt     dof, off, d;
5517     PetscScalar *varr;
5518 
5519     ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
5520     ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
5521     varr = &vArray[off];
5522     for (d = 0; d < dof; ++d, ++offset) {
5523       array[offset] = varr[d];
5524     }
5525     size += dof;
5526   }
5527   for (p = 0; p < numPoints; ++p) {
5528     const PetscInt cp = cone[p];
5529     PetscInt       o  = coneO[p];
5530     PetscInt       dof, off, d;
5531     PetscScalar   *varr;
5532 
5533     if ((cp < pStart) || (cp >= pEnd)) continue;
5534     ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
5535     ierr = PetscSectionGetOffset(section, cp, &off);CHKERRQ(ierr);
5536     varr = &vArray[off];
5537     if (o >= 0) {
5538       for (d = 0; d < dof; ++d, ++offset) {
5539         array[offset] = varr[d];
5540       }
5541     } else {
5542       for (d = dof-1; d >= 0; --d, ++offset) {
5543         array[offset] = varr[d];
5544       }
5545     }
5546     size += dof;
5547   }
5548   ierr = VecRestoreArray(v, &vArray);CHKERRQ(ierr);
5549   if (!*values) {
5550     if (csize) *csize = size;
5551     *values = array;
5552   } else {
5553     PetscAssertFalse(size > *csize,PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Size of input array %D < actual size %D", *csize, size);
5554     *csize = size;
5555   }
5556   PetscFunctionReturn(0);
5557 }
5558 
5559 /* Compress out points not in the section */
5560 static inline PetscErrorCode CompressPoints_Private(PetscSection section, PetscInt *numPoints, PetscInt points[])
5561 {
5562   const PetscInt np = *numPoints;
5563   PetscInt       pStart, pEnd, p, q;
5564   PetscErrorCode ierr;
5565 
5566   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
5567   for (p = 0, q = 0; p < np; ++p) {
5568     const PetscInt r = points[p*2];
5569     if ((r >= pStart) && (r < pEnd)) {
5570       points[q*2]   = r;
5571       points[q*2+1] = points[p*2+1];
5572       ++q;
5573     }
5574   }
5575   *numPoints = q;
5576   return 0;
5577 }
5578 
5579 /* Compressed closure does not apply closure permutation */
5580 PetscErrorCode DMPlexGetCompressedClosure(DM dm, PetscSection section, PetscInt point, PetscInt *numPoints, PetscInt **points, PetscSection *clSec, IS *clPoints, const PetscInt **clp)
5581 {
5582   const PetscInt *cla = NULL;
5583   PetscInt       np, *pts = NULL;
5584   PetscErrorCode ierr;
5585 
5586   PetscFunctionBeginHot;
5587   ierr = PetscSectionGetClosureIndex(section, (PetscObject) dm, clSec, clPoints);CHKERRQ(ierr);
5588   if (*clPoints) {
5589     PetscInt dof, off;
5590 
5591     ierr = PetscSectionGetDof(*clSec, point, &dof);CHKERRQ(ierr);
5592     ierr = PetscSectionGetOffset(*clSec, point, &off);CHKERRQ(ierr);
5593     ierr = ISGetIndices(*clPoints, &cla);CHKERRQ(ierr);
5594     np   = dof/2;
5595     pts  = (PetscInt *) &cla[off];
5596   } else {
5597     ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &np, &pts);CHKERRQ(ierr);
5598     ierr = CompressPoints_Private(section, &np, pts);CHKERRQ(ierr);
5599   }
5600   *numPoints = np;
5601   *points    = pts;
5602   *clp       = cla;
5603   PetscFunctionReturn(0);
5604 }
5605 
5606 PetscErrorCode DMPlexRestoreCompressedClosure(DM dm, PetscSection section, PetscInt point, PetscInt *numPoints, PetscInt **points, PetscSection *clSec, IS *clPoints, const PetscInt **clp)
5607 {
5608   PetscErrorCode ierr;
5609 
5610   PetscFunctionBeginHot;
5611   if (!*clPoints) {
5612     ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, numPoints, points);CHKERRQ(ierr);
5613   } else {
5614     ierr = ISRestoreIndices(*clPoints, clp);CHKERRQ(ierr);
5615   }
5616   *numPoints = 0;
5617   *points    = NULL;
5618   *clSec     = NULL;
5619   *clPoints  = NULL;
5620   *clp       = NULL;
5621   PetscFunctionReturn(0);
5622 }
5623 
5624 static inline PetscErrorCode DMPlexVecGetClosure_Static(DM dm, PetscSection section, PetscInt numPoints, const PetscInt points[], const PetscInt clperm[], const PetscScalar vArray[], PetscInt *size, PetscScalar array[])
5625 {
5626   PetscInt          offset = 0, p;
5627   const PetscInt    **perms = NULL;
5628   const PetscScalar **flips = NULL;
5629   PetscErrorCode    ierr;
5630 
5631   PetscFunctionBeginHot;
5632   *size = 0;
5633   ierr = PetscSectionGetPointSyms(section,numPoints,points,&perms,&flips);CHKERRQ(ierr);
5634   for (p = 0; p < numPoints; p++) {
5635     const PetscInt    point = points[2*p];
5636     const PetscInt    *perm = perms ? perms[p] : NULL;
5637     const PetscScalar *flip = flips ? flips[p] : NULL;
5638     PetscInt          dof, off, d;
5639     const PetscScalar *varr;
5640 
5641     ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
5642     ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
5643     varr = &vArray[off];
5644     if (clperm) {
5645       if (perm) {
5646         for (d = 0; d < dof; d++) array[clperm[offset + perm[d]]]  = varr[d];
5647       } else {
5648         for (d = 0; d < dof; d++) array[clperm[offset +      d ]]  = varr[d];
5649       }
5650       if (flip) {
5651         for (d = 0; d < dof; d++) array[clperm[offset +      d ]] *= flip[d];
5652       }
5653     } else {
5654       if (perm) {
5655         for (d = 0; d < dof; d++) array[offset + perm[d]]  = varr[d];
5656       } else {
5657         for (d = 0; d < dof; d++) array[offset +      d ]  = varr[d];
5658       }
5659       if (flip) {
5660         for (d = 0; d < dof; d++) array[offset +      d ] *= flip[d];
5661       }
5662     }
5663     offset += dof;
5664   }
5665   ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms,&flips);CHKERRQ(ierr);
5666   *size = offset;
5667   PetscFunctionReturn(0);
5668 }
5669 
5670 static inline PetscErrorCode DMPlexVecGetClosure_Fields_Static(DM dm, PetscSection section, PetscInt numPoints, const PetscInt points[], PetscInt numFields, const PetscInt clperm[], const PetscScalar vArray[], PetscInt *size, PetscScalar array[])
5671 {
5672   PetscInt          offset = 0, f;
5673   PetscErrorCode    ierr;
5674 
5675   PetscFunctionBeginHot;
5676   *size = 0;
5677   for (f = 0; f < numFields; ++f) {
5678     PetscInt          p;
5679     const PetscInt    **perms = NULL;
5680     const PetscScalar **flips = NULL;
5681 
5682     ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
5683     for (p = 0; p < numPoints; p++) {
5684       const PetscInt    point = points[2*p];
5685       PetscInt          fdof, foff, b;
5686       const PetscScalar *varr;
5687       const PetscInt    *perm = perms ? perms[p] : NULL;
5688       const PetscScalar *flip = flips ? flips[p] : NULL;
5689 
5690       ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
5691       ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
5692       varr = &vArray[foff];
5693       if (clperm) {
5694         if (perm) {for (b = 0; b < fdof; b++) {array[clperm[offset + perm[b]]]  = varr[b];}}
5695         else      {for (b = 0; b < fdof; b++) {array[clperm[offset +      b ]]  = varr[b];}}
5696         if (flip) {for (b = 0; b < fdof; b++) {array[clperm[offset +      b ]] *= flip[b];}}
5697       } else {
5698         if (perm) {for (b = 0; b < fdof; b++) {array[offset + perm[b]]  = varr[b];}}
5699         else      {for (b = 0; b < fdof; b++) {array[offset +      b ]  = varr[b];}}
5700         if (flip) {for (b = 0; b < fdof; b++) {array[offset +      b ] *= flip[b];}}
5701       }
5702       offset += fdof;
5703     }
5704     ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
5705   }
5706   *size = offset;
5707   PetscFunctionReturn(0);
5708 }
5709 
5710 /*@C
5711   DMPlexVecGetClosure - Get an array of the values on the closure of 'point'
5712 
5713   Not collective
5714 
5715   Input Parameters:
5716 + dm - The DM
5717 . section - The section describing the layout in v, or NULL to use the default section
5718 . v - The local vector
5719 - point - The point in the DM
5720 
5721   Input/Output Parameters:
5722 + csize  - The size of the input values array, or NULL; on output the number of values in the closure
5723 - values - An array to use for the values, or NULL to have it allocated automatically;
5724            if the user provided NULL, it is a borrowed array and should not be freed
5725 
5726 $ Note that DMPlexVecGetClosure/DMPlexVecRestoreClosure only allocates the values array if it set to NULL in the
5727 $ calling function. This is because DMPlexVecGetClosure() is typically called in the inner loop of a Vec or Mat
5728 $ assembly function, and a user may already have allocated storage for this operation.
5729 $
5730 $ A typical use could be
5731 $
5732 $  values = NULL;
5733 $  ierr = DMPlexVecGetClosure(dm, NULL, v, p, &clSize, &values);CHKERRQ(ierr);
5734 $  for (cl = 0; cl < clSize; ++cl) {
5735 $    <Compute on closure>
5736 $  }
5737 $  ierr = DMPlexVecRestoreClosure(dm, NULL, v, p, &clSize, &values);CHKERRQ(ierr);
5738 $
5739 $ or
5740 $
5741 $  PetscMalloc1(clMaxSize, &values);
5742 $  for (p = pStart; p < pEnd; ++p) {
5743 $    clSize = clMaxSize;
5744 $    ierr = DMPlexVecGetClosure(dm, NULL, v, p, &clSize, &values);CHKERRQ(ierr);
5745 $    for (cl = 0; cl < clSize; ++cl) {
5746 $      <Compute on closure>
5747 $    }
5748 $  }
5749 $  PetscFree(values);
5750 
5751   Fortran Notes:
5752   Since it returns an array, this routine is only available in Fortran 90, and you must
5753   include petsc.h90 in your code.
5754 
5755   The csize argument is not present in the Fortran 90 binding since it is internal to the array.
5756 
5757   Level: intermediate
5758 
5759 .seealso DMPlexVecRestoreClosure(), DMPlexVecSetClosure(), DMPlexMatSetClosure()
5760 @*/
5761 PetscErrorCode DMPlexVecGetClosure(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
5762 {
5763   PetscSection       clSection;
5764   IS                 clPoints;
5765   PetscInt          *points = NULL;
5766   const PetscInt    *clp, *perm;
5767   PetscInt           depth, numFields, numPoints, asize;
5768   PetscErrorCode     ierr;
5769 
5770   PetscFunctionBeginHot;
5771   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5772   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
5773   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
5774   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
5775   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
5776   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5777   if (depth == 1 && numFields < 2) {
5778     ierr = DMPlexVecGetClosure_Depth1_Static(dm, section, v, point, csize, values);CHKERRQ(ierr);
5779     PetscFunctionReturn(0);
5780   }
5781   /* Get points */
5782   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5783   /* Get sizes */
5784   asize = 0;
5785   for (PetscInt p = 0; p < numPoints*2; p += 2) {
5786     PetscInt dof;
5787     ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5788     asize += dof;
5789   }
5790   if (values) {
5791     const PetscScalar *vArray;
5792     PetscInt          size;
5793 
5794     if (*values) {
5795       PetscAssertFalse(*csize < asize,PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Provided array size %D not sufficient to hold closure size %D", *csize, asize);
5796     } else {ierr = DMGetWorkArray(dm, asize, MPIU_SCALAR, values);CHKERRQ(ierr);}
5797     ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, depth, asize, &perm);CHKERRQ(ierr);
5798     ierr = VecGetArrayRead(v, &vArray);CHKERRQ(ierr);
5799     /* Get values */
5800     if (numFields > 0) {ierr = DMPlexVecGetClosure_Fields_Static(dm, section, numPoints, points, numFields, perm, vArray, &size, *values);CHKERRQ(ierr);}
5801     else               {ierr = DMPlexVecGetClosure_Static(dm, section, numPoints, points, perm, vArray, &size, *values);CHKERRQ(ierr);}
5802     PetscAssertFalse(asize != size,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Section size %D does not match Vec closure size %D", asize, size);
5803     /* Cleanup array */
5804     ierr = VecRestoreArrayRead(v, &vArray);CHKERRQ(ierr);
5805   }
5806   if (csize) *csize = asize;
5807   /* Cleanup points */
5808   ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5809   PetscFunctionReturn(0);
5810 }
5811 
5812 PetscErrorCode DMPlexVecGetClosureAtDepth_Internal(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt depth, PetscInt *csize, PetscScalar *values[])
5813 {
5814   DMLabel            depthLabel;
5815   PetscSection       clSection;
5816   IS                 clPoints;
5817   PetscScalar       *array;
5818   const PetscScalar *vArray;
5819   PetscInt          *points = NULL;
5820   const PetscInt    *clp, *perm = NULL;
5821   PetscInt           mdepth, numFields, numPoints, Np = 0, p, clsize, size;
5822   PetscErrorCode     ierr;
5823 
5824   PetscFunctionBeginHot;
5825   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5826   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
5827   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
5828   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
5829   ierr = DMPlexGetDepth(dm, &mdepth);CHKERRQ(ierr);
5830   ierr = DMPlexGetDepthLabel(dm, &depthLabel);CHKERRQ(ierr);
5831   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5832   if (mdepth == 1 && numFields < 2) {
5833     ierr = DMPlexVecGetClosure_Depth1_Static(dm, section, v, point, csize, values);CHKERRQ(ierr);
5834     PetscFunctionReturn(0);
5835   }
5836   /* Get points */
5837   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5838   for (clsize=0,p=0; p<Np; p++) {
5839     PetscInt dof;
5840     ierr = PetscSectionGetDof(section, points[2*p], &dof);CHKERRQ(ierr);
5841     clsize += dof;
5842   }
5843   ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, depth, clsize, &perm);CHKERRQ(ierr);
5844   /* Filter points */
5845   for (p = 0; p < numPoints*2; p += 2) {
5846     PetscInt dep;
5847 
5848     ierr = DMLabelGetValue(depthLabel, points[p], &dep);CHKERRQ(ierr);
5849     if (dep != depth) continue;
5850     points[Np*2+0] = points[p];
5851     points[Np*2+1] = points[p+1];
5852     ++Np;
5853   }
5854   /* Get array */
5855   if (!values || !*values) {
5856     PetscInt asize = 0, dof;
5857 
5858     for (p = 0; p < Np*2; p += 2) {
5859       ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5860       asize += dof;
5861     }
5862     if (!values) {
5863       ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5864       if (csize) *csize = asize;
5865       PetscFunctionReturn(0);
5866     }
5867     ierr = DMGetWorkArray(dm, asize, MPIU_SCALAR, &array);CHKERRQ(ierr);
5868   } else {
5869     array = *values;
5870   }
5871   ierr = VecGetArrayRead(v, &vArray);CHKERRQ(ierr);
5872   /* Get values */
5873   if (numFields > 0) {ierr = DMPlexVecGetClosure_Fields_Static(dm, section, Np, points, numFields, perm, vArray, &size, array);CHKERRQ(ierr);}
5874   else               {ierr = DMPlexVecGetClosure_Static(dm, section, Np, points, perm, vArray, &size, array);CHKERRQ(ierr);}
5875   /* Cleanup points */
5876   ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5877   /* Cleanup array */
5878   ierr = VecRestoreArrayRead(v, &vArray);CHKERRQ(ierr);
5879   if (!*values) {
5880     if (csize) *csize = size;
5881     *values = array;
5882   } else {
5883     PetscAssertFalse(size > *csize,PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Size of input array %D < actual size %D", *csize, size);
5884     *csize = size;
5885   }
5886   PetscFunctionReturn(0);
5887 }
5888 
5889 /*@C
5890   DMPlexVecRestoreClosure - Restore the array of the values on the closure of 'point'
5891 
5892   Not collective
5893 
5894   Input Parameters:
5895 + dm - The DM
5896 . section - The section describing the layout in v, or NULL to use the default section
5897 . v - The local vector
5898 . point - The point in the DM
5899 . csize - The number of values in the closure, or NULL
5900 - values - The array of values, which is a borrowed array and should not be freed
5901 
5902   Note that the array values are discarded and not copied back into v. In order to copy values back to v, use DMPlexVecSetClosure()
5903 
5904   Fortran Notes:
5905   Since it returns an array, this routine is only available in Fortran 90, and you must
5906   include petsc.h90 in your code.
5907 
5908   The csize argument is not present in the Fortran 90 binding since it is internal to the array.
5909 
5910   Level: intermediate
5911 
5912 .seealso DMPlexVecGetClosure(), DMPlexVecSetClosure(), DMPlexMatSetClosure()
5913 @*/
5914 PetscErrorCode DMPlexVecRestoreClosure(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
5915 {
5916   PetscInt       size = 0;
5917   PetscErrorCode ierr;
5918 
5919   PetscFunctionBegin;
5920   /* Should work without recalculating size */
5921   ierr = DMRestoreWorkArray(dm, size, MPIU_SCALAR, (void*) values);CHKERRQ(ierr);
5922   *values = NULL;
5923   PetscFunctionReturn(0);
5924 }
5925 
5926 static inline void add   (PetscScalar *x, PetscScalar y) {*x += y;}
5927 static inline void insert(PetscScalar *x, PetscScalar y) {*x  = y;}
5928 
5929 static inline PetscErrorCode updatePoint_private(PetscSection section, PetscInt point, PetscInt dof, void (*fuse)(PetscScalar*, PetscScalar), PetscBool setBC, const PetscInt perm[], const PetscScalar flip[], const PetscInt clperm[], const PetscScalar values[], PetscInt offset, PetscScalar array[])
5930 {
5931   PetscInt        cdof;   /* The number of constraints on this point */
5932   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
5933   PetscScalar    *a;
5934   PetscInt        off, cind = 0, k;
5935   PetscErrorCode  ierr;
5936 
5937   PetscFunctionBegin;
5938   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
5939   ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
5940   a    = &array[off];
5941   if (!cdof || setBC) {
5942     if (clperm) {
5943       if (perm) {for (k = 0; k < dof; ++k) {fuse(&a[k], values[clperm[offset+perm[k]]] * (flip ? flip[perm[k]] : 1.));}}
5944       else      {for (k = 0; k < dof; ++k) {fuse(&a[k], values[clperm[offset+     k ]] * (flip ? flip[     k ] : 1.));}}
5945     } else {
5946       if (perm) {for (k = 0; k < dof; ++k) {fuse(&a[k], values[offset+perm[k]] * (flip ? flip[perm[k]] : 1.));}}
5947       else      {for (k = 0; k < dof; ++k) {fuse(&a[k], values[offset+     k ] * (flip ? flip[     k ] : 1.));}}
5948     }
5949   } else {
5950     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
5951     if (clperm) {
5952       if (perm) {for (k = 0; k < dof; ++k) {
5953           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
5954           fuse(&a[k], values[clperm[offset+perm[k]]] * (flip ? flip[perm[k]] : 1.));
5955         }
5956       } else {
5957         for (k = 0; k < dof; ++k) {
5958           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
5959           fuse(&a[k], values[clperm[offset+     k ]] * (flip ? flip[     k ] : 1.));
5960         }
5961       }
5962     } else {
5963       if (perm) {
5964         for (k = 0; k < dof; ++k) {
5965           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
5966           fuse(&a[k], values[offset+perm[k]] * (flip ? flip[perm[k]] : 1.));
5967         }
5968       } else {
5969         for (k = 0; k < dof; ++k) {
5970           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
5971           fuse(&a[k], values[offset+     k ] * (flip ? flip[     k ] : 1.));
5972         }
5973       }
5974     }
5975   }
5976   PetscFunctionReturn(0);
5977 }
5978 
5979 static inline PetscErrorCode updatePointBC_private(PetscSection section, PetscInt point, PetscInt dof, void (*fuse)(PetscScalar*, PetscScalar), const PetscInt perm[], const PetscScalar flip[], const PetscInt clperm[], const PetscScalar values[], PetscInt offset, PetscScalar array[])
5980 {
5981   PetscInt        cdof;   /* The number of constraints on this point */
5982   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
5983   PetscScalar    *a;
5984   PetscInt        off, cind = 0, k;
5985   PetscErrorCode  ierr;
5986 
5987   PetscFunctionBegin;
5988   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
5989   ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
5990   a    = &array[off];
5991   if (cdof) {
5992     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
5993     if (clperm) {
5994       if (perm) {
5995         for (k = 0; k < dof; ++k) {
5996           if ((cind < cdof) && (k == cdofs[cind])) {
5997             fuse(&a[k], values[clperm[offset+perm[k]]] * (flip ? flip[perm[k]] : 1.));
5998             cind++;
5999           }
6000         }
6001       } else {
6002         for (k = 0; k < dof; ++k) {
6003           if ((cind < cdof) && (k == cdofs[cind])) {
6004             fuse(&a[k], values[clperm[offset+     k ]] * (flip ? flip[     k ] : 1.));
6005             cind++;
6006           }
6007         }
6008       }
6009     } else {
6010       if (perm) {
6011         for (k = 0; k < dof; ++k) {
6012           if ((cind < cdof) && (k == cdofs[cind])) {
6013             fuse(&a[k], values[offset+perm[k]] * (flip ? flip[perm[k]] : 1.));
6014             cind++;
6015           }
6016         }
6017       } else {
6018         for (k = 0; k < dof; ++k) {
6019           if ((cind < cdof) && (k == cdofs[cind])) {
6020             fuse(&a[k], values[offset+     k ] * (flip ? flip[     k ] : 1.));
6021             cind++;
6022           }
6023         }
6024       }
6025     }
6026   }
6027   PetscFunctionReturn(0);
6028 }
6029 
6030 static inline PetscErrorCode updatePointFields_private(PetscSection section, PetscInt point, const PetscInt *perm, const PetscScalar *flip, PetscInt f, void (*fuse)(PetscScalar*, PetscScalar), PetscBool setBC, const PetscInt clperm[], const PetscScalar values[], PetscInt *offset, PetscScalar array[])
6031 {
6032   PetscScalar    *a;
6033   PetscInt        fdof, foff, fcdof, foffset = *offset;
6034   const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
6035   PetscInt        cind = 0, b;
6036   PetscErrorCode  ierr;
6037 
6038   PetscFunctionBegin;
6039   ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
6040   ierr = PetscSectionGetFieldConstraintDof(section, point, f, &fcdof);CHKERRQ(ierr);
6041   ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
6042   a    = &array[foff];
6043   if (!fcdof || setBC) {
6044     if (clperm) {
6045       if (perm) {for (b = 0; b < fdof; b++) {fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));}}
6046       else      {for (b = 0; b < fdof; b++) {fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));}}
6047     } else {
6048       if (perm) {for (b = 0; b < fdof; b++) {fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));}}
6049       else      {for (b = 0; b < fdof; b++) {fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));}}
6050     }
6051   } else {
6052     ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
6053     if (clperm) {
6054       if (perm) {
6055         for (b = 0; b < fdof; b++) {
6056           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
6057           fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));
6058         }
6059       } else {
6060         for (b = 0; b < fdof; b++) {
6061           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
6062           fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));
6063         }
6064       }
6065     } else {
6066       if (perm) {
6067         for (b = 0; b < fdof; b++) {
6068           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
6069           fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));
6070         }
6071       } else {
6072         for (b = 0; b < fdof; b++) {
6073           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
6074           fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));
6075         }
6076       }
6077     }
6078   }
6079   *offset += fdof;
6080   PetscFunctionReturn(0);
6081 }
6082 
6083 static inline PetscErrorCode updatePointFieldsBC_private(PetscSection section, PetscInt point, const PetscInt perm[], const PetscScalar flip[], PetscInt f, PetscInt Ncc, const PetscInt comps[], void (*fuse)(PetscScalar*, PetscScalar), const PetscInt clperm[], const PetscScalar values[], PetscInt *offset, PetscScalar array[])
6084 {
6085   PetscScalar    *a;
6086   PetscInt        fdof, foff, fcdof, foffset = *offset;
6087   const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
6088   PetscInt        Nc, cind = 0, ncind = 0, b;
6089   PetscBool       ncSet, fcSet;
6090   PetscErrorCode  ierr;
6091 
6092   PetscFunctionBegin;
6093   ierr = PetscSectionGetFieldComponents(section, f, &Nc);CHKERRQ(ierr);
6094   ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
6095   ierr = PetscSectionGetFieldConstraintDof(section, point, f, &fcdof);CHKERRQ(ierr);
6096   ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
6097   a    = &array[foff];
6098   if (fcdof) {
6099     /* We just override fcdof and fcdofs with Ncc and comps */
6100     ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
6101     if (clperm) {
6102       if (perm) {
6103         if (comps) {
6104           for (b = 0; b < fdof; b++) {
6105             ncSet = fcSet = PETSC_FALSE;
6106             if (b%Nc == comps[ncind]) {ncind = (ncind+1)%Ncc; ncSet = PETSC_TRUE;}
6107             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
6108             if (ncSet && fcSet) {fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));}
6109           }
6110         } else {
6111           for (b = 0; b < fdof; b++) {
6112             if ((cind < fcdof) && (b == fcdofs[cind])) {
6113               fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));
6114               ++cind;
6115             }
6116           }
6117         }
6118       } else {
6119         if (comps) {
6120           for (b = 0; b < fdof; b++) {
6121             ncSet = fcSet = PETSC_FALSE;
6122             if (b%Nc == comps[ncind]) {ncind = (ncind+1)%Ncc; ncSet = PETSC_TRUE;}
6123             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
6124             if (ncSet && fcSet) {fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));}
6125           }
6126         } else {
6127           for (b = 0; b < fdof; b++) {
6128             if ((cind < fcdof) && (b == fcdofs[cind])) {
6129               fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));
6130               ++cind;
6131             }
6132           }
6133         }
6134       }
6135     } else {
6136       if (perm) {
6137         if (comps) {
6138           for (b = 0; b < fdof; b++) {
6139             ncSet = fcSet = PETSC_FALSE;
6140             if (b%Nc == comps[ncind]) {ncind = (ncind+1)%Ncc; ncSet = PETSC_TRUE;}
6141             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
6142             if (ncSet && fcSet) {fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));}
6143           }
6144         } else {
6145           for (b = 0; b < fdof; b++) {
6146             if ((cind < fcdof) && (b == fcdofs[cind])) {
6147               fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));
6148               ++cind;
6149             }
6150           }
6151         }
6152       } else {
6153         if (comps) {
6154           for (b = 0; b < fdof; b++) {
6155             ncSet = fcSet = PETSC_FALSE;
6156             if (b%Nc == comps[ncind]) {ncind = (ncind+1)%Ncc; ncSet = PETSC_TRUE;}
6157             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
6158             if (ncSet && fcSet) {fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));}
6159           }
6160         } else {
6161           for (b = 0; b < fdof; b++) {
6162             if ((cind < fcdof) && (b == fcdofs[cind])) {
6163               fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));
6164               ++cind;
6165             }
6166           }
6167         }
6168       }
6169     }
6170   }
6171   *offset += fdof;
6172   PetscFunctionReturn(0);
6173 }
6174 
6175 static inline PetscErrorCode DMPlexVecSetClosure_Depth1_Static(DM dm, PetscSection section, Vec v, PetscInt point, const PetscScalar values[], InsertMode mode)
6176 {
6177   PetscScalar    *array;
6178   const PetscInt *cone, *coneO;
6179   PetscInt        pStart, pEnd, p, numPoints, off, dof;
6180   PetscErrorCode  ierr;
6181 
6182   PetscFunctionBeginHot;
6183   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
6184   ierr = DMPlexGetConeSize(dm, point, &numPoints);CHKERRQ(ierr);
6185   ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
6186   ierr = DMPlexGetConeOrientation(dm, point, &coneO);CHKERRQ(ierr);
6187   ierr = VecGetArray(v, &array);CHKERRQ(ierr);
6188   for (p = 0, off = 0; p <= numPoints; ++p, off += dof) {
6189     const PetscInt cp = !p ? point : cone[p-1];
6190     const PetscInt o  = !p ? 0     : coneO[p-1];
6191 
6192     if ((cp < pStart) || (cp >= pEnd)) {dof = 0; continue;}
6193     ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
6194     /* ADD_VALUES */
6195     {
6196       const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
6197       PetscScalar    *a;
6198       PetscInt        cdof, coff, cind = 0, k;
6199 
6200       ierr = PetscSectionGetConstraintDof(section, cp, &cdof);CHKERRQ(ierr);
6201       ierr = PetscSectionGetOffset(section, cp, &coff);CHKERRQ(ierr);
6202       a    = &array[coff];
6203       if (!cdof) {
6204         if (o >= 0) {
6205           for (k = 0; k < dof; ++k) {
6206             a[k] += values[off+k];
6207           }
6208         } else {
6209           for (k = 0; k < dof; ++k) {
6210             a[k] += values[off+dof-k-1];
6211           }
6212         }
6213       } else {
6214         ierr = PetscSectionGetConstraintIndices(section, cp, &cdofs);CHKERRQ(ierr);
6215         if (o >= 0) {
6216           for (k = 0; k < dof; ++k) {
6217             if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
6218             a[k] += values[off+k];
6219           }
6220         } else {
6221           for (k = 0; k < dof; ++k) {
6222             if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
6223             a[k] += values[off+dof-k-1];
6224           }
6225         }
6226       }
6227     }
6228   }
6229   ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
6230   PetscFunctionReturn(0);
6231 }
6232 
6233 /*@C
6234   DMPlexVecSetClosure - Set an array of the values on the closure of 'point'
6235 
6236   Not collective
6237 
6238   Input Parameters:
6239 + dm - The DM
6240 . section - The section describing the layout in v, or NULL to use the default section
6241 . v - The local vector
6242 . point - The point in the DM
6243 . values - The array of values
6244 - mode - The insert mode. One of INSERT_ALL_VALUES, ADD_ALL_VALUES, INSERT_VALUES, ADD_VALUES, INSERT_BC_VALUES, and ADD_BC_VALUES,
6245          where INSERT_ALL_VALUES and ADD_ALL_VALUES also overwrite boundary conditions.
6246 
6247   Fortran Notes:
6248   This routine is only available in Fortran 90, and you must include petsc.h90 in your code.
6249 
6250   Level: intermediate
6251 
6252 .seealso DMPlexVecGetClosure(), DMPlexMatSetClosure()
6253 @*/
6254 PetscErrorCode DMPlexVecSetClosure(DM dm, PetscSection section, Vec v, PetscInt point, const PetscScalar values[], InsertMode mode)
6255 {
6256   PetscSection    clSection;
6257   IS              clPoints;
6258   PetscScalar    *array;
6259   PetscInt       *points = NULL;
6260   const PetscInt *clp, *clperm = NULL;
6261   PetscInt        depth, numFields, numPoints, p, clsize;
6262   PetscErrorCode  ierr;
6263 
6264   PetscFunctionBeginHot;
6265   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6266   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
6267   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
6268   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
6269   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
6270   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
6271   if (depth == 1 && numFields < 2 && mode == ADD_VALUES) {
6272     ierr = DMPlexVecSetClosure_Depth1_Static(dm, section, v, point, values, mode);CHKERRQ(ierr);
6273     PetscFunctionReturn(0);
6274   }
6275   /* Get points */
6276   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
6277   for (clsize=0,p=0; p<numPoints; p++) {
6278     PetscInt dof;
6279     ierr = PetscSectionGetDof(section, points[2*p], &dof);CHKERRQ(ierr);
6280     clsize += dof;
6281   }
6282   ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, depth, clsize, &clperm);CHKERRQ(ierr);
6283   /* Get array */
6284   ierr = VecGetArray(v, &array);CHKERRQ(ierr);
6285   /* Get values */
6286   if (numFields > 0) {
6287     PetscInt offset = 0, f;
6288     for (f = 0; f < numFields; ++f) {
6289       const PetscInt    **perms = NULL;
6290       const PetscScalar **flips = NULL;
6291 
6292       ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
6293       switch (mode) {
6294       case INSERT_VALUES:
6295         for (p = 0; p < numPoints; p++) {
6296           const PetscInt    point = points[2*p];
6297           const PetscInt    *perm = perms ? perms[p] : NULL;
6298           const PetscScalar *flip = flips ? flips[p] : NULL;
6299           updatePointFields_private(section, point, perm, flip, f, insert, PETSC_FALSE, clperm, values, &offset, array);
6300         } break;
6301       case INSERT_ALL_VALUES:
6302         for (p = 0; p < numPoints; p++) {
6303           const PetscInt    point = points[2*p];
6304           const PetscInt    *perm = perms ? perms[p] : NULL;
6305           const PetscScalar *flip = flips ? flips[p] : NULL;
6306           updatePointFields_private(section, point, perm, flip, f, insert, PETSC_TRUE, clperm, values, &offset, array);
6307         } break;
6308       case INSERT_BC_VALUES:
6309         for (p = 0; p < numPoints; p++) {
6310           const PetscInt    point = points[2*p];
6311           const PetscInt    *perm = perms ? perms[p] : NULL;
6312           const PetscScalar *flip = flips ? flips[p] : NULL;
6313           updatePointFieldsBC_private(section, point, perm, flip, f, -1, NULL, insert, clperm, values, &offset, array);
6314         } break;
6315       case ADD_VALUES:
6316         for (p = 0; p < numPoints; p++) {
6317           const PetscInt    point = points[2*p];
6318           const PetscInt    *perm = perms ? perms[p] : NULL;
6319           const PetscScalar *flip = flips ? flips[p] : NULL;
6320           updatePointFields_private(section, point, perm, flip, f, add, PETSC_FALSE, clperm, values, &offset, array);
6321         } break;
6322       case ADD_ALL_VALUES:
6323         for (p = 0; p < numPoints; p++) {
6324           const PetscInt    point = points[2*p];
6325           const PetscInt    *perm = perms ? perms[p] : NULL;
6326           const PetscScalar *flip = flips ? flips[p] : NULL;
6327           updatePointFields_private(section, point, perm, flip, f, add, PETSC_TRUE, clperm, values, &offset, array);
6328         } break;
6329       case ADD_BC_VALUES:
6330         for (p = 0; p < numPoints; p++) {
6331           const PetscInt    point = points[2*p];
6332           const PetscInt    *perm = perms ? perms[p] : NULL;
6333           const PetscScalar *flip = flips ? flips[p] : NULL;
6334           updatePointFieldsBC_private(section, point, perm, flip, f, -1, NULL, add, clperm, values, &offset, array);
6335         } break;
6336       default:
6337         SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %d", mode);
6338       }
6339       ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
6340     }
6341   } else {
6342     PetscInt dof, off;
6343     const PetscInt    **perms = NULL;
6344     const PetscScalar **flips = NULL;
6345 
6346     ierr = PetscSectionGetPointSyms(section,numPoints,points,&perms,&flips);CHKERRQ(ierr);
6347     switch (mode) {
6348     case INSERT_VALUES:
6349       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
6350         const PetscInt    point = points[2*p];
6351         const PetscInt    *perm = perms ? perms[p] : NULL;
6352         const PetscScalar *flip = flips ? flips[p] : NULL;
6353         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
6354         updatePoint_private(section, point, dof, insert, PETSC_FALSE, perm, flip, clperm, values, off, array);
6355       } break;
6356     case INSERT_ALL_VALUES:
6357       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
6358         const PetscInt    point = points[2*p];
6359         const PetscInt    *perm = perms ? perms[p] : NULL;
6360         const PetscScalar *flip = flips ? flips[p] : NULL;
6361         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
6362         updatePoint_private(section, point, dof, insert, PETSC_TRUE,  perm, flip, clperm, values, off, array);
6363       } break;
6364     case INSERT_BC_VALUES:
6365       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
6366         const PetscInt    point = points[2*p];
6367         const PetscInt    *perm = perms ? perms[p] : NULL;
6368         const PetscScalar *flip = flips ? flips[p] : NULL;
6369         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
6370         updatePointBC_private(section, point, dof, insert,  perm, flip, clperm, values, off, array);
6371       } break;
6372     case ADD_VALUES:
6373       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
6374         const PetscInt    point = points[2*p];
6375         const PetscInt    *perm = perms ? perms[p] : NULL;
6376         const PetscScalar *flip = flips ? flips[p] : NULL;
6377         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
6378         updatePoint_private(section, point, dof, add,    PETSC_FALSE, perm, flip, clperm, values, off, array);
6379       } break;
6380     case ADD_ALL_VALUES:
6381       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
6382         const PetscInt    point = points[2*p];
6383         const PetscInt    *perm = perms ? perms[p] : NULL;
6384         const PetscScalar *flip = flips ? flips[p] : NULL;
6385         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
6386         updatePoint_private(section, point, dof, add,    PETSC_TRUE,  perm, flip, clperm, values, off, array);
6387       } break;
6388     case ADD_BC_VALUES:
6389       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
6390         const PetscInt    point = points[2*p];
6391         const PetscInt    *perm = perms ? perms[p] : NULL;
6392         const PetscScalar *flip = flips ? flips[p] : NULL;
6393         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
6394         updatePointBC_private(section, point, dof, add,  perm, flip, clperm, values, off, array);
6395       } break;
6396     default:
6397       SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %d", mode);
6398     }
6399     ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms,&flips);CHKERRQ(ierr);
6400   }
6401   /* Cleanup points */
6402   ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
6403   /* Cleanup array */
6404   ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
6405   PetscFunctionReturn(0);
6406 }
6407 
6408 /* Check whether the given point is in the label. If not, update the offset to skip this point */
6409 static inline PetscErrorCode CheckPoint_Private(DMLabel label, PetscInt labelId, PetscSection section, PetscInt point, PetscInt f, PetscInt *offset)
6410 {
6411   PetscFunctionBegin;
6412   if (label) {
6413     PetscInt       val, fdof;
6414     PetscErrorCode ierr;
6415 
6416     /* There is a problem with this:
6417          Suppose we have two label values, defining surfaces, interecting along a line in 3D. When we add cells to the label, the cells that
6418        touch both surfaces must pick a label value. Thus we miss setting values for the surface with that other value intersecting that cell.
6419        Thus I am only going to check val != -1, not val != labelId
6420     */
6421     ierr = DMLabelGetValue(label, point, &val);CHKERRQ(ierr);
6422     if (val < 0) {
6423       ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
6424       *offset += fdof;
6425       PetscFunctionReturn(1);
6426     }
6427   }
6428   PetscFunctionReturn(0);
6429 }
6430 
6431 /* Unlike DMPlexVecSetClosure(), this uses plex-native closure permutation, not a user-specified permutation such as DMPlexSetClosurePermutationTensor(). */
6432 PetscErrorCode DMPlexVecSetFieldClosure_Internal(DM dm, PetscSection section, Vec v, PetscBool fieldActive[], PetscInt point, PetscInt Ncc, const PetscInt comps[], DMLabel label, PetscInt labelId, const PetscScalar values[], InsertMode mode)
6433 {
6434   PetscSection      clSection;
6435   IS                clPoints;
6436   PetscScalar       *array;
6437   PetscInt          *points = NULL;
6438   const PetscInt    *clp;
6439   PetscInt          numFields, numPoints, p;
6440   PetscInt          offset = 0, f;
6441   PetscErrorCode    ierr;
6442 
6443   PetscFunctionBeginHot;
6444   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6445   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
6446   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
6447   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
6448   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
6449   /* Get points */
6450   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
6451   /* Get array */
6452   ierr = VecGetArray(v, &array);CHKERRQ(ierr);
6453   /* Get values */
6454   for (f = 0; f < numFields; ++f) {
6455     const PetscInt    **perms = NULL;
6456     const PetscScalar **flips = NULL;
6457 
6458     if (!fieldActive[f]) {
6459       for (p = 0; p < numPoints*2; p += 2) {
6460         PetscInt fdof;
6461         ierr = PetscSectionGetFieldDof(section, points[p], f, &fdof);CHKERRQ(ierr);
6462         offset += fdof;
6463       }
6464       continue;
6465     }
6466     ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
6467     switch (mode) {
6468     case INSERT_VALUES:
6469       for (p = 0; p < numPoints; p++) {
6470         const PetscInt    point = points[2*p];
6471         const PetscInt    *perm = perms ? perms[p] : NULL;
6472         const PetscScalar *flip = flips ? flips[p] : NULL;
6473         ierr = CheckPoint_Private(label, labelId, section, point, f, &offset); if (ierr) continue;
6474         updatePointFields_private(section, point, perm, flip, f, insert, PETSC_FALSE, NULL, values, &offset, array);
6475       } break;
6476     case INSERT_ALL_VALUES:
6477       for (p = 0; p < numPoints; p++) {
6478         const PetscInt    point = points[2*p];
6479         const PetscInt    *perm = perms ? perms[p] : NULL;
6480         const PetscScalar *flip = flips ? flips[p] : NULL;
6481         ierr = CheckPoint_Private(label, labelId, section, point, f, &offset); if (ierr) continue;
6482         updatePointFields_private(section, point, perm, flip, f, insert, PETSC_TRUE, NULL, values, &offset, array);
6483       } break;
6484     case INSERT_BC_VALUES:
6485       for (p = 0; p < numPoints; p++) {
6486         const PetscInt    point = points[2*p];
6487         const PetscInt    *perm = perms ? perms[p] : NULL;
6488         const PetscScalar *flip = flips ? flips[p] : NULL;
6489         ierr = CheckPoint_Private(label, labelId, section, point, f, &offset); if (ierr) continue;
6490         updatePointFieldsBC_private(section, point, perm, flip, f, Ncc, comps, insert, NULL, values, &offset, array);
6491       } break;
6492     case ADD_VALUES:
6493       for (p = 0; p < numPoints; p++) {
6494         const PetscInt    point = points[2*p];
6495         const PetscInt    *perm = perms ? perms[p] : NULL;
6496         const PetscScalar *flip = flips ? flips[p] : NULL;
6497         ierr = CheckPoint_Private(label, labelId, section, point, f, &offset); if (ierr) continue;
6498         updatePointFields_private(section, point, perm, flip, f, add, PETSC_FALSE, NULL, values, &offset, array);
6499       } break;
6500     case ADD_ALL_VALUES:
6501       for (p = 0; p < numPoints; p++) {
6502         const PetscInt    point = points[2*p];
6503         const PetscInt    *perm = perms ? perms[p] : NULL;
6504         const PetscScalar *flip = flips ? flips[p] : NULL;
6505         ierr = CheckPoint_Private(label, labelId, section, point, f, &offset); if (ierr) continue;
6506         updatePointFields_private(section, point, perm, flip, f, add, PETSC_TRUE, NULL, values, &offset, array);
6507       } break;
6508     default:
6509       SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %d", mode);
6510     }
6511     ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
6512   }
6513   /* Cleanup points */
6514   ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
6515   /* Cleanup array */
6516   ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
6517   PetscFunctionReturn(0);
6518 }
6519 
6520 static PetscErrorCode DMPlexPrintMatSetValues(PetscViewer viewer, Mat A, PetscInt point, PetscInt numRIndices, const PetscInt rindices[], PetscInt numCIndices, const PetscInt cindices[], const PetscScalar values[])
6521 {
6522   PetscMPIInt    rank;
6523   PetscInt       i, j;
6524   PetscErrorCode ierr;
6525 
6526   PetscFunctionBegin;
6527   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRMPI(ierr);
6528   ierr = PetscViewerASCIIPrintf(viewer, "[%d]mat for point %D\n", rank, point);CHKERRQ(ierr);
6529   for (i = 0; i < numRIndices; i++) {ierr = PetscViewerASCIIPrintf(viewer, "[%d]mat row indices[%D] = %D\n", rank, i, rindices[i]);CHKERRQ(ierr);}
6530   for (i = 0; i < numCIndices; i++) {ierr = PetscViewerASCIIPrintf(viewer, "[%d]mat col indices[%D] = %D\n", rank, i, cindices[i]);CHKERRQ(ierr);}
6531   numCIndices = numCIndices ? numCIndices : numRIndices;
6532   if (!values) PetscFunctionReturn(0);
6533   for (i = 0; i < numRIndices; i++) {
6534     ierr = PetscViewerASCIIPrintf(viewer, "[%d]", rank);CHKERRQ(ierr);
6535     for (j = 0; j < numCIndices; j++) {
6536 #if defined(PETSC_USE_COMPLEX)
6537       ierr = PetscViewerASCIIPrintf(viewer, " (%g,%g)", (double)PetscRealPart(values[i*numCIndices+j]), (double)PetscImaginaryPart(values[i*numCIndices+j]));CHKERRQ(ierr);
6538 #else
6539       ierr = PetscViewerASCIIPrintf(viewer, " %g", (double)values[i*numCIndices+j]);CHKERRQ(ierr);
6540 #endif
6541     }
6542     ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
6543   }
6544   PetscFunctionReturn(0);
6545 }
6546 
6547 /*
6548   DMPlexGetIndicesPoint_Internal - Add the indices for dofs on a point to an index array
6549 
6550   Input Parameters:
6551 + section - The section for this data layout
6552 . islocal - Is the section (and thus indices being requested) local or global?
6553 . point   - The point contributing dofs with these indices
6554 . off     - The global offset of this point
6555 . loff    - The local offset of each field
6556 . setBC   - The flag determining whether to include indices of boundary values
6557 . perm    - A permutation of the dofs on this point, or NULL
6558 - indperm - A permutation of the entire indices array, or NULL
6559 
6560   Output Parameter:
6561 . indices - Indices for dofs on this point
6562 
6563   Level: developer
6564 
6565   Note: The indices could be local or global, depending on the value of 'off'.
6566 */
6567 PetscErrorCode DMPlexGetIndicesPoint_Internal(PetscSection section, PetscBool islocal,PetscInt point, PetscInt off, PetscInt *loff, PetscBool setBC, const PetscInt perm[], const PetscInt indperm[], PetscInt indices[])
6568 {
6569   PetscInt        dof;   /* The number of unknowns on this point */
6570   PetscInt        cdof;  /* The number of constraints on this point */
6571   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
6572   PetscInt        cind = 0, k;
6573   PetscErrorCode  ierr;
6574 
6575   PetscFunctionBegin;
6576   PetscAssertFalse(!islocal && setBC,PetscObjectComm((PetscObject)section),PETSC_ERR_ARG_INCOMP,"setBC incompatible with global indices; use a local section or disable setBC");
6577   ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
6578   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
6579   if (!cdof || setBC) {
6580     for (k = 0; k < dof; ++k) {
6581       const PetscInt preind = perm ? *loff+perm[k] : *loff+k;
6582       const PetscInt ind    = indperm ? indperm[preind] : preind;
6583 
6584       indices[ind] = off + k;
6585     }
6586   } else {
6587     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
6588     for (k = 0; k < dof; ++k) {
6589       const PetscInt preind = perm ? *loff+perm[k] : *loff+k;
6590       const PetscInt ind    = indperm ? indperm[preind] : preind;
6591 
6592       if ((cind < cdof) && (k == cdofs[cind])) {
6593         /* Insert check for returning constrained indices */
6594         indices[ind] = -(off+k+1);
6595         ++cind;
6596       } else {
6597         indices[ind] = off + k - (islocal ? 0 : cind);
6598       }
6599     }
6600   }
6601   *loff += dof;
6602   PetscFunctionReturn(0);
6603 }
6604 
6605 /*
6606  DMPlexGetIndicesPointFields_Internal - gets section indices for a point in its canonical ordering.
6607 
6608  Input Parameters:
6609 + section - a section (global or local)
6610 - islocal - PETSC_TRUE if requesting local indices (i.e., section is local); PETSC_FALSE for global
6611 . point - point within section
6612 . off - The offset of this point in the (local or global) indexed space - should match islocal and (usually) the section
6613 . foffs - array of length numFields containing the offset in canonical point ordering (the location in indices) of each field
6614 . setBC - identify constrained (boundary condition) points via involution.
6615 . perms - perms[f][permsoff][:] is a permutation of dofs within each field
6616 . permsoff - offset
6617 - indperm - index permutation
6618 
6619  Output Parameter:
6620 . foffs - each entry is incremented by the number of (unconstrained if setBC=FALSE) dofs in that field
6621 . indices - array to hold indices (as defined by section) of each dof associated with point
6622 
6623  Notes:
6624  If section is local and setBC=true, there is no distinction between constrained and unconstrained dofs.
6625  If section is local and setBC=false, the indices for constrained points are the involution -(i+1) of their position
6626  in the local vector.
6627 
6628  If section is global and setBC=false, the indices for constrained points are negative (and their value is not
6629  significant).  It is invalid to call with a global section and setBC=true.
6630 
6631  Developer Note:
6632  The section is only used for field layout, so islocal is technically a statement about the offset (off).  At some point
6633  in the future, global sections may have fields set, in which case we could pass the global section and obtain the
6634  offset could be obtained from the section instead of passing it explicitly as we do now.
6635 
6636  Example:
6637  Suppose a point contains one field with three components, and for which the unconstrained indices are {10, 11, 12}.
6638  When the middle component is constrained, we get the array {10, -12, 12} for (islocal=TRUE, setBC=FALSE).
6639  Note that -12 is the involution of 11, so the user can involute negative indices to recover local indices.
6640  The global vector does not store constrained dofs, so when this function returns global indices, say {110, -112, 111}, the value of -112 is an arbitrary flag that should not be interpreted beyond its sign.
6641 
6642  Level: developer
6643 */
6644 PetscErrorCode DMPlexGetIndicesPointFields_Internal(PetscSection section, PetscBool islocal, PetscInt point, PetscInt off, PetscInt foffs[], PetscBool setBC, const PetscInt ***perms, PetscInt permsoff, const PetscInt indperm[], PetscInt indices[])
6645 {
6646   PetscInt       numFields, foff, f;
6647   PetscErrorCode ierr;
6648 
6649   PetscFunctionBegin;
6650   PetscAssertFalse(!islocal && setBC,PetscObjectComm((PetscObject)section),PETSC_ERR_ARG_INCOMP,"setBC incompatible with global indices; use a local section or disable setBC");
6651   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
6652   for (f = 0, foff = 0; f < numFields; ++f) {
6653     PetscInt        fdof, cfdof;
6654     const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
6655     PetscInt        cind = 0, b;
6656     const PetscInt  *perm = (perms && perms[f]) ? perms[f][permsoff] : NULL;
6657 
6658     ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
6659     ierr = PetscSectionGetFieldConstraintDof(section, point, f, &cfdof);CHKERRQ(ierr);
6660     if (!cfdof || setBC) {
6661       for (b = 0; b < fdof; ++b) {
6662         const PetscInt preind = perm ? foffs[f]+perm[b] : foffs[f]+b;
6663         const PetscInt ind    = indperm ? indperm[preind] : preind;
6664 
6665         indices[ind] = off+foff+b;
6666       }
6667     } else {
6668       ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
6669       for (b = 0; b < fdof; ++b) {
6670         const PetscInt preind = perm ? foffs[f]+perm[b] : foffs[f]+b;
6671         const PetscInt ind    = indperm ? indperm[preind] : preind;
6672 
6673         if ((cind < cfdof) && (b == fcdofs[cind])) {
6674           indices[ind] = -(off+foff+b+1);
6675           ++cind;
6676         } else {
6677           indices[ind] = off + foff + b - (islocal ? 0 : cind);
6678         }
6679       }
6680     }
6681     foff     += (setBC || islocal ? fdof : (fdof - cfdof));
6682     foffs[f] += fdof;
6683   }
6684   PetscFunctionReturn(0);
6685 }
6686 
6687 /*
6688   This version believes the globalSection offsets for each field, rather than just the point offset
6689 
6690  . foffs - The offset into 'indices' for each field, since it is segregated by field
6691 
6692  Notes:
6693  The semantics of this function relate to that of setBC=FALSE in DMPlexGetIndicesPointFields_Internal.
6694  Since this function uses global indices, setBC=TRUE would be invalid, so no such argument exists.
6695 */
6696 static PetscErrorCode DMPlexGetIndicesPointFieldsSplit_Internal(PetscSection section, PetscSection globalSection, PetscInt point, PetscInt foffs[], const PetscInt ***perms, PetscInt permsoff, const PetscInt indperm[], PetscInt indices[])
6697 {
6698   PetscInt       numFields, foff, f;
6699   PetscErrorCode ierr;
6700 
6701   PetscFunctionBegin;
6702   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
6703   for (f = 0; f < numFields; ++f) {
6704     PetscInt        fdof, cfdof;
6705     const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
6706     PetscInt        cind = 0, b;
6707     const PetscInt  *perm = (perms && perms[f]) ? perms[f][permsoff] : NULL;
6708 
6709     ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
6710     ierr = PetscSectionGetFieldConstraintDof(section, point, f, &cfdof);CHKERRQ(ierr);
6711     ierr = PetscSectionGetFieldOffset(globalSection, point, f, &foff);CHKERRQ(ierr);
6712     if (!cfdof) {
6713       for (b = 0; b < fdof; ++b) {
6714         const PetscInt preind = perm ? foffs[f]+perm[b] : foffs[f]+b;
6715         const PetscInt ind    = indperm ? indperm[preind] : preind;
6716 
6717         indices[ind] = foff+b;
6718       }
6719     } else {
6720       ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
6721       for (b = 0; b < fdof; ++b) {
6722         const PetscInt preind = perm ? foffs[f]+perm[b] : foffs[f]+b;
6723         const PetscInt ind    = indperm ? indperm[preind] : preind;
6724 
6725         if ((cind < cfdof) && (b == fcdofs[cind])) {
6726           indices[ind] = -(foff+b+1);
6727           ++cind;
6728         } else {
6729           indices[ind] = foff+b-cind;
6730         }
6731       }
6732     }
6733     foffs[f] += fdof;
6734   }
6735   PetscFunctionReturn(0);
6736 }
6737 
6738 PetscErrorCode DMPlexAnchorsModifyMat(DM dm, PetscSection section, PetscInt numPoints, PetscInt numIndices, const PetscInt points[], const PetscInt ***perms, const PetscScalar values[], PetscInt *outNumPoints, PetscInt *outNumIndices, PetscInt *outPoints[], PetscScalar *outValues[], PetscInt offsets[], PetscBool multiplyLeft)
6739 {
6740   Mat             cMat;
6741   PetscSection    aSec, cSec;
6742   IS              aIS;
6743   PetscInt        aStart = -1, aEnd = -1;
6744   const PetscInt  *anchors;
6745   PetscInt        numFields, f, p, q, newP = 0;
6746   PetscInt        newNumPoints = 0, newNumIndices = 0;
6747   PetscInt        *newPoints, *indices, *newIndices;
6748   PetscInt        maxAnchor, maxDof;
6749   PetscInt        newOffsets[32];
6750   PetscInt        *pointMatOffsets[32];
6751   PetscInt        *newPointOffsets[32];
6752   PetscScalar     *pointMat[32];
6753   PetscScalar     *newValues=NULL,*tmpValues;
6754   PetscBool       anyConstrained = PETSC_FALSE;
6755   PetscErrorCode  ierr;
6756 
6757   PetscFunctionBegin;
6758   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6759   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
6760   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
6761 
6762   ierr = DMPlexGetAnchors(dm,&aSec,&aIS);CHKERRQ(ierr);
6763   /* if there are point-to-point constraints */
6764   if (aSec) {
6765     ierr = PetscArrayzero(newOffsets, 32);CHKERRQ(ierr);
6766     ierr = ISGetIndices(aIS,&anchors);CHKERRQ(ierr);
6767     ierr = PetscSectionGetChart(aSec,&aStart,&aEnd);CHKERRQ(ierr);
6768     /* figure out how many points are going to be in the new element matrix
6769      * (we allow double counting, because it's all just going to be summed
6770      * into the global matrix anyway) */
6771     for (p = 0; p < 2*numPoints; p+=2) {
6772       PetscInt b    = points[p];
6773       PetscInt bDof = 0, bSecDof;
6774 
6775       ierr = PetscSectionGetDof(section,b,&bSecDof);CHKERRQ(ierr);
6776       if (!bSecDof) {
6777         continue;
6778       }
6779       if (b >= aStart && b < aEnd) {
6780         ierr = PetscSectionGetDof(aSec,b,&bDof);CHKERRQ(ierr);
6781       }
6782       if (bDof) {
6783         /* this point is constrained */
6784         /* it is going to be replaced by its anchors */
6785         PetscInt bOff, q;
6786 
6787         anyConstrained = PETSC_TRUE;
6788         newNumPoints  += bDof;
6789         ierr = PetscSectionGetOffset(aSec,b,&bOff);CHKERRQ(ierr);
6790         for (q = 0; q < bDof; q++) {
6791           PetscInt a = anchors[bOff + q];
6792           PetscInt aDof;
6793 
6794           ierr           = PetscSectionGetDof(section,a,&aDof);CHKERRQ(ierr);
6795           newNumIndices += aDof;
6796           for (f = 0; f < numFields; ++f) {
6797             PetscInt fDof;
6798 
6799             ierr             = PetscSectionGetFieldDof(section, a, f, &fDof);CHKERRQ(ierr);
6800             newOffsets[f+1] += fDof;
6801           }
6802         }
6803       }
6804       else {
6805         /* this point is not constrained */
6806         newNumPoints++;
6807         newNumIndices += bSecDof;
6808         for (f = 0; f < numFields; ++f) {
6809           PetscInt fDof;
6810 
6811           ierr = PetscSectionGetFieldDof(section, b, f, &fDof);CHKERRQ(ierr);
6812           newOffsets[f+1] += fDof;
6813         }
6814       }
6815     }
6816   }
6817   if (!anyConstrained) {
6818     if (outNumPoints)  *outNumPoints  = 0;
6819     if (outNumIndices) *outNumIndices = 0;
6820     if (outPoints)     *outPoints     = NULL;
6821     if (outValues)     *outValues     = NULL;
6822     if (aSec) {ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);}
6823     PetscFunctionReturn(0);
6824   }
6825 
6826   if (outNumPoints)  *outNumPoints  = newNumPoints;
6827   if (outNumIndices) *outNumIndices = newNumIndices;
6828 
6829   for (f = 0; f < numFields; ++f) newOffsets[f+1] += newOffsets[f];
6830 
6831   if (!outPoints && !outValues) {
6832     if (offsets) {
6833       for (f = 0; f <= numFields; f++) {
6834         offsets[f] = newOffsets[f];
6835       }
6836     }
6837     if (aSec) {ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);}
6838     PetscFunctionReturn(0);
6839   }
6840 
6841   PetscAssertFalse(numFields && newOffsets[numFields] != newNumIndices,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", newOffsets[numFields], newNumIndices);
6842 
6843   ierr = DMGetDefaultConstraints(dm, &cSec, &cMat);CHKERRQ(ierr);
6844 
6845   /* workspaces */
6846   if (numFields) {
6847     for (f = 0; f < numFields; f++) {
6848       ierr = DMGetWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[f]);CHKERRQ(ierr);
6849       ierr = DMGetWorkArray(dm,numPoints+1,MPIU_INT,&newPointOffsets[f]);CHKERRQ(ierr);
6850     }
6851   }
6852   else {
6853     ierr = DMGetWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[0]);CHKERRQ(ierr);
6854     ierr = DMGetWorkArray(dm,numPoints,MPIU_INT,&newPointOffsets[0]);CHKERRQ(ierr);
6855   }
6856 
6857   /* get workspaces for the point-to-point matrices */
6858   if (numFields) {
6859     PetscInt totalOffset, totalMatOffset;
6860 
6861     for (p = 0; p < numPoints; p++) {
6862       PetscInt b    = points[2*p];
6863       PetscInt bDof = 0, bSecDof;
6864 
6865       ierr = PetscSectionGetDof(section,b,&bSecDof);CHKERRQ(ierr);
6866       if (!bSecDof) {
6867         for (f = 0; f < numFields; f++) {
6868           newPointOffsets[f][p + 1] = 0;
6869           pointMatOffsets[f][p + 1] = 0;
6870         }
6871         continue;
6872       }
6873       if (b >= aStart && b < aEnd) {
6874         ierr = PetscSectionGetDof(aSec, b, &bDof);CHKERRQ(ierr);
6875       }
6876       if (bDof) {
6877         for (f = 0; f < numFields; f++) {
6878           PetscInt fDof, q, bOff, allFDof = 0;
6879 
6880           ierr = PetscSectionGetFieldDof(section, b, f, &fDof);CHKERRQ(ierr);
6881           ierr = PetscSectionGetOffset(aSec, b, &bOff);CHKERRQ(ierr);
6882           for (q = 0; q < bDof; q++) {
6883             PetscInt a = anchors[bOff + q];
6884             PetscInt aFDof;
6885 
6886             ierr     = PetscSectionGetFieldDof(section, a, f, &aFDof);CHKERRQ(ierr);
6887             allFDof += aFDof;
6888           }
6889           newPointOffsets[f][p+1] = allFDof;
6890           pointMatOffsets[f][p+1] = fDof * allFDof;
6891         }
6892       }
6893       else {
6894         for (f = 0; f < numFields; f++) {
6895           PetscInt fDof;
6896 
6897           ierr = PetscSectionGetFieldDof(section, b, f, &fDof);CHKERRQ(ierr);
6898           newPointOffsets[f][p+1] = fDof;
6899           pointMatOffsets[f][p+1] = 0;
6900         }
6901       }
6902     }
6903     for (f = 0, totalOffset = 0, totalMatOffset = 0; f < numFields; f++) {
6904       newPointOffsets[f][0] = totalOffset;
6905       pointMatOffsets[f][0] = totalMatOffset;
6906       for (p = 0; p < numPoints; p++) {
6907         newPointOffsets[f][p+1] += newPointOffsets[f][p];
6908         pointMatOffsets[f][p+1] += pointMatOffsets[f][p];
6909       }
6910       totalOffset    = newPointOffsets[f][numPoints];
6911       totalMatOffset = pointMatOffsets[f][numPoints];
6912       ierr = DMGetWorkArray(dm,pointMatOffsets[f][numPoints],MPIU_SCALAR,&pointMat[f]);CHKERRQ(ierr);
6913     }
6914   }
6915   else {
6916     for (p = 0; p < numPoints; p++) {
6917       PetscInt b    = points[2*p];
6918       PetscInt bDof = 0, bSecDof;
6919 
6920       ierr = PetscSectionGetDof(section,b,&bSecDof);CHKERRQ(ierr);
6921       if (!bSecDof) {
6922         newPointOffsets[0][p + 1] = 0;
6923         pointMatOffsets[0][p + 1] = 0;
6924         continue;
6925       }
6926       if (b >= aStart && b < aEnd) {
6927         ierr = PetscSectionGetDof(aSec, b, &bDof);CHKERRQ(ierr);
6928       }
6929       if (bDof) {
6930         PetscInt bOff, q, allDof = 0;
6931 
6932         ierr = PetscSectionGetOffset(aSec, b, &bOff);CHKERRQ(ierr);
6933         for (q = 0; q < bDof; q++) {
6934           PetscInt a = anchors[bOff + q], aDof;
6935 
6936           ierr    = PetscSectionGetDof(section, a, &aDof);CHKERRQ(ierr);
6937           allDof += aDof;
6938         }
6939         newPointOffsets[0][p+1] = allDof;
6940         pointMatOffsets[0][p+1] = bSecDof * allDof;
6941       }
6942       else {
6943         newPointOffsets[0][p+1] = bSecDof;
6944         pointMatOffsets[0][p+1] = 0;
6945       }
6946     }
6947     newPointOffsets[0][0] = 0;
6948     pointMatOffsets[0][0] = 0;
6949     for (p = 0; p < numPoints; p++) {
6950       newPointOffsets[0][p+1] += newPointOffsets[0][p];
6951       pointMatOffsets[0][p+1] += pointMatOffsets[0][p];
6952     }
6953     ierr = DMGetWorkArray(dm,pointMatOffsets[0][numPoints],MPIU_SCALAR,&pointMat[0]);CHKERRQ(ierr);
6954   }
6955 
6956   /* output arrays */
6957   ierr = DMGetWorkArray(dm,2*newNumPoints,MPIU_INT,&newPoints);CHKERRQ(ierr);
6958 
6959   /* get the point-to-point matrices; construct newPoints */
6960   ierr = PetscSectionGetMaxDof(aSec, &maxAnchor);CHKERRQ(ierr);
6961   ierr = PetscSectionGetMaxDof(section, &maxDof);CHKERRQ(ierr);
6962   ierr = DMGetWorkArray(dm,maxDof,MPIU_INT,&indices);CHKERRQ(ierr);
6963   ierr = DMGetWorkArray(dm,maxAnchor*maxDof,MPIU_INT,&newIndices);CHKERRQ(ierr);
6964   if (numFields) {
6965     for (p = 0, newP = 0; p < numPoints; p++) {
6966       PetscInt b    = points[2*p];
6967       PetscInt o    = points[2*p+1];
6968       PetscInt bDof = 0, bSecDof;
6969 
6970       ierr = PetscSectionGetDof(section, b, &bSecDof);CHKERRQ(ierr);
6971       if (!bSecDof) {
6972         continue;
6973       }
6974       if (b >= aStart && b < aEnd) {
6975         ierr = PetscSectionGetDof(aSec, b, &bDof);CHKERRQ(ierr);
6976       }
6977       if (bDof) {
6978         PetscInt fStart[32], fEnd[32], fAnchorStart[32], fAnchorEnd[32], bOff, q;
6979 
6980         fStart[0] = 0;
6981         fEnd[0]   = 0;
6982         for (f = 0; f < numFields; f++) {
6983           PetscInt fDof;
6984 
6985           ierr        = PetscSectionGetFieldDof(cSec, b, f, &fDof);CHKERRQ(ierr);
6986           fStart[f+1] = fStart[f] + fDof;
6987           fEnd[f+1]   = fStart[f+1];
6988         }
6989         ierr = PetscSectionGetOffset(cSec, b, &bOff);CHKERRQ(ierr);
6990         ierr = DMPlexGetIndicesPointFields_Internal(cSec, PETSC_TRUE, b, bOff, fEnd, PETSC_TRUE, perms, p, NULL, indices);CHKERRQ(ierr);
6991 
6992         fAnchorStart[0] = 0;
6993         fAnchorEnd[0]   = 0;
6994         for (f = 0; f < numFields; f++) {
6995           PetscInt fDof = newPointOffsets[f][p + 1] - newPointOffsets[f][p];
6996 
6997           fAnchorStart[f+1] = fAnchorStart[f] + fDof;
6998           fAnchorEnd[f+1]   = fAnchorStart[f + 1];
6999         }
7000         ierr = PetscSectionGetOffset(aSec, b, &bOff);CHKERRQ(ierr);
7001         for (q = 0; q < bDof; q++) {
7002           PetscInt a = anchors[bOff + q], aOff;
7003 
7004           /* we take the orientation of ap into account in the order that we constructed the indices above: the newly added points have no orientation */
7005           newPoints[2*(newP + q)]     = a;
7006           newPoints[2*(newP + q) + 1] = 0;
7007           ierr = PetscSectionGetOffset(section, a, &aOff);CHKERRQ(ierr);
7008           ierr = DMPlexGetIndicesPointFields_Internal(section, PETSC_TRUE, a, aOff, fAnchorEnd, PETSC_TRUE, NULL, -1, NULL, newIndices);CHKERRQ(ierr);
7009         }
7010         newP += bDof;
7011 
7012         if (outValues) {
7013           /* get the point-to-point submatrix */
7014           for (f = 0; f < numFields; f++) {
7015             ierr = MatGetValues(cMat,fEnd[f]-fStart[f],indices + fStart[f],fAnchorEnd[f] - fAnchorStart[f],newIndices + fAnchorStart[f],pointMat[f] + pointMatOffsets[f][p]);CHKERRQ(ierr);
7016           }
7017         }
7018       }
7019       else {
7020         newPoints[2 * newP]     = b;
7021         newPoints[2 * newP + 1] = o;
7022         newP++;
7023       }
7024     }
7025   } else {
7026     for (p = 0; p < numPoints; p++) {
7027       PetscInt b    = points[2*p];
7028       PetscInt o    = points[2*p+1];
7029       PetscInt bDof = 0, bSecDof;
7030 
7031       ierr = PetscSectionGetDof(section, b, &bSecDof);CHKERRQ(ierr);
7032       if (!bSecDof) {
7033         continue;
7034       }
7035       if (b >= aStart && b < aEnd) {
7036         ierr = PetscSectionGetDof(aSec, b, &bDof);CHKERRQ(ierr);
7037       }
7038       if (bDof) {
7039         PetscInt bEnd = 0, bAnchorEnd = 0, bOff;
7040 
7041         ierr = PetscSectionGetOffset(cSec, b, &bOff);CHKERRQ(ierr);
7042         ierr = DMPlexGetIndicesPoint_Internal(cSec, PETSC_TRUE, b, bOff, &bEnd, PETSC_TRUE, (perms && perms[0]) ? perms[0][p] : NULL, NULL, indices);CHKERRQ(ierr);
7043 
7044         ierr = PetscSectionGetOffset (aSec, b, &bOff);CHKERRQ(ierr);
7045         for (q = 0; q < bDof; q++) {
7046           PetscInt a = anchors[bOff + q], aOff;
7047 
7048           /* we take the orientation of ap into account in the order that we constructed the indices above: the newly added points have no orientation */
7049 
7050           newPoints[2*(newP + q)]     = a;
7051           newPoints[2*(newP + q) + 1] = 0;
7052           ierr = PetscSectionGetOffset(section, a, &aOff);CHKERRQ(ierr);
7053           ierr = DMPlexGetIndicesPoint_Internal(section, PETSC_TRUE, a, aOff, &bAnchorEnd, PETSC_TRUE, NULL, NULL, newIndices);CHKERRQ(ierr);
7054         }
7055         newP += bDof;
7056 
7057         /* get the point-to-point submatrix */
7058         if (outValues) {
7059           ierr = MatGetValues(cMat,bEnd,indices,bAnchorEnd,newIndices,pointMat[0] + pointMatOffsets[0][p]);CHKERRQ(ierr);
7060         }
7061       }
7062       else {
7063         newPoints[2 * newP]     = b;
7064         newPoints[2 * newP + 1] = o;
7065         newP++;
7066       }
7067     }
7068   }
7069 
7070   if (outValues) {
7071     ierr = DMGetWorkArray(dm,newNumIndices*numIndices,MPIU_SCALAR,&tmpValues);CHKERRQ(ierr);
7072     ierr = PetscArrayzero(tmpValues,newNumIndices*numIndices);CHKERRQ(ierr);
7073     /* multiply constraints on the right */
7074     if (numFields) {
7075       for (f = 0; f < numFields; f++) {
7076         PetscInt oldOff = offsets[f];
7077 
7078         for (p = 0; p < numPoints; p++) {
7079           PetscInt cStart = newPointOffsets[f][p];
7080           PetscInt b      = points[2 * p];
7081           PetscInt c, r, k;
7082           PetscInt dof;
7083 
7084           ierr = PetscSectionGetFieldDof(section,b,f,&dof);CHKERRQ(ierr);
7085           if (!dof) {
7086             continue;
7087           }
7088           if (pointMatOffsets[f][p] < pointMatOffsets[f][p + 1]) {
7089             PetscInt nCols         = newPointOffsets[f][p+1]-cStart;
7090             const PetscScalar *mat = pointMat[f] + pointMatOffsets[f][p];
7091 
7092             for (r = 0; r < numIndices; r++) {
7093               for (c = 0; c < nCols; c++) {
7094                 for (k = 0; k < dof; k++) {
7095                   tmpValues[r * newNumIndices + cStart + c] += values[r * numIndices + oldOff + k] * mat[k * nCols + c];
7096                 }
7097               }
7098             }
7099           }
7100           else {
7101             /* copy this column as is */
7102             for (r = 0; r < numIndices; r++) {
7103               for (c = 0; c < dof; c++) {
7104                 tmpValues[r * newNumIndices + cStart + c] = values[r * numIndices + oldOff + c];
7105               }
7106             }
7107           }
7108           oldOff += dof;
7109         }
7110       }
7111     }
7112     else {
7113       PetscInt oldOff = 0;
7114       for (p = 0; p < numPoints; p++) {
7115         PetscInt cStart = newPointOffsets[0][p];
7116         PetscInt b      = points[2 * p];
7117         PetscInt c, r, k;
7118         PetscInt dof;
7119 
7120         ierr = PetscSectionGetDof(section,b,&dof);CHKERRQ(ierr);
7121         if (!dof) {
7122           continue;
7123         }
7124         if (pointMatOffsets[0][p] < pointMatOffsets[0][p + 1]) {
7125           PetscInt nCols         = newPointOffsets[0][p+1]-cStart;
7126           const PetscScalar *mat = pointMat[0] + pointMatOffsets[0][p];
7127 
7128           for (r = 0; r < numIndices; r++) {
7129             for (c = 0; c < nCols; c++) {
7130               for (k = 0; k < dof; k++) {
7131                 tmpValues[r * newNumIndices + cStart + c] += mat[k * nCols + c] * values[r * numIndices + oldOff + k];
7132               }
7133             }
7134           }
7135         }
7136         else {
7137           /* copy this column as is */
7138           for (r = 0; r < numIndices; r++) {
7139             for (c = 0; c < dof; c++) {
7140               tmpValues[r * newNumIndices + cStart + c] = values[r * numIndices + oldOff + c];
7141             }
7142           }
7143         }
7144         oldOff += dof;
7145       }
7146     }
7147 
7148     if (multiplyLeft) {
7149       ierr = DMGetWorkArray(dm,newNumIndices*newNumIndices,MPIU_SCALAR,&newValues);CHKERRQ(ierr);
7150       ierr = PetscArrayzero(newValues,newNumIndices*newNumIndices);CHKERRQ(ierr);
7151       /* multiply constraints transpose on the left */
7152       if (numFields) {
7153         for (f = 0; f < numFields; f++) {
7154           PetscInt oldOff = offsets[f];
7155 
7156           for (p = 0; p < numPoints; p++) {
7157             PetscInt rStart = newPointOffsets[f][p];
7158             PetscInt b      = points[2 * p];
7159             PetscInt c, r, k;
7160             PetscInt dof;
7161 
7162             ierr = PetscSectionGetFieldDof(section,b,f,&dof);CHKERRQ(ierr);
7163             if (pointMatOffsets[f][p] < pointMatOffsets[f][p + 1]) {
7164               PetscInt nRows                        = newPointOffsets[f][p+1]-rStart;
7165               const PetscScalar *PETSC_RESTRICT mat = pointMat[f] + pointMatOffsets[f][p];
7166 
7167               for (r = 0; r < nRows; r++) {
7168                 for (c = 0; c < newNumIndices; c++) {
7169                   for (k = 0; k < dof; k++) {
7170                     newValues[(rStart + r) * newNumIndices + c] += mat[k * nRows + r] * tmpValues[(oldOff + k) * newNumIndices + c];
7171                   }
7172                 }
7173               }
7174             }
7175             else {
7176               /* copy this row as is */
7177               for (r = 0; r < dof; r++) {
7178                 for (c = 0; c < newNumIndices; c++) {
7179                   newValues[(rStart + r) * newNumIndices + c] = tmpValues[(oldOff + r) * newNumIndices + c];
7180                 }
7181               }
7182             }
7183             oldOff += dof;
7184           }
7185         }
7186       }
7187       else {
7188         PetscInt oldOff = 0;
7189 
7190         for (p = 0; p < numPoints; p++) {
7191           PetscInt rStart = newPointOffsets[0][p];
7192           PetscInt b      = points[2 * p];
7193           PetscInt c, r, k;
7194           PetscInt dof;
7195 
7196           ierr = PetscSectionGetDof(section,b,&dof);CHKERRQ(ierr);
7197           if (pointMatOffsets[0][p] < pointMatOffsets[0][p + 1]) {
7198             PetscInt nRows                        = newPointOffsets[0][p+1]-rStart;
7199             const PetscScalar *PETSC_RESTRICT mat = pointMat[0] + pointMatOffsets[0][p];
7200 
7201             for (r = 0; r < nRows; r++) {
7202               for (c = 0; c < newNumIndices; c++) {
7203                 for (k = 0; k < dof; k++) {
7204                   newValues[(rStart + r) * newNumIndices + c] += mat[k * nRows + r] * tmpValues[(oldOff + k) * newNumIndices + c];
7205                 }
7206               }
7207             }
7208           }
7209           else {
7210             /* copy this row as is */
7211             for (r = 0; r < dof; r++) {
7212               for (c = 0; c < newNumIndices; c++) {
7213                 newValues[(rStart + r) * newNumIndices + c] = tmpValues[(oldOff + r) * newNumIndices + c];
7214               }
7215             }
7216           }
7217           oldOff += dof;
7218         }
7219       }
7220 
7221       ierr = DMRestoreWorkArray(dm,newNumIndices*numIndices,MPIU_SCALAR,&tmpValues);CHKERRQ(ierr);
7222     }
7223     else {
7224       newValues = tmpValues;
7225     }
7226   }
7227 
7228   /* clean up */
7229   ierr = DMRestoreWorkArray(dm,maxDof,MPIU_INT,&indices);CHKERRQ(ierr);
7230   ierr = DMRestoreWorkArray(dm,maxAnchor*maxDof,MPIU_INT,&newIndices);CHKERRQ(ierr);
7231 
7232   if (numFields) {
7233     for (f = 0; f < numFields; f++) {
7234       ierr = DMRestoreWorkArray(dm,pointMatOffsets[f][numPoints],MPIU_SCALAR,&pointMat[f]);CHKERRQ(ierr);
7235       ierr = DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[f]);CHKERRQ(ierr);
7236       ierr = DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&newPointOffsets[f]);CHKERRQ(ierr);
7237     }
7238   }
7239   else {
7240     ierr = DMRestoreWorkArray(dm,pointMatOffsets[0][numPoints],MPIU_SCALAR,&pointMat[0]);CHKERRQ(ierr);
7241     ierr = DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[0]);CHKERRQ(ierr);
7242     ierr = DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&newPointOffsets[0]);CHKERRQ(ierr);
7243   }
7244   ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);
7245 
7246   /* output */
7247   if (outPoints) {
7248     *outPoints = newPoints;
7249   }
7250   else {
7251     ierr = DMRestoreWorkArray(dm,2*newNumPoints,MPIU_INT,&newPoints);CHKERRQ(ierr);
7252   }
7253   if (outValues) {
7254     *outValues = newValues;
7255   }
7256   for (f = 0; f <= numFields; f++) {
7257     offsets[f] = newOffsets[f];
7258   }
7259   PetscFunctionReturn(0);
7260 }
7261 
7262 /*@C
7263   DMPlexGetClosureIndices - Gets the global dof indices associated with the closure of the given point within the provided sections.
7264 
7265   Not collective
7266 
7267   Input Parameters:
7268 + dm         - The DM
7269 . section    - The PetscSection describing the points (a local section)
7270 . idxSection - The PetscSection from which to obtain indices (may be local or global)
7271 . point      - The point defining the closure
7272 - useClPerm  - Use the closure point permutation if available
7273 
7274   Output Parameters:
7275 + numIndices - The number of dof indices in the closure of point with the input sections
7276 . indices    - The dof indices
7277 . outOffsets - Array to write the field offsets into, or NULL
7278 - values     - The input values, which may be modified if sign flips are induced by the point symmetries, or NULL
7279 
7280   Notes:
7281   Must call DMPlexRestoreClosureIndices() to free allocated memory
7282 
7283   If idxSection is global, any constrained dofs (see DMAddBoundary(), for example) will get negative indices.  The value
7284   of those indices is not significant.  If idxSection is local, the constrained dofs will yield the involution -(idx+1)
7285   of their index in a local vector.  A caller who does not wish to distinguish those points may recover the nonnegative
7286   indices via involution, -(-(idx+1)+1)==idx.  Local indices are provided when idxSection == section, otherwise global
7287   indices (with the above semantics) are implied.
7288 
7289   Level: advanced
7290 
7291 .seealso DMPlexRestoreClosureIndices(), DMPlexVecGetClosure(), DMPlexMatSetClosure(), DMGetLocalSection(), DMGetGlobalSection()
7292 @*/
7293 PetscErrorCode DMPlexGetClosureIndices(DM dm, PetscSection section, PetscSection idxSection, PetscInt point, PetscBool useClPerm,
7294                                        PetscInt *numIndices, PetscInt *indices[], PetscInt outOffsets[], PetscScalar *values[])
7295 {
7296   /* Closure ordering */
7297   PetscSection        clSection;
7298   IS                  clPoints;
7299   const PetscInt     *clp;
7300   PetscInt           *points;
7301   const PetscInt     *clperm = NULL;
7302   /* Dof permutation and sign flips */
7303   const PetscInt    **perms[32] = {NULL};
7304   const PetscScalar **flips[32] = {NULL};
7305   PetscScalar        *valCopy   = NULL;
7306   /* Hanging node constraints */
7307   PetscInt           *pointsC = NULL;
7308   PetscScalar        *valuesC = NULL;
7309   PetscInt            NclC, NiC;
7310 
7311   PetscInt           *idx;
7312   PetscInt            Nf, Ncl, Ni = 0, offsets[32], p, f;
7313   PetscBool           isLocal = (section == idxSection) ? PETSC_TRUE : PETSC_FALSE;
7314   PetscErrorCode      ierr;
7315 
7316   PetscFunctionBeginHot;
7317   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7318   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
7319   PetscValidHeaderSpecific(idxSection, PETSC_SECTION_CLASSID, 3);
7320   if (numIndices) PetscValidPointer(numIndices, 6);
7321   if (indices)    PetscValidPointer(indices, 7);
7322   if (outOffsets) PetscValidPointer(outOffsets, 8);
7323   if (values)     PetscValidPointer(values, 9);
7324   ierr = PetscSectionGetNumFields(section, &Nf);CHKERRQ(ierr);
7325   PetscAssertFalse(Nf > 31,PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", Nf);
7326   ierr = PetscArrayzero(offsets, 32);CHKERRQ(ierr);
7327   /* 1) Get points in closure */
7328   ierr = DMPlexGetCompressedClosure(dm, section, point, &Ncl, &points, &clSection, &clPoints, &clp);CHKERRQ(ierr);
7329   if (useClPerm) {
7330     PetscInt depth, clsize;
7331     ierr = DMPlexGetPointDepth(dm, point, &depth);CHKERRQ(ierr);
7332     for (clsize=0,p=0; p<Ncl; p++) {
7333       PetscInt dof;
7334       ierr = PetscSectionGetDof(section, points[2*p], &dof);CHKERRQ(ierr);
7335       clsize += dof;
7336     }
7337     ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, depth, clsize, &clperm);CHKERRQ(ierr);
7338   }
7339   /* 2) Get number of indices on these points and field offsets from section */
7340   for (p = 0; p < Ncl*2; p += 2) {
7341     PetscInt dof, fdof;
7342 
7343     ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
7344     for (f = 0; f < Nf; ++f) {
7345       ierr = PetscSectionGetFieldDof(section, points[p], f, &fdof);CHKERRQ(ierr);
7346       offsets[f+1] += fdof;
7347     }
7348     Ni += dof;
7349   }
7350   for (f = 1; f < Nf; ++f) offsets[f+1] += offsets[f];
7351   PetscAssertFalse(Nf && offsets[Nf] != Ni,PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", offsets[Nf], Ni);
7352   /* 3) Get symmetries and sign flips. Apply sign flips to values if passed in (only works for square values matrix) */
7353   for (f = 0; f < PetscMax(1, Nf); ++f) {
7354     if (Nf) {ierr = PetscSectionGetFieldPointSyms(section, f, Ncl, points, &perms[f], &flips[f]);CHKERRQ(ierr);}
7355     else    {ierr = PetscSectionGetPointSyms(section, Ncl, points, &perms[f], &flips[f]);CHKERRQ(ierr);}
7356     /* may need to apply sign changes to the element matrix */
7357     if (values && flips[f]) {
7358       PetscInt foffset = offsets[f];
7359 
7360       for (p = 0; p < Ncl; ++p) {
7361         PetscInt           pnt  = points[2*p], fdof;
7362         const PetscScalar *flip = flips[f] ? flips[f][p] : NULL;
7363 
7364         if (!Nf) {ierr = PetscSectionGetDof(section, pnt, &fdof);CHKERRQ(ierr);}
7365         else     {ierr = PetscSectionGetFieldDof(section, pnt, f, &fdof);CHKERRQ(ierr);}
7366         if (flip) {
7367           PetscInt i, j, k;
7368 
7369           if (!valCopy) {
7370             ierr = DMGetWorkArray(dm, Ni*Ni, MPIU_SCALAR, &valCopy);CHKERRQ(ierr);
7371             for (j = 0; j < Ni * Ni; ++j) valCopy[j] = (*values)[j];
7372             *values = valCopy;
7373           }
7374           for (i = 0; i < fdof; ++i) {
7375             PetscScalar fval = flip[i];
7376 
7377             for (k = 0; k < Ni; ++k) {
7378               valCopy[Ni * (foffset + i) + k] *= fval;
7379               valCopy[Ni * k + (foffset + i)] *= fval;
7380             }
7381           }
7382         }
7383         foffset += fdof;
7384       }
7385     }
7386   }
7387   /* 4) Apply hanging node constraints. Get new symmetries and replace all storage with constrained storage */
7388   ierr = DMPlexAnchorsModifyMat(dm, section, Ncl, Ni, points, perms, values ? *values : NULL, &NclC, &NiC, &pointsC, values ? &valuesC : NULL, offsets, PETSC_TRUE);CHKERRQ(ierr);
7389   if (NclC) {
7390     if (valCopy) {ierr = DMRestoreWorkArray(dm, Ni*Ni, MPIU_SCALAR, &valCopy);CHKERRQ(ierr);}
7391     for (f = 0; f < PetscMax(1, Nf); ++f) {
7392       if (Nf) {ierr = PetscSectionRestoreFieldPointSyms(section, f, Ncl, points, &perms[f], &flips[f]);CHKERRQ(ierr);}
7393       else    {ierr = PetscSectionRestorePointSyms(section, Ncl, points, &perms[f], &flips[f]);CHKERRQ(ierr);}
7394     }
7395     for (f = 0; f < PetscMax(1, Nf); ++f) {
7396       if (Nf) {ierr = PetscSectionGetFieldPointSyms(section, f, NclC, pointsC, &perms[f], &flips[f]);CHKERRQ(ierr);}
7397       else    {ierr = PetscSectionGetPointSyms(section, NclC, pointsC, &perms[f], &flips[f]);CHKERRQ(ierr);}
7398     }
7399     ierr = DMPlexRestoreCompressedClosure(dm, section, point, &Ncl, &points, &clSection, &clPoints, &clp);CHKERRQ(ierr);
7400     Ncl     = NclC;
7401     Ni      = NiC;
7402     points  = pointsC;
7403     if (values) *values = valuesC;
7404   }
7405   /* 5) Calculate indices */
7406   ierr = DMGetWorkArray(dm, Ni, MPIU_INT, &idx);CHKERRQ(ierr);
7407   if (Nf) {
7408     PetscInt  idxOff;
7409     PetscBool useFieldOffsets;
7410 
7411     if (outOffsets) {for (f = 0; f <= Nf; f++) outOffsets[f] = offsets[f];}
7412     ierr = PetscSectionGetUseFieldOffsets(idxSection, &useFieldOffsets);CHKERRQ(ierr);
7413     if (useFieldOffsets) {
7414       for (p = 0; p < Ncl; ++p) {
7415         const PetscInt pnt = points[p*2];
7416 
7417         ierr = DMPlexGetIndicesPointFieldsSplit_Internal(section, idxSection, pnt, offsets, perms, p, clperm, idx);CHKERRQ(ierr);
7418       }
7419     } else {
7420       for (p = 0; p < Ncl; ++p) {
7421         const PetscInt pnt = points[p*2];
7422 
7423         ierr = PetscSectionGetOffset(idxSection, pnt, &idxOff);CHKERRQ(ierr);
7424         /* Note that we pass a local section even though we're using global offsets.  This is because global sections do
7425          * not (at the time of this writing) have fields set. They probably should, in which case we would pass the
7426          * global section. */
7427         ierr = DMPlexGetIndicesPointFields_Internal(section, isLocal, pnt, idxOff < 0 ? -(idxOff+1) : idxOff, offsets, PETSC_FALSE, perms, p, clperm, idx);CHKERRQ(ierr);
7428       }
7429     }
7430   } else {
7431     PetscInt off = 0, idxOff;
7432 
7433     for (p = 0; p < Ncl; ++p) {
7434       const PetscInt  pnt  = points[p*2];
7435       const PetscInt *perm = perms[0] ? perms[0][p] : NULL;
7436 
7437       ierr = PetscSectionGetOffset(idxSection, pnt, &idxOff);CHKERRQ(ierr);
7438       /* Note that we pass a local section even though we're using global offsets.  This is because global sections do
7439        * not (at the time of this writing) have fields set. They probably should, in which case we would pass the global section. */
7440       ierr = DMPlexGetIndicesPoint_Internal(section, isLocal, pnt, idxOff < 0 ? -(idxOff+1) : idxOff, &off, PETSC_FALSE, perm, clperm, idx);CHKERRQ(ierr);
7441     }
7442   }
7443   /* 6) Cleanup */
7444   for (f = 0; f < PetscMax(1, Nf); ++f) {
7445     if (Nf) {ierr = PetscSectionRestoreFieldPointSyms(section, f, Ncl, points, &perms[f], &flips[f]);CHKERRQ(ierr);}
7446     else    {ierr = PetscSectionRestorePointSyms(section, Ncl, points, &perms[f], &flips[f]);CHKERRQ(ierr);}
7447   }
7448   if (NclC) {
7449     ierr = DMRestoreWorkArray(dm, NclC*2, MPIU_INT, &pointsC);CHKERRQ(ierr);
7450   } else {
7451     ierr = DMPlexRestoreCompressedClosure(dm, section, point, &Ncl, &points, &clSection, &clPoints, &clp);CHKERRQ(ierr);
7452   }
7453 
7454   if (numIndices) *numIndices = Ni;
7455   if (indices)    *indices    = idx;
7456   PetscFunctionReturn(0);
7457 }
7458 
7459 /*@C
7460   DMPlexRestoreClosureIndices - Restores the global dof indices associated with the closure of the given point within the provided sections.
7461 
7462   Not collective
7463 
7464   Input Parameters:
7465 + dm         - The DM
7466 . section    - The PetscSection describing the points (a local section)
7467 . idxSection - The PetscSection from which to obtain indices (may be local or global)
7468 . point      - The point defining the closure
7469 - useClPerm  - Use the closure point permutation if available
7470 
7471   Output Parameters:
7472 + numIndices - The number of dof indices in the closure of point with the input sections
7473 . indices    - The dof indices
7474 . outOffsets - Array to write the field offsets into, or NULL
7475 - values     - The input values, which may be modified if sign flips are induced by the point symmetries, or NULL
7476 
7477   Notes:
7478   If values were modified, the user is responsible for calling DMRestoreWorkArray(dm, 0, MPIU_SCALAR, &values).
7479 
7480   If idxSection is global, any constrained dofs (see DMAddBoundary(), for example) will get negative indices.  The value
7481   of those indices is not significant.  If idxSection is local, the constrained dofs will yield the involution -(idx+1)
7482   of their index in a local vector.  A caller who does not wish to distinguish those points may recover the nonnegative
7483   indices via involution, -(-(idx+1)+1)==idx.  Local indices are provided when idxSection == section, otherwise global
7484   indices (with the above semantics) are implied.
7485 
7486   Level: advanced
7487 
7488 .seealso DMPlexGetClosureIndices(), DMPlexVecGetClosure(), DMPlexMatSetClosure(), DMGetLocalSection(), DMGetGlobalSection()
7489 @*/
7490 PetscErrorCode DMPlexRestoreClosureIndices(DM dm, PetscSection section, PetscSection idxSection, PetscInt point, PetscBool useClPerm,
7491                                            PetscInt *numIndices, PetscInt *indices[], PetscInt outOffsets[], PetscScalar *values[])
7492 {
7493   PetscErrorCode ierr;
7494 
7495   PetscFunctionBegin;
7496   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7497   PetscValidPointer(indices, 7);
7498   ierr = DMRestoreWorkArray(dm, 0, MPIU_INT, indices);CHKERRQ(ierr);
7499   PetscFunctionReturn(0);
7500 }
7501 
7502 /*@C
7503   DMPlexMatSetClosure - Set an array of the values on the closure of 'point'
7504 
7505   Not collective
7506 
7507   Input Parameters:
7508 + dm - The DM
7509 . section - The section describing the layout in v, or NULL to use the default section
7510 . globalSection - The section describing the layout in v, or NULL to use the default global section
7511 . A - The matrix
7512 . point - The point in the DM
7513 . values - The array of values
7514 - mode - The insert mode, where INSERT_ALL_VALUES and ADD_ALL_VALUES also overwrite boundary conditions
7515 
7516   Fortran Notes:
7517   This routine is only available in Fortran 90, and you must include petsc.h90 in your code.
7518 
7519   Level: intermediate
7520 
7521 .seealso DMPlexMatSetClosureGeneral(), DMPlexVecGetClosure(), DMPlexVecSetClosure()
7522 @*/
7523 PetscErrorCode DMPlexMatSetClosure(DM dm, PetscSection section, PetscSection globalSection, Mat A, PetscInt point, const PetscScalar values[], InsertMode mode)
7524 {
7525   DM_Plex           *mesh = (DM_Plex*) dm->data;
7526   PetscInt          *indices;
7527   PetscInt           numIndices;
7528   const PetscScalar *valuesOrig = values;
7529   PetscErrorCode     ierr;
7530 
7531   PetscFunctionBegin;
7532   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7533   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
7534   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
7535   if (!globalSection) {ierr = DMGetGlobalSection(dm, &globalSection);CHKERRQ(ierr);}
7536   PetscValidHeaderSpecific(globalSection, PETSC_SECTION_CLASSID, 3);
7537   PetscValidHeaderSpecific(A, MAT_CLASSID, 4);
7538 
7539   ierr = DMPlexGetClosureIndices(dm, section, globalSection, point, PETSC_TRUE, &numIndices, &indices, NULL, (PetscScalar **) &values);CHKERRQ(ierr);
7540 
7541   if (mesh->printSetValues) {ierr = DMPlexPrintMatSetValues(PETSC_VIEWER_STDOUT_SELF, A, point, numIndices, indices, 0, NULL, values);CHKERRQ(ierr);}
7542   ierr = MatSetValues(A, numIndices, indices, numIndices, indices, values, mode);
7543   if (ierr) {
7544     PetscMPIInt    rank;
7545     PetscErrorCode ierr2;
7546 
7547     ierr2 = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRMPI(ierr2);
7548     ierr2 = (*PetscErrorPrintf)("[%d]ERROR in DMPlexMatSetClosure\n", rank);CHKERRQ(ierr2);
7549     ierr2 = DMPlexPrintMatSetValues(PETSC_VIEWER_STDERR_SELF, A, point, numIndices, indices, 0, NULL, values);CHKERRQ(ierr2);
7550     ierr2 = DMPlexRestoreClosureIndices(dm, section, globalSection, point, PETSC_TRUE, &numIndices, &indices, NULL, (PetscScalar **) &values);CHKERRQ(ierr2);
7551     if (values != valuesOrig) {ierr2 = DMRestoreWorkArray(dm, 0, MPIU_SCALAR, &values);CHKERRQ(ierr2);}
7552     SETERRQ(PetscObjectComm((PetscObject)dm),ierr,"Not possible to set matrix values");
7553   }
7554   if (mesh->printFEM > 1) {
7555     PetscInt i;
7556     ierr = PetscPrintf(PETSC_COMM_SELF, "  Indices:");CHKERRQ(ierr);
7557     for (i = 0; i < numIndices; ++i) {ierr = PetscPrintf(PETSC_COMM_SELF, " %D", indices[i]);CHKERRQ(ierr);}
7558     ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
7559   }
7560 
7561   ierr = DMPlexRestoreClosureIndices(dm, section, globalSection, point, PETSC_TRUE, &numIndices, &indices, NULL, (PetscScalar **) &values);CHKERRQ(ierr);
7562   if (values != valuesOrig) {ierr = DMRestoreWorkArray(dm, 0, MPIU_SCALAR, &values);CHKERRQ(ierr);}
7563   PetscFunctionReturn(0);
7564 }
7565 
7566 /*@C
7567   DMPlexMatSetClosure - Set an array of the values on the closure of 'point' using a different row and column section
7568 
7569   Not collective
7570 
7571   Input Parameters:
7572 + dmRow - The DM for the row fields
7573 . sectionRow - The section describing the layout, or NULL to use the default section in dmRow
7574 . globalSectionRow - The section describing the layout, or NULL to use the default global section in dmRow
7575 . dmCol - The DM for the column fields
7576 . sectionCol - The section describing the layout, or NULL to use the default section in dmCol
7577 . globalSectionCol - The section describing the layout, or NULL to use the default global section in dmCol
7578 . A - The matrix
7579 . point - The point in the DMs
7580 . values - The array of values
7581 - mode - The insert mode, where INSERT_ALL_VALUES and ADD_ALL_VALUES also overwrite boundary conditions
7582 
7583   Level: intermediate
7584 
7585 .seealso DMPlexMatSetClosure(), DMPlexVecGetClosure(), DMPlexVecSetClosure()
7586 @*/
7587 PetscErrorCode DMPlexMatSetClosureGeneral(DM dmRow, PetscSection sectionRow, PetscSection globalSectionRow, DM dmCol, PetscSection sectionCol, PetscSection globalSectionCol, Mat A, PetscInt point, const PetscScalar values[], InsertMode mode)
7588 {
7589   DM_Plex           *mesh = (DM_Plex*) dmRow->data;
7590   PetscInt          *indicesRow, *indicesCol;
7591   PetscInt           numIndicesRow, numIndicesCol;
7592   const PetscScalar *valuesOrig = values;
7593   PetscErrorCode     ierr;
7594 
7595   PetscFunctionBegin;
7596   PetscValidHeaderSpecific(dmRow, DM_CLASSID, 1);
7597   if (!sectionRow) {ierr = DMGetLocalSection(dmRow, &sectionRow);CHKERRQ(ierr);}
7598   PetscValidHeaderSpecific(sectionRow, PETSC_SECTION_CLASSID, 2);
7599   if (!globalSectionRow) {ierr = DMGetGlobalSection(dmRow, &globalSectionRow);CHKERRQ(ierr);}
7600   PetscValidHeaderSpecific(globalSectionRow, PETSC_SECTION_CLASSID, 3);
7601   PetscValidHeaderSpecific(dmCol, DM_CLASSID, 4);
7602   if (!sectionCol) {ierr = DMGetLocalSection(dmCol, &sectionCol);CHKERRQ(ierr);}
7603   PetscValidHeaderSpecific(sectionCol, PETSC_SECTION_CLASSID, 5);
7604   if (!globalSectionCol) {ierr = DMGetGlobalSection(dmCol, &globalSectionCol);CHKERRQ(ierr);}
7605   PetscValidHeaderSpecific(globalSectionCol, PETSC_SECTION_CLASSID, 6);
7606   PetscValidHeaderSpecific(A, MAT_CLASSID, 7);
7607 
7608   ierr = DMPlexGetClosureIndices(dmRow, sectionRow, globalSectionRow, point, PETSC_TRUE, &numIndicesRow, &indicesRow, NULL, (PetscScalar **) &values);CHKERRQ(ierr);
7609   ierr = DMPlexGetClosureIndices(dmCol, sectionCol, globalSectionCol, point, PETSC_TRUE, &numIndicesCol, &indicesCol, NULL, (PetscScalar **) &values);CHKERRQ(ierr);
7610 
7611   if (mesh->printSetValues) {ierr = DMPlexPrintMatSetValues(PETSC_VIEWER_STDOUT_SELF, A, point, numIndicesRow, indicesRow, numIndicesCol, indicesCol, values);CHKERRQ(ierr);}
7612   ierr = MatSetValues(A, numIndicesRow, indicesRow, numIndicesCol, indicesCol, values, mode);
7613   if (ierr) {
7614     PetscMPIInt    rank;
7615     PetscErrorCode ierr2;
7616 
7617     ierr2 = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRMPI(ierr2);
7618     ierr2 = (*PetscErrorPrintf)("[%d]ERROR in DMPlexMatSetClosure\n", rank);CHKERRQ(ierr2);
7619     ierr2 = DMPlexPrintMatSetValues(PETSC_VIEWER_STDERR_SELF, A, point, numIndicesRow, indicesRow, numIndicesCol, indicesCol, values);CHKERRQ(ierr2);
7620     ierr2 = DMPlexRestoreClosureIndices(dmRow, sectionRow, globalSectionRow, point, PETSC_TRUE, &numIndicesRow, &indicesRow, NULL, (PetscScalar **) &values);CHKERRQ(ierr2);
7621     ierr2 = DMPlexRestoreClosureIndices(dmCol, sectionCol, globalSectionCol, point, PETSC_TRUE, &numIndicesCol, &indicesRow, NULL, (PetscScalar **) &values);CHKERRQ(ierr2);
7622     if (values != valuesOrig) {ierr2 = DMRestoreWorkArray(dmRow, 0, MPIU_SCALAR, &values);CHKERRQ(ierr2);}
7623     CHKERRQ(ierr);
7624   }
7625 
7626   ierr = DMPlexRestoreClosureIndices(dmRow, sectionRow, globalSectionRow, point, PETSC_TRUE, &numIndicesRow, &indicesRow, NULL, (PetscScalar **) &values);CHKERRQ(ierr);
7627   ierr = DMPlexRestoreClosureIndices(dmCol, sectionCol, globalSectionCol, point, PETSC_TRUE, &numIndicesCol, &indicesCol, NULL, (PetscScalar **) &values);CHKERRQ(ierr);
7628   if (values != valuesOrig) {ierr = DMRestoreWorkArray(dmRow, 0, MPIU_SCALAR, &values);CHKERRQ(ierr);}
7629   PetscFunctionReturn(0);
7630 }
7631 
7632 PetscErrorCode DMPlexMatSetClosureRefined(DM dmf, PetscSection fsection, PetscSection globalFSection, DM dmc, PetscSection csection, PetscSection globalCSection, Mat A, PetscInt point, const PetscScalar values[], InsertMode mode)
7633 {
7634   DM_Plex        *mesh   = (DM_Plex*) dmf->data;
7635   PetscInt       *fpoints = NULL, *ftotpoints = NULL;
7636   PetscInt       *cpoints = NULL;
7637   PetscInt       *findices, *cindices;
7638   const PetscInt *fclperm = NULL, *cclperm = NULL; /* Closure permutations cannot work here */
7639   PetscInt        foffsets[32], coffsets[32];
7640   DMPolytopeType  ct;
7641   PetscInt        numFields, numSubcells, maxFPoints, numFPoints, numCPoints, numFIndices, numCIndices, dof, off, globalOff, pStart, pEnd, p, q, r, s, f;
7642   PetscErrorCode  ierr;
7643 
7644   PetscFunctionBegin;
7645   PetscValidHeaderSpecific(dmf, DM_CLASSID, 1);
7646   PetscValidHeaderSpecific(dmc, DM_CLASSID, 4);
7647   if (!fsection) {ierr = DMGetLocalSection(dmf, &fsection);CHKERRQ(ierr);}
7648   PetscValidHeaderSpecific(fsection, PETSC_SECTION_CLASSID, 2);
7649   if (!csection) {ierr = DMGetLocalSection(dmc, &csection);CHKERRQ(ierr);}
7650   PetscValidHeaderSpecific(csection, PETSC_SECTION_CLASSID, 5);
7651   if (!globalFSection) {ierr = DMGetGlobalSection(dmf, &globalFSection);CHKERRQ(ierr);}
7652   PetscValidHeaderSpecific(globalFSection, PETSC_SECTION_CLASSID, 3);
7653   if (!globalCSection) {ierr = DMGetGlobalSection(dmc, &globalCSection);CHKERRQ(ierr);}
7654   PetscValidHeaderSpecific(globalCSection, PETSC_SECTION_CLASSID, 6);
7655   PetscValidHeaderSpecific(A, MAT_CLASSID, 7);
7656   ierr = PetscSectionGetNumFields(fsection, &numFields);CHKERRQ(ierr);
7657   PetscAssertFalse(numFields > 31,PetscObjectComm((PetscObject)dmf), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", numFields);
7658   ierr = PetscArrayzero(foffsets, 32);CHKERRQ(ierr);
7659   ierr = PetscArrayzero(coffsets, 32);CHKERRQ(ierr);
7660   /* Column indices */
7661   ierr = DMPlexGetTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
7662   maxFPoints = numCPoints;
7663   /* Compress out points not in the section */
7664   /*   TODO: Squeeze out points with 0 dof as well */
7665   ierr = PetscSectionGetChart(csection, &pStart, &pEnd);CHKERRQ(ierr);
7666   for (p = 0, q = 0; p < numCPoints*2; p += 2) {
7667     if ((cpoints[p] >= pStart) && (cpoints[p] < pEnd)) {
7668       cpoints[q*2]   = cpoints[p];
7669       cpoints[q*2+1] = cpoints[p+1];
7670       ++q;
7671     }
7672   }
7673   numCPoints = q;
7674   for (p = 0, numCIndices = 0; p < numCPoints*2; p += 2) {
7675     PetscInt fdof;
7676 
7677     ierr = PetscSectionGetDof(csection, cpoints[p], &dof);CHKERRQ(ierr);
7678     if (!dof) continue;
7679     for (f = 0; f < numFields; ++f) {
7680       ierr           = PetscSectionGetFieldDof(csection, cpoints[p], f, &fdof);CHKERRQ(ierr);
7681       coffsets[f+1] += fdof;
7682     }
7683     numCIndices += dof;
7684   }
7685   for (f = 1; f < numFields; ++f) coffsets[f+1] += coffsets[f];
7686   /* Row indices */
7687   ierr = DMPlexGetCellType(dmc, point, &ct);CHKERRQ(ierr);
7688   {
7689     DMPlexTransform tr;
7690     DMPolytopeType *rct;
7691     PetscInt       *rsize, *rcone, *rornt, Nt;
7692 
7693     ierr = DMPlexTransformCreate(PETSC_COMM_SELF, &tr);CHKERRQ(ierr);
7694     ierr = DMPlexTransformSetType(tr, DMPLEXREFINEREGULAR);CHKERRQ(ierr);
7695     ierr = DMPlexTransformCellTransform(tr, ct, point, NULL, &Nt, &rct, &rsize, &rcone, &rornt);CHKERRQ(ierr);
7696     numSubcells = rsize[Nt-1];
7697     ierr = DMPlexTransformDestroy(&tr);CHKERRQ(ierr);
7698   }
7699   ierr = DMGetWorkArray(dmf, maxFPoints*2*numSubcells, MPIU_INT, &ftotpoints);CHKERRQ(ierr);
7700   for (r = 0, q = 0; r < numSubcells; ++r) {
7701     /* TODO Map from coarse to fine cells */
7702     ierr = DMPlexGetTransitiveClosure(dmf, point*numSubcells + r, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
7703     /* Compress out points not in the section */
7704     ierr = PetscSectionGetChart(fsection, &pStart, &pEnd);CHKERRQ(ierr);
7705     for (p = 0; p < numFPoints*2; p += 2) {
7706       if ((fpoints[p] >= pStart) && (fpoints[p] < pEnd)) {
7707         ierr = PetscSectionGetDof(fsection, fpoints[p], &dof);CHKERRQ(ierr);
7708         if (!dof) continue;
7709         for (s = 0; s < q; ++s) if (fpoints[p] == ftotpoints[s*2]) break;
7710         if (s < q) continue;
7711         ftotpoints[q*2]   = fpoints[p];
7712         ftotpoints[q*2+1] = fpoints[p+1];
7713         ++q;
7714       }
7715     }
7716     ierr = DMPlexRestoreTransitiveClosure(dmf, point, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
7717   }
7718   numFPoints = q;
7719   for (p = 0, numFIndices = 0; p < numFPoints*2; p += 2) {
7720     PetscInt fdof;
7721 
7722     ierr = PetscSectionGetDof(fsection, ftotpoints[p], &dof);CHKERRQ(ierr);
7723     if (!dof) continue;
7724     for (f = 0; f < numFields; ++f) {
7725       ierr           = PetscSectionGetFieldDof(fsection, ftotpoints[p], f, &fdof);CHKERRQ(ierr);
7726       foffsets[f+1] += fdof;
7727     }
7728     numFIndices += dof;
7729   }
7730   for (f = 1; f < numFields; ++f) foffsets[f+1] += foffsets[f];
7731 
7732   PetscAssertFalse(numFields && foffsets[numFields] != numFIndices,PetscObjectComm((PetscObject)dmf), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", foffsets[numFields], numFIndices);
7733   PetscAssertFalse(numFields && coffsets[numFields] != numCIndices,PetscObjectComm((PetscObject)dmc), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", coffsets[numFields], numCIndices);
7734   ierr = DMGetWorkArray(dmf, numFIndices, MPIU_INT, &findices);CHKERRQ(ierr);
7735   ierr = DMGetWorkArray(dmc, numCIndices, MPIU_INT, &cindices);CHKERRQ(ierr);
7736   if (numFields) {
7737     const PetscInt **permsF[32] = {NULL};
7738     const PetscInt **permsC[32] = {NULL};
7739 
7740     for (f = 0; f < numFields; f++) {
7741       ierr = PetscSectionGetFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL);CHKERRQ(ierr);
7742       ierr = PetscSectionGetFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL);CHKERRQ(ierr);
7743     }
7744     for (p = 0; p < numFPoints; p++) {
7745       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff);CHKERRQ(ierr);
7746       ierr = DMPlexGetIndicesPointFields_Internal(fsection, PETSC_FALSE, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, foffsets, PETSC_FALSE, permsF, p, fclperm, findices);CHKERRQ(ierr);
7747     }
7748     for (p = 0; p < numCPoints; p++) {
7749       ierr = PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff);CHKERRQ(ierr);
7750       ierr = DMPlexGetIndicesPointFields_Internal(csection, PETSC_FALSE, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, coffsets, PETSC_FALSE, permsC, p, cclperm, cindices);CHKERRQ(ierr);
7751     }
7752     for (f = 0; f < numFields; f++) {
7753       ierr = PetscSectionRestoreFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL);CHKERRQ(ierr);
7754       ierr = PetscSectionRestoreFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL);CHKERRQ(ierr);
7755     }
7756   } else {
7757     const PetscInt **permsF = NULL;
7758     const PetscInt **permsC = NULL;
7759 
7760     ierr = PetscSectionGetPointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL);CHKERRQ(ierr);
7761     ierr = PetscSectionGetPointSyms(csection,numCPoints,cpoints,&permsC,NULL);CHKERRQ(ierr);
7762     for (p = 0, off = 0; p < numFPoints; p++) {
7763       const PetscInt *perm = permsF ? permsF[p] : NULL;
7764 
7765       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff);CHKERRQ(ierr);
7766       ierr = DMPlexGetIndicesPoint_Internal(fsection, PETSC_FALSE, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, fclperm, findices);CHKERRQ(ierr);
7767     }
7768     for (p = 0, off = 0; p < numCPoints; p++) {
7769       const PetscInt *perm = permsC ? permsC[p] : NULL;
7770 
7771       ierr = PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff);CHKERRQ(ierr);
7772       ierr = DMPlexGetIndicesPoint_Internal(csection, PETSC_FALSE, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, cclperm, cindices);CHKERRQ(ierr);
7773     }
7774     ierr = PetscSectionRestorePointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL);CHKERRQ(ierr);
7775     ierr = PetscSectionRestorePointSyms(csection,numCPoints,cpoints,&permsC,NULL);CHKERRQ(ierr);
7776   }
7777   if (mesh->printSetValues) {ierr = DMPlexPrintMatSetValues(PETSC_VIEWER_STDOUT_SELF, A, point, numFIndices, findices, numCIndices, cindices, values);CHKERRQ(ierr);}
7778   /* TODO: flips */
7779   ierr = MatSetValues(A, numFIndices, findices, numCIndices, cindices, values, mode);
7780   if (ierr) {
7781     PetscMPIInt    rank;
7782     PetscErrorCode ierr2;
7783 
7784     ierr2 = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRMPI(ierr2);
7785     ierr2 = (*PetscErrorPrintf)("[%d]ERROR in DMPlexMatSetClosure\n", rank);CHKERRQ(ierr2);
7786     ierr2 = DMPlexPrintMatSetValues(PETSC_VIEWER_STDERR_SELF, A, point, numFIndices, findices, numCIndices, cindices, values);CHKERRQ(ierr2);
7787     ierr2 = DMRestoreWorkArray(dmf, numFIndices, MPIU_INT, &findices);CHKERRQ(ierr2);
7788     ierr2 = DMRestoreWorkArray(dmc, numCIndices, MPIU_INT, &cindices);CHKERRQ(ierr2);
7789     CHKERRQ(ierr);
7790   }
7791   ierr = DMRestoreWorkArray(dmf, numCPoints*2*4, MPIU_INT, &ftotpoints);CHKERRQ(ierr);
7792   ierr = DMPlexRestoreTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
7793   ierr = DMRestoreWorkArray(dmf, numFIndices, MPIU_INT, &findices);CHKERRQ(ierr);
7794   ierr = DMRestoreWorkArray(dmc, numCIndices, MPIU_INT, &cindices);CHKERRQ(ierr);
7795   PetscFunctionReturn(0);
7796 }
7797 
7798 PetscErrorCode DMPlexMatGetClosureIndicesRefined(DM dmf, PetscSection fsection, PetscSection globalFSection, DM dmc, PetscSection csection, PetscSection globalCSection, PetscInt point, PetscInt cindices[], PetscInt findices[])
7799 {
7800   PetscInt      *fpoints = NULL, *ftotpoints = NULL;
7801   PetscInt      *cpoints = NULL;
7802   PetscInt       foffsets[32], coffsets[32];
7803   const PetscInt *fclperm = NULL, *cclperm = NULL; /* Closure permutations cannot work here */
7804   DMPolytopeType ct;
7805   PetscInt       numFields, numSubcells, maxFPoints, numFPoints, numCPoints, numFIndices, numCIndices, dof, off, globalOff, pStart, pEnd, p, q, r, s, f;
7806   PetscErrorCode ierr;
7807 
7808   PetscFunctionBegin;
7809   PetscValidHeaderSpecific(dmf, DM_CLASSID, 1);
7810   PetscValidHeaderSpecific(dmc, DM_CLASSID, 4);
7811   if (!fsection) {ierr = DMGetLocalSection(dmf, &fsection);CHKERRQ(ierr);}
7812   PetscValidHeaderSpecific(fsection, PETSC_SECTION_CLASSID, 2);
7813   if (!csection) {ierr = DMGetLocalSection(dmc, &csection);CHKERRQ(ierr);}
7814   PetscValidHeaderSpecific(csection, PETSC_SECTION_CLASSID, 5);
7815   if (!globalFSection) {ierr = DMGetGlobalSection(dmf, &globalFSection);CHKERRQ(ierr);}
7816   PetscValidHeaderSpecific(globalFSection, PETSC_SECTION_CLASSID, 3);
7817   if (!globalCSection) {ierr = DMGetGlobalSection(dmc, &globalCSection);CHKERRQ(ierr);}
7818   PetscValidHeaderSpecific(globalCSection, PETSC_SECTION_CLASSID, 6);
7819   ierr = PetscSectionGetNumFields(fsection, &numFields);CHKERRQ(ierr);
7820   PetscAssertFalse(numFields > 31,PetscObjectComm((PetscObject)dmf), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", numFields);
7821   ierr = PetscArrayzero(foffsets, 32);CHKERRQ(ierr);
7822   ierr = PetscArrayzero(coffsets, 32);CHKERRQ(ierr);
7823   /* Column indices */
7824   ierr = DMPlexGetTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
7825   maxFPoints = numCPoints;
7826   /* Compress out points not in the section */
7827   /*   TODO: Squeeze out points with 0 dof as well */
7828   ierr = PetscSectionGetChart(csection, &pStart, &pEnd);CHKERRQ(ierr);
7829   for (p = 0, q = 0; p < numCPoints*2; p += 2) {
7830     if ((cpoints[p] >= pStart) && (cpoints[p] < pEnd)) {
7831       cpoints[q*2]   = cpoints[p];
7832       cpoints[q*2+1] = cpoints[p+1];
7833       ++q;
7834     }
7835   }
7836   numCPoints = q;
7837   for (p = 0, numCIndices = 0; p < numCPoints*2; p += 2) {
7838     PetscInt fdof;
7839 
7840     ierr = PetscSectionGetDof(csection, cpoints[p], &dof);CHKERRQ(ierr);
7841     if (!dof) continue;
7842     for (f = 0; f < numFields; ++f) {
7843       ierr           = PetscSectionGetFieldDof(csection, cpoints[p], f, &fdof);CHKERRQ(ierr);
7844       coffsets[f+1] += fdof;
7845     }
7846     numCIndices += dof;
7847   }
7848   for (f = 1; f < numFields; ++f) coffsets[f+1] += coffsets[f];
7849   /* Row indices */
7850   ierr = DMPlexGetCellType(dmc, point, &ct);CHKERRQ(ierr);
7851   {
7852     DMPlexTransform tr;
7853     DMPolytopeType *rct;
7854     PetscInt       *rsize, *rcone, *rornt, Nt;
7855 
7856     ierr = DMPlexTransformCreate(PETSC_COMM_SELF, &tr);CHKERRQ(ierr);
7857     ierr = DMPlexTransformSetType(tr, DMPLEXREFINEREGULAR);CHKERRQ(ierr);
7858     ierr = DMPlexTransformCellTransform(tr, ct, point, NULL, &Nt, &rct, &rsize, &rcone, &rornt);CHKERRQ(ierr);
7859     numSubcells = rsize[Nt-1];
7860     ierr = DMPlexTransformDestroy(&tr);CHKERRQ(ierr);
7861   }
7862   ierr = DMGetWorkArray(dmf, maxFPoints*2*numSubcells, MPIU_INT, &ftotpoints);CHKERRQ(ierr);
7863   for (r = 0, q = 0; r < numSubcells; ++r) {
7864     /* TODO Map from coarse to fine cells */
7865     ierr = DMPlexGetTransitiveClosure(dmf, point*numSubcells + r, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
7866     /* Compress out points not in the section */
7867     ierr = PetscSectionGetChart(fsection, &pStart, &pEnd);CHKERRQ(ierr);
7868     for (p = 0; p < numFPoints*2; p += 2) {
7869       if ((fpoints[p] >= pStart) && (fpoints[p] < pEnd)) {
7870         ierr = PetscSectionGetDof(fsection, fpoints[p], &dof);CHKERRQ(ierr);
7871         if (!dof) continue;
7872         for (s = 0; s < q; ++s) if (fpoints[p] == ftotpoints[s*2]) break;
7873         if (s < q) continue;
7874         ftotpoints[q*2]   = fpoints[p];
7875         ftotpoints[q*2+1] = fpoints[p+1];
7876         ++q;
7877       }
7878     }
7879     ierr = DMPlexRestoreTransitiveClosure(dmf, point, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
7880   }
7881   numFPoints = q;
7882   for (p = 0, numFIndices = 0; p < numFPoints*2; p += 2) {
7883     PetscInt fdof;
7884 
7885     ierr = PetscSectionGetDof(fsection, ftotpoints[p], &dof);CHKERRQ(ierr);
7886     if (!dof) continue;
7887     for (f = 0; f < numFields; ++f) {
7888       ierr           = PetscSectionGetFieldDof(fsection, ftotpoints[p], f, &fdof);CHKERRQ(ierr);
7889       foffsets[f+1] += fdof;
7890     }
7891     numFIndices += dof;
7892   }
7893   for (f = 1; f < numFields; ++f) foffsets[f+1] += foffsets[f];
7894 
7895   PetscAssertFalse(numFields && foffsets[numFields] != numFIndices,PetscObjectComm((PetscObject)dmf), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", foffsets[numFields], numFIndices);
7896   PetscAssertFalse(numFields && coffsets[numFields] != numCIndices,PetscObjectComm((PetscObject)dmc), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", coffsets[numFields], numCIndices);
7897   if (numFields) {
7898     const PetscInt **permsF[32] = {NULL};
7899     const PetscInt **permsC[32] = {NULL};
7900 
7901     for (f = 0; f < numFields; f++) {
7902       ierr = PetscSectionGetFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL);CHKERRQ(ierr);
7903       ierr = PetscSectionGetFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL);CHKERRQ(ierr);
7904     }
7905     for (p = 0; p < numFPoints; p++) {
7906       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff);CHKERRQ(ierr);
7907       ierr = DMPlexGetIndicesPointFields_Internal(fsection, PETSC_FALSE, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, foffsets, PETSC_FALSE, permsF, p, fclperm, findices);CHKERRQ(ierr);
7908     }
7909     for (p = 0; p < numCPoints; p++) {
7910       ierr = PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff);CHKERRQ(ierr);
7911       ierr = DMPlexGetIndicesPointFields_Internal(csection, PETSC_FALSE, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, coffsets, PETSC_FALSE, permsC, p, cclperm, cindices);CHKERRQ(ierr);
7912     }
7913     for (f = 0; f < numFields; f++) {
7914       ierr = PetscSectionRestoreFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL);CHKERRQ(ierr);
7915       ierr = PetscSectionRestoreFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL);CHKERRQ(ierr);
7916     }
7917   } else {
7918     const PetscInt **permsF = NULL;
7919     const PetscInt **permsC = NULL;
7920 
7921     ierr = PetscSectionGetPointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL);CHKERRQ(ierr);
7922     ierr = PetscSectionGetPointSyms(csection,numCPoints,cpoints,&permsC,NULL);CHKERRQ(ierr);
7923     for (p = 0, off = 0; p < numFPoints; p++) {
7924       const PetscInt *perm = permsF ? permsF[p] : NULL;
7925 
7926       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff);CHKERRQ(ierr);
7927       ierr = DMPlexGetIndicesPoint_Internal(fsection, PETSC_FALSE, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, fclperm, findices);CHKERRQ(ierr);
7928     }
7929     for (p = 0, off = 0; p < numCPoints; p++) {
7930       const PetscInt *perm = permsC ? permsC[p] : NULL;
7931 
7932       ierr = PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff);CHKERRQ(ierr);
7933       ierr = DMPlexGetIndicesPoint_Internal(csection, PETSC_FALSE, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, cclperm, cindices);CHKERRQ(ierr);
7934     }
7935     ierr = PetscSectionRestorePointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL);CHKERRQ(ierr);
7936     ierr = PetscSectionRestorePointSyms(csection,numCPoints,cpoints,&permsC,NULL);CHKERRQ(ierr);
7937   }
7938   ierr = DMRestoreWorkArray(dmf, numCPoints*2*4, MPIU_INT, &ftotpoints);CHKERRQ(ierr);
7939   ierr = DMPlexRestoreTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
7940   PetscFunctionReturn(0);
7941 }
7942 
7943 /*@C
7944   DMPlexGetVTKCellHeight - Returns the height in the DAG used to determine which points are cells (normally 0)
7945 
7946   Input Parameter:
7947 . dm   - The DMPlex object
7948 
7949   Output Parameter:
7950 . cellHeight - The height of a cell
7951 
7952   Level: developer
7953 
7954 .seealso DMPlexSetVTKCellHeight()
7955 @*/
7956 PetscErrorCode DMPlexGetVTKCellHeight(DM dm, PetscInt *cellHeight)
7957 {
7958   DM_Plex *mesh = (DM_Plex*) dm->data;
7959 
7960   PetscFunctionBegin;
7961   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7962   PetscValidPointer(cellHeight, 2);
7963   *cellHeight = mesh->vtkCellHeight;
7964   PetscFunctionReturn(0);
7965 }
7966 
7967 /*@C
7968   DMPlexSetVTKCellHeight - Sets the height in the DAG used to determine which points are cells (normally 0)
7969 
7970   Input Parameters:
7971 + dm   - The DMPlex object
7972 - cellHeight - The height of a cell
7973 
7974   Level: developer
7975 
7976 .seealso DMPlexGetVTKCellHeight()
7977 @*/
7978 PetscErrorCode DMPlexSetVTKCellHeight(DM dm, PetscInt cellHeight)
7979 {
7980   DM_Plex *mesh = (DM_Plex*) dm->data;
7981 
7982   PetscFunctionBegin;
7983   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7984   mesh->vtkCellHeight = cellHeight;
7985   PetscFunctionReturn(0);
7986 }
7987 
7988 /*@
7989   DMPlexGetGhostCellStratum - Get the range of cells which are used to enforce FV boundary conditions
7990 
7991   Input Parameter:
7992 . dm - The DMPlex object
7993 
7994   Output Parameters:
7995 + gcStart - The first ghost cell, or NULL
7996 - gcEnd   - The upper bound on ghost cells, or NULL
7997 
7998   Level: advanced
7999 
8000 .seealso DMPlexConstructGhostCells(), DMPlexGetGhostCellStratum()
8001 @*/
8002 PetscErrorCode DMPlexGetGhostCellStratum(DM dm, PetscInt *gcStart, PetscInt *gcEnd)
8003 {
8004   DMLabel        ctLabel;
8005   PetscErrorCode ierr;
8006 
8007   PetscFunctionBegin;
8008   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8009   ierr = DMPlexGetCellTypeLabel(dm, &ctLabel);CHKERRQ(ierr);
8010   ierr = DMLabelGetStratumBounds(ctLabel, DM_POLYTOPE_FV_GHOST, gcStart, gcEnd);CHKERRQ(ierr);
8011   PetscFunctionReturn(0);
8012 }
8013 
8014 /* We can easily have a form that takes an IS instead */
8015 PetscErrorCode DMPlexCreateNumbering_Plex(DM dm, PetscInt pStart, PetscInt pEnd, PetscInt shift, PetscInt *globalSize, PetscSF sf, IS *numbering)
8016 {
8017   PetscSection   section, globalSection;
8018   PetscInt      *numbers, p;
8019   PetscErrorCode ierr;
8020 
8021   PetscFunctionBegin;
8022   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &section);CHKERRQ(ierr);
8023   ierr = PetscSectionSetChart(section, pStart, pEnd);CHKERRQ(ierr);
8024   for (p = pStart; p < pEnd; ++p) {
8025     ierr = PetscSectionSetDof(section, p, 1);CHKERRQ(ierr);
8026   }
8027   ierr = PetscSectionSetUp(section);CHKERRQ(ierr);
8028   ierr = PetscSectionCreateGlobalSection(section, sf, PETSC_FALSE, PETSC_FALSE, &globalSection);CHKERRQ(ierr);
8029   ierr = PetscMalloc1(pEnd - pStart, &numbers);CHKERRQ(ierr);
8030   for (p = pStart; p < pEnd; ++p) {
8031     ierr = PetscSectionGetOffset(globalSection, p, &numbers[p-pStart]);CHKERRQ(ierr);
8032     if (numbers[p-pStart] < 0) numbers[p-pStart] -= shift;
8033     else                       numbers[p-pStart] += shift;
8034   }
8035   ierr = ISCreateGeneral(PetscObjectComm((PetscObject) dm), pEnd - pStart, numbers, PETSC_OWN_POINTER, numbering);CHKERRQ(ierr);
8036   if (globalSize) {
8037     PetscLayout layout;
8038     ierr = PetscSectionGetPointLayout(PetscObjectComm((PetscObject) dm), globalSection, &layout);CHKERRQ(ierr);
8039     ierr = PetscLayoutGetSize(layout, globalSize);CHKERRQ(ierr);
8040     ierr = PetscLayoutDestroy(&layout);CHKERRQ(ierr);
8041   }
8042   ierr = PetscSectionDestroy(&section);CHKERRQ(ierr);
8043   ierr = PetscSectionDestroy(&globalSection);CHKERRQ(ierr);
8044   PetscFunctionReturn(0);
8045 }
8046 
8047 PetscErrorCode DMPlexCreateCellNumbering_Internal(DM dm, PetscBool includeHybrid, IS *globalCellNumbers)
8048 {
8049   PetscInt       cellHeight, cStart, cEnd;
8050   PetscErrorCode ierr;
8051 
8052   PetscFunctionBegin;
8053   ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
8054   if (includeHybrid) {ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);}
8055   else               {ierr = DMPlexGetSimplexOrBoxCells(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);}
8056   ierr = DMPlexCreateNumbering_Plex(dm, cStart, cEnd, 0, NULL, dm->sf, globalCellNumbers);CHKERRQ(ierr);
8057   PetscFunctionReturn(0);
8058 }
8059 
8060 /*@
8061   DMPlexGetCellNumbering - Get a global cell numbering for all cells on this process
8062 
8063   Input Parameter:
8064 . dm   - The DMPlex object
8065 
8066   Output Parameter:
8067 . globalCellNumbers - Global cell numbers for all cells on this process
8068 
8069   Level: developer
8070 
8071 .seealso DMPlexGetVertexNumbering()
8072 @*/
8073 PetscErrorCode DMPlexGetCellNumbering(DM dm, IS *globalCellNumbers)
8074 {
8075   DM_Plex       *mesh = (DM_Plex*) dm->data;
8076   PetscErrorCode ierr;
8077 
8078   PetscFunctionBegin;
8079   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8080   if (!mesh->globalCellNumbers) {ierr = DMPlexCreateCellNumbering_Internal(dm, PETSC_FALSE, &mesh->globalCellNumbers);CHKERRQ(ierr);}
8081   *globalCellNumbers = mesh->globalCellNumbers;
8082   PetscFunctionReturn(0);
8083 }
8084 
8085 PetscErrorCode DMPlexCreateVertexNumbering_Internal(DM dm, PetscBool includeHybrid, IS *globalVertexNumbers)
8086 {
8087   PetscInt       vStart, vEnd;
8088   PetscErrorCode ierr;
8089 
8090   PetscFunctionBegin;
8091   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8092   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
8093   ierr = DMPlexCreateNumbering_Plex(dm, vStart, vEnd, 0, NULL, dm->sf, globalVertexNumbers);CHKERRQ(ierr);
8094   PetscFunctionReturn(0);
8095 }
8096 
8097 /*@
8098   DMPlexGetVertexNumbering - Get a global vertex numbering for all vertices on this process
8099 
8100   Input Parameter:
8101 . dm   - The DMPlex object
8102 
8103   Output Parameter:
8104 . globalVertexNumbers - Global vertex numbers for all vertices on this process
8105 
8106   Level: developer
8107 
8108 .seealso DMPlexGetCellNumbering()
8109 @*/
8110 PetscErrorCode DMPlexGetVertexNumbering(DM dm, IS *globalVertexNumbers)
8111 {
8112   DM_Plex       *mesh = (DM_Plex*) dm->data;
8113   PetscErrorCode ierr;
8114 
8115   PetscFunctionBegin;
8116   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8117   if (!mesh->globalVertexNumbers) {ierr = DMPlexCreateVertexNumbering_Internal(dm, PETSC_FALSE, &mesh->globalVertexNumbers);CHKERRQ(ierr);}
8118   *globalVertexNumbers = mesh->globalVertexNumbers;
8119   PetscFunctionReturn(0);
8120 }
8121 
8122 /*@
8123   DMPlexCreatePointNumbering - Create a global numbering for all points on this process
8124 
8125   Input Parameter:
8126 . dm   - The DMPlex object
8127 
8128   Output Parameter:
8129 . globalPointNumbers - Global numbers for all points on this process
8130 
8131   Level: developer
8132 
8133 .seealso DMPlexGetCellNumbering()
8134 @*/
8135 PetscErrorCode DMPlexCreatePointNumbering(DM dm, IS *globalPointNumbers)
8136 {
8137   IS             nums[4];
8138   PetscInt       depths[4], gdepths[4], starts[4];
8139   PetscInt       depth, d, shift = 0;
8140   PetscErrorCode ierr;
8141 
8142   PetscFunctionBegin;
8143   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8144   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
8145   /* For unstratified meshes use dim instead of depth */
8146   if (depth < 0) {ierr = DMGetDimension(dm, &depth);CHKERRQ(ierr);}
8147   for (d = 0; d <= depth; ++d) {
8148     PetscInt end;
8149 
8150     depths[d] = depth-d;
8151     ierr = DMPlexGetDepthStratum(dm, depths[d], &starts[d], &end);CHKERRQ(ierr);
8152     if (!(starts[d]-end)) { starts[d] = depths[d] = -1; }
8153   }
8154   ierr = PetscSortIntWithArray(depth+1, starts, depths);CHKERRQ(ierr);
8155   ierr = MPIU_Allreduce(depths, gdepths, depth+1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject) dm));CHKERRMPI(ierr);
8156   for (d = 0; d <= depth; ++d) {
8157     PetscAssertFalse(starts[d] >= 0 && depths[d] != gdepths[d],PETSC_COMM_SELF,PETSC_ERR_PLIB,"Expected depth %D, found %D",depths[d],gdepths[d]);
8158   }
8159   for (d = 0; d <= depth; ++d) {
8160     PetscInt pStart, pEnd, gsize;
8161 
8162     ierr = DMPlexGetDepthStratum(dm, gdepths[d], &pStart, &pEnd);CHKERRQ(ierr);
8163     ierr = DMPlexCreateNumbering_Plex(dm, pStart, pEnd, shift, &gsize, dm->sf, &nums[d]);CHKERRQ(ierr);
8164     shift += gsize;
8165   }
8166   ierr = ISConcatenate(PetscObjectComm((PetscObject) dm), depth+1, nums, globalPointNumbers);CHKERRQ(ierr);
8167   for (d = 0; d <= depth; ++d) {ierr = ISDestroy(&nums[d]);CHKERRQ(ierr);}
8168   PetscFunctionReturn(0);
8169 }
8170 
8171 /*@
8172   DMPlexCreateRankField - Create a cell field whose value is the rank of the owner
8173 
8174   Input Parameter:
8175 . dm - The DMPlex object
8176 
8177   Output Parameter:
8178 . ranks - The rank field
8179 
8180   Options Database Keys:
8181 . -dm_partition_view - Adds the rank field into the DM output from -dm_view using the same viewer
8182 
8183   Level: intermediate
8184 
8185 .seealso: DMView()
8186 @*/
8187 PetscErrorCode DMPlexCreateRankField(DM dm, Vec *ranks)
8188 {
8189   DM             rdm;
8190   PetscFE        fe;
8191   PetscScalar   *r;
8192   PetscMPIInt    rank;
8193   DMPolytopeType ct;
8194   PetscInt       dim, cStart, cEnd, c;
8195   PetscBool      simplex;
8196   PetscErrorCode ierr;
8197 
8198   PetscFunctionBeginUser;
8199   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8200   PetscValidPointer(ranks, 2);
8201   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRMPI(ierr);
8202   ierr = DMClone(dm, &rdm);CHKERRQ(ierr);
8203   ierr = DMGetDimension(rdm, &dim);CHKERRQ(ierr);
8204   ierr = DMPlexGetHeightStratum(rdm, 0, &cStart, &cEnd);CHKERRQ(ierr);
8205   ierr = DMPlexGetCellType(dm, cStart, &ct);CHKERRQ(ierr);
8206   simplex = DMPolytopeTypeGetNumVertices(ct) == DMPolytopeTypeGetDim(ct)+1 ? PETSC_TRUE : PETSC_FALSE;
8207   ierr = PetscFECreateDefault(PETSC_COMM_SELF, dim, 1, simplex, "PETSc___rank_", -1, &fe);CHKERRQ(ierr);
8208   ierr = PetscObjectSetName((PetscObject) fe, "rank");CHKERRQ(ierr);
8209   ierr = DMSetField(rdm, 0, NULL, (PetscObject) fe);CHKERRQ(ierr);
8210   ierr = PetscFEDestroy(&fe);CHKERRQ(ierr);
8211   ierr = DMCreateDS(rdm);CHKERRQ(ierr);
8212   ierr = DMCreateGlobalVector(rdm, ranks);CHKERRQ(ierr);
8213   ierr = PetscObjectSetName((PetscObject) *ranks, "partition");CHKERRQ(ierr);
8214   ierr = VecGetArray(*ranks, &r);CHKERRQ(ierr);
8215   for (c = cStart; c < cEnd; ++c) {
8216     PetscScalar *lr;
8217 
8218     ierr = DMPlexPointGlobalRef(rdm, c, r, &lr);CHKERRQ(ierr);
8219     if (lr) *lr = rank;
8220   }
8221   ierr = VecRestoreArray(*ranks, &r);CHKERRQ(ierr);
8222   ierr = DMDestroy(&rdm);CHKERRQ(ierr);
8223   PetscFunctionReturn(0);
8224 }
8225 
8226 /*@
8227   DMPlexCreateLabelField - Create a cell field whose value is the label value for that cell
8228 
8229   Input Parameters:
8230 + dm    - The DMPlex
8231 - label - The DMLabel
8232 
8233   Output Parameter:
8234 . val - The label value field
8235 
8236   Options Database Keys:
8237 . -dm_label_view - Adds the label value field into the DM output from -dm_view using the same viewer
8238 
8239   Level: intermediate
8240 
8241 .seealso: DMView()
8242 @*/
8243 PetscErrorCode DMPlexCreateLabelField(DM dm, DMLabel label, Vec *val)
8244 {
8245   DM             rdm;
8246   PetscFE        fe;
8247   PetscScalar   *v;
8248   PetscInt       dim, cStart, cEnd, c;
8249   PetscErrorCode ierr;
8250 
8251   PetscFunctionBeginUser;
8252   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8253   PetscValidPointer(label, 2);
8254   PetscValidPointer(val, 3);
8255   ierr = DMClone(dm, &rdm);CHKERRQ(ierr);
8256   ierr = DMGetDimension(rdm, &dim);CHKERRQ(ierr);
8257   ierr = PetscFECreateDefault(PetscObjectComm((PetscObject) rdm), dim, 1, PETSC_TRUE, "PETSc___label_value_", -1, &fe);CHKERRQ(ierr);
8258   ierr = PetscObjectSetName((PetscObject) fe, "label_value");CHKERRQ(ierr);
8259   ierr = DMSetField(rdm, 0, NULL, (PetscObject) fe);CHKERRQ(ierr);
8260   ierr = PetscFEDestroy(&fe);CHKERRQ(ierr);
8261   ierr = DMCreateDS(rdm);CHKERRQ(ierr);
8262   ierr = DMPlexGetHeightStratum(rdm, 0, &cStart, &cEnd);CHKERRQ(ierr);
8263   ierr = DMCreateGlobalVector(rdm, val);CHKERRQ(ierr);
8264   ierr = PetscObjectSetName((PetscObject) *val, "label_value");CHKERRQ(ierr);
8265   ierr = VecGetArray(*val, &v);CHKERRQ(ierr);
8266   for (c = cStart; c < cEnd; ++c) {
8267     PetscScalar *lv;
8268     PetscInt     cval;
8269 
8270     ierr = DMPlexPointGlobalRef(rdm, c, v, &lv);CHKERRQ(ierr);
8271     ierr = DMLabelGetValue(label, c, &cval);CHKERRQ(ierr);
8272     *lv = cval;
8273   }
8274   ierr = VecRestoreArray(*val, &v);CHKERRQ(ierr);
8275   ierr = DMDestroy(&rdm);CHKERRQ(ierr);
8276   PetscFunctionReturn(0);
8277 }
8278 
8279 /*@
8280   DMPlexCheckSymmetry - Check that the adjacency information in the mesh is symmetric.
8281 
8282   Input Parameter:
8283 . dm - The DMPlex object
8284 
8285   Notes:
8286   This is a useful diagnostic when creating meshes programmatically.
8287 
8288   For the complete list of DMPlexCheck* functions, see DMSetFromOptions().
8289 
8290   Level: developer
8291 
8292 .seealso: DMCreate(), DMSetFromOptions()
8293 @*/
8294 PetscErrorCode DMPlexCheckSymmetry(DM dm)
8295 {
8296   PetscSection    coneSection, supportSection;
8297   const PetscInt *cone, *support;
8298   PetscInt        coneSize, c, supportSize, s;
8299   PetscInt        pStart, pEnd, p, pp, csize, ssize;
8300   PetscBool       storagecheck = PETSC_TRUE;
8301   PetscErrorCode  ierr;
8302 
8303   PetscFunctionBegin;
8304   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8305   ierr = DMViewFromOptions(dm, NULL, "-sym_dm_view");CHKERRQ(ierr);
8306   ierr = DMPlexGetConeSection(dm, &coneSection);CHKERRQ(ierr);
8307   ierr = DMPlexGetSupportSection(dm, &supportSection);CHKERRQ(ierr);
8308   /* Check that point p is found in the support of its cone points, and vice versa */
8309   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
8310   for (p = pStart; p < pEnd; ++p) {
8311     ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
8312     ierr = DMPlexGetCone(dm, p, &cone);CHKERRQ(ierr);
8313     for (c = 0; c < coneSize; ++c) {
8314       PetscBool dup = PETSC_FALSE;
8315       PetscInt  d;
8316       for (d = c-1; d >= 0; --d) {
8317         if (cone[c] == cone[d]) {dup = PETSC_TRUE; break;}
8318       }
8319       ierr = DMPlexGetSupportSize(dm, cone[c], &supportSize);CHKERRQ(ierr);
8320       ierr = DMPlexGetSupport(dm, cone[c], &support);CHKERRQ(ierr);
8321       for (s = 0; s < supportSize; ++s) {
8322         if (support[s] == p) break;
8323       }
8324       if ((s >= supportSize) || (dup && (support[s+1] != p))) {
8325         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %D cone: ", p);CHKERRQ(ierr);
8326         for (s = 0; s < coneSize; ++s) {
8327           ierr = PetscPrintf(PETSC_COMM_SELF, "%D, ", cone[s]);CHKERRQ(ierr);
8328         }
8329         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
8330         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %D support: ", cone[c]);CHKERRQ(ierr);
8331         for (s = 0; s < supportSize; ++s) {
8332           ierr = PetscPrintf(PETSC_COMM_SELF, "%D, ", support[s]);CHKERRQ(ierr);
8333         }
8334         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
8335         PetscAssertFalse(dup,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %D not repeatedly found in support of repeated cone point %D", p, cone[c]);
8336         else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %D not found in support of cone point %D", p, cone[c]);
8337       }
8338     }
8339     ierr = DMPlexGetTreeParent(dm, p, &pp, NULL);CHKERRQ(ierr);
8340     if (p != pp) { storagecheck = PETSC_FALSE; continue; }
8341     ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
8342     ierr = DMPlexGetSupport(dm, p, &support);CHKERRQ(ierr);
8343     for (s = 0; s < supportSize; ++s) {
8344       ierr = DMPlexGetConeSize(dm, support[s], &coneSize);CHKERRQ(ierr);
8345       ierr = DMPlexGetCone(dm, support[s], &cone);CHKERRQ(ierr);
8346       for (c = 0; c < coneSize; ++c) {
8347         ierr = DMPlexGetTreeParent(dm, cone[c], &pp, NULL);CHKERRQ(ierr);
8348         if (cone[c] != pp) { c = 0; break; }
8349         if (cone[c] == p) break;
8350       }
8351       if (c >= coneSize) {
8352         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %D support: ", p);CHKERRQ(ierr);
8353         for (c = 0; c < supportSize; ++c) {
8354           ierr = PetscPrintf(PETSC_COMM_SELF, "%D, ", support[c]);CHKERRQ(ierr);
8355         }
8356         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
8357         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %D cone: ", support[s]);CHKERRQ(ierr);
8358         for (c = 0; c < coneSize; ++c) {
8359           ierr = PetscPrintf(PETSC_COMM_SELF, "%D, ", cone[c]);CHKERRQ(ierr);
8360         }
8361         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
8362         SETERRQ(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %D not found in cone of support point %D", p, support[s]);
8363       }
8364     }
8365   }
8366   if (storagecheck) {
8367     ierr = PetscSectionGetStorageSize(coneSection, &csize);CHKERRQ(ierr);
8368     ierr = PetscSectionGetStorageSize(supportSection, &ssize);CHKERRQ(ierr);
8369     PetscAssertFalse(csize != ssize,PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "Total cone size %D != Total support size %D", csize, ssize);
8370   }
8371   PetscFunctionReturn(0);
8372 }
8373 
8374 /*
8375   For submeshes with cohesive cells (see DMPlexConstructCohesiveCells()), we allow a special case where some of the boundary of a face (edges and vertices) are not duplicated. We call these special boundary points "unsplit", since the same edge or vertex appears in both copies of the face. These unsplit points throw off our counting, so we have to explicitly account for them here.
8376 */
8377 static PetscErrorCode DMPlexCellUnsplitVertices_Private(DM dm, PetscInt c, DMPolytopeType ct, PetscInt *unsplit)
8378 {
8379   DMPolytopeType  cct;
8380   PetscInt        ptpoints[4];
8381   const PetscInt *cone, *ccone, *ptcone;
8382   PetscInt        coneSize, cp, cconeSize, ccp, npt = 0, pt;
8383   PetscErrorCode  ierr;
8384 
8385   PetscFunctionBegin;
8386   *unsplit = 0;
8387   switch (ct) {
8388     case DM_POLYTOPE_POINT_PRISM_TENSOR:
8389       ptpoints[npt++] = c;
8390       break;
8391     case DM_POLYTOPE_SEG_PRISM_TENSOR:
8392       ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
8393       ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
8394       for (cp = 0; cp < coneSize; ++cp) {
8395         ierr = DMPlexGetCellType(dm, cone[cp], &cct);CHKERRQ(ierr);
8396         if (cct == DM_POLYTOPE_POINT_PRISM_TENSOR) ptpoints[npt++] = cone[cp];
8397       }
8398       break;
8399     case DM_POLYTOPE_TRI_PRISM_TENSOR:
8400     case DM_POLYTOPE_QUAD_PRISM_TENSOR:
8401       ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
8402       ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
8403       for (cp = 0; cp < coneSize; ++cp) {
8404         ierr = DMPlexGetCone(dm, cone[cp], &ccone);CHKERRQ(ierr);
8405         ierr = DMPlexGetConeSize(dm, cone[cp], &cconeSize);CHKERRQ(ierr);
8406         for (ccp = 0; ccp < cconeSize; ++ccp) {
8407           ierr = DMPlexGetCellType(dm, ccone[ccp], &cct);CHKERRQ(ierr);
8408           if (cct == DM_POLYTOPE_POINT_PRISM_TENSOR) {
8409             PetscInt p;
8410             for (p = 0; p < npt; ++p) if (ptpoints[p] == ccone[ccp]) break;
8411             if (p == npt) ptpoints[npt++] = ccone[ccp];
8412           }
8413         }
8414       }
8415       break;
8416     default: break;
8417   }
8418   for (pt = 0; pt < npt; ++pt) {
8419     ierr = DMPlexGetCone(dm, ptpoints[pt], &ptcone);CHKERRQ(ierr);
8420     if (ptcone[0] == ptcone[1]) ++(*unsplit);
8421   }
8422   PetscFunctionReturn(0);
8423 }
8424 
8425 /*@
8426   DMPlexCheckSkeleton - Check that each cell has the correct number of vertices
8427 
8428   Input Parameters:
8429 + dm - The DMPlex object
8430 - cellHeight - Normally 0
8431 
8432   Notes:
8433   This is a useful diagnostic when creating meshes programmatically.
8434   Currently applicable only to homogeneous simplex or tensor meshes.
8435 
8436   For the complete list of DMPlexCheck* functions, see DMSetFromOptions().
8437 
8438   Level: developer
8439 
8440 .seealso: DMCreate(), DMSetFromOptions()
8441 @*/
8442 PetscErrorCode DMPlexCheckSkeleton(DM dm, PetscInt cellHeight)
8443 {
8444   DMPlexInterpolatedFlag interp;
8445   DMPolytopeType         ct;
8446   PetscInt               vStart, vEnd, cStart, cEnd, c;
8447   PetscErrorCode         ierr;
8448 
8449   PetscFunctionBegin;
8450   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8451   ierr = DMPlexIsInterpolated(dm, &interp);CHKERRQ(ierr);
8452   ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
8453   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
8454   for (c = cStart; c < cEnd; ++c) {
8455     PetscInt *closure = NULL;
8456     PetscInt  coneSize, closureSize, cl, Nv = 0;
8457 
8458     ierr = DMPlexGetCellType(dm, c, &ct);CHKERRQ(ierr);
8459     PetscAssertFalse((PetscInt) ct < 0,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %D has no cell type", c);
8460     if (ct == DM_POLYTOPE_UNKNOWN) continue;
8461     if (interp == DMPLEX_INTERPOLATED_FULL) {
8462       ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
8463       PetscAssertFalse(coneSize != DMPolytopeTypeGetConeSize(ct),PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %D of type %s has cone size %D != %D", c, DMPolytopeTypes[ct], coneSize, DMPolytopeTypeGetConeSize(ct));
8464     }
8465     ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
8466     for (cl = 0; cl < closureSize*2; cl += 2) {
8467       const PetscInt p = closure[cl];
8468       if ((p >= vStart) && (p < vEnd)) ++Nv;
8469     }
8470     ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
8471     /* Special Case: Tensor faces with identified vertices */
8472     if (Nv < DMPolytopeTypeGetNumVertices(ct)) {
8473       PetscInt unsplit;
8474 
8475       ierr = DMPlexCellUnsplitVertices_Private(dm, c, ct, &unsplit);CHKERRQ(ierr);
8476       if (Nv + unsplit == DMPolytopeTypeGetNumVertices(ct)) continue;
8477     }
8478     PetscAssertFalse(Nv != DMPolytopeTypeGetNumVertices(ct),PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %D of type %s has %D vertices != %D", c, DMPolytopeTypes[ct], Nv, DMPolytopeTypeGetNumVertices(ct));
8479   }
8480   PetscFunctionReturn(0);
8481 }
8482 
8483 /*@
8484   DMPlexCheckFaces - Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type
8485 
8486   Not Collective
8487 
8488   Input Parameters:
8489 + dm - The DMPlex object
8490 - cellHeight - Normally 0
8491 
8492   Notes:
8493   This is a useful diagnostic when creating meshes programmatically.
8494   This routine is only relevant for meshes that are fully interpolated across all ranks.
8495   It will error out if a partially interpolated mesh is given on some rank.
8496   It will do nothing for locally uninterpolated mesh (as there is nothing to check).
8497 
8498   For the complete list of DMPlexCheck* functions, see DMSetFromOptions().
8499 
8500   Level: developer
8501 
8502 .seealso: DMCreate(), DMPlexGetVTKCellHeight(), DMSetFromOptions()
8503 @*/
8504 PetscErrorCode DMPlexCheckFaces(DM dm, PetscInt cellHeight)
8505 {
8506   PetscInt       dim, depth, vStart, vEnd, cStart, cEnd, c, h;
8507   PetscErrorCode ierr;
8508   DMPlexInterpolatedFlag interpEnum;
8509 
8510   PetscFunctionBegin;
8511   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8512   ierr = DMPlexIsInterpolated(dm, &interpEnum);CHKERRQ(ierr);
8513   if (interpEnum == DMPLEX_INTERPOLATED_NONE) PetscFunctionReturn(0);
8514   if (interpEnum == DMPLEX_INTERPOLATED_PARTIAL) {
8515     PetscMPIInt rank;
8516     MPI_Comm    comm;
8517 
8518     ierr = PetscObjectGetComm((PetscObject) dm, &comm);CHKERRQ(ierr);
8519     ierr = MPI_Comm_rank(comm, &rank);CHKERRMPI(ierr);
8520     SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Mesh is only partially interpolated on rank %d, this is currently not supported", rank);
8521   }
8522 
8523   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
8524   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
8525   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
8526   for (h = cellHeight; h < PetscMin(depth, dim); ++h) {
8527     ierr = DMPlexGetHeightStratum(dm, h, &cStart, &cEnd);CHKERRQ(ierr);
8528     for (c = cStart; c < cEnd; ++c) {
8529       const PetscInt      *cone, *ornt, *faceSizes, *faces;
8530       const DMPolytopeType *faceTypes;
8531       DMPolytopeType        ct;
8532       PetscInt              numFaces, coneSize, f;
8533       PetscInt             *closure = NULL, closureSize, cl, numCorners = 0, fOff = 0, unsplit;
8534 
8535       ierr = DMPlexGetCellType(dm, c, &ct);CHKERRQ(ierr);
8536       ierr = DMPlexCellUnsplitVertices_Private(dm, c, ct, &unsplit);CHKERRQ(ierr);
8537       if (unsplit) continue;
8538       ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
8539       ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
8540       ierr = DMPlexGetConeOrientation(dm, c, &ornt);CHKERRQ(ierr);
8541       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
8542       for (cl = 0; cl < closureSize*2; cl += 2) {
8543         const PetscInt p = closure[cl];
8544         if ((p >= vStart) && (p < vEnd)) closure[numCorners++] = p;
8545       }
8546       ierr = DMPlexGetRawFaces_Internal(dm, ct, closure, &numFaces, &faceTypes, &faceSizes, &faces);CHKERRQ(ierr);
8547       PetscAssertFalse(coneSize != numFaces,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %D of type %s has %D faces but should have %D", c, DMPolytopeTypes[ct], coneSize, numFaces);
8548       for (f = 0; f < numFaces; ++f) {
8549         DMPolytopeType fct;
8550         PetscInt       *fclosure = NULL, fclosureSize, cl, fnumCorners = 0, v;
8551 
8552         ierr = DMPlexGetCellType(dm, cone[f], &fct);CHKERRQ(ierr);
8553         ierr = DMPlexGetTransitiveClosure_Internal(dm, cone[f], ornt[f], PETSC_TRUE, &fclosureSize, &fclosure);CHKERRQ(ierr);
8554         for (cl = 0; cl < fclosureSize*2; cl += 2) {
8555           const PetscInt p = fclosure[cl];
8556           if ((p >= vStart) && (p < vEnd)) fclosure[fnumCorners++] = p;
8557         }
8558         PetscAssertFalse(fnumCorners != faceSizes[f],PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %D of type %s (cone idx %D) of cell %D of type %s has %D vertices but should have %D", cone[f], DMPolytopeTypes[fct], f, c, DMPolytopeTypes[ct], fnumCorners, faceSizes[f]);
8559         for (v = 0; v < fnumCorners; ++v) {
8560           if (fclosure[v] != faces[fOff+v]) {
8561             PetscInt v1;
8562 
8563             ierr = PetscPrintf(PETSC_COMM_SELF, "face closure:");CHKERRQ(ierr);
8564             for (v1 = 0; v1 < fnumCorners; ++v1) {ierr = PetscPrintf(PETSC_COMM_SELF, " %D", fclosure[v1]);CHKERRQ(ierr);}
8565             ierr = PetscPrintf(PETSC_COMM_SELF, "\ncell face:");CHKERRQ(ierr);
8566             for (v1 = 0; v1 < fnumCorners; ++v1) {ierr = PetscPrintf(PETSC_COMM_SELF, " %D", faces[fOff+v1]);CHKERRQ(ierr);}
8567             ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
8568             SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %D of type %s (cone idx %d, ornt %D) of cell %D of type %s vertex %D, %D != %D", cone[f], DMPolytopeTypes[fct], f, ornt[f], c, DMPolytopeTypes[ct], v, fclosure[v], faces[fOff+v]);
8569           }
8570         }
8571         ierr = DMPlexRestoreTransitiveClosure(dm, cone[f], PETSC_TRUE, &fclosureSize, &fclosure);CHKERRQ(ierr);
8572         fOff += faceSizes[f];
8573       }
8574       ierr = DMPlexRestoreRawFaces_Internal(dm, ct, closure, &numFaces, &faceTypes, &faceSizes, &faces);CHKERRQ(ierr);
8575       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
8576     }
8577   }
8578   PetscFunctionReturn(0);
8579 }
8580 
8581 /*@
8582   DMPlexCheckGeometry - Check the geometry of mesh cells
8583 
8584   Input Parameter:
8585 . dm - The DMPlex object
8586 
8587   Notes:
8588   This is a useful diagnostic when creating meshes programmatically.
8589 
8590   For the complete list of DMPlexCheck* functions, see DMSetFromOptions().
8591 
8592   Level: developer
8593 
8594 .seealso: DMCreate(), DMSetFromOptions()
8595 @*/
8596 PetscErrorCode DMPlexCheckGeometry(DM dm)
8597 {
8598   Vec            coordinates;
8599   PetscReal      detJ, J[9], refVol = 1.0;
8600   PetscReal      vol;
8601   PetscBool      periodic;
8602   PetscInt       dim, depth, dE, d, cStart, cEnd, c;
8603   PetscErrorCode ierr;
8604 
8605   PetscFunctionBegin;
8606   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
8607   ierr = DMGetCoordinateDim(dm, &dE);CHKERRQ(ierr);
8608   if (dim != dE) PetscFunctionReturn(0);
8609   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
8610   ierr = DMGetPeriodicity(dm, &periodic, NULL, NULL, NULL);CHKERRQ(ierr);
8611   for (d = 0; d < dim; ++d) refVol *= 2.0;
8612   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
8613   /* Make sure local coordinates are created, because that step is collective */
8614   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
8615   for (c = cStart; c < cEnd; ++c) {
8616     DMPolytopeType ct;
8617     PetscInt       unsplit;
8618     PetscBool      ignoreZeroVol = PETSC_FALSE;
8619 
8620     ierr = DMPlexGetCellType(dm, c, &ct);CHKERRQ(ierr);
8621     switch (ct) {
8622       case DM_POLYTOPE_SEG_PRISM_TENSOR:
8623       case DM_POLYTOPE_TRI_PRISM_TENSOR:
8624       case DM_POLYTOPE_QUAD_PRISM_TENSOR:
8625         ignoreZeroVol = PETSC_TRUE; break;
8626       default: break;
8627     }
8628     switch (ct) {
8629       case DM_POLYTOPE_TRI_PRISM:
8630       case DM_POLYTOPE_TRI_PRISM_TENSOR:
8631       case DM_POLYTOPE_QUAD_PRISM_TENSOR:
8632       case DM_POLYTOPE_PYRAMID:
8633         continue;
8634       default: break;
8635     }
8636     ierr = DMPlexCellUnsplitVertices_Private(dm, c, ct, &unsplit);CHKERRQ(ierr);
8637     if (unsplit) continue;
8638     ierr = DMPlexComputeCellGeometryFEM(dm, c, NULL, NULL, J, NULL, &detJ);CHKERRQ(ierr);
8639     PetscAssertFalse(detJ < -PETSC_SMALL || (detJ <= 0.0 && !ignoreZeroVol),PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Mesh cell %D of type %s is inverted, |J| = %g", c, DMPolytopeTypes[ct], (double) detJ);
8640     ierr = PetscInfo(dm, "Cell %D FEM Volume %g\n", c, (double) detJ*refVol);CHKERRQ(ierr);
8641     if (depth > 1 && !periodic) {
8642       ierr = DMPlexComputeCellGeometryFVM(dm, c, &vol, NULL, NULL);CHKERRQ(ierr);
8643       PetscAssertFalse(vol < -PETSC_SMALL || (vol <= 0.0 && !ignoreZeroVol),PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Mesh cell %D of type %s is inverted, vol = %g", c, DMPolytopeTypes[ct], (double) vol);
8644       ierr = PetscInfo(dm, "Cell %D FVM Volume %g\n", c, (double) vol);CHKERRQ(ierr);
8645     }
8646   }
8647   PetscFunctionReturn(0);
8648 }
8649 
8650 /*@
8651   DMPlexCheckPointSF - Check that several necessary conditions are met for the point SF of this plex.
8652 
8653   Input Parameters:
8654 . dm - The DMPlex object
8655 
8656   Notes:
8657   This is mainly intended for debugging/testing purposes.
8658   It currently checks only meshes with no partition overlapping.
8659 
8660   For the complete list of DMPlexCheck* functions, see DMSetFromOptions().
8661 
8662   Level: developer
8663 
8664 .seealso: DMGetPointSF(), DMSetFromOptions()
8665 @*/
8666 PetscErrorCode DMPlexCheckPointSF(DM dm)
8667 {
8668   PetscSF         pointSF;
8669   PetscInt        cellHeight, cStart, cEnd, l, nleaves, nroots, overlap;
8670   const PetscInt *locals, *rootdegree;
8671   PetscBool       distributed;
8672   PetscErrorCode  ierr;
8673 
8674   PetscFunctionBegin;
8675   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8676   ierr = DMGetPointSF(dm, &pointSF);CHKERRQ(ierr);
8677   ierr = DMPlexIsDistributed(dm, &distributed);CHKERRQ(ierr);
8678   if (!distributed) PetscFunctionReturn(0);
8679   ierr = DMPlexGetOverlap(dm, &overlap);CHKERRQ(ierr);
8680   if (overlap) {
8681     ierr = PetscPrintf(PetscObjectComm((PetscObject)dm), "Warning: DMPlexCheckPointSF() is currently not implemented for meshes with partition overlapping");CHKERRQ(ierr);
8682     PetscFunctionReturn(0);
8683   }
8684   PetscAssertFalse(!pointSF,PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONGSTATE, "This DMPlex is distributed but does not have PointSF attached");
8685   ierr = PetscSFGetGraph(pointSF, &nroots, &nleaves, &locals, NULL);CHKERRQ(ierr);
8686   PetscAssertFalse(nroots < 0,PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONGSTATE, "This DMPlex is distributed but its PointSF has no graph set");
8687   ierr = PetscSFComputeDegreeBegin(pointSF, &rootdegree);CHKERRQ(ierr);
8688   ierr = PetscSFComputeDegreeEnd(pointSF, &rootdegree);CHKERRQ(ierr);
8689 
8690   /* 1) check there are no faces in 2D, cells in 3D, in interface */
8691   ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
8692   ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
8693   for (l = 0; l < nleaves; ++l) {
8694     const PetscInt point = locals[l];
8695 
8696     PetscAssertFalse(point >= cStart && point < cEnd,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point SF contains %D which is a cell", point);
8697   }
8698 
8699   /* 2) if some point is in interface, then all its cone points must be also in interface (either as leaves or roots) */
8700   for (l = 0; l < nleaves; ++l) {
8701     const PetscInt  point = locals[l];
8702     const PetscInt *cone;
8703     PetscInt        coneSize, c, idx;
8704 
8705     ierr = DMPlexGetConeSize(dm, point, &coneSize);CHKERRQ(ierr);
8706     ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
8707     for (c = 0; c < coneSize; ++c) {
8708       if (!rootdegree[cone[c]]) {
8709         ierr = PetscFindInt(cone[c], nleaves, locals, &idx);CHKERRQ(ierr);
8710         PetscAssertFalse(idx < 0,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point SF contains %D but not %D from its cone", point, cone[c]);
8711       }
8712     }
8713   }
8714   PetscFunctionReturn(0);
8715 }
8716 
8717 PetscErrorCode DMPlexCheckAll_Internal(DM dm, PetscInt cellHeight)
8718 {
8719   PetscErrorCode ierr;
8720 
8721   PetscFunctionBegin;
8722   ierr = DMPlexCheckSymmetry(dm);CHKERRQ(ierr);
8723   ierr = DMPlexCheckSkeleton(dm, cellHeight);CHKERRQ(ierr);
8724   ierr = DMPlexCheckFaces(dm, cellHeight);CHKERRQ(ierr);
8725   ierr = DMPlexCheckGeometry(dm);CHKERRQ(ierr);
8726   ierr = DMPlexCheckPointSF(dm);CHKERRQ(ierr);
8727   ierr = DMPlexCheckInterfaceCones(dm);CHKERRQ(ierr);
8728   PetscFunctionReturn(0);
8729 }
8730 
8731 typedef struct cell_stats
8732 {
8733   PetscReal min, max, sum, squaresum;
8734   PetscInt  count;
8735 } cell_stats_t;
8736 
8737 static void MPIAPI cell_stats_reduce(void *a, void *b, int * len, MPI_Datatype *datatype)
8738 {
8739   PetscInt i, N = *len;
8740 
8741   for (i = 0; i < N; i++) {
8742     cell_stats_t *A = (cell_stats_t *) a;
8743     cell_stats_t *B = (cell_stats_t *) b;
8744 
8745     B->min = PetscMin(A->min,B->min);
8746     B->max = PetscMax(A->max,B->max);
8747     B->sum += A->sum;
8748     B->squaresum += A->squaresum;
8749     B->count += A->count;
8750   }
8751 }
8752 
8753 /*@
8754   DMPlexCheckCellShape - Checks the Jacobian of the mapping from reference to real cells and computes some minimal statistics.
8755 
8756   Collective on dm
8757 
8758   Input Parameters:
8759 + dm        - The DMPlex object
8760 . output    - If true, statistics will be displayed on stdout
8761 - condLimit - Display all cells above this condition number, or PETSC_DETERMINE for no cell output
8762 
8763   Notes:
8764   This is mainly intended for debugging/testing purposes.
8765 
8766   For the complete list of DMPlexCheck* functions, see DMSetFromOptions().
8767 
8768   Level: developer
8769 
8770 .seealso: DMSetFromOptions(), DMPlexComputeOrthogonalQuality()
8771 @*/
8772 PetscErrorCode DMPlexCheckCellShape(DM dm, PetscBool output, PetscReal condLimit)
8773 {
8774   DM             dmCoarse;
8775   cell_stats_t   stats, globalStats;
8776   MPI_Comm       comm = PetscObjectComm((PetscObject)dm);
8777   PetscReal      *J, *invJ, min = 0, max = 0, mean = 0, stdev = 0;
8778   PetscReal      limit = condLimit > 0 ? condLimit : PETSC_MAX_REAL;
8779   PetscInt       cdim, cStart, cEnd, c, eStart, eEnd, count = 0;
8780   PetscMPIInt    rank,size;
8781   PetscErrorCode ierr;
8782 
8783   PetscFunctionBegin;
8784   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8785   stats.min   = PETSC_MAX_REAL;
8786   stats.max   = PETSC_MIN_REAL;
8787   stats.sum   = stats.squaresum = 0.;
8788   stats.count = 0;
8789 
8790   ierr = MPI_Comm_size(comm, &size);CHKERRMPI(ierr);
8791   ierr = MPI_Comm_rank(comm, &rank);CHKERRMPI(ierr);
8792   ierr = DMGetCoordinateDim(dm,&cdim);CHKERRQ(ierr);
8793   ierr = PetscMalloc2(PetscSqr(cdim), &J, PetscSqr(cdim), &invJ);CHKERRQ(ierr);
8794   ierr = DMPlexGetSimplexOrBoxCells(dm,0,&cStart,&cEnd);CHKERRQ(ierr);
8795   ierr = DMPlexGetDepthStratum(dm,1,&eStart,&eEnd);CHKERRQ(ierr);
8796   for (c = cStart; c < cEnd; c++) {
8797     PetscInt  i;
8798     PetscReal frobJ = 0., frobInvJ = 0., cond2, cond, detJ;
8799 
8800     ierr = DMPlexComputeCellGeometryAffineFEM(dm,c,NULL,J,invJ,&detJ);CHKERRQ(ierr);
8801     PetscAssertFalse(detJ < 0.0,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Mesh cell %D is inverted", c);
8802     for (i = 0; i < PetscSqr(cdim); ++i) {
8803       frobJ    += J[i] * J[i];
8804       frobInvJ += invJ[i] * invJ[i];
8805     }
8806     cond2 = frobJ * frobInvJ;
8807     cond  = PetscSqrtReal(cond2);
8808 
8809     stats.min        = PetscMin(stats.min,cond);
8810     stats.max        = PetscMax(stats.max,cond);
8811     stats.sum       += cond;
8812     stats.squaresum += cond2;
8813     stats.count++;
8814     if (output && cond > limit) {
8815       PetscSection coordSection;
8816       Vec          coordsLocal;
8817       PetscScalar *coords = NULL;
8818       PetscInt     Nv, d, clSize, cl, *closure = NULL;
8819 
8820       ierr = DMGetCoordinatesLocal(dm, &coordsLocal);CHKERRQ(ierr);
8821       ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
8822       ierr = DMPlexVecGetClosure(dm, coordSection, coordsLocal, c, &Nv, &coords);CHKERRQ(ierr);
8823       ierr = PetscSynchronizedPrintf(comm, "[%d] Cell %D cond %g\n", rank, c, (double) cond);CHKERRQ(ierr);
8824       for (i = 0; i < Nv/cdim; ++i) {
8825         ierr = PetscSynchronizedPrintf(comm, "  Vertex %D: (", i);CHKERRQ(ierr);
8826         for (d = 0; d < cdim; ++d) {
8827           if (d > 0) {ierr = PetscSynchronizedPrintf(comm, ", ");CHKERRQ(ierr);}
8828           ierr = PetscSynchronizedPrintf(comm, "%g", (double) PetscRealPart(coords[i*cdim+d]));CHKERRQ(ierr);
8829         }
8830         ierr = PetscSynchronizedPrintf(comm, ")\n");CHKERRQ(ierr);
8831       }
8832       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &clSize, &closure);CHKERRQ(ierr);
8833       for (cl = 0; cl < clSize*2; cl += 2) {
8834         const PetscInt edge = closure[cl];
8835 
8836         if ((edge >= eStart) && (edge < eEnd)) {
8837           PetscReal len;
8838 
8839           ierr = DMPlexComputeCellGeometryFVM(dm, edge, &len, NULL, NULL);CHKERRQ(ierr);
8840           ierr = PetscSynchronizedPrintf(comm, "  Edge %D: length %g\n", edge, (double) len);CHKERRQ(ierr);
8841         }
8842       }
8843       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &clSize, &closure);CHKERRQ(ierr);
8844       ierr = DMPlexVecRestoreClosure(dm, coordSection, coordsLocal, c, &Nv, &coords);CHKERRQ(ierr);
8845     }
8846   }
8847   if (output) {ierr = PetscSynchronizedFlush(comm, NULL);CHKERRQ(ierr);}
8848 
8849   if (size > 1) {
8850     PetscMPIInt   blockLengths[2] = {4,1};
8851     MPI_Aint      blockOffsets[2] = {offsetof(cell_stats_t,min),offsetof(cell_stats_t,count)};
8852     MPI_Datatype  blockTypes[2]   = {MPIU_REAL,MPIU_INT}, statType;
8853     MPI_Op        statReduce;
8854 
8855     ierr = MPI_Type_create_struct(2,blockLengths,blockOffsets,blockTypes,&statType);CHKERRMPI(ierr);
8856     ierr = MPI_Type_commit(&statType);CHKERRMPI(ierr);
8857     ierr = MPI_Op_create(cell_stats_reduce, PETSC_TRUE, &statReduce);CHKERRMPI(ierr);
8858     ierr = MPI_Reduce(&stats,&globalStats,1,statType,statReduce,0,comm);CHKERRMPI(ierr);
8859     ierr = MPI_Op_free(&statReduce);CHKERRMPI(ierr);
8860     ierr = MPI_Type_free(&statType);CHKERRMPI(ierr);
8861   } else {
8862     ierr = PetscArraycpy(&globalStats,&stats,1);CHKERRQ(ierr);
8863   }
8864   if (rank == 0) {
8865     count = globalStats.count;
8866     min   = globalStats.min;
8867     max   = globalStats.max;
8868     mean  = globalStats.sum / globalStats.count;
8869     stdev = globalStats.count > 1 ? PetscSqrtReal(PetscMax((globalStats.squaresum - globalStats.count * mean * mean) / (globalStats.count - 1),0)) : 0.0;
8870   }
8871 
8872   if (output) {
8873     ierr = PetscPrintf(comm,"Mesh with %D cells, shape condition numbers: min = %g, max = %g, mean = %g, stddev = %g\n", count, (double) min, (double) max, (double) mean, (double) stdev);CHKERRQ(ierr);
8874   }
8875   ierr = PetscFree2(J,invJ);CHKERRQ(ierr);
8876 
8877   ierr = DMGetCoarseDM(dm,&dmCoarse);CHKERRQ(ierr);
8878   if (dmCoarse) {
8879     PetscBool isplex;
8880 
8881     ierr = PetscObjectTypeCompare((PetscObject)dmCoarse,DMPLEX,&isplex);CHKERRQ(ierr);
8882     if (isplex) {
8883       ierr = DMPlexCheckCellShape(dmCoarse,output,condLimit);CHKERRQ(ierr);
8884     }
8885   }
8886   PetscFunctionReturn(0);
8887 }
8888 
8889 /*@
8890   DMPlexComputeOrthogonalQuality - Compute cell-wise orthogonal quality mesh statistic. Optionally tags all cells with
8891   orthogonal quality below given tolerance.
8892 
8893   Collective on dm
8894 
8895   Input Parameters:
8896 + dm   - The DMPlex object
8897 . fv   - Optional PetscFV object for pre-computed cell/face centroid information
8898 - atol - [0, 1] Absolute tolerance for tagging cells.
8899 
8900   Output Parameters:
8901 + OrthQual      - Vec containing orthogonal quality per cell
8902 - OrthQualLabel - DMLabel tagging cells below atol with DM_ADAPT_REFINE
8903 
8904   Options Database Keys:
8905 + -dm_plex_orthogonal_quality_label_view - view OrthQualLabel if label is requested. Currently only PETSCVIEWERASCII is
8906 supported.
8907 - -dm_plex_orthogonal_quality_vec_view - view OrthQual vector.
8908 
8909   Notes:
8910   Orthogonal quality is given by the following formula:
8911 
8912   \min \left[ \frac{A_i \cdot f_i}{\|A_i\| \|f_i\|} , \frac{A_i \cdot c_i}{\|A_i\| \|c_i\|} \right]
8913 
8914   Where A_i is the i'th face-normal vector, f_i is the vector from the cell centroid to the i'th face centroid, and c_i
8915   is the vector from the current cells centroid to the centroid of its i'th neighbor (which shares a face with the
8916   current cell). This computes the vector similarity between each cell face and its corresponding neighbor centroid by
8917   calculating the cosine of the angle between these vectors.
8918 
8919   Orthogonal quality ranges from 1 (best) to 0 (worst).
8920 
8921   This routine is mainly useful for FVM, however is not restricted to only FVM. The PetscFV object is optionally used to check for
8922   pre-computed FVM cell data, but if it is not passed in then this data will be computed.
8923 
8924   Cells are tagged if they have an orthogonal quality less than or equal to the absolute tolerance.
8925 
8926   Level: intermediate
8927 
8928 .seealso: DMPlexCheckCellShape(), DMCreateLabel()
8929 @*/
8930 PetscErrorCode DMPlexComputeOrthogonalQuality(DM dm, PetscFV fv, PetscReal atol, Vec *OrthQual, DMLabel *OrthQualLabel)
8931 {
8932   PetscInt                nc, cellHeight, cStart, cEnd, cell, cellIter = 0;
8933   PetscInt                *idx;
8934   PetscScalar             *oqVals;
8935   const PetscScalar       *cellGeomArr, *faceGeomArr;
8936   PetscReal               *ci, *fi, *Ai;
8937   MPI_Comm                comm;
8938   Vec                     cellgeom, facegeom;
8939   DM                      dmFace, dmCell;
8940   IS                      glob;
8941   ISLocalToGlobalMapping  ltog;
8942   PetscViewer             vwr;
8943   PetscErrorCode          ierr;
8944 
8945   PetscFunctionBegin;
8946   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8947   if (fv) {PetscValidHeaderSpecific(fv, PETSCFV_CLASSID, 2);}
8948   PetscValidPointer(OrthQual, 4);
8949   PetscAssertFalseDebug(atol < 0.0 || atol > 1.0,PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"Absolute tolerance %g not in [0,1]",(double)atol);
8950   ierr = PetscObjectGetComm((PetscObject) dm, &comm);CHKERRQ(ierr);
8951   ierr = DMGetDimension(dm, &nc);CHKERRQ(ierr);
8952   PetscAssertFalse(nc < 2,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONGSTATE, "DM must have dimension >= 2 (current %D)", nc);
8953   {
8954     DMPlexInterpolatedFlag interpFlag;
8955 
8956     ierr = DMPlexIsInterpolated(dm, &interpFlag);CHKERRQ(ierr);
8957     if (interpFlag != DMPLEX_INTERPOLATED_FULL) {
8958       PetscMPIInt rank;
8959 
8960       ierr = MPI_Comm_rank(comm, &rank);CHKERRMPI(ierr);
8961       SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONGSTATE, "DM must be fully interpolated, DM on rank %d is not fully interpolated", rank);
8962     }
8963   }
8964   if (OrthQualLabel) {
8965     PetscValidPointer(OrthQualLabel, 5);
8966     ierr = DMCreateLabel(dm, "Orthogonal_Quality");CHKERRQ(ierr);
8967     ierr = DMGetLabel(dm, "Orthogonal_Quality", OrthQualLabel);CHKERRQ(ierr);
8968   } else {*OrthQualLabel = NULL;}
8969   ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
8970   ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
8971   ierr = DMPlexCreateCellNumbering_Internal(dm, PETSC_TRUE, &glob);CHKERRQ(ierr);
8972   ierr = ISLocalToGlobalMappingCreateIS(glob, &ltog);CHKERRQ(ierr);
8973   ierr = ISLocalToGlobalMappingSetType(ltog, ISLOCALTOGLOBALMAPPINGHASH);CHKERRQ(ierr);
8974   ierr = VecCreate(comm, OrthQual);CHKERRQ(ierr);
8975   ierr = VecSetType(*OrthQual, VECSTANDARD);CHKERRQ(ierr);
8976   ierr = VecSetSizes(*OrthQual, cEnd-cStart, PETSC_DETERMINE);CHKERRQ(ierr);
8977   ierr = VecSetLocalToGlobalMapping(*OrthQual, ltog);CHKERRQ(ierr);
8978   ierr = VecSetUp(*OrthQual);CHKERRQ(ierr);
8979   ierr = ISDestroy(&glob);CHKERRQ(ierr);
8980   ierr = ISLocalToGlobalMappingDestroy(&ltog);CHKERRQ(ierr);
8981   ierr = DMPlexGetDataFVM(dm, fv, &cellgeom, &facegeom, NULL);CHKERRQ(ierr);
8982   ierr = VecGetArrayRead(cellgeom, &cellGeomArr);CHKERRQ(ierr);
8983   ierr = VecGetArrayRead(facegeom, &faceGeomArr);CHKERRQ(ierr);
8984   ierr = VecGetDM(cellgeom, &dmCell);CHKERRQ(ierr);
8985   ierr = VecGetDM(facegeom, &dmFace);CHKERRQ(ierr);
8986   ierr = PetscMalloc5(cEnd-cStart, &idx, cEnd-cStart, &oqVals, nc, &ci, nc, &fi, nc, &Ai);CHKERRQ(ierr);
8987   for (cell = cStart; cell < cEnd; cellIter++,cell++) {
8988     PetscInt           cellneigh, cellneighiter = 0, adjSize = PETSC_DETERMINE;
8989     PetscInt           cellarr[2], *adj = NULL;
8990     PetscScalar        *cArr, *fArr;
8991     PetscReal          minvalc = 1.0, minvalf = 1.0;
8992     PetscFVCellGeom    *cg;
8993 
8994     idx[cellIter] = cell-cStart;
8995     cellarr[0] = cell;
8996     /* Make indexing into cellGeom easier */
8997     ierr = DMPlexPointLocalRead(dmCell, cell, cellGeomArr, &cg);CHKERRQ(ierr);
8998     ierr = DMPlexGetAdjacency_Internal(dm, cell, PETSC_TRUE, PETSC_FALSE, PETSC_FALSE, &adjSize, &adj);CHKERRQ(ierr);
8999     /* Technically 1 too big, but easier than fiddling with empty adjacency array */
9000     ierr = PetscCalloc2(adjSize, &cArr, adjSize, &fArr);CHKERRQ(ierr);
9001     for (cellneigh = 0; cellneigh < adjSize; cellneighiter++,cellneigh++) {
9002       PetscInt         i;
9003       const PetscInt   neigh = adj[cellneigh];
9004       PetscReal        normci = 0, normfi = 0, normai = 0;
9005       PetscFVCellGeom  *cgneigh;
9006       PetscFVFaceGeom  *fg;
9007 
9008       /* Don't count ourselves in the neighbor list */
9009       if (neigh == cell) continue;
9010       ierr = DMPlexPointLocalRead(dmCell, neigh, cellGeomArr, &cgneigh);CHKERRQ(ierr);
9011       cellarr[1] = neigh;
9012       {
9013         PetscInt       numcovpts;
9014         const PetscInt *covpts;
9015 
9016         ierr = DMPlexGetMeet(dm, 2, cellarr, &numcovpts, &covpts);CHKERRQ(ierr);
9017         ierr = DMPlexPointLocalRead(dmFace, covpts[0], faceGeomArr, &fg);CHKERRQ(ierr);
9018         ierr = DMPlexRestoreMeet(dm, 2, cellarr, &numcovpts, &covpts);CHKERRQ(ierr);
9019       }
9020 
9021       /* Compute c_i, f_i and their norms */
9022       for (i = 0; i < nc; i++) {
9023         ci[i] = cgneigh->centroid[i] - cg->centroid[i];
9024         fi[i] = fg->centroid[i] - cg->centroid[i];
9025         Ai[i] = fg->normal[i];
9026         normci += PetscPowReal(ci[i], 2);
9027         normfi += PetscPowReal(fi[i], 2);
9028         normai += PetscPowReal(Ai[i], 2);
9029       }
9030       normci = PetscSqrtReal(normci);
9031       normfi = PetscSqrtReal(normfi);
9032       normai = PetscSqrtReal(normai);
9033 
9034       /* Normalize and compute for each face-cell-normal pair */
9035       for (i = 0; i < nc; i++) {
9036         ci[i] = ci[i]/normci;
9037         fi[i] = fi[i]/normfi;
9038         Ai[i] = Ai[i]/normai;
9039         /* PetscAbs because I don't know if normals are guaranteed to point out */
9040         cArr[cellneighiter] += PetscAbs(Ai[i]*ci[i]);
9041         fArr[cellneighiter] += PetscAbs(Ai[i]*fi[i]);
9042       }
9043       if (PetscRealPart(cArr[cellneighiter]) < minvalc) {
9044         minvalc = PetscRealPart(cArr[cellneighiter]);
9045       }
9046       if (PetscRealPart(fArr[cellneighiter]) < minvalf) {
9047         minvalf = PetscRealPart(fArr[cellneighiter]);
9048       }
9049     }
9050     ierr = PetscFree(adj);CHKERRQ(ierr);
9051     ierr = PetscFree2(cArr, fArr);CHKERRQ(ierr);
9052     /* Defer to cell if they're equal */
9053     oqVals[cellIter] = PetscMin(minvalf, minvalc);
9054     if (OrthQualLabel) {
9055       if (PetscRealPart(oqVals[cellIter]) <= atol) {ierr = DMLabelSetValue(*OrthQualLabel, cell, DM_ADAPT_REFINE);CHKERRQ(ierr);}
9056     }
9057   }
9058   ierr = VecSetValuesLocal(*OrthQual, cEnd-cStart, idx, oqVals, INSERT_VALUES);CHKERRQ(ierr);
9059   ierr = VecAssemblyBegin(*OrthQual);CHKERRQ(ierr);
9060   ierr = VecAssemblyEnd(*OrthQual);CHKERRQ(ierr);
9061   ierr = VecRestoreArrayRead(cellgeom, &cellGeomArr);CHKERRQ(ierr);
9062   ierr = VecRestoreArrayRead(facegeom, &faceGeomArr);CHKERRQ(ierr);
9063   ierr = PetscOptionsGetViewer(comm, NULL, NULL, "-dm_plex_orthogonal_quality_label_view", &vwr, NULL, NULL);CHKERRQ(ierr);
9064   if (OrthQualLabel) {
9065     if (vwr) {ierr = DMLabelView(*OrthQualLabel, vwr);CHKERRQ(ierr);}
9066   }
9067   ierr = PetscFree5(idx, oqVals, ci, fi, Ai);CHKERRQ(ierr);
9068   ierr = PetscViewerDestroy(&vwr);CHKERRQ(ierr);
9069   ierr = VecViewFromOptions(*OrthQual, NULL, "-dm_plex_orthogonal_quality_vec_view");CHKERRQ(ierr);
9070   PetscFunctionReturn(0);
9071 }
9072 
9073 /* this is here insead of DMGetOutputDM because output DM still has constraints in the local indices that affect
9074  * interpolator construction */
9075 static PetscErrorCode DMGetFullDM(DM dm, DM *odm)
9076 {
9077   PetscSection   section, newSection, gsection;
9078   PetscSF        sf;
9079   PetscBool      hasConstraints, ghasConstraints;
9080   PetscErrorCode ierr;
9081 
9082   PetscFunctionBegin;
9083   PetscValidHeaderSpecific(dm,DM_CLASSID,1);
9084   PetscValidPointer(odm,2);
9085   ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);
9086   ierr = PetscSectionHasConstraints(section, &hasConstraints);CHKERRQ(ierr);
9087   ierr = MPI_Allreduce(&hasConstraints, &ghasConstraints, 1, MPIU_BOOL, MPI_LOR, PetscObjectComm((PetscObject) dm));CHKERRMPI(ierr);
9088   if (!ghasConstraints) {
9089     ierr = PetscObjectReference((PetscObject)dm);CHKERRQ(ierr);
9090     *odm = dm;
9091     PetscFunctionReturn(0);
9092   }
9093   ierr = DMClone(dm, odm);CHKERRQ(ierr);
9094   ierr = DMCopyFields(dm, *odm);CHKERRQ(ierr);
9095   ierr = DMGetLocalSection(*odm, &newSection);CHKERRQ(ierr);
9096   ierr = DMGetPointSF(*odm, &sf);CHKERRQ(ierr);
9097   ierr = PetscSectionCreateGlobalSection(newSection, sf, PETSC_TRUE, PETSC_FALSE, &gsection);CHKERRQ(ierr);
9098   ierr = DMSetGlobalSection(*odm, gsection);CHKERRQ(ierr);
9099   ierr = PetscSectionDestroy(&gsection);CHKERRQ(ierr);
9100   PetscFunctionReturn(0);
9101 }
9102 
9103 static PetscErrorCode DMCreateAffineInterpolationCorrection_Plex(DM dmc, DM dmf, Vec *shift)
9104 {
9105   DM             dmco, dmfo;
9106   Mat            interpo;
9107   Vec            rscale;
9108   Vec            cglobalo, clocal;
9109   Vec            fglobal, fglobalo, flocal;
9110   PetscBool      regular;
9111   PetscErrorCode ierr;
9112 
9113   PetscFunctionBegin;
9114   ierr = DMGetFullDM(dmc, &dmco);CHKERRQ(ierr);
9115   ierr = DMGetFullDM(dmf, &dmfo);CHKERRQ(ierr);
9116   ierr = DMSetCoarseDM(dmfo, dmco);CHKERRQ(ierr);
9117   ierr = DMPlexGetRegularRefinement(dmf, &regular);CHKERRQ(ierr);
9118   ierr = DMPlexSetRegularRefinement(dmfo, regular);CHKERRQ(ierr);
9119   ierr = DMCreateInterpolation(dmco, dmfo, &interpo, &rscale);CHKERRQ(ierr);
9120   ierr = DMCreateGlobalVector(dmco, &cglobalo);CHKERRQ(ierr);
9121   ierr = DMCreateLocalVector(dmc, &clocal);CHKERRQ(ierr);
9122   ierr = VecSet(cglobalo, 0.);CHKERRQ(ierr);
9123   ierr = VecSet(clocal, 0.);CHKERRQ(ierr);
9124   ierr = DMCreateGlobalVector(dmf, &fglobal);CHKERRQ(ierr);
9125   ierr = DMCreateGlobalVector(dmfo, &fglobalo);CHKERRQ(ierr);
9126   ierr = DMCreateLocalVector(dmf, &flocal);CHKERRQ(ierr);
9127   ierr = VecSet(fglobal, 0.);CHKERRQ(ierr);
9128   ierr = VecSet(fglobalo, 0.);CHKERRQ(ierr);
9129   ierr = VecSet(flocal, 0.);CHKERRQ(ierr);
9130   ierr = DMPlexInsertBoundaryValues(dmc, PETSC_TRUE, clocal, 0., NULL, NULL, NULL);CHKERRQ(ierr);
9131   ierr = DMLocalToGlobalBegin(dmco, clocal, INSERT_VALUES, cglobalo);CHKERRQ(ierr);
9132   ierr = DMLocalToGlobalEnd(dmco, clocal, INSERT_VALUES, cglobalo);CHKERRQ(ierr);
9133   ierr = MatMult(interpo, cglobalo, fglobalo);CHKERRQ(ierr);
9134   ierr = DMGlobalToLocalBegin(dmfo, fglobalo, INSERT_VALUES, flocal);CHKERRQ(ierr);
9135   ierr = DMGlobalToLocalEnd(dmfo, fglobalo, INSERT_VALUES, flocal);CHKERRQ(ierr);
9136   ierr = DMLocalToGlobalBegin(dmf, flocal, INSERT_VALUES, fglobal);CHKERRQ(ierr);
9137   ierr = DMLocalToGlobalEnd(dmf, flocal, INSERT_VALUES, fglobal);CHKERRQ(ierr);
9138   *shift = fglobal;
9139   ierr = VecDestroy(&flocal);CHKERRQ(ierr);
9140   ierr = VecDestroy(&fglobalo);CHKERRQ(ierr);
9141   ierr = VecDestroy(&clocal);CHKERRQ(ierr);
9142   ierr = VecDestroy(&cglobalo);CHKERRQ(ierr);
9143   ierr = VecDestroy(&rscale);CHKERRQ(ierr);
9144   ierr = MatDestroy(&interpo);CHKERRQ(ierr);
9145   ierr = DMDestroy(&dmfo);CHKERRQ(ierr);
9146   ierr = DMDestroy(&dmco);CHKERRQ(ierr);
9147   PetscFunctionReturn(0);
9148 }
9149 
9150 PETSC_INTERN PetscErrorCode DMInterpolateSolution_Plex(DM coarse, DM fine, Mat interp, Vec coarseSol, Vec fineSol)
9151 {
9152   PetscObject    shifto;
9153   Vec            shift;
9154 
9155   PetscErrorCode ierr;
9156 
9157   PetscFunctionBegin;
9158   if (!interp) {
9159     Vec rscale;
9160 
9161     ierr = DMCreateInterpolation(coarse, fine, &interp, &rscale);CHKERRQ(ierr);
9162     ierr = VecDestroy(&rscale);CHKERRQ(ierr);
9163   } else {
9164     ierr = PetscObjectReference((PetscObject)interp);CHKERRQ(ierr);
9165   }
9166   ierr = PetscObjectQuery((PetscObject)interp, "_DMInterpolateSolution_Plex_Vec", &shifto);CHKERRQ(ierr);
9167   if (!shifto) {
9168     ierr = DMCreateAffineInterpolationCorrection_Plex(coarse, fine, &shift);CHKERRQ(ierr);
9169     ierr = PetscObjectCompose((PetscObject)interp, "_DMInterpolateSolution_Plex_Vec", (PetscObject) shift);CHKERRQ(ierr);
9170     shifto = (PetscObject) shift;
9171     ierr = VecDestroy(&shift);CHKERRQ(ierr);
9172   }
9173   shift = (Vec) shifto;
9174   ierr = MatInterpolate(interp, coarseSol, fineSol);CHKERRQ(ierr);
9175   ierr = VecAXPY(fineSol, 1.0, shift);CHKERRQ(ierr);
9176   ierr = MatDestroy(&interp);CHKERRQ(ierr);
9177   PetscFunctionReturn(0);
9178 }
9179 
9180 /* Pointwise interpolation
9181      Just code FEM for now
9182      u^f = I u^c
9183      sum_k u^f_k phi^f_k = I sum_j u^c_j phi^c_j
9184      u^f_i = sum_j psi^f_i I phi^c_j u^c_j
9185      I_{ij} = psi^f_i phi^c_j
9186 */
9187 PetscErrorCode DMCreateInterpolation_Plex(DM dmCoarse, DM dmFine, Mat *interpolation, Vec *scaling)
9188 {
9189   PetscSection   gsc, gsf;
9190   PetscInt       m, n;
9191   void          *ctx;
9192   DM             cdm;
9193   PetscBool      regular, ismatis, isRefined = dmCoarse->data == dmFine->data ? PETSC_FALSE : PETSC_TRUE;
9194   PetscErrorCode ierr;
9195 
9196   PetscFunctionBegin;
9197   ierr = DMGetGlobalSection(dmFine, &gsf);CHKERRQ(ierr);
9198   ierr = PetscSectionGetConstrainedStorageSize(gsf, &m);CHKERRQ(ierr);
9199   ierr = DMGetGlobalSection(dmCoarse, &gsc);CHKERRQ(ierr);
9200   ierr = PetscSectionGetConstrainedStorageSize(gsc, &n);CHKERRQ(ierr);
9201 
9202   ierr = PetscStrcmp(dmCoarse->mattype, MATIS, &ismatis);CHKERRQ(ierr);
9203   ierr = MatCreate(PetscObjectComm((PetscObject) dmCoarse), interpolation);CHKERRQ(ierr);
9204   ierr = MatSetSizes(*interpolation, m, n, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
9205   ierr = MatSetType(*interpolation, ismatis ? MATAIJ : dmCoarse->mattype);CHKERRQ(ierr);
9206   ierr = DMGetApplicationContext(dmFine, &ctx);CHKERRQ(ierr);
9207 
9208   ierr = DMGetCoarseDM(dmFine, &cdm);CHKERRQ(ierr);
9209   ierr = DMPlexGetRegularRefinement(dmFine, &regular);CHKERRQ(ierr);
9210   if (!isRefined || (regular && cdm == dmCoarse)) {ierr = DMPlexComputeInterpolatorNested(dmCoarse, dmFine, isRefined, *interpolation, ctx);CHKERRQ(ierr);}
9211   else                                            {ierr = DMPlexComputeInterpolatorGeneral(dmCoarse, dmFine, *interpolation, ctx);CHKERRQ(ierr);}
9212   ierr = MatViewFromOptions(*interpolation, NULL, "-interp_mat_view");CHKERRQ(ierr);
9213   if (scaling) {
9214     /* Use naive scaling */
9215     ierr = DMCreateInterpolationScale(dmCoarse, dmFine, *interpolation, scaling);CHKERRQ(ierr);
9216   }
9217   PetscFunctionReturn(0);
9218 }
9219 
9220 PetscErrorCode DMCreateInjection_Plex(DM dmCoarse, DM dmFine, Mat *mat)
9221 {
9222   PetscErrorCode ierr;
9223   VecScatter     ctx;
9224 
9225   PetscFunctionBegin;
9226   ierr = DMPlexComputeInjectorFEM(dmCoarse, dmFine, &ctx, NULL);CHKERRQ(ierr);
9227   ierr = MatCreateScatter(PetscObjectComm((PetscObject)ctx), ctx, mat);CHKERRQ(ierr);
9228   ierr = VecScatterDestroy(&ctx);CHKERRQ(ierr);
9229   PetscFunctionReturn(0);
9230 }
9231 
9232 static void g0_identity_private(PetscInt dim, PetscInt Nf, PetscInt NfAux,
9233                                 const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[],
9234                                 const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[],
9235                                 PetscReal t, PetscReal u_tShift, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar g0[])
9236 {
9237   g0[0] = 1.0;
9238 }
9239 
9240 PetscErrorCode DMCreateMassMatrixLumped_Plex(DM dm, Vec *mass)
9241 {
9242   DM             dmc;
9243   PetscDS        ds;
9244   Vec            ones, locmass;
9245   IS             cellIS;
9246   PetscFormKey   key;
9247   PetscInt       depth;
9248   PetscErrorCode ierr;
9249 
9250   PetscFunctionBegin;
9251   ierr = DMClone(dm, &dmc);CHKERRQ(ierr);
9252   ierr = DMCopyDisc(dm, dmc);CHKERRQ(ierr);
9253   ierr = DMGetDS(dmc, &ds);CHKERRQ(ierr);
9254   ierr = PetscDSSetJacobian(ds, 0, 0, g0_identity_private, NULL, NULL, NULL);CHKERRQ(ierr);
9255   ierr = DMCreateGlobalVector(dmc, mass);CHKERRQ(ierr);
9256   ierr = DMGetLocalVector(dmc, &ones);CHKERRQ(ierr);
9257   ierr = DMGetLocalVector(dmc, &locmass);CHKERRQ(ierr);
9258   ierr = DMPlexGetDepth(dmc, &depth);CHKERRQ(ierr);
9259   ierr = DMGetStratumIS(dmc, "depth", depth, &cellIS);CHKERRQ(ierr);
9260   ierr = VecSet(locmass, 0.0);CHKERRQ(ierr);
9261   ierr = VecSet(ones, 1.0);CHKERRQ(ierr);
9262   key.label = NULL;
9263   key.value = 0;
9264   key.field = 0;
9265   key.part  = 0;
9266   ierr = DMPlexComputeJacobian_Action_Internal(dmc, key, cellIS, 0.0, 0.0, ones, NULL, ones, locmass, NULL);CHKERRQ(ierr);
9267   ierr = ISDestroy(&cellIS);CHKERRQ(ierr);
9268   ierr = VecSet(*mass, 0.0);CHKERRQ(ierr);
9269   ierr = DMLocalToGlobalBegin(dmc, locmass, ADD_VALUES, *mass);CHKERRQ(ierr);
9270   ierr = DMLocalToGlobalEnd(dmc, locmass, ADD_VALUES, *mass);CHKERRQ(ierr);
9271   ierr = DMRestoreLocalVector(dmc, &ones);CHKERRQ(ierr);
9272   ierr = DMRestoreLocalVector(dmc, &locmass);CHKERRQ(ierr);
9273   ierr = DMDestroy(&dmc);CHKERRQ(ierr);
9274   PetscFunctionReturn(0);
9275 }
9276 
9277 PetscErrorCode DMCreateMassMatrix_Plex(DM dmCoarse, DM dmFine, Mat *mass)
9278 {
9279   PetscSection   gsc, gsf;
9280   PetscInt       m, n;
9281   void          *ctx;
9282   DM             cdm;
9283   PetscBool      regular;
9284   PetscErrorCode ierr;
9285 
9286   PetscFunctionBegin;
9287   if (dmFine == dmCoarse) {
9288     DM            dmc;
9289     PetscDS       ds;
9290     PetscWeakForm wf;
9291     Vec           u;
9292     IS            cellIS;
9293     PetscFormKey  key;
9294     PetscInt      depth;
9295 
9296     ierr = DMClone(dmFine, &dmc);CHKERRQ(ierr);
9297     ierr = DMCopyDisc(dmFine, dmc);CHKERRQ(ierr);
9298     ierr = DMGetDS(dmc, &ds);CHKERRQ(ierr);
9299     ierr = PetscDSGetWeakForm(ds, &wf);CHKERRQ(ierr);
9300     ierr = PetscWeakFormClear(wf);CHKERRQ(ierr);
9301     ierr = PetscDSSetJacobian(ds, 0, 0, g0_identity_private, NULL, NULL, NULL);CHKERRQ(ierr);
9302     ierr = DMCreateMatrix(dmc, mass);CHKERRQ(ierr);
9303     ierr = DMGetGlobalVector(dmc, &u);CHKERRQ(ierr);
9304     ierr = DMPlexGetDepth(dmc, &depth);CHKERRQ(ierr);
9305     ierr = DMGetStratumIS(dmc, "depth", depth, &cellIS);CHKERRQ(ierr);
9306     ierr = MatZeroEntries(*mass);CHKERRQ(ierr);
9307     key.label = NULL;
9308     key.value = 0;
9309     key.field = 0;
9310     key.part  = 0;
9311     ierr = DMPlexComputeJacobian_Internal(dmc, key, cellIS, 0.0, 0.0, u, NULL, *mass, *mass, NULL);CHKERRQ(ierr);
9312     ierr = ISDestroy(&cellIS);CHKERRQ(ierr);
9313     ierr = DMRestoreGlobalVector(dmc, &u);CHKERRQ(ierr);
9314     ierr = DMDestroy(&dmc);CHKERRQ(ierr);
9315   } else {
9316     ierr = DMGetGlobalSection(dmFine, &gsf);CHKERRQ(ierr);
9317     ierr = PetscSectionGetConstrainedStorageSize(gsf, &m);CHKERRQ(ierr);
9318     ierr = DMGetGlobalSection(dmCoarse, &gsc);CHKERRQ(ierr);
9319     ierr = PetscSectionGetConstrainedStorageSize(gsc, &n);CHKERRQ(ierr);
9320 
9321     ierr = MatCreate(PetscObjectComm((PetscObject) dmCoarse), mass);CHKERRQ(ierr);
9322     ierr = MatSetSizes(*mass, m, n, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
9323     ierr = MatSetType(*mass, dmCoarse->mattype);CHKERRQ(ierr);
9324     ierr = DMGetApplicationContext(dmFine, &ctx);CHKERRQ(ierr);
9325 
9326     ierr = DMGetCoarseDM(dmFine, &cdm);CHKERRQ(ierr);
9327     ierr = DMPlexGetRegularRefinement(dmFine, &regular);CHKERRQ(ierr);
9328     if (regular && cdm == dmCoarse) {ierr = DMPlexComputeMassMatrixNested(dmCoarse, dmFine, *mass, ctx);CHKERRQ(ierr);}
9329     else                            {ierr = DMPlexComputeMassMatrixGeneral(dmCoarse, dmFine, *mass, ctx);CHKERRQ(ierr);}
9330   }
9331   ierr = MatViewFromOptions(*mass, NULL, "-mass_mat_view");CHKERRQ(ierr);
9332   PetscFunctionReturn(0);
9333 }
9334 
9335 /*@
9336   DMPlexGetRegularRefinement - Get the flag indicating that this mesh was obtained by regular refinement from its coarse mesh
9337 
9338   Input Parameter:
9339 . dm - The DMPlex object
9340 
9341   Output Parameter:
9342 . regular - The flag
9343 
9344   Level: intermediate
9345 
9346 .seealso: DMPlexSetRegularRefinement()
9347 @*/
9348 PetscErrorCode DMPlexGetRegularRefinement(DM dm, PetscBool *regular)
9349 {
9350   PetscFunctionBegin;
9351   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9352   PetscValidPointer(regular, 2);
9353   *regular = ((DM_Plex *) dm->data)->regularRefinement;
9354   PetscFunctionReturn(0);
9355 }
9356 
9357 /*@
9358   DMPlexSetRegularRefinement - Set the flag indicating that this mesh was obtained by regular refinement from its coarse mesh
9359 
9360   Input Parameters:
9361 + dm - The DMPlex object
9362 - regular - The flag
9363 
9364   Level: intermediate
9365 
9366 .seealso: DMPlexGetRegularRefinement()
9367 @*/
9368 PetscErrorCode DMPlexSetRegularRefinement(DM dm, PetscBool regular)
9369 {
9370   PetscFunctionBegin;
9371   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9372   ((DM_Plex *) dm->data)->regularRefinement = regular;
9373   PetscFunctionReturn(0);
9374 }
9375 
9376 /* anchors */
9377 /*@
9378   DMPlexGetAnchors - Get the layout of the anchor (point-to-point) constraints.  Typically, the user will not have to
9379   call DMPlexGetAnchors() directly: if there are anchors, then DMPlexGetAnchors() is called during DMGetConstraints().
9380 
9381   not collective
9382 
9383   Input Parameter:
9384 . dm - The DMPlex object
9385 
9386   Output Parameters:
9387 + anchorSection - If not NULL, set to the section describing which points anchor the constrained points.
9388 - anchorIS - If not NULL, set to the list of anchors indexed by anchorSection
9389 
9390   Level: intermediate
9391 
9392 .seealso: DMPlexSetAnchors(), DMGetConstraints(), DMSetConstraints()
9393 @*/
9394 PetscErrorCode DMPlexGetAnchors(DM dm, PetscSection *anchorSection, IS *anchorIS)
9395 {
9396   DM_Plex *plex = (DM_Plex *)dm->data;
9397   PetscErrorCode ierr;
9398 
9399   PetscFunctionBegin;
9400   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9401   if (!plex->anchorSection && !plex->anchorIS && plex->createanchors) {ierr = (*plex->createanchors)(dm);CHKERRQ(ierr);}
9402   if (anchorSection) *anchorSection = plex->anchorSection;
9403   if (anchorIS) *anchorIS = plex->anchorIS;
9404   PetscFunctionReturn(0);
9405 }
9406 
9407 /*@
9408   DMPlexSetAnchors - Set the layout of the local anchor (point-to-point) constraints.  Unlike boundary conditions,
9409   when a point's degrees of freedom in a section are constrained to an outside value, the anchor constraints set a
9410   point's degrees of freedom to be a linear combination of other points' degrees of freedom.
9411 
9412   After specifying the layout of constraints with DMPlexSetAnchors(), one specifies the constraints by calling
9413   DMGetConstraints() and filling in the entries in the constraint matrix.
9414 
9415   collective on dm
9416 
9417   Input Parameters:
9418 + dm - The DMPlex object
9419 . anchorSection - The section that describes the mapping from constrained points to the anchor points listed in anchorIS.  Must have a local communicator (PETSC_COMM_SELF or derivative).
9420 - anchorIS - The list of all anchor points.  Must have a local communicator (PETSC_COMM_SELF or derivative).
9421 
9422   The reference counts of anchorSection and anchorIS are incremented.
9423 
9424   Level: intermediate
9425 
9426 .seealso: DMPlexGetAnchors(), DMGetConstraints(), DMSetConstraints()
9427 @*/
9428 PetscErrorCode DMPlexSetAnchors(DM dm, PetscSection anchorSection, IS anchorIS)
9429 {
9430   DM_Plex        *plex = (DM_Plex *)dm->data;
9431   PetscMPIInt    result;
9432   PetscErrorCode ierr;
9433 
9434   PetscFunctionBegin;
9435   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9436   if (anchorSection) {
9437     PetscValidHeaderSpecific(anchorSection,PETSC_SECTION_CLASSID,2);
9438     ierr = MPI_Comm_compare(PETSC_COMM_SELF,PetscObjectComm((PetscObject)anchorSection),&result);CHKERRMPI(ierr);
9439     PetscAssertFalse(result != MPI_CONGRUENT && result != MPI_IDENT,PETSC_COMM_SELF,PETSC_ERR_ARG_NOTSAMECOMM,"anchor section must have local communicator");
9440   }
9441   if (anchorIS) {
9442     PetscValidHeaderSpecific(anchorIS,IS_CLASSID,3);
9443     ierr = MPI_Comm_compare(PETSC_COMM_SELF,PetscObjectComm((PetscObject)anchorIS),&result);CHKERRMPI(ierr);
9444     PetscAssertFalse(result != MPI_CONGRUENT && result != MPI_IDENT,PETSC_COMM_SELF,PETSC_ERR_ARG_NOTSAMECOMM,"anchor IS must have local communicator");
9445   }
9446 
9447   ierr = PetscObjectReference((PetscObject)anchorSection);CHKERRQ(ierr);
9448   ierr = PetscSectionDestroy(&plex->anchorSection);CHKERRQ(ierr);
9449   plex->anchorSection = anchorSection;
9450 
9451   ierr = PetscObjectReference((PetscObject)anchorIS);CHKERRQ(ierr);
9452   ierr = ISDestroy(&plex->anchorIS);CHKERRQ(ierr);
9453   plex->anchorIS = anchorIS;
9454 
9455   if (PetscUnlikelyDebug(anchorIS && anchorSection)) {
9456     PetscInt size, a, pStart, pEnd;
9457     const PetscInt *anchors;
9458 
9459     ierr = PetscSectionGetChart(anchorSection,&pStart,&pEnd);CHKERRQ(ierr);
9460     ierr = ISGetLocalSize(anchorIS,&size);CHKERRQ(ierr);
9461     ierr = ISGetIndices(anchorIS,&anchors);CHKERRQ(ierr);
9462     for (a = 0; a < size; a++) {
9463       PetscInt p;
9464 
9465       p = anchors[a];
9466       if (p >= pStart && p < pEnd) {
9467         PetscInt dof;
9468 
9469         ierr = PetscSectionGetDof(anchorSection,p,&dof);CHKERRQ(ierr);
9470         if (dof) {
9471           PetscErrorCode ierr2;
9472 
9473           ierr2 = ISRestoreIndices(anchorIS,&anchors);CHKERRQ(ierr2);
9474           SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_INCOMP,"Point %D cannot be constrained and an anchor",p);
9475         }
9476       }
9477     }
9478     ierr = ISRestoreIndices(anchorIS,&anchors);CHKERRQ(ierr);
9479   }
9480   /* reset the generic constraints */
9481   ierr = DMSetDefaultConstraints(dm,NULL,NULL);CHKERRQ(ierr);
9482   PetscFunctionReturn(0);
9483 }
9484 
9485 static PetscErrorCode DMPlexCreateConstraintSection_Anchors(DM dm, PetscSection section, PetscSection *cSec)
9486 {
9487   PetscSection anchorSection;
9488   PetscInt pStart, pEnd, sStart, sEnd, p, dof, numFields, f;
9489   PetscErrorCode ierr;
9490 
9491   PetscFunctionBegin;
9492   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9493   ierr = DMPlexGetAnchors(dm,&anchorSection,NULL);CHKERRQ(ierr);
9494   ierr = PetscSectionCreate(PETSC_COMM_SELF,cSec);CHKERRQ(ierr);
9495   ierr = PetscSectionGetNumFields(section,&numFields);CHKERRQ(ierr);
9496   if (numFields) {
9497     PetscInt f;
9498     ierr = PetscSectionSetNumFields(*cSec,numFields);CHKERRQ(ierr);
9499 
9500     for (f = 0; f < numFields; f++) {
9501       PetscInt numComp;
9502 
9503       ierr = PetscSectionGetFieldComponents(section,f,&numComp);CHKERRQ(ierr);
9504       ierr = PetscSectionSetFieldComponents(*cSec,f,numComp);CHKERRQ(ierr);
9505     }
9506   }
9507   ierr = PetscSectionGetChart(anchorSection,&pStart,&pEnd);CHKERRQ(ierr);
9508   ierr = PetscSectionGetChart(section,&sStart,&sEnd);CHKERRQ(ierr);
9509   pStart = PetscMax(pStart,sStart);
9510   pEnd   = PetscMin(pEnd,sEnd);
9511   pEnd   = PetscMax(pStart,pEnd);
9512   ierr = PetscSectionSetChart(*cSec,pStart,pEnd);CHKERRQ(ierr);
9513   for (p = pStart; p < pEnd; p++) {
9514     ierr = PetscSectionGetDof(anchorSection,p,&dof);CHKERRQ(ierr);
9515     if (dof) {
9516       ierr = PetscSectionGetDof(section,p,&dof);CHKERRQ(ierr);
9517       ierr = PetscSectionSetDof(*cSec,p,dof);CHKERRQ(ierr);
9518       for (f = 0; f < numFields; f++) {
9519         ierr = PetscSectionGetFieldDof(section,p,f,&dof);CHKERRQ(ierr);
9520         ierr = PetscSectionSetFieldDof(*cSec,p,f,dof);CHKERRQ(ierr);
9521       }
9522     }
9523   }
9524   ierr = PetscSectionSetUp(*cSec);CHKERRQ(ierr);
9525   ierr = PetscObjectSetName((PetscObject) *cSec, "Constraint Section");CHKERRQ(ierr);
9526   PetscFunctionReturn(0);
9527 }
9528 
9529 static PetscErrorCode DMPlexCreateConstraintMatrix_Anchors(DM dm, PetscSection section, PetscSection cSec, Mat *cMat)
9530 {
9531   PetscSection   aSec;
9532   PetscInt       pStart, pEnd, p, sStart, sEnd, dof, aDof, aOff, off, nnz, annz, m, n, q, a, offset, *i, *j;
9533   const PetscInt *anchors;
9534   PetscInt       numFields, f;
9535   IS             aIS;
9536   PetscErrorCode ierr;
9537   MatType        mtype;
9538   PetscBool      iscuda,iskokkos;
9539 
9540   PetscFunctionBegin;
9541   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9542   ierr = PetscSectionGetStorageSize(cSec, &m);CHKERRQ(ierr);
9543   ierr = PetscSectionGetStorageSize(section, &n);CHKERRQ(ierr);
9544   ierr = MatCreate(PETSC_COMM_SELF,cMat);CHKERRQ(ierr);
9545   ierr = MatSetSizes(*cMat,m,n,m,n);CHKERRQ(ierr);
9546   ierr = PetscStrcmp(dm->mattype,MATSEQAIJCUSPARSE,&iscuda);CHKERRQ(ierr);
9547   if (!iscuda) { ierr = PetscStrcmp(dm->mattype,MATMPIAIJCUSPARSE,&iscuda);CHKERRQ(ierr); }
9548   ierr = PetscStrcmp(dm->mattype,MATSEQAIJKOKKOS,&iskokkos);CHKERRQ(ierr);
9549   if (!iskokkos) { ierr = PetscStrcmp(dm->mattype,MATMPIAIJKOKKOS,&iskokkos);CHKERRQ(ierr); }
9550   if (iscuda) mtype = MATSEQAIJCUSPARSE;
9551   else if (iskokkos) mtype = MATSEQAIJKOKKOS;
9552   else mtype = MATSEQAIJ;
9553   ierr = MatSetType(*cMat,mtype);CHKERRQ(ierr);
9554   ierr = DMPlexGetAnchors(dm,&aSec,&aIS);CHKERRQ(ierr);
9555   ierr = ISGetIndices(aIS,&anchors);CHKERRQ(ierr);
9556   /* cSec will be a subset of aSec and section */
9557   ierr = PetscSectionGetChart(cSec,&pStart,&pEnd);CHKERRQ(ierr);
9558   ierr = PetscSectionGetChart(section,&sStart,&sEnd);CHKERRQ(ierr);
9559   ierr = PetscMalloc1(m+1,&i);CHKERRQ(ierr);
9560   i[0] = 0;
9561   ierr = PetscSectionGetNumFields(section,&numFields);CHKERRQ(ierr);
9562   for (p = pStart; p < pEnd; p++) {
9563     PetscInt rDof, rOff, r;
9564 
9565     ierr = PetscSectionGetDof(aSec,p,&rDof);CHKERRQ(ierr);
9566     if (!rDof) continue;
9567     ierr = PetscSectionGetOffset(aSec,p,&rOff);CHKERRQ(ierr);
9568     if (numFields) {
9569       for (f = 0; f < numFields; f++) {
9570         annz = 0;
9571         for (r = 0; r < rDof; r++) {
9572           a = anchors[rOff + r];
9573           if (a < sStart || a >= sEnd) continue;
9574           ierr = PetscSectionGetFieldDof(section,a,f,&aDof);CHKERRQ(ierr);
9575           annz += aDof;
9576         }
9577         ierr = PetscSectionGetFieldDof(cSec,p,f,&dof);CHKERRQ(ierr);
9578         ierr = PetscSectionGetFieldOffset(cSec,p,f,&off);CHKERRQ(ierr);
9579         for (q = 0; q < dof; q++) {
9580           i[off + q + 1] = i[off + q] + annz;
9581         }
9582       }
9583     } else {
9584       annz = 0;
9585       ierr = PetscSectionGetDof(cSec,p,&dof);CHKERRQ(ierr);
9586       for (q = 0; q < dof; q++) {
9587         a = anchors[rOff + q];
9588         if (a < sStart || a >= sEnd) continue;
9589         ierr = PetscSectionGetDof(section,a,&aDof);CHKERRQ(ierr);
9590         annz += aDof;
9591       }
9592       ierr = PetscSectionGetDof(cSec,p,&dof);CHKERRQ(ierr);
9593       ierr = PetscSectionGetOffset(cSec,p,&off);CHKERRQ(ierr);
9594       for (q = 0; q < dof; q++) {
9595         i[off + q + 1] = i[off + q] + annz;
9596       }
9597     }
9598   }
9599   nnz = i[m];
9600   ierr = PetscMalloc1(nnz,&j);CHKERRQ(ierr);
9601   offset = 0;
9602   for (p = pStart; p < pEnd; p++) {
9603     if (numFields) {
9604       for (f = 0; f < numFields; f++) {
9605         ierr = PetscSectionGetFieldDof(cSec,p,f,&dof);CHKERRQ(ierr);
9606         for (q = 0; q < dof; q++) {
9607           PetscInt rDof, rOff, r;
9608           ierr = PetscSectionGetDof(aSec,p,&rDof);CHKERRQ(ierr);
9609           ierr = PetscSectionGetOffset(aSec,p,&rOff);CHKERRQ(ierr);
9610           for (r = 0; r < rDof; r++) {
9611             PetscInt s;
9612 
9613             a = anchors[rOff + r];
9614             if (a < sStart || a >= sEnd) continue;
9615             ierr = PetscSectionGetFieldDof(section,a,f,&aDof);CHKERRQ(ierr);
9616             ierr = PetscSectionGetFieldOffset(section,a,f,&aOff);CHKERRQ(ierr);
9617             for (s = 0; s < aDof; s++) {
9618               j[offset++] = aOff + s;
9619             }
9620           }
9621         }
9622       }
9623     } else {
9624       ierr = PetscSectionGetDof(cSec,p,&dof);CHKERRQ(ierr);
9625       for (q = 0; q < dof; q++) {
9626         PetscInt rDof, rOff, r;
9627         ierr = PetscSectionGetDof(aSec,p,&rDof);CHKERRQ(ierr);
9628         ierr = PetscSectionGetOffset(aSec,p,&rOff);CHKERRQ(ierr);
9629         for (r = 0; r < rDof; r++) {
9630           PetscInt s;
9631 
9632           a = anchors[rOff + r];
9633           if (a < sStart || a >= sEnd) continue;
9634           ierr = PetscSectionGetDof(section,a,&aDof);CHKERRQ(ierr);
9635           ierr = PetscSectionGetOffset(section,a,&aOff);CHKERRQ(ierr);
9636           for (s = 0; s < aDof; s++) {
9637             j[offset++] = aOff + s;
9638           }
9639         }
9640       }
9641     }
9642   }
9643   ierr = MatSeqAIJSetPreallocationCSR(*cMat,i,j,NULL);CHKERRQ(ierr);
9644   ierr = PetscFree(i);CHKERRQ(ierr);
9645   ierr = PetscFree(j);CHKERRQ(ierr);
9646   ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);
9647   PetscFunctionReturn(0);
9648 }
9649 
9650 PetscErrorCode DMCreateDefaultConstraints_Plex(DM dm)
9651 {
9652   DM_Plex        *plex = (DM_Plex *)dm->data;
9653   PetscSection   anchorSection, section, cSec;
9654   Mat            cMat;
9655   PetscErrorCode ierr;
9656 
9657   PetscFunctionBegin;
9658   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9659   ierr = DMPlexGetAnchors(dm,&anchorSection,NULL);CHKERRQ(ierr);
9660   if (anchorSection) {
9661     PetscInt Nf;
9662 
9663     ierr = DMGetLocalSection(dm,&section);CHKERRQ(ierr);
9664     ierr = DMPlexCreateConstraintSection_Anchors(dm,section,&cSec);CHKERRQ(ierr);
9665     ierr = DMPlexCreateConstraintMatrix_Anchors(dm,section,cSec,&cMat);CHKERRQ(ierr);
9666     ierr = DMGetNumFields(dm,&Nf);CHKERRQ(ierr);
9667     if (Nf && plex->computeanchormatrix) {ierr = (*plex->computeanchormatrix)(dm,section,cSec,cMat);CHKERRQ(ierr);}
9668     ierr = DMSetDefaultConstraints(dm,cSec,cMat);CHKERRQ(ierr);
9669     ierr = PetscSectionDestroy(&cSec);CHKERRQ(ierr);
9670     ierr = MatDestroy(&cMat);CHKERRQ(ierr);
9671   }
9672   PetscFunctionReturn(0);
9673 }
9674 
9675 PetscErrorCode DMCreateSubDomainDM_Plex(DM dm, DMLabel label, PetscInt value, IS *is, DM *subdm)
9676 {
9677   IS             subis;
9678   PetscSection   section, subsection;
9679   PetscErrorCode ierr;
9680 
9681   PetscFunctionBegin;
9682   ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);
9683   PetscAssertFalse(!section,PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Must set default section for DM before splitting subdomain");
9684   PetscAssertFalse(!subdm,PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Must set output subDM for splitting subdomain");
9685   /* Create subdomain */
9686   ierr = DMPlexFilter(dm, label, value, subdm);CHKERRQ(ierr);
9687   /* Create submodel */
9688   ierr = DMPlexGetSubpointIS(*subdm, &subis);CHKERRQ(ierr);
9689   ierr = PetscSectionCreateSubmeshSection(section, subis, &subsection);CHKERRQ(ierr);
9690   ierr = DMSetLocalSection(*subdm, subsection);CHKERRQ(ierr);
9691   ierr = PetscSectionDestroy(&subsection);CHKERRQ(ierr);
9692   ierr = DMCopyDisc(dm, *subdm);CHKERRQ(ierr);
9693   /* Create map from submodel to global model */
9694   if (is) {
9695     PetscSection    sectionGlobal, subsectionGlobal;
9696     IS              spIS;
9697     const PetscInt *spmap;
9698     PetscInt       *subIndices;
9699     PetscInt        subSize = 0, subOff = 0, pStart, pEnd, p;
9700     PetscInt        Nf, f, bs = -1, bsLocal[2], bsMinMax[2];
9701 
9702     ierr = DMPlexGetSubpointIS(*subdm, &spIS);CHKERRQ(ierr);
9703     ierr = ISGetIndices(spIS, &spmap);CHKERRQ(ierr);
9704     ierr = PetscSectionGetNumFields(section, &Nf);CHKERRQ(ierr);
9705     ierr = DMGetGlobalSection(dm, &sectionGlobal);CHKERRQ(ierr);
9706     ierr = DMGetGlobalSection(*subdm, &subsectionGlobal);CHKERRQ(ierr);
9707     ierr = PetscSectionGetChart(subsection, &pStart, &pEnd);CHKERRQ(ierr);
9708     for (p = pStart; p < pEnd; ++p) {
9709       PetscInt gdof, pSubSize  = 0;
9710 
9711       ierr = PetscSectionGetDof(sectionGlobal, p, &gdof);CHKERRQ(ierr);
9712       if (gdof > 0) {
9713         for (f = 0; f < Nf; ++f) {
9714           PetscInt fdof, fcdof;
9715 
9716           ierr     = PetscSectionGetFieldDof(subsection, p, f, &fdof);CHKERRQ(ierr);
9717           ierr     = PetscSectionGetFieldConstraintDof(subsection, p, f, &fcdof);CHKERRQ(ierr);
9718           pSubSize += fdof-fcdof;
9719         }
9720         subSize += pSubSize;
9721         if (pSubSize) {
9722           if (bs < 0) {
9723             bs = pSubSize;
9724           } else if (bs != pSubSize) {
9725             /* Layout does not admit a pointwise block size */
9726             bs = 1;
9727           }
9728         }
9729       }
9730     }
9731     /* Must have same blocksize on all procs (some might have no points) */
9732     bsLocal[0] = bs < 0 ? PETSC_MAX_INT : bs; bsLocal[1] = bs;
9733     ierr = PetscGlobalMinMaxInt(PetscObjectComm((PetscObject) dm), bsLocal, bsMinMax);CHKERRQ(ierr);
9734     if (bsMinMax[0] != bsMinMax[1]) {bs = 1;}
9735     else                            {bs = bsMinMax[0];}
9736     ierr = PetscMalloc1(subSize, &subIndices);CHKERRQ(ierr);
9737     for (p = pStart; p < pEnd; ++p) {
9738       PetscInt gdof, goff;
9739 
9740       ierr = PetscSectionGetDof(subsectionGlobal, p, &gdof);CHKERRQ(ierr);
9741       if (gdof > 0) {
9742         const PetscInt point = spmap[p];
9743 
9744         ierr = PetscSectionGetOffset(sectionGlobal, point, &goff);CHKERRQ(ierr);
9745         for (f = 0; f < Nf; ++f) {
9746           PetscInt fdof, fcdof, fc, f2, poff = 0;
9747 
9748           /* Can get rid of this loop by storing field information in the global section */
9749           for (f2 = 0; f2 < f; ++f2) {
9750             ierr  = PetscSectionGetFieldDof(section, p, f2, &fdof);CHKERRQ(ierr);
9751             ierr  = PetscSectionGetFieldConstraintDof(section, p, f2, &fcdof);CHKERRQ(ierr);
9752             poff += fdof-fcdof;
9753           }
9754           ierr = PetscSectionGetFieldDof(section, p, f, &fdof);CHKERRQ(ierr);
9755           ierr = PetscSectionGetFieldConstraintDof(section, p, f, &fcdof);CHKERRQ(ierr);
9756           for (fc = 0; fc < fdof-fcdof; ++fc, ++subOff) {
9757             subIndices[subOff] = goff+poff+fc;
9758           }
9759         }
9760       }
9761     }
9762     ierr = ISRestoreIndices(spIS, &spmap);CHKERRQ(ierr);
9763     ierr = ISCreateGeneral(PetscObjectComm((PetscObject)dm), subSize, subIndices, PETSC_OWN_POINTER, is);CHKERRQ(ierr);
9764     if (bs > 1) {
9765       /* We need to check that the block size does not come from non-contiguous fields */
9766       PetscInt i, j, set = 1;
9767       for (i = 0; i < subSize; i += bs) {
9768         for (j = 0; j < bs; ++j) {
9769           if (subIndices[i+j] != subIndices[i]+j) {set = 0; break;}
9770         }
9771       }
9772       if (set) {ierr = ISSetBlockSize(*is, bs);CHKERRQ(ierr);}
9773     }
9774     /* Attach nullspace */
9775     for (f = 0; f < Nf; ++f) {
9776       (*subdm)->nullspaceConstructors[f] = dm->nullspaceConstructors[f];
9777       if ((*subdm)->nullspaceConstructors[f]) break;
9778     }
9779     if (f < Nf) {
9780       MatNullSpace nullSpace;
9781       ierr = (*(*subdm)->nullspaceConstructors[f])(*subdm, f, f, &nullSpace);CHKERRQ(ierr);
9782 
9783       ierr = PetscObjectCompose((PetscObject) *is, "nullspace", (PetscObject) nullSpace);CHKERRQ(ierr);
9784       ierr = MatNullSpaceDestroy(&nullSpace);CHKERRQ(ierr);
9785     }
9786   }
9787   PetscFunctionReturn(0);
9788 }
9789 
9790 /*@
9791   DMPlexMonitorThroughput - Report the cell throughput of FE integration
9792 
9793   Input Parameter:
9794 - dm - The DM
9795 
9796   Level: developer
9797 
9798   Options Database Keys:
9799 . -dm_plex_monitor_throughput - Activate the monitor
9800 
9801 .seealso: DMSetFromOptions(), DMPlexCreate()
9802 @*/
9803 PetscErrorCode DMPlexMonitorThroughput(DM dm, void *dummy)
9804 {
9805 #if defined(PETSC_USE_LOG)
9806   PetscStageLog      stageLog;
9807   PetscLogEvent      event;
9808   PetscLogStage      stage;
9809   PetscEventPerfInfo eventInfo;
9810   PetscReal          cellRate, flopRate;
9811   PetscInt           cStart, cEnd, Nf, N;
9812   const char        *name;
9813   PetscErrorCode     ierr;
9814 #endif
9815 
9816   PetscFunctionBegin;
9817   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9818 #if defined(PETSC_USE_LOG)
9819   ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
9820   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
9821   ierr = DMGetNumFields(dm, &Nf);CHKERRQ(ierr);
9822   ierr = PetscLogGetStageLog(&stageLog);CHKERRQ(ierr);
9823   ierr = PetscStageLogGetCurrent(stageLog, &stage);CHKERRQ(ierr);
9824   ierr = PetscLogEventGetId("DMPlexResidualFE", &event);CHKERRQ(ierr);
9825   ierr = PetscLogEventGetPerfInfo(stage, event, &eventInfo);CHKERRQ(ierr);
9826   N        = (cEnd - cStart)*Nf*eventInfo.count;
9827   flopRate = eventInfo.flops/eventInfo.time;
9828   cellRate = N/eventInfo.time;
9829   ierr = PetscPrintf(PetscObjectComm((PetscObject) dm), "DM (%s) FE Residual Integration: %D integrals %D reps\n  Cell rate: %.2g/s flop rate: %.2g MF/s\n", name ? name : "unknown", N, eventInfo.count, (double) cellRate, (double) (flopRate/1.e6));CHKERRQ(ierr);
9830 #else
9831   SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Plex Throughput Monitor is not supported if logging is turned off. Reconfigure using --with-log.");
9832 #endif
9833   PetscFunctionReturn(0);
9834 }
9835