xref: /petsc/src/dm/impls/plex/plex.c (revision c5f5e4256b5371f4ad8269da27ce9e5aa970ec1e) 
1 #include <petsc/private/dmpleximpl.h>   /*I      "petscdmplex.h"   I*/
2 #include <petsc/private/isimpl.h>
3 #include <petsc/private/vecimpl.h>
4 #include <petsc/private/glvisvecimpl.h>
5 #include <petscsf.h>
6 #include <petscds.h>
7 #include <petscdraw.h>
8 #include <petscdmfield.h>
9 #include <petscdmplextransform.h>
10 
11 /* Logging support */
12 PetscLogEvent DMPLEX_Interpolate, DMPLEX_Partition, DMPLEX_Distribute, DMPLEX_DistributeCones, DMPLEX_DistributeLabels, DMPLEX_DistributeSF, DMPLEX_DistributeOverlap, DMPLEX_DistributeField, DMPLEX_DistributeData, DMPLEX_Migrate, DMPLEX_InterpolateSF, DMPLEX_GlobalToNaturalBegin, DMPLEX_GlobalToNaturalEnd, DMPLEX_NaturalToGlobalBegin, DMPLEX_NaturalToGlobalEnd, DMPLEX_Stratify, DMPLEX_Symmetrize, DMPLEX_Preallocate, DMPLEX_ResidualFEM, DMPLEX_JacobianFEM, DMPLEX_InterpolatorFEM, DMPLEX_InjectorFEM, DMPLEX_IntegralFEM, DMPLEX_CreateGmsh, DMPLEX_RebalanceSharedPoints, DMPLEX_PartSelf, DMPLEX_PartLabelInvert, DMPLEX_PartLabelCreateSF, DMPLEX_PartStratSF, DMPLEX_CreatePointSF,DMPLEX_LocatePoints,DMPLEX_TopologyView,DMPLEX_LabelsView,DMPLEX_CoordinatesView,DMPLEX_SectionView,DMPLEX_GlobalVectorView,DMPLEX_LocalVectorView,DMPLEX_TopologyLoad,DMPLEX_LabelsLoad,DMPLEX_CoordinatesLoad,DMPLEX_SectionLoad,DMPLEX_GlobalVectorLoad,DMPLEX_LocalVectorLoad;
13 
14 PETSC_EXTERN PetscErrorCode VecView_MPI(Vec, PetscViewer);
15 
16 /*@
17   DMPlexIsSimplex - Is the first cell in this mesh a simplex?
18 
19   Input Parameter:
20 . dm      - The DMPlex object
21 
22   Output Parameter:
23 . simplex - Flag checking for a simplex
24 
25   Note: This just gives the first range of cells found. If the mesh has several cell types, it will only give the first.
26   If the mesh has no cells, this returns PETSC_FALSE.
27 
28   Level: intermediate
29 
30 .seealso DMPlexGetSimplexOrBoxCells(), DMPlexGetCellType(), DMPlexGetHeightStratum(), DMPolytopeTypeGetNumVertices()
31 @*/
32 PetscErrorCode DMPlexIsSimplex(DM dm, PetscBool *simplex)
33 {
34   DMPolytopeType ct;
35   PetscInt       cStart, cEnd;
36   PetscErrorCode ierr;
37 
38   PetscFunctionBegin;
39   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
40   if (cEnd <= cStart) {*simplex = PETSC_FALSE; PetscFunctionReturn(0);}
41   ierr = DMPlexGetCellType(dm, cStart, &ct);CHKERRQ(ierr);
42   *simplex = DMPolytopeTypeGetNumVertices(ct) == DMPolytopeTypeGetDim(ct)+1 ? PETSC_TRUE : PETSC_FALSE;
43   PetscFunctionReturn(0);
44 }
45 
46 /*@
47   DMPlexGetSimplexOrBoxCells - Get the range of cells which are neither prisms nor ghost FV cells
48 
49   Input Parameters:
50 + dm     - The DMPlex object
51 - height - The cell height in the Plex, 0 is the default
52 
53   Output Parameters:
54 + cStart - The first "normal" cell
55 - cEnd   - The upper bound on "normal"" cells
56 
57   Note: This just gives the first range of cells found. If the mesh has several cell types, it will only give the first.
58 
59   Level: developer
60 
61 .seealso DMPlexConstructGhostCells(), DMPlexGetGhostCellStratum()
62 @*/
63 PetscErrorCode DMPlexGetSimplexOrBoxCells(DM dm, PetscInt height, PetscInt *cStart, PetscInt *cEnd)
64 {
65   DMPolytopeType ct = DM_POLYTOPE_UNKNOWN;
66   PetscInt       cS, cE, c;
67   PetscErrorCode ierr;
68 
69   PetscFunctionBegin;
70   ierr = DMPlexGetHeightStratum(dm, PetscMax(height, 0), &cS, &cE);CHKERRQ(ierr);
71   for (c = cS; c < cE; ++c) {
72     DMPolytopeType cct;
73 
74     ierr = DMPlexGetCellType(dm, c, &cct);CHKERRQ(ierr);
75     if ((PetscInt) cct < 0) break;
76     switch (cct) {
77       case DM_POLYTOPE_POINT:
78       case DM_POLYTOPE_SEGMENT:
79       case DM_POLYTOPE_TRIANGLE:
80       case DM_POLYTOPE_QUADRILATERAL:
81       case DM_POLYTOPE_TETRAHEDRON:
82       case DM_POLYTOPE_HEXAHEDRON:
83         ct = cct;
84         break;
85       default: break;
86     }
87     if (ct != DM_POLYTOPE_UNKNOWN) break;
88   }
89   if (ct != DM_POLYTOPE_UNKNOWN) {
90     DMLabel ctLabel;
91 
92     ierr = DMPlexGetCellTypeLabel(dm, &ctLabel);CHKERRQ(ierr);
93     ierr = DMLabelGetStratumBounds(ctLabel, ct, &cS, &cE);CHKERRQ(ierr);
94   }
95   if (cStart) *cStart = cS;
96   if (cEnd)   *cEnd   = cE;
97   PetscFunctionReturn(0);
98 }
99 
100 PetscErrorCode DMPlexGetFieldType_Internal(DM dm, PetscSection section, PetscInt field, PetscInt *sStart, PetscInt *sEnd, PetscViewerVTKFieldType *ft)
101 {
102   PetscInt       cdim, pStart, pEnd, vStart, vEnd, cStart, cEnd;
103   PetscInt       vcdof[2] = {0,0}, globalvcdof[2];
104   PetscErrorCode ierr;
105 
106   PetscFunctionBegin;
107   *ft  = PETSC_VTK_INVALID;
108   ierr = DMGetCoordinateDim(dm, &cdim);CHKERRQ(ierr);
109   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
110   ierr = DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
111   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
112   if (field >= 0) {
113     if ((vStart >= pStart) && (vStart < pEnd)) {ierr = PetscSectionGetFieldDof(section, vStart, field, &vcdof[0]);CHKERRQ(ierr);}
114     if ((cStart >= pStart) && (cStart < pEnd)) {ierr = PetscSectionGetFieldDof(section, cStart, field, &vcdof[1]);CHKERRQ(ierr);}
115   } else {
116     if ((vStart >= pStart) && (vStart < pEnd)) {ierr = PetscSectionGetDof(section, vStart, &vcdof[0]);CHKERRQ(ierr);}
117     if ((cStart >= pStart) && (cStart < pEnd)) {ierr = PetscSectionGetDof(section, cStart, &vcdof[1]);CHKERRQ(ierr);}
118   }
119   ierr = MPI_Allreduce(vcdof, globalvcdof, 2, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject)dm));CHKERRMPI(ierr);
120   if (globalvcdof[0]) {
121     *sStart = vStart;
122     *sEnd   = vEnd;
123     if (globalvcdof[0] == cdim) *ft = PETSC_VTK_POINT_VECTOR_FIELD;
124     else                        *ft = PETSC_VTK_POINT_FIELD;
125   } else if (globalvcdof[1]) {
126     *sStart = cStart;
127     *sEnd   = cEnd;
128     if (globalvcdof[1] == cdim) *ft = PETSC_VTK_CELL_VECTOR_FIELD;
129     else                        *ft = PETSC_VTK_CELL_FIELD;
130   } else {
131     if (field >= 0) {
132       const char *fieldname;
133 
134       ierr = PetscSectionGetFieldName(section, field, &fieldname);CHKERRQ(ierr);
135       ierr = PetscInfo2((PetscObject) dm, "Could not classify VTK output type of section field %D \"%s\"\n", field, fieldname);CHKERRQ(ierr);
136     } else {
137       ierr = PetscInfo((PetscObject) dm, "Could not classify VTK output type of section\"%s\"\n");CHKERRQ(ierr);
138     }
139   }
140   PetscFunctionReturn(0);
141 }
142 
143 static PetscErrorCode VecView_Plex_Local_Draw(Vec v, PetscViewer viewer)
144 {
145   DM                 dm;
146   PetscSection       s;
147   PetscDraw          draw, popup;
148   DM                 cdm;
149   PetscSection       coordSection;
150   Vec                coordinates;
151   const PetscScalar *coords, *array;
152   PetscReal          bound[4] = {PETSC_MAX_REAL, PETSC_MAX_REAL, PETSC_MIN_REAL, PETSC_MIN_REAL};
153   PetscReal          vbound[2], time;
154   PetscBool          isnull, flg;
155   PetscInt           dim, Nf, f, Nc, comp, vStart, vEnd, cStart, cEnd, c, N, level, step, w = 0;
156   const char        *name;
157   char               title[PETSC_MAX_PATH_LEN];
158   PetscErrorCode     ierr;
159 
160   PetscFunctionBegin;
161   ierr = PetscViewerDrawGetDraw(viewer, 0, &draw);CHKERRQ(ierr);
162   ierr = PetscDrawIsNull(draw, &isnull);CHKERRQ(ierr);
163   if (isnull) PetscFunctionReturn(0);
164 
165   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
166   ierr = DMGetCoordinateDim(dm, &dim);CHKERRQ(ierr);
167   if (dim != 2) SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Cannot draw meshes of dimension %D. Use PETSCVIEWERGLVIS", dim);
168   ierr = DMGetLocalSection(dm, &s);CHKERRQ(ierr);
169   ierr = PetscSectionGetNumFields(s, &Nf);CHKERRQ(ierr);
170   ierr = DMGetCoarsenLevel(dm, &level);CHKERRQ(ierr);
171   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
172   ierr = DMGetLocalSection(cdm, &coordSection);CHKERRQ(ierr);
173   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
174   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
175   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
176 
177   ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
178   ierr = DMGetOutputSequenceNumber(dm, &step, &time);CHKERRQ(ierr);
179 
180   ierr = VecGetLocalSize(coordinates, &N);CHKERRQ(ierr);
181   ierr = VecGetArrayRead(coordinates, &coords);CHKERRQ(ierr);
182   for (c = 0; c < N; c += dim) {
183     bound[0] = PetscMin(bound[0], PetscRealPart(coords[c]));   bound[2] = PetscMax(bound[2], PetscRealPart(coords[c]));
184     bound[1] = PetscMin(bound[1], PetscRealPart(coords[c+1])); bound[3] = PetscMax(bound[3], PetscRealPart(coords[c+1]));
185   }
186   ierr = VecRestoreArrayRead(coordinates, &coords);CHKERRQ(ierr);
187   ierr = PetscDrawClear(draw);CHKERRQ(ierr);
188 
189   /* Could implement something like DMDASelectFields() */
190   for (f = 0; f < Nf; ++f) {
191     DM   fdm = dm;
192     Vec  fv  = v;
193     IS   fis;
194     char prefix[PETSC_MAX_PATH_LEN];
195     const char *fname;
196 
197     ierr = PetscSectionGetFieldComponents(s, f, &Nc);CHKERRQ(ierr);
198     ierr = PetscSectionGetFieldName(s, f, &fname);CHKERRQ(ierr);
199 
200     if (v->hdr.prefix) {ierr = PetscStrncpy(prefix, v->hdr.prefix,sizeof(prefix));CHKERRQ(ierr);}
201     else               {prefix[0] = '\0';}
202     if (Nf > 1) {
203       ierr = DMCreateSubDM(dm, 1, &f, &fis, &fdm);CHKERRQ(ierr);
204       ierr = VecGetSubVector(v, fis, &fv);CHKERRQ(ierr);
205       ierr = PetscStrlcat(prefix, fname,sizeof(prefix));CHKERRQ(ierr);
206       ierr = PetscStrlcat(prefix, "_",sizeof(prefix));CHKERRQ(ierr);
207     }
208     for (comp = 0; comp < Nc; ++comp, ++w) {
209       PetscInt nmax = 2;
210 
211       ierr = PetscViewerDrawGetDraw(viewer, w, &draw);CHKERRQ(ierr);
212       if (Nc > 1) {ierr = PetscSNPrintf(title, sizeof(title), "%s:%s_%D Step: %D Time: %.4g", name, fname, comp, step, time);CHKERRQ(ierr);}
213       else        {ierr = PetscSNPrintf(title, sizeof(title), "%s:%s Step: %D Time: %.4g", name, fname, step, time);CHKERRQ(ierr);}
214       ierr = PetscDrawSetTitle(draw, title);CHKERRQ(ierr);
215 
216       /* TODO Get max and min only for this component */
217       ierr = PetscOptionsGetRealArray(NULL, prefix, "-vec_view_bounds", vbound, &nmax, &flg);CHKERRQ(ierr);
218       if (!flg) {
219         ierr = VecMin(fv, NULL, &vbound[0]);CHKERRQ(ierr);
220         ierr = VecMax(fv, NULL, &vbound[1]);CHKERRQ(ierr);
221         if (vbound[1] <= vbound[0]) vbound[1] = vbound[0] + 1.0;
222       }
223       ierr = PetscDrawGetPopup(draw, &popup);CHKERRQ(ierr);
224       ierr = PetscDrawScalePopup(popup, vbound[0], vbound[1]);CHKERRQ(ierr);
225       ierr = PetscDrawSetCoordinates(draw, bound[0], bound[1], bound[2], bound[3]);CHKERRQ(ierr);
226 
227       ierr = VecGetArrayRead(fv, &array);CHKERRQ(ierr);
228       for (c = cStart; c < cEnd; ++c) {
229         PetscScalar *coords = NULL, *a = NULL;
230         PetscInt     numCoords, color[4] = {-1,-1,-1,-1};
231 
232         ierr = DMPlexPointLocalRead(fdm, c, array, &a);CHKERRQ(ierr);
233         if (a) {
234           color[0] = PetscDrawRealToColor(PetscRealPart(a[comp]), vbound[0], vbound[1]);
235           color[1] = color[2] = color[3] = color[0];
236         } else {
237           PetscScalar *vals = NULL;
238           PetscInt     numVals, va;
239 
240           ierr = DMPlexVecGetClosure(fdm, NULL, fv, c, &numVals, &vals);CHKERRQ(ierr);
241           if (numVals % Nc) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "The number of components %D does not divide the number of values in the closure %D", Nc, numVals);
242           switch (numVals/Nc) {
243           case 3: /* P1 Triangle */
244           case 4: /* P1 Quadrangle */
245             for (va = 0; va < numVals/Nc; ++va) color[va] = PetscDrawRealToColor(PetscRealPart(vals[va*Nc+comp]), vbound[0], vbound[1]);
246             break;
247           case 6: /* P2 Triangle */
248           case 8: /* P2 Quadrangle */
249             for (va = 0; va < numVals/(Nc*2); ++va) color[va] = PetscDrawRealToColor(PetscRealPart(vals[va*Nc+comp + numVals/(Nc*2)]), vbound[0], vbound[1]);
250             break;
251           default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of values for cell closure %D cannot be handled", numVals/Nc);
252           }
253           ierr = DMPlexVecRestoreClosure(fdm, NULL, fv, c, &numVals, &vals);CHKERRQ(ierr);
254         }
255         ierr = DMPlexVecGetClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
256         switch (numCoords) {
257         case 6:
258         case 12: /* Localized triangle */
259           ierr = PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), color[0], color[1], color[2]);CHKERRQ(ierr);
260           break;
261         case 8:
262         case 16: /* Localized quadrilateral */
263           ierr = PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), color[0], color[1], color[2]);CHKERRQ(ierr);
264           ierr = PetscDrawTriangle(draw, PetscRealPart(coords[4]), PetscRealPart(coords[5]), PetscRealPart(coords[6]), PetscRealPart(coords[7]), PetscRealPart(coords[0]), PetscRealPart(coords[1]), color[2], color[3], color[0]);CHKERRQ(ierr);
265           break;
266         default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_SUP, "Cannot draw cells with %D coordinates", numCoords);
267         }
268         ierr = DMPlexVecRestoreClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
269       }
270       ierr = VecRestoreArrayRead(fv, &array);CHKERRQ(ierr);
271       ierr = PetscDrawFlush(draw);CHKERRQ(ierr);
272       ierr = PetscDrawPause(draw);CHKERRQ(ierr);
273       ierr = PetscDrawSave(draw);CHKERRQ(ierr);
274     }
275     if (Nf > 1) {
276       ierr = VecRestoreSubVector(v, fis, &fv);CHKERRQ(ierr);
277       ierr = ISDestroy(&fis);CHKERRQ(ierr);
278       ierr = DMDestroy(&fdm);CHKERRQ(ierr);
279     }
280   }
281   PetscFunctionReturn(0);
282 }
283 
284 static PetscErrorCode VecView_Plex_Local_VTK(Vec v, PetscViewer viewer)
285 {
286   DM                      dm;
287   Vec                     locv;
288   const char              *name;
289   PetscSection            section;
290   PetscInt                pStart, pEnd;
291   PetscInt                numFields;
292   PetscViewerVTKFieldType ft;
293   PetscErrorCode          ierr;
294 
295   PetscFunctionBegin;
296   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
297   ierr = DMCreateLocalVector(dm, &locv);CHKERRQ(ierr); /* VTK viewer requires exclusive ownership of the vector */
298   ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
299   ierr = PetscObjectSetName((PetscObject) locv, name);CHKERRQ(ierr);
300   ierr = VecCopy(v, locv);CHKERRQ(ierr);
301   ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);
302   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
303   if (!numFields) {
304     ierr = DMPlexGetFieldType_Internal(dm, section, PETSC_DETERMINE, &pStart, &pEnd, &ft);CHKERRQ(ierr);
305     ierr = PetscViewerVTKAddField(viewer, (PetscObject) dm, DMPlexVTKWriteAll, PETSC_DEFAULT, ft, PETSC_TRUE,(PetscObject) locv);CHKERRQ(ierr);
306   } else {
307     PetscInt f;
308 
309     for (f = 0; f < numFields; f++) {
310       ierr = DMPlexGetFieldType_Internal(dm, section, f, &pStart, &pEnd, &ft);CHKERRQ(ierr);
311       if (ft == PETSC_VTK_INVALID) continue;
312       ierr = PetscObjectReference((PetscObject)locv);CHKERRQ(ierr);
313       ierr = PetscViewerVTKAddField(viewer, (PetscObject) dm, DMPlexVTKWriteAll, f, ft, PETSC_TRUE,(PetscObject) locv);CHKERRQ(ierr);
314     }
315     ierr = VecDestroy(&locv);CHKERRQ(ierr);
316   }
317   PetscFunctionReturn(0);
318 }
319 
320 PetscErrorCode VecView_Plex_Local(Vec v, PetscViewer viewer)
321 {
322   DM             dm;
323   PetscBool      isvtk, ishdf5, isdraw, isglvis;
324   PetscErrorCode ierr;
325 
326   PetscFunctionBegin;
327   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
328   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
329   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,   &isvtk);CHKERRQ(ierr);
330   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,  &ishdf5);CHKERRQ(ierr);
331   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERDRAW,  &isdraw);CHKERRQ(ierr);
332   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERGLVIS, &isglvis);CHKERRQ(ierr);
333   if (isvtk || ishdf5 || isdraw || isglvis) {
334     PetscInt    i,numFields;
335     PetscObject fe;
336     PetscBool   fem = PETSC_FALSE;
337     Vec         locv = v;
338     const char  *name;
339     PetscInt    step;
340     PetscReal   time;
341 
342     ierr = DMGetNumFields(dm, &numFields);CHKERRQ(ierr);
343     for (i=0; i<numFields; i++) {
344       ierr = DMGetField(dm, i, NULL, &fe);CHKERRQ(ierr);
345       if (fe->classid == PETSCFE_CLASSID) { fem = PETSC_TRUE; break; }
346     }
347     if (fem) {
348       PetscObject isZero;
349 
350       ierr = DMGetLocalVector(dm, &locv);CHKERRQ(ierr);
351       ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
352       ierr = PetscObjectSetName((PetscObject) locv, name);CHKERRQ(ierr);
353       ierr = PetscObjectQuery((PetscObject) v, "__Vec_bc_zero__", &isZero);CHKERRQ(ierr);
354       ierr = PetscObjectCompose((PetscObject) locv, "__Vec_bc_zero__", isZero);CHKERRQ(ierr);
355       ierr = VecCopy(v, locv);CHKERRQ(ierr);
356       ierr = DMGetOutputSequenceNumber(dm, NULL, &time);CHKERRQ(ierr);
357       ierr = DMPlexInsertBoundaryValues(dm, PETSC_TRUE, locv, time, NULL, NULL, NULL);CHKERRQ(ierr);
358     }
359     if (isvtk) {
360       ierr = VecView_Plex_Local_VTK(locv, viewer);CHKERRQ(ierr);
361     } else if (ishdf5) {
362 #if defined(PETSC_HAVE_HDF5)
363       ierr = VecView_Plex_Local_HDF5_Internal(locv, viewer);CHKERRQ(ierr);
364 #else
365       SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
366 #endif
367     } else if (isdraw) {
368       ierr = VecView_Plex_Local_Draw(locv, viewer);CHKERRQ(ierr);
369     } else if (isglvis) {
370       ierr = DMGetOutputSequenceNumber(dm, &step, NULL);CHKERRQ(ierr);
371       ierr = PetscViewerGLVisSetSnapId(viewer, step);CHKERRQ(ierr);
372       ierr = VecView_GLVis(locv, viewer);CHKERRQ(ierr);
373     }
374     if (fem) {
375       ierr = PetscObjectCompose((PetscObject) locv, "__Vec_bc_zero__", NULL);CHKERRQ(ierr);
376       ierr = DMRestoreLocalVector(dm, &locv);CHKERRQ(ierr);
377     }
378   } else {
379     PetscBool isseq;
380 
381     ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
382     if (isseq) {ierr = VecView_Seq(v, viewer);CHKERRQ(ierr);}
383     else       {ierr = VecView_MPI(v, viewer);CHKERRQ(ierr);}
384   }
385   PetscFunctionReturn(0);
386 }
387 
388 PetscErrorCode VecView_Plex(Vec v, PetscViewer viewer)
389 {
390   DM             dm;
391   PetscBool      isvtk, ishdf5, isdraw, isglvis, isexodusii;
392   PetscErrorCode ierr;
393 
394   PetscFunctionBegin;
395   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
396   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
397   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,      &isvtk);CHKERRQ(ierr);
398   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,     &ishdf5);CHKERRQ(ierr);
399   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERDRAW,     &isdraw);CHKERRQ(ierr);
400   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERGLVIS,    &isglvis);CHKERRQ(ierr);
401   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWEREXODUSII, &isexodusii);CHKERRQ(ierr);
402   if (isvtk || isdraw || isglvis) {
403     Vec         locv;
404     PetscObject isZero;
405     const char *name;
406 
407     ierr = DMGetLocalVector(dm, &locv);CHKERRQ(ierr);
408     ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
409     ierr = PetscObjectSetName((PetscObject) locv, name);CHKERRQ(ierr);
410     ierr = DMGlobalToLocalBegin(dm, v, INSERT_VALUES, locv);CHKERRQ(ierr);
411     ierr = DMGlobalToLocalEnd(dm, v, INSERT_VALUES, locv);CHKERRQ(ierr);
412     ierr = PetscObjectQuery((PetscObject) v, "__Vec_bc_zero__", &isZero);CHKERRQ(ierr);
413     ierr = PetscObjectCompose((PetscObject) locv, "__Vec_bc_zero__", isZero);CHKERRQ(ierr);
414     ierr = VecView_Plex_Local(locv, viewer);CHKERRQ(ierr);
415     ierr = PetscObjectCompose((PetscObject) locv, "__Vec_bc_zero__", NULL);CHKERRQ(ierr);
416     ierr = DMRestoreLocalVector(dm, &locv);CHKERRQ(ierr);
417   } else if (ishdf5) {
418 #if defined(PETSC_HAVE_HDF5)
419     ierr = VecView_Plex_HDF5_Internal(v, viewer);CHKERRQ(ierr);
420 #else
421     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
422 #endif
423   } else if (isexodusii) {
424 #if defined(PETSC_HAVE_EXODUSII)
425     ierr = VecView_PlexExodusII_Internal(v, viewer);CHKERRQ(ierr);
426 #else
427     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "ExodusII not supported in this build.\nPlease reconfigure using --download-exodusii");
428 #endif
429   } else {
430     PetscBool isseq;
431 
432     ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
433     if (isseq) {ierr = VecView_Seq(v, viewer);CHKERRQ(ierr);}
434     else       {ierr = VecView_MPI(v, viewer);CHKERRQ(ierr);}
435   }
436   PetscFunctionReturn(0);
437 }
438 
439 PetscErrorCode VecView_Plex_Native(Vec originalv, PetscViewer viewer)
440 {
441   DM                dm;
442   MPI_Comm          comm;
443   PetscViewerFormat format;
444   Vec               v;
445   PetscBool         isvtk, ishdf5;
446   PetscErrorCode    ierr;
447 
448   PetscFunctionBegin;
449   ierr = VecGetDM(originalv, &dm);CHKERRQ(ierr);
450   ierr = PetscObjectGetComm((PetscObject) originalv, &comm);CHKERRQ(ierr);
451   if (!dm) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
452   ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
453   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
454   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,  &isvtk);CHKERRQ(ierr);
455   if (format == PETSC_VIEWER_NATIVE) {
456     /* Natural ordering is the common case for DMDA, NATIVE means plain vector, for PLEX is the opposite */
457     /* this need a better fix */
458     if (dm->useNatural) {
459       if (dm->sfNatural) {
460         const char *vecname;
461         PetscInt    n, nroots;
462 
463         ierr = VecGetLocalSize(originalv, &n);CHKERRQ(ierr);
464         ierr = PetscSFGetGraph(dm->sfNatural, &nroots, NULL, NULL, NULL);CHKERRQ(ierr);
465         if (n == nroots) {
466           ierr = DMGetGlobalVector(dm, &v);CHKERRQ(ierr);
467           ierr = DMPlexGlobalToNaturalBegin(dm, originalv, v);CHKERRQ(ierr);
468           ierr = DMPlexGlobalToNaturalEnd(dm, originalv, v);CHKERRQ(ierr);
469           ierr = PetscObjectGetName((PetscObject) originalv, &vecname);CHKERRQ(ierr);
470           ierr = PetscObjectSetName((PetscObject) v, vecname);CHKERRQ(ierr);
471         } else SETERRQ(comm, PETSC_ERR_ARG_WRONG, "DM global to natural SF only handles global vectors");
472       } else SETERRQ(comm, PETSC_ERR_ARG_WRONGSTATE, "DM global to natural SF was not created");
473     } else v = originalv;
474   } else v = originalv;
475 
476   if (ishdf5) {
477 #if defined(PETSC_HAVE_HDF5)
478     ierr = VecView_Plex_HDF5_Native_Internal(v, viewer);CHKERRQ(ierr);
479 #else
480     SETERRQ(comm, PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
481 #endif
482   } else if (isvtk) {
483     SETERRQ(comm, PETSC_ERR_SUP, "VTK format does not support viewing in natural order. Please switch to HDF5.");
484   } else {
485     PetscBool isseq;
486 
487     ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
488     if (isseq) {ierr = VecView_Seq(v, viewer);CHKERRQ(ierr);}
489     else       {ierr = VecView_MPI(v, viewer);CHKERRQ(ierr);}
490   }
491   if (v != originalv) {ierr = DMRestoreGlobalVector(dm, &v);CHKERRQ(ierr);}
492   PetscFunctionReturn(0);
493 }
494 
495 PetscErrorCode VecLoad_Plex_Local(Vec v, PetscViewer viewer)
496 {
497   DM             dm;
498   PetscBool      ishdf5;
499   PetscErrorCode ierr;
500 
501   PetscFunctionBegin;
502   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
503   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
504   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
505   if (ishdf5) {
506     DM          dmBC;
507     Vec         gv;
508     const char *name;
509 
510     ierr = DMGetOutputDM(dm, &dmBC);CHKERRQ(ierr);
511     ierr = DMGetGlobalVector(dmBC, &gv);CHKERRQ(ierr);
512     ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
513     ierr = PetscObjectSetName((PetscObject) gv, name);CHKERRQ(ierr);
514     ierr = VecLoad_Default(gv, viewer);CHKERRQ(ierr);
515     ierr = DMGlobalToLocalBegin(dmBC, gv, INSERT_VALUES, v);CHKERRQ(ierr);
516     ierr = DMGlobalToLocalEnd(dmBC, gv, INSERT_VALUES, v);CHKERRQ(ierr);
517     ierr = DMRestoreGlobalVector(dmBC, &gv);CHKERRQ(ierr);
518   } else {
519     ierr = VecLoad_Default(v, viewer);CHKERRQ(ierr);
520   }
521   PetscFunctionReturn(0);
522 }
523 
524 PetscErrorCode VecLoad_Plex(Vec v, PetscViewer viewer)
525 {
526   DM             dm;
527   PetscBool      ishdf5,isexodusii;
528   PetscErrorCode ierr;
529 
530   PetscFunctionBegin;
531   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
532   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
533   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,     &ishdf5);CHKERRQ(ierr);
534   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWEREXODUSII, &isexodusii);CHKERRQ(ierr);
535   if (ishdf5) {
536 #if defined(PETSC_HAVE_HDF5)
537     ierr = VecLoad_Plex_HDF5_Internal(v, viewer);CHKERRQ(ierr);
538 #else
539     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
540 #endif
541   } else if (isexodusii) {
542 #if defined(PETSC_HAVE_EXODUSII)
543     ierr = VecLoad_PlexExodusII_Internal(v, viewer);CHKERRQ(ierr);
544 #else
545     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "ExodusII not supported in this build.\nPlease reconfigure using --download-exodusii");
546 #endif
547   } else {
548     ierr = VecLoad_Default(v, viewer);CHKERRQ(ierr);
549   }
550   PetscFunctionReturn(0);
551 }
552 
553 PetscErrorCode VecLoad_Plex_Native(Vec originalv, PetscViewer viewer)
554 {
555   DM                dm;
556   PetscViewerFormat format;
557   PetscBool         ishdf5;
558   PetscErrorCode    ierr;
559 
560   PetscFunctionBegin;
561   ierr = VecGetDM(originalv, &dm);CHKERRQ(ierr);
562   if (!dm) SETERRQ(PetscObjectComm((PetscObject) originalv), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
563   ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
564   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
565   if (format == PETSC_VIEWER_NATIVE) {
566     if (dm->useNatural) {
567       if (dm->sfNatural) {
568         if (ishdf5) {
569 #if defined(PETSC_HAVE_HDF5)
570           Vec         v;
571           const char *vecname;
572 
573           ierr = DMGetGlobalVector(dm, &v);CHKERRQ(ierr);
574           ierr = PetscObjectGetName((PetscObject) originalv, &vecname);CHKERRQ(ierr);
575           ierr = PetscObjectSetName((PetscObject) v, vecname);CHKERRQ(ierr);
576           ierr = VecLoad_Plex_HDF5_Native_Internal(v, viewer);CHKERRQ(ierr);
577           ierr = DMPlexNaturalToGlobalBegin(dm, v, originalv);CHKERRQ(ierr);
578           ierr = DMPlexNaturalToGlobalEnd(dm, v, originalv);CHKERRQ(ierr);
579           ierr = DMRestoreGlobalVector(dm, &v);CHKERRQ(ierr);
580 #else
581           SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
582 #endif
583         } else SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Reading in natural order is not supported for anything but HDF5.");
584       }
585     } else {
586       ierr = VecLoad_Default(originalv, viewer);CHKERRQ(ierr);
587     }
588   }
589   PetscFunctionReturn(0);
590 }
591 
592 PETSC_UNUSED static PetscErrorCode DMPlexView_Ascii_Geometry(DM dm, PetscViewer viewer)
593 {
594   PetscSection       coordSection;
595   Vec                coordinates;
596   DMLabel            depthLabel, celltypeLabel;
597   const char        *name[4];
598   const PetscScalar *a;
599   PetscInt           dim, pStart, pEnd, cStart, cEnd, c;
600   PetscErrorCode     ierr;
601 
602   PetscFunctionBegin;
603   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
604   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
605   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
606   ierr = DMPlexGetDepthLabel(dm, &depthLabel);CHKERRQ(ierr);
607   ierr = DMPlexGetCellTypeLabel(dm, &celltypeLabel);CHKERRQ(ierr);
608   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
609   ierr = PetscSectionGetChart(coordSection, &pStart, &pEnd);CHKERRQ(ierr);
610   ierr = VecGetArrayRead(coordinates, &a);CHKERRQ(ierr);
611   name[0]     = "vertex";
612   name[1]     = "edge";
613   name[dim-1] = "face";
614   name[dim]   = "cell";
615   for (c = cStart; c < cEnd; ++c) {
616     PetscInt *closure = NULL;
617     PetscInt  closureSize, cl, ct;
618 
619     ierr = DMLabelGetValue(celltypeLabel, c, &ct);CHKERRQ(ierr);
620     ierr = PetscViewerASCIIPrintf(viewer, "Geometry for cell %D polytope type %s:\n", c, DMPolytopeTypes[ct]);CHKERRQ(ierr);
621     ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
622     ierr = PetscViewerASCIIPushTab(viewer);CHKERRQ(ierr);
623     for (cl = 0; cl < closureSize*2; cl += 2) {
624       PetscInt point = closure[cl], depth, dof, off, d, p;
625 
626       if ((point < pStart) || (point >= pEnd)) continue;
627       ierr = PetscSectionGetDof(coordSection, point, &dof);CHKERRQ(ierr);
628       if (!dof) continue;
629       ierr = DMLabelGetValue(depthLabel, point, &depth);CHKERRQ(ierr);
630       ierr = PetscSectionGetOffset(coordSection, point, &off);CHKERRQ(ierr);
631       ierr = PetscViewerASCIIPrintf(viewer, "%s %D coords:", name[depth], point);CHKERRQ(ierr);
632       for (p = 0; p < dof/dim; ++p) {
633         ierr = PetscViewerASCIIPrintf(viewer, " (");CHKERRQ(ierr);
634         for (d = 0; d < dim; ++d) {
635           if (d > 0) {ierr = PetscViewerASCIIPrintf(viewer, ", ");CHKERRQ(ierr);}
636           ierr = PetscViewerASCIIPrintf(viewer, "%g", (double) PetscRealPart(a[off+p*dim+d]));CHKERRQ(ierr);
637         }
638         ierr = PetscViewerASCIIPrintf(viewer, ")");CHKERRQ(ierr);
639       }
640       ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
641     }
642     ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
643     ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr);
644   }
645   ierr = VecRestoreArrayRead(coordinates, &a);CHKERRQ(ierr);
646   PetscFunctionReturn(0);
647 }
648 
649 typedef enum {CS_CARTESIAN, CS_POLAR, CS_CYLINDRICAL, CS_SPHERICAL} CoordSystem;
650 const char *CoordSystems[] = {"cartesian", "polar", "cylindrical", "spherical", "CoordSystem", "CS_", NULL};
651 
652 static PetscErrorCode DMPlexView_Ascii_Coordinates(PetscViewer viewer, CoordSystem cs, PetscInt dim, const PetscScalar x[])
653 {
654   PetscInt       i;
655   PetscErrorCode ierr;
656 
657   PetscFunctionBegin;
658   if (dim > 3) {
659     for (i = 0; i < dim; ++i) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, " %g", (double) PetscRealPart(x[i]));CHKERRQ(ierr);}
660   } else {
661     PetscReal coords[3], trcoords[3];
662 
663     for (i = 0; i < dim; ++i) coords[i] = PetscRealPart(x[i]);
664     switch (cs) {
665       case CS_CARTESIAN: for (i = 0; i < dim; ++i) trcoords[i] = coords[i];break;
666       case CS_POLAR:
667         if (dim != 2) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Polar coordinates are for 2 dimension, not %D", dim);
668         trcoords[0] = PetscSqrtReal(PetscSqr(coords[0]) + PetscSqr(coords[1]));
669         trcoords[1] = PetscAtan2Real(coords[1], coords[0]);
670         break;
671       case CS_CYLINDRICAL:
672         if (dim != 3) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cylindrical coordinates are for 3 dimension, not %D", dim);
673         trcoords[0] = PetscSqrtReal(PetscSqr(coords[0]) + PetscSqr(coords[1]));
674         trcoords[1] = PetscAtan2Real(coords[1], coords[0]);
675         trcoords[2] = coords[2];
676         break;
677       case CS_SPHERICAL:
678         if (dim != 3) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Spherical coordinates are for 3 dimension, not %D", dim);
679         trcoords[0] = PetscSqrtReal(PetscSqr(coords[0]) + PetscSqr(coords[1]) + PetscSqr(coords[2]));
680         trcoords[1] = PetscAtan2Real(PetscSqrtReal(PetscSqr(coords[0]) + PetscSqr(coords[1])), coords[2]);
681         trcoords[2] = PetscAtan2Real(coords[1], coords[0]);
682         break;
683     }
684     for (i = 0; i < dim; ++i) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, " %g", (double) trcoords[i]);CHKERRQ(ierr);}
685   }
686   PetscFunctionReturn(0);
687 }
688 
689 static PetscErrorCode DMPlexView_Ascii(DM dm, PetscViewer viewer)
690 {
691   DM_Plex          *mesh = (DM_Plex*) dm->data;
692   DM                cdm;
693   PetscSection      coordSection;
694   Vec               coordinates;
695   PetscViewerFormat format;
696   PetscErrorCode    ierr;
697 
698   PetscFunctionBegin;
699   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
700   ierr = DMGetLocalSection(cdm, &coordSection);CHKERRQ(ierr);
701   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
702   ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
703   if (format == PETSC_VIEWER_ASCII_INFO_DETAIL) {
704     const char *name;
705     PetscInt    dim, cellHeight, maxConeSize, maxSupportSize;
706     PetscInt    pStart, pEnd, p, numLabels, l;
707     PetscMPIInt rank, size;
708 
709     ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRMPI(ierr);
710     ierr = MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size);CHKERRMPI(ierr);
711     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
712     ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
713     ierr = DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize);CHKERRQ(ierr);
714     ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
715     ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
716     if (name) {ierr = PetscViewerASCIIPrintf(viewer, "%s in %D dimension%s:\n", name, dim, dim == 1 ? "" : "s");CHKERRQ(ierr);}
717     else      {ierr = PetscViewerASCIIPrintf(viewer, "Mesh in %D dimension%s:\n", dim, dim == 1 ? "" : "s");CHKERRQ(ierr);}
718     if (cellHeight) {ierr = PetscViewerASCIIPrintf(viewer, "  Cells are at height %D\n", cellHeight);CHKERRQ(ierr);}
719     ierr = PetscViewerASCIIPrintf(viewer, "Supports:\n", name);CHKERRQ(ierr);
720     ierr = PetscViewerASCIIPushSynchronized(viewer);CHKERRQ(ierr);
721     ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%d] Max support size: %D\n", rank, maxSupportSize);CHKERRQ(ierr);
722     for (p = pStart; p < pEnd; ++p) {
723       PetscInt dof, off, s;
724 
725       ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
726       ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
727       for (s = off; s < off+dof; ++s) {
728         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%d]: %D ----> %D\n", rank, p, mesh->supports[s]);CHKERRQ(ierr);
729       }
730     }
731     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
732     ierr = PetscViewerASCIIPrintf(viewer, "Cones:\n", name);CHKERRQ(ierr);
733     ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%d] Max cone size: %D\n", rank, maxConeSize);CHKERRQ(ierr);
734     for (p = pStart; p < pEnd; ++p) {
735       PetscInt dof, off, c;
736 
737       ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
738       ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
739       for (c = off; c < off+dof; ++c) {
740         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%d]: %D <---- %D (%D)\n", rank, p, mesh->cones[c], mesh->coneOrientations[c]);CHKERRQ(ierr);
741       }
742     }
743     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
744     ierr = PetscViewerASCIIPopSynchronized(viewer);CHKERRQ(ierr);
745     if (coordSection && coordinates) {
746       CoordSystem        cs = CS_CARTESIAN;
747       const PetscScalar *array;
748       PetscInt           Nf, Nc, pStart, pEnd, p;
749       PetscMPIInt        rank;
750       const char        *name;
751 
752       ierr = PetscOptionsGetEnum(((PetscObject) viewer)->options, ((PetscObject) viewer)->prefix, "-dm_plex_view_coord_system", CoordSystems, (PetscEnum *) &cs, NULL);CHKERRQ(ierr);
753       ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)viewer), &rank);CHKERRMPI(ierr);
754       ierr = PetscSectionGetNumFields(coordSection, &Nf);CHKERRQ(ierr);
755       if (Nf != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Coordinate section should have 1 field, not %D", Nf);
756       ierr = PetscSectionGetFieldComponents(coordSection, 0, &Nc);CHKERRQ(ierr);
757       ierr = PetscSectionGetChart(coordSection, &pStart, &pEnd);CHKERRQ(ierr);
758       ierr = PetscObjectGetName((PetscObject) coordinates, &name);CHKERRQ(ierr);
759       ierr = PetscViewerASCIIPrintf(viewer, "%s with %D fields\n", name, Nf);CHKERRQ(ierr);
760       ierr = PetscViewerASCIIPrintf(viewer, "  field 0 with %D components\n", Nc);CHKERRQ(ierr);
761       if (cs != CS_CARTESIAN) {ierr = PetscViewerASCIIPrintf(viewer, "  output coordinate system: %s\n", CoordSystems[cs]);CHKERRQ(ierr);}
762 
763       ierr = VecGetArrayRead(coordinates, &array);CHKERRQ(ierr);
764       ierr = PetscViewerASCIIPushSynchronized(viewer);CHKERRQ(ierr);
765       ierr = PetscViewerASCIISynchronizedPrintf(viewer, "Process %d:\n", rank);CHKERRQ(ierr);
766       for (p = pStart; p < pEnd; ++p) {
767         PetscInt dof, off;
768 
769         ierr = PetscSectionGetDof(coordSection, p, &dof);CHKERRQ(ierr);
770         ierr = PetscSectionGetOffset(coordSection, p, &off);CHKERRQ(ierr);
771         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "  (%4D) dim %2D offset %3D", p, dof, off);CHKERRQ(ierr);
772         ierr = DMPlexView_Ascii_Coordinates(viewer, cs, dof, &array[off]);CHKERRQ(ierr);
773         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\n");CHKERRQ(ierr);
774       }
775       ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
776       ierr = PetscViewerASCIIPopSynchronized(viewer);CHKERRQ(ierr);
777       ierr = VecRestoreArrayRead(coordinates, &array);CHKERRQ(ierr);
778     }
779     ierr = DMGetNumLabels(dm, &numLabels);CHKERRQ(ierr);
780     if (numLabels) {ierr = PetscViewerASCIIPrintf(viewer, "Labels:\n");CHKERRQ(ierr);}
781     for (l = 0; l < numLabels; ++l) {
782       DMLabel     label;
783       PetscBool   isdepth;
784       const char *name;
785 
786       ierr = DMGetLabelName(dm, l, &name);CHKERRQ(ierr);
787       ierr = PetscStrcmp(name, "depth", &isdepth);CHKERRQ(ierr);
788       if (isdepth) continue;
789       ierr = DMGetLabel(dm, name, &label);CHKERRQ(ierr);
790       ierr = DMLabelView(label, viewer);CHKERRQ(ierr);
791     }
792     if (size > 1) {
793       PetscSF sf;
794 
795       ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr);
796       ierr = PetscSFView(sf, viewer);CHKERRQ(ierr);
797     }
798     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
799   } else if (format == PETSC_VIEWER_ASCII_LATEX) {
800     const char  *name, *color;
801     const char  *defcolors[3]  = {"gray", "orange", "green"};
802     const char  *deflcolors[4] = {"blue", "cyan", "red", "magenta"};
803     char         lname[PETSC_MAX_PATH_LEN];
804     PetscReal    scale         = 2.0;
805     PetscReal    tikzscale     = 1.0;
806     PetscBool    useNumbers    = PETSC_TRUE, drawNumbers[4], drawColors[4], useLabels, useColors, plotEdges, drawHasse = PETSC_FALSE;
807     double       tcoords[3];
808     PetscScalar *coords;
809     PetscInt     numLabels, l, numColors, numLColors, dim, d, depth, cStart, cEnd, c, vStart, vEnd, v, eStart = 0, eEnd = 0, e, p, n;
810     PetscMPIInt  rank, size;
811     char         **names, **colors, **lcolors;
812     PetscBool    flg, lflg;
813     PetscBT      wp = NULL;
814     PetscInt     pEnd, pStart;
815 
816     ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
817     ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
818     ierr = DMGetNumLabels(dm, &numLabels);CHKERRQ(ierr);
819     numLabels  = PetscMax(numLabels, 10);
820     numColors  = 10;
821     numLColors = 10;
822     ierr = PetscCalloc3(numLabels, &names, numColors, &colors, numLColors, &lcolors);CHKERRQ(ierr);
823     ierr = PetscOptionsGetReal(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_scale", &scale, NULL);CHKERRQ(ierr);
824     ierr = PetscOptionsGetReal(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_tikzscale", &tikzscale, NULL);CHKERRQ(ierr);
825     ierr = PetscOptionsGetBool(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_numbers", &useNumbers, NULL);CHKERRQ(ierr);
826     for (d = 0; d < 4; ++d) drawNumbers[d] = useNumbers;
827     for (d = 0; d < 4; ++d) drawColors[d]  = PETSC_TRUE;
828     n = 4;
829     ierr = PetscOptionsGetBoolArray(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_numbers_depth", drawNumbers, &n, &flg);CHKERRQ(ierr);
830     if (flg && n != dim+1) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_SIZ, "Number of flags %D != %D dim+1", n, dim+1);
831     ierr = PetscOptionsGetBoolArray(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_colors_depth", drawColors, &n, &flg);CHKERRQ(ierr);
832     if (flg && n != dim+1) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_SIZ, "Number of flags %D != %D dim+1", n, dim+1);
833     ierr = PetscOptionsGetStringArray(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_labels", names, &numLabels, &useLabels);CHKERRQ(ierr);
834     if (!useLabels) numLabels = 0;
835     ierr = PetscOptionsGetStringArray(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_colors", colors, &numColors, &useColors);CHKERRQ(ierr);
836     if (!useColors) {
837       numColors = 3;
838       for (c = 0; c < numColors; ++c) {ierr = PetscStrallocpy(defcolors[c], &colors[c]);CHKERRQ(ierr);}
839     }
840     ierr = PetscOptionsGetStringArray(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_lcolors", lcolors, &numLColors, &useColors);CHKERRQ(ierr);
841     if (!useColors) {
842       numLColors = 4;
843       for (c = 0; c < numLColors; ++c) {ierr = PetscStrallocpy(deflcolors[c], &lcolors[c]);CHKERRQ(ierr);}
844     }
845     ierr = PetscOptionsGetString(((PetscObject) viewer)->options, ((PetscObject) viewer)->prefix, "-dm_plex_view_label_filter", lname, sizeof(lname), &lflg);CHKERRQ(ierr);
846     plotEdges = (PetscBool)(depth > 1 && drawNumbers[1] && dim < 3);
847     ierr = PetscOptionsGetBool(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_edges", &plotEdges, &flg);CHKERRQ(ierr);
848     if (flg && plotEdges && depth < dim) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Mesh must be interpolated");
849     if (depth < dim) plotEdges = PETSC_FALSE;
850     ierr = PetscOptionsGetBool(((PetscObject) viewer)->options, ((PetscObject) viewer)->prefix, "-dm_plex_view_hasse", &drawHasse, NULL);CHKERRQ(ierr);
851 
852     /* filter points with labelvalue != labeldefaultvalue */
853     ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
854     ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
855     ierr = DMPlexGetDepthStratum(dm, 1, &eStart, &eEnd);CHKERRQ(ierr);
856     ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
857     if (lflg) {
858       DMLabel lbl;
859 
860       ierr = DMGetLabel(dm, lname, &lbl);CHKERRQ(ierr);
861       if (lbl) {
862         PetscInt val, defval;
863 
864         ierr = DMLabelGetDefaultValue(lbl, &defval);CHKERRQ(ierr);
865         ierr = PetscBTCreate(pEnd-pStart, &wp);CHKERRQ(ierr);
866         for (c = pStart;  c < pEnd; c++) {
867           PetscInt *closure = NULL;
868           PetscInt  closureSize;
869 
870           ierr = DMLabelGetValue(lbl, c, &val);CHKERRQ(ierr);
871           if (val == defval) continue;
872 
873           ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
874           for (p = 0; p < closureSize*2; p += 2) {
875             ierr = PetscBTSet(wp, closure[p] - pStart);CHKERRQ(ierr);
876           }
877           ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
878         }
879       }
880     }
881 
882     ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRMPI(ierr);
883     ierr = MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size);CHKERRMPI(ierr);
884     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
885     ierr = PetscViewerASCIIPrintf(viewer, "\
886 \\documentclass[tikz]{standalone}\n\n\
887 \\usepackage{pgflibraryshapes}\n\
888 \\usetikzlibrary{backgrounds}\n\
889 \\usetikzlibrary{arrows}\n\
890 \\begin{document}\n");CHKERRQ(ierr);
891     if (size > 1) {
892       ierr = PetscViewerASCIIPrintf(viewer, "%s for process ", name);CHKERRQ(ierr);
893       for (p = 0; p < size; ++p) {
894         if (p > 0 && p == size-1) {
895           ierr = PetscViewerASCIIPrintf(viewer, ", and ", colors[p%numColors], p);CHKERRQ(ierr);
896         } else if (p > 0) {
897           ierr = PetscViewerASCIIPrintf(viewer, ", ", colors[p%numColors], p);CHKERRQ(ierr);
898         }
899         ierr = PetscViewerASCIIPrintf(viewer, "{\\textcolor{%s}%D}", colors[p%numColors], p);CHKERRQ(ierr);
900       }
901       ierr = PetscViewerASCIIPrintf(viewer, ".\n\n\n");CHKERRQ(ierr);
902     }
903     if (drawHasse) {
904       PetscInt maxStratum = PetscMax(vEnd-vStart, PetscMax(eEnd-eStart, cEnd-cStart));
905 
906       ierr = PetscViewerASCIIPrintf(viewer, "\\newcommand{\\vStart}{%D}\n", vStart);CHKERRQ(ierr);
907       ierr = PetscViewerASCIIPrintf(viewer, "\\newcommand{\\vEnd}{%D}\n", vEnd-1);CHKERRQ(ierr);
908       ierr = PetscViewerASCIIPrintf(viewer, "\\newcommand{\\numVertices}{%D}\n", vEnd-vStart);CHKERRQ(ierr);
909       ierr = PetscViewerASCIIPrintf(viewer, "\\newcommand{\\vShift}{%.2f}\n", 3 + (maxStratum-(vEnd-vStart))/2.);CHKERRQ(ierr);
910       ierr = PetscViewerASCIIPrintf(viewer, "\\newcommand{\\eStart}{%D}\n", eStart);CHKERRQ(ierr);
911       ierr = PetscViewerASCIIPrintf(viewer, "\\newcommand{\\eEnd}{%D}\n", eEnd-1);CHKERRQ(ierr);
912       ierr = PetscViewerASCIIPrintf(viewer, "\\newcommand{\\eShift}{%.2f}\n", 3 + (maxStratum-(eEnd-eStart))/2.);CHKERRQ(ierr);
913       ierr = PetscViewerASCIIPrintf(viewer, "\\newcommand{\\numEdges}{%D}\n", eEnd-eStart);CHKERRQ(ierr);
914       ierr = PetscViewerASCIIPrintf(viewer, "\\newcommand{\\cStart}{%D}\n", cStart);CHKERRQ(ierr);
915       ierr = PetscViewerASCIIPrintf(viewer, "\\newcommand{\\cEnd}{%D}\n", cEnd-1);CHKERRQ(ierr);
916       ierr = PetscViewerASCIIPrintf(viewer, "\\newcommand{\\numCells}{%D}\n", cEnd-cStart);CHKERRQ(ierr);
917       ierr = PetscViewerASCIIPrintf(viewer, "\\newcommand{\\cShift}{%.2f}\n", 3 + (maxStratum-(cEnd-cStart))/2.);CHKERRQ(ierr);
918     }
919     ierr = PetscViewerASCIIPrintf(viewer, "\\begin{tikzpicture}[scale = %g,font=\\fontsize{8}{8}\\selectfont]\n", (double) tikzscale);CHKERRQ(ierr);
920 
921     /* Plot vertices */
922     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
923     ierr = PetscViewerASCIIPushSynchronized(viewer);CHKERRQ(ierr);
924     for (v = vStart; v < vEnd; ++v) {
925       PetscInt  off, dof, d;
926       PetscBool isLabeled = PETSC_FALSE;
927 
928       if (wp && !PetscBTLookup(wp,v - pStart)) continue;
929       ierr = PetscSectionGetDof(coordSection, v, &dof);CHKERRQ(ierr);
930       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
931       ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\path (");CHKERRQ(ierr);
932       if (PetscUnlikely(dof > 3)) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"coordSection vertex %D has dof %D > 3",v,dof);
933       for (d = 0; d < dof; ++d) {
934         tcoords[d] = (double) (scale*PetscRealPart(coords[off+d]));
935         tcoords[d] = PetscAbs(tcoords[d]) < 1e-10 ? 0.0 : tcoords[d];
936       }
937       /* Rotate coordinates since PGF makes z point out of the page instead of up */
938       if (dim == 3) {PetscReal tmp = tcoords[1]; tcoords[1] = tcoords[2]; tcoords[2] = -tmp;}
939       for (d = 0; d < dof; ++d) {
940         if (d > 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, ",");CHKERRQ(ierr);}
941         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "%g", (double) tcoords[d]);CHKERRQ(ierr);
942       }
943       if (drawHasse) color = colors[0%numColors];
944       else           color = colors[rank%numColors];
945       for (l = 0; l < numLabels; ++l) {
946         PetscInt val;
947         ierr = DMGetLabelValue(dm, names[l], v, &val);CHKERRQ(ierr);
948         if (val >= 0) {color = lcolors[l%numLColors]; isLabeled = PETSC_TRUE; break;}
949       }
950       if (drawNumbers[0]) {
951         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [draw,shape=circle,color=%s] {%D};\n", v, rank, color, v);CHKERRQ(ierr);
952       } else if (drawColors[0]) {
953         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [fill,inner sep=%dpt,shape=circle,color=%s] {};\n", v, rank, !isLabeled ? 1 : 2, color);CHKERRQ(ierr);
954       } else {
955         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [] {};\n", v, rank);CHKERRQ(ierr);
956       }
957     }
958     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
959     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
960     /* Plot edges */
961     if (plotEdges) {
962       ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
963       ierr = PetscViewerASCIIPrintf(viewer, "\\path\n");CHKERRQ(ierr);
964       for (e = eStart; e < eEnd; ++e) {
965         const PetscInt *cone;
966         PetscInt        coneSize, offA, offB, dof, d;
967 
968         if (wp && !PetscBTLookup(wp,e - pStart)) continue;
969         ierr = DMPlexGetConeSize(dm, e, &coneSize);CHKERRQ(ierr);
970         if (coneSize != 2) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Edge %D cone should have two vertices, not %D", e, coneSize);
971         ierr = DMPlexGetCone(dm, e, &cone);CHKERRQ(ierr);
972         ierr = PetscSectionGetDof(coordSection, cone[0], &dof);CHKERRQ(ierr);
973         ierr = PetscSectionGetOffset(coordSection, cone[0], &offA);CHKERRQ(ierr);
974         ierr = PetscSectionGetOffset(coordSection, cone[1], &offB);CHKERRQ(ierr);
975         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "(");CHKERRQ(ierr);
976         for (d = 0; d < dof; ++d) {
977           tcoords[d] = (double) (0.5*scale*PetscRealPart(coords[offA+d]+coords[offB+d]));
978           tcoords[d] = PetscAbs(tcoords[d]) < 1e-10 ? 0.0 : tcoords[d];
979         }
980         /* Rotate coordinates since PGF makes z point out of the page instead of up */
981         if (dim == 3) {PetscReal tmp = tcoords[1]; tcoords[1] = tcoords[2]; tcoords[2] = -tmp;}
982         for (d = 0; d < dof; ++d) {
983           if (d > 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, ",");CHKERRQ(ierr);}
984           ierr = PetscViewerASCIISynchronizedPrintf(viewer, "%g", (double)tcoords[d]);CHKERRQ(ierr);
985         }
986         if (drawHasse) color = colors[1%numColors];
987         else           color = colors[rank%numColors];
988         for (l = 0; l < numLabels; ++l) {
989           PetscInt val;
990           ierr = DMGetLabelValue(dm, names[l], v, &val);CHKERRQ(ierr);
991           if (val >= 0) {color = lcolors[l%numLColors]; break;}
992         }
993         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [draw,shape=circle,color=%s] {%D} --\n", e, rank, color, e);CHKERRQ(ierr);
994       }
995       ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
996       ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
997       ierr = PetscViewerASCIIPrintf(viewer, "(0,0);\n");CHKERRQ(ierr);
998     }
999     /* Plot cells */
1000     if (dim == 3 || !drawNumbers[1]) {
1001       for (e = eStart; e < eEnd; ++e) {
1002         const PetscInt *cone;
1003 
1004         if (wp && !PetscBTLookup(wp,e - pStart)) continue;
1005         color = colors[rank%numColors];
1006         for (l = 0; l < numLabels; ++l) {
1007           PetscInt val;
1008           ierr = DMGetLabelValue(dm, names[l], e, &val);CHKERRQ(ierr);
1009           if (val >= 0) {color = lcolors[l%numLColors]; break;}
1010         }
1011         ierr = DMPlexGetCone(dm, e, &cone);CHKERRQ(ierr);
1012         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\draw[color=%s] (%D_%d) -- (%D_%d);\n", color, cone[0], rank, cone[1], rank);CHKERRQ(ierr);
1013       }
1014     } else {
1015        DMPolytopeType ct;
1016 
1017       /* Drawing a 2D polygon */
1018       for (c = cStart; c < cEnd; ++c) {
1019         if (wp && !PetscBTLookup(wp, c - pStart)) continue;
1020         ierr = DMPlexGetCellType(dm, c, &ct);CHKERRQ(ierr);
1021         if (ct == DM_POLYTOPE_SEG_PRISM_TENSOR ||
1022             ct == DM_POLYTOPE_TRI_PRISM_TENSOR ||
1023             ct == DM_POLYTOPE_QUAD_PRISM_TENSOR) {
1024           const PetscInt *cone;
1025           PetscInt        coneSize, e;
1026 
1027           ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
1028           ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
1029           for (e = 0; e < coneSize; ++e) {
1030             const PetscInt *econe;
1031 
1032             ierr = DMPlexGetCone(dm, cone[e], &econe);CHKERRQ(ierr);
1033             ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\draw[color=%s] (%D_%d) -- (%D_%d) -- (%D_%d);\n", colors[rank%numColors], econe[0], rank, cone[e], rank, econe[1], rank);CHKERRQ(ierr);
1034           }
1035         } else {
1036           PetscInt *closure = NULL;
1037           PetscInt  closureSize, Nv = 0, v;
1038 
1039           ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1040           for (p = 0; p < closureSize*2; p += 2) {
1041             const PetscInt point = closure[p];
1042 
1043             if ((point >= vStart) && (point < vEnd)) closure[Nv++] = point;
1044           }
1045           ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\draw[color=%s] ", colors[rank%numColors]);CHKERRQ(ierr);
1046           for (v = 0; v <= Nv; ++v) {
1047             const PetscInt vertex = closure[v%Nv];
1048 
1049             if (v > 0) {
1050               if (plotEdges) {
1051                 const PetscInt *edge;
1052                 PetscInt        endpoints[2], ne;
1053 
1054                 endpoints[0] = closure[v-1]; endpoints[1] = vertex;
1055                 ierr = DMPlexGetJoin(dm, 2, endpoints, &ne, &edge);CHKERRQ(ierr);
1056                 if (ne != 1) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Could not find edge for vertices %D, %D", endpoints[0], endpoints[1]);
1057                 ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- (%D_%d) -- ", edge[0], rank);CHKERRQ(ierr);
1058                 ierr = DMPlexRestoreJoin(dm, 2, endpoints, &ne, &edge);CHKERRQ(ierr);
1059               } else {
1060                 ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- ");CHKERRQ(ierr);
1061               }
1062             }
1063             ierr = PetscViewerASCIISynchronizedPrintf(viewer, "(%D_%d)", vertex, rank);CHKERRQ(ierr);
1064           }
1065           ierr = PetscViewerASCIISynchronizedPrintf(viewer, ";\n");CHKERRQ(ierr);
1066           ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1067         }
1068       }
1069     }
1070     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
1071     for (c = cStart; c < cEnd; ++c) {
1072       double    ccoords[3] = {0.0, 0.0, 0.0};
1073       PetscBool isLabeled  = PETSC_FALSE;
1074       PetscInt *closure    = NULL;
1075       PetscInt  closureSize, dof, d, n = 0;
1076 
1077       if (wp && !PetscBTLookup(wp,c - pStart)) continue;
1078       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1079       ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\path (");CHKERRQ(ierr);
1080       for (p = 0; p < closureSize*2; p += 2) {
1081         const PetscInt point = closure[p];
1082         PetscInt       off;
1083 
1084         if ((point < vStart) || (point >= vEnd)) continue;
1085         ierr = PetscSectionGetDof(coordSection, point, &dof);CHKERRQ(ierr);
1086         ierr = PetscSectionGetOffset(coordSection, point, &off);CHKERRQ(ierr);
1087         for (d = 0; d < dof; ++d) {
1088           tcoords[d] = (double) (scale*PetscRealPart(coords[off+d]));
1089           tcoords[d] = PetscAbs(tcoords[d]) < 1e-10 ? 0.0 : tcoords[d];
1090         }
1091         /* Rotate coordinates since PGF makes z point out of the page instead of up */
1092         if (dof == 3) {PetscReal tmp = tcoords[1]; tcoords[1] = tcoords[2]; tcoords[2] = -tmp;}
1093         for (d = 0; d < dof; ++d) {ccoords[d] += tcoords[d];}
1094         ++n;
1095       }
1096       for (d = 0; d < dof; ++d) {ccoords[d] /= n;}
1097       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1098       for (d = 0; d < dof; ++d) {
1099         if (d > 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, ",");CHKERRQ(ierr);}
1100         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "%g", (double) ccoords[d]);CHKERRQ(ierr);
1101       }
1102       if (drawHasse) color = colors[depth%numColors];
1103       else           color = colors[rank%numColors];
1104       for (l = 0; l < numLabels; ++l) {
1105         PetscInt val;
1106         ierr = DMGetLabelValue(dm, names[l], c, &val);CHKERRQ(ierr);
1107         if (val >= 0) {color = lcolors[l%numLColors]; isLabeled = PETSC_TRUE; break;}
1108       }
1109       if (drawNumbers[dim]) {
1110         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [draw,shape=circle,color=%s] {%D};\n", c, rank, color, c);CHKERRQ(ierr);
1111       } else if (drawColors[dim]) {
1112         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [fill,inner sep=%dpt,shape=circle,color=%s] {};\n", c, rank, !isLabeled ? 1 : 2, color);CHKERRQ(ierr);
1113       } else {
1114         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [] {};\n", c, rank);CHKERRQ(ierr);
1115       }
1116     }
1117     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
1118     if (drawHasse) {
1119       color = colors[depth%numColors];
1120       ierr = PetscViewerASCIIPrintf(viewer, "%% Cells\n");CHKERRQ(ierr);
1121       ierr = PetscViewerASCIIPrintf(viewer, "\\foreach \\c in {\\cStart,...,\\cEnd}\n");CHKERRQ(ierr);
1122       ierr = PetscViewerASCIIPrintf(viewer, "{\n");CHKERRQ(ierr);
1123       ierr = PetscViewerASCIIPrintf(viewer, "  \\node(\\c_%d) [draw,shape=circle,color=%s,minimum size = 6mm] at (\\cShift+\\c-\\cStart,0) {\\c};\n", rank, color);CHKERRQ(ierr);
1124       ierr = PetscViewerASCIIPrintf(viewer, "}\n");CHKERRQ(ierr);
1125 
1126       color = colors[1%numColors];
1127       ierr = PetscViewerASCIIPrintf(viewer, "%% Edges\n");CHKERRQ(ierr);
1128       ierr = PetscViewerASCIIPrintf(viewer, "\\foreach \\e in {\\eStart,...,\\eEnd}\n");CHKERRQ(ierr);
1129       ierr = PetscViewerASCIIPrintf(viewer, "{\n");CHKERRQ(ierr);
1130       ierr = PetscViewerASCIIPrintf(viewer, "  \\node(\\e_%d) [draw,shape=circle,color=%s,minimum size = 6mm] at (\\eShift+\\e-\\eStart,1) {\\e};\n", rank, color);CHKERRQ(ierr);
1131       ierr = PetscViewerASCIIPrintf(viewer, "}\n");CHKERRQ(ierr);
1132 
1133       color = colors[0%numColors];
1134       ierr = PetscViewerASCIIPrintf(viewer, "%% Vertices\n");CHKERRQ(ierr);
1135       ierr = PetscViewerASCIIPrintf(viewer, "\\foreach \\v in {\\vStart,...,\\vEnd}\n");CHKERRQ(ierr);
1136       ierr = PetscViewerASCIIPrintf(viewer, "{\n");CHKERRQ(ierr);
1137       ierr = PetscViewerASCIIPrintf(viewer, "  \\node(\\v_%d) [draw,shape=circle,color=%s,minimum size = 6mm] at (\\vShift+\\v-\\vStart,2) {\\v};\n", rank, color);CHKERRQ(ierr);
1138       ierr = PetscViewerASCIIPrintf(viewer, "}\n");CHKERRQ(ierr);
1139 
1140       for (p = pStart; p < pEnd; ++p) {
1141         const PetscInt *cone;
1142         PetscInt        coneSize, cp;
1143 
1144         ierr = DMPlexGetCone(dm, p, &cone);CHKERRQ(ierr);
1145         ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
1146         for (cp = 0; cp < coneSize; ++cp) {
1147           ierr = PetscViewerASCIIPrintf(viewer, "\\draw[->, shorten >=1pt] (%D_%d) -- (%D_%d);\n", cone[cp], rank, p, rank);CHKERRQ(ierr);
1148         }
1149       }
1150     }
1151     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
1152     ierr = PetscViewerASCIIPopSynchronized(viewer);CHKERRQ(ierr);
1153     ierr = PetscViewerASCIIPrintf(viewer, "\\end{tikzpicture}\n");CHKERRQ(ierr);
1154     ierr = PetscViewerASCIIPrintf(viewer, "\\end{document}\n", name);CHKERRQ(ierr);
1155     for (l = 0; l < numLabels;  ++l) {ierr = PetscFree(names[l]);CHKERRQ(ierr);}
1156     for (c = 0; c < numColors;  ++c) {ierr = PetscFree(colors[c]);CHKERRQ(ierr);}
1157     for (c = 0; c < numLColors; ++c) {ierr = PetscFree(lcolors[c]);CHKERRQ(ierr);}
1158     ierr = PetscFree3(names, colors, lcolors);CHKERRQ(ierr);
1159     ierr = PetscBTDestroy(&wp);CHKERRQ(ierr);
1160   } else if (format == PETSC_VIEWER_LOAD_BALANCE) {
1161     Vec                    cown,acown;
1162     VecScatter             sct;
1163     ISLocalToGlobalMapping g2l;
1164     IS                     gid,acis;
1165     MPI_Comm               comm,ncomm = MPI_COMM_NULL;
1166     MPI_Group              ggroup,ngroup;
1167     PetscScalar            *array,nid;
1168     const PetscInt         *idxs;
1169     PetscInt               *idxs2,*start,*adjacency,*work;
1170     PetscInt64             lm[3],gm[3];
1171     PetscInt               i,c,cStart,cEnd,cum,numVertices,ect,ectn,cellHeight;
1172     PetscMPIInt            d1,d2,rank;
1173 
1174     ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
1175     ierr = MPI_Comm_rank(comm,&rank);CHKERRMPI(ierr);
1176 #if defined(PETSC_HAVE_MPI_PROCESS_SHARED_MEMORY)
1177     ierr = MPI_Comm_split_type(comm,MPI_COMM_TYPE_SHARED,rank,MPI_INFO_NULL,&ncomm);CHKERRMPI(ierr);
1178 #endif
1179     if (ncomm != MPI_COMM_NULL) {
1180       ierr = MPI_Comm_group(comm,&ggroup);CHKERRMPI(ierr);
1181       ierr = MPI_Comm_group(ncomm,&ngroup);CHKERRMPI(ierr);
1182       d1   = 0;
1183       ierr = MPI_Group_translate_ranks(ngroup,1,&d1,ggroup,&d2);CHKERRMPI(ierr);
1184       nid  = d2;
1185       ierr = MPI_Group_free(&ggroup);CHKERRMPI(ierr);
1186       ierr = MPI_Group_free(&ngroup);CHKERRMPI(ierr);
1187       ierr = MPI_Comm_free(&ncomm);CHKERRMPI(ierr);
1188     } else nid = 0.0;
1189 
1190     /* Get connectivity */
1191     ierr = DMPlexGetVTKCellHeight(dm,&cellHeight);CHKERRQ(ierr);
1192     ierr = DMPlexCreatePartitionerGraph(dm,cellHeight,&numVertices,&start,&adjacency,&gid);CHKERRQ(ierr);
1193 
1194     /* filter overlapped local cells */
1195     ierr = DMPlexGetHeightStratum(dm,cellHeight,&cStart,&cEnd);CHKERRQ(ierr);
1196     ierr = ISGetIndices(gid,&idxs);CHKERRQ(ierr);
1197     ierr = ISGetLocalSize(gid,&cum);CHKERRQ(ierr);
1198     ierr = PetscMalloc1(cum,&idxs2);CHKERRQ(ierr);
1199     for (c = cStart, cum = 0; c < cEnd; c++) {
1200       if (idxs[c-cStart] < 0) continue;
1201       idxs2[cum++] = idxs[c-cStart];
1202     }
1203     ierr = ISRestoreIndices(gid,&idxs);CHKERRQ(ierr);
1204     if (numVertices != cum) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Unexpected %D != %D",numVertices,cum);
1205     ierr = ISDestroy(&gid);CHKERRQ(ierr);
1206     ierr = ISCreateGeneral(comm,numVertices,idxs2,PETSC_OWN_POINTER,&gid);CHKERRQ(ierr);
1207 
1208     /* support for node-aware cell locality */
1209     ierr = ISCreateGeneral(comm,start[numVertices],adjacency,PETSC_USE_POINTER,&acis);CHKERRQ(ierr);
1210     ierr = VecCreateSeq(PETSC_COMM_SELF,start[numVertices],&acown);CHKERRQ(ierr);
1211     ierr = VecCreateMPI(comm,numVertices,PETSC_DECIDE,&cown);CHKERRQ(ierr);
1212     ierr = VecGetArray(cown,&array);CHKERRQ(ierr);
1213     for (c = 0; c < numVertices; c++) array[c] = nid;
1214     ierr = VecRestoreArray(cown,&array);CHKERRQ(ierr);
1215     ierr = VecScatterCreate(cown,acis,acown,NULL,&sct);CHKERRQ(ierr);
1216     ierr = VecScatterBegin(sct,cown,acown,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
1217     ierr = VecScatterEnd(sct,cown,acown,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
1218     ierr = ISDestroy(&acis);CHKERRQ(ierr);
1219     ierr = VecScatterDestroy(&sct);CHKERRQ(ierr);
1220     ierr = VecDestroy(&cown);CHKERRQ(ierr);
1221 
1222     /* compute edgeCut */
1223     for (c = 0, cum = 0; c < numVertices; c++) cum = PetscMax(cum,start[c+1]-start[c]);
1224     ierr = PetscMalloc1(cum,&work);CHKERRQ(ierr);
1225     ierr = ISLocalToGlobalMappingCreateIS(gid,&g2l);CHKERRQ(ierr);
1226     ierr = ISLocalToGlobalMappingSetType(g2l,ISLOCALTOGLOBALMAPPINGHASH);CHKERRQ(ierr);
1227     ierr = ISDestroy(&gid);CHKERRQ(ierr);
1228     ierr = VecGetArray(acown,&array);CHKERRQ(ierr);
1229     for (c = 0, ect = 0, ectn = 0; c < numVertices; c++) {
1230       PetscInt totl;
1231 
1232       totl = start[c+1]-start[c];
1233       ierr = ISGlobalToLocalMappingApply(g2l,IS_GTOLM_MASK,totl,adjacency+start[c],NULL,work);CHKERRQ(ierr);
1234       for (i = 0; i < totl; i++) {
1235         if (work[i] < 0) {
1236           ect  += 1;
1237           ectn += (array[i + start[c]] != nid) ? 0 : 1;
1238         }
1239       }
1240     }
1241     ierr  = PetscFree(work);CHKERRQ(ierr);
1242     ierr  = VecRestoreArray(acown,&array);CHKERRQ(ierr);
1243     lm[0] = numVertices > 0 ?  numVertices : PETSC_MAX_INT;
1244     lm[1] = -numVertices;
1245     ierr  = MPIU_Allreduce(lm,gm,2,MPIU_INT64,MPI_MIN,comm);CHKERRMPI(ierr);
1246     ierr  = PetscViewerASCIIPrintf(viewer,"  Cell balance: %.2f (max %D, min %D",-((double)gm[1])/((double)gm[0]),-(PetscInt)gm[1],(PetscInt)gm[0]);CHKERRQ(ierr);
1247     lm[0] = ect; /* edgeCut */
1248     lm[1] = ectn; /* node-aware edgeCut */
1249     lm[2] = numVertices > 0 ? 0 : 1; /* empty processes */
1250     ierr  = MPIU_Allreduce(lm,gm,3,MPIU_INT64,MPI_SUM,comm);CHKERRMPI(ierr);
1251     ierr  = PetscViewerASCIIPrintf(viewer,", empty %D)\n",(PetscInt)gm[2]);CHKERRQ(ierr);
1252 #if defined(PETSC_HAVE_MPI_PROCESS_SHARED_MEMORY)
1253     ierr  = PetscViewerASCIIPrintf(viewer,"  Edge Cut: %D (on node %.3f)\n",(PetscInt)(gm[0]/2),gm[0] ? ((double)(gm[1]))/((double)gm[0]) : 1.);CHKERRQ(ierr);
1254 #else
1255     ierr  = PetscViewerASCIIPrintf(viewer,"  Edge Cut: %D (on node %.3f)\n",(PetscInt)(gm[0]/2),0.0);CHKERRQ(ierr);
1256 #endif
1257     ierr  = ISLocalToGlobalMappingDestroy(&g2l);CHKERRQ(ierr);
1258     ierr  = PetscFree(start);CHKERRQ(ierr);
1259     ierr  = PetscFree(adjacency);CHKERRQ(ierr);
1260     ierr  = VecDestroy(&acown);CHKERRQ(ierr);
1261   } else {
1262     const char    *name;
1263     PetscInt      *sizes, *hybsizes, *ghostsizes;
1264     PetscInt       locDepth, depth, cellHeight, dim, d;
1265     PetscInt       pStart, pEnd, p, gcStart, gcEnd, gcNum;
1266     PetscInt       numLabels, l;
1267     DMPolytopeType ct0 = DM_POLYTOPE_UNKNOWN;
1268     MPI_Comm       comm;
1269     PetscMPIInt    size, rank;
1270 
1271     ierr = PetscObjectGetComm((PetscObject) dm, &comm);CHKERRQ(ierr);
1272     ierr = MPI_Comm_size(comm, &size);CHKERRMPI(ierr);
1273     ierr = MPI_Comm_rank(comm, &rank);CHKERRMPI(ierr);
1274     ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
1275     ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
1276     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
1277     if (name) {ierr = PetscViewerASCIIPrintf(viewer, "%s in %D dimension%s:\n", name, dim, dim == 1 ? "" : "s");CHKERRQ(ierr);}
1278     else      {ierr = PetscViewerASCIIPrintf(viewer, "Mesh in %D dimension%s:\n", dim, dim == 1 ? "" : "s");CHKERRQ(ierr);}
1279     if (cellHeight) {ierr = PetscViewerASCIIPrintf(viewer, "  Cells are at height %D\n", cellHeight);CHKERRQ(ierr);}
1280     ierr = DMPlexGetDepth(dm, &locDepth);CHKERRQ(ierr);
1281     ierr = MPIU_Allreduce(&locDepth, &depth, 1, MPIU_INT, MPI_MAX, comm);CHKERRMPI(ierr);
1282     ierr = DMPlexGetGhostCellStratum(dm, &gcStart, &gcEnd);CHKERRQ(ierr);
1283     gcNum = gcEnd - gcStart;
1284     ierr = PetscCalloc3(size,&sizes,size,&hybsizes,size,&ghostsizes);CHKERRQ(ierr);
1285     for (d = 0; d <= depth; d++) {
1286       PetscInt Nc[2] = {0, 0}, ict;
1287 
1288       ierr = DMPlexGetDepthStratum(dm, d, &pStart, &pEnd);CHKERRQ(ierr);
1289       if (pStart < pEnd) {ierr = DMPlexGetCellType(dm, pStart, &ct0);CHKERRQ(ierr);}
1290       ict  = ct0;
1291       ierr = MPI_Bcast(&ict, 1, MPIU_INT, 0, comm);CHKERRMPI(ierr);
1292       ct0  = (DMPolytopeType) ict;
1293       for (p = pStart; p < pEnd; ++p) {
1294         DMPolytopeType ct;
1295 
1296         ierr = DMPlexGetCellType(dm, p, &ct);CHKERRQ(ierr);
1297         if (ct == ct0) ++Nc[0];
1298         else           ++Nc[1];
1299       }
1300       ierr = MPI_Gather(&Nc[0], 1, MPIU_INT, sizes,    1, MPIU_INT, 0, comm);CHKERRMPI(ierr);
1301       ierr = MPI_Gather(&Nc[1], 1, MPIU_INT, hybsizes, 1, MPIU_INT, 0, comm);CHKERRMPI(ierr);
1302       if (d == depth) {ierr = MPI_Gather(&gcNum, 1, MPIU_INT, ghostsizes, 1, MPIU_INT, 0, comm);CHKERRMPI(ierr);}
1303       ierr = PetscViewerASCIIPrintf(viewer, "  %D-cells:", (depth == 1) && d ? dim : d);CHKERRQ(ierr);
1304       for (p = 0; p < size; ++p) {
1305         if (rank == 0) {
1306           ierr = PetscViewerASCIIPrintf(viewer, " %D", sizes[p]+hybsizes[p]);CHKERRQ(ierr);
1307           if (hybsizes[p]   > 0) {ierr = PetscViewerASCIIPrintf(viewer, " (%D)", hybsizes[p]);CHKERRQ(ierr);}
1308           if (ghostsizes[p] > 0) {ierr = PetscViewerASCIIPrintf(viewer, " [%D]", ghostsizes[p]);CHKERRQ(ierr);}
1309         }
1310       }
1311       ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
1312     }
1313     ierr = PetscFree3(sizes,hybsizes,ghostsizes);CHKERRQ(ierr);
1314     {
1315       const PetscReal      *maxCell;
1316       const PetscReal      *L;
1317       const DMBoundaryType *bd;
1318       PetscBool             per, localized;
1319 
1320       ierr = DMGetPeriodicity(dm, &per, &maxCell, &L, &bd);CHKERRQ(ierr);
1321       ierr = DMGetCoordinatesLocalized(dm, &localized);CHKERRQ(ierr);
1322       if (per) {
1323         ierr = PetscViewerASCIIPrintf(viewer, "Periodic mesh (");CHKERRQ(ierr);
1324         ierr = PetscViewerASCIIUseTabs(viewer, PETSC_FALSE);CHKERRQ(ierr);
1325         for (d = 0; d < dim; ++d) {
1326           if (bd && d > 0) {ierr = PetscViewerASCIIPrintf(viewer, ", ");CHKERRQ(ierr);}
1327           if (bd)    {ierr = PetscViewerASCIIPrintf(viewer, "%s", DMBoundaryTypes[bd[d]]);CHKERRQ(ierr);}
1328         }
1329         ierr = PetscViewerASCIIPrintf(viewer, ") coordinates %s\n", localized ? "localized" : "not localized");CHKERRQ(ierr);
1330         ierr = PetscViewerASCIIUseTabs(viewer, PETSC_TRUE);CHKERRQ(ierr);
1331       }
1332     }
1333     ierr = DMGetNumLabels(dm, &numLabels);CHKERRQ(ierr);
1334     if (numLabels) {ierr = PetscViewerASCIIPrintf(viewer, "Labels:\n");CHKERRQ(ierr);}
1335     for (l = 0; l < numLabels; ++l) {
1336       DMLabel         label;
1337       const char     *name;
1338       IS              valueIS;
1339       const PetscInt *values;
1340       PetscInt        numValues, v;
1341 
1342       ierr = DMGetLabelName(dm, l, &name);CHKERRQ(ierr);
1343       ierr = DMGetLabel(dm, name, &label);CHKERRQ(ierr);
1344       ierr = DMLabelGetNumValues(label, &numValues);CHKERRQ(ierr);
1345       ierr = PetscViewerASCIIPrintf(viewer, "  %s: %D strata with value/size (", name, numValues);CHKERRQ(ierr);
1346       ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
1347       ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
1348       ierr = PetscViewerASCIIUseTabs(viewer, PETSC_FALSE);CHKERRQ(ierr);
1349       for (v = 0; v < numValues; ++v) {
1350         PetscInt size;
1351 
1352         ierr = DMLabelGetStratumSize(label, values[v], &size);CHKERRQ(ierr);
1353         if (v > 0) {ierr = PetscViewerASCIIPrintf(viewer, ", ");CHKERRQ(ierr);}
1354         ierr = PetscViewerASCIIPrintf(viewer, "%D (%D)", values[v], size);CHKERRQ(ierr);
1355       }
1356       ierr = PetscViewerASCIIPrintf(viewer, ")\n");CHKERRQ(ierr);
1357       ierr = PetscViewerASCIIUseTabs(viewer, PETSC_TRUE);CHKERRQ(ierr);
1358       ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
1359       ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
1360     }
1361     {
1362       char    **labelNames;
1363       PetscInt  Nl = numLabels;
1364       PetscBool flg;
1365 
1366       ierr = PetscMalloc1(Nl, &labelNames);CHKERRQ(ierr);
1367       ierr = PetscOptionsGetStringArray(((PetscObject) dm)->options, ((PetscObject) dm)->prefix, "-dm_plex_view_labels", labelNames, &Nl, &flg);CHKERRQ(ierr);
1368       for (l = 0; l < Nl; ++l) {
1369         DMLabel label;
1370 
1371         ierr = DMHasLabel(dm, labelNames[l], &flg);CHKERRQ(ierr);
1372         if (flg) {
1373           ierr = DMGetLabel(dm, labelNames[l], &label);CHKERRQ(ierr);
1374           ierr = DMLabelView(label, viewer);CHKERRQ(ierr);
1375         }
1376         ierr = PetscFree(labelNames[l]);CHKERRQ(ierr);
1377       }
1378       ierr = PetscFree(labelNames);CHKERRQ(ierr);
1379     }
1380     /* If no fields are specified, people do not want to see adjacency */
1381     if (dm->Nf) {
1382       PetscInt f;
1383 
1384       for (f = 0; f < dm->Nf; ++f) {
1385         const char *name;
1386 
1387         ierr = PetscObjectGetName(dm->fields[f].disc, &name);CHKERRQ(ierr);
1388         if (numLabels) {ierr = PetscViewerASCIIPrintf(viewer, "Field %s:\n", name);CHKERRQ(ierr);}
1389         ierr = PetscViewerASCIIPushTab(viewer);CHKERRQ(ierr);
1390         if (dm->fields[f].label) {ierr = DMLabelView(dm->fields[f].label, viewer);CHKERRQ(ierr);}
1391         if (dm->fields[f].adjacency[0]) {
1392           if (dm->fields[f].adjacency[1]) {ierr = PetscViewerASCIIPrintf(viewer, "adjacency FVM++\n");CHKERRQ(ierr);}
1393           else                            {ierr = PetscViewerASCIIPrintf(viewer, "adjacency FVM\n");CHKERRQ(ierr);}
1394         } else {
1395           if (dm->fields[f].adjacency[1]) {ierr = PetscViewerASCIIPrintf(viewer, "adjacency FEM\n");CHKERRQ(ierr);}
1396           else                            {ierr = PetscViewerASCIIPrintf(viewer, "adjacency FUNKY\n");CHKERRQ(ierr);}
1397         }
1398         ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr);
1399       }
1400     }
1401     ierr = DMGetCoarseDM(dm, &cdm);CHKERRQ(ierr);
1402     if (cdm) {
1403       ierr = PetscViewerASCIIPushTab(viewer);CHKERRQ(ierr);
1404       ierr = DMPlexView_Ascii(cdm, viewer);CHKERRQ(ierr);
1405       ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr);
1406     }
1407   }
1408   PetscFunctionReturn(0);
1409 }
1410 
1411 static PetscErrorCode DMPlexDrawCell(DM dm, PetscDraw draw, PetscInt cell, const PetscScalar coords[])
1412 {
1413   DMPolytopeType ct;
1414   PetscMPIInt    rank;
1415   PetscInt       cdim;
1416   PetscErrorCode ierr;
1417 
1418   PetscFunctionBegin;
1419   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRMPI(ierr);
1420   ierr = DMPlexGetCellType(dm, cell, &ct);CHKERRQ(ierr);
1421   ierr = DMGetCoordinateDim(dm, &cdim);CHKERRQ(ierr);
1422   switch (ct) {
1423   case DM_POLYTOPE_SEGMENT:
1424   case DM_POLYTOPE_POINT_PRISM_TENSOR:
1425     switch (cdim) {
1426     case 1:
1427     {
1428       const PetscReal y  = 0.5;  /* TODO Put it in the middle of the viewport */
1429       const PetscReal dy = 0.05; /* TODO Make it a fraction of the total length */
1430 
1431       ierr = PetscDrawLine(draw, PetscRealPart(coords[0]), y,    PetscRealPart(coords[1]), y,    PETSC_DRAW_BLACK);CHKERRQ(ierr);
1432       ierr = PetscDrawLine(draw, PetscRealPart(coords[0]), y+dy, PetscRealPart(coords[0]), y-dy, PETSC_DRAW_BLACK);CHKERRQ(ierr);
1433       ierr = PetscDrawLine(draw, PetscRealPart(coords[1]), y+dy, PetscRealPart(coords[1]), y-dy, PETSC_DRAW_BLACK);CHKERRQ(ierr);
1434     }
1435     break;
1436     case 2:
1437     {
1438       const PetscReal dx = (PetscRealPart(coords[3]) - PetscRealPart(coords[1]));
1439       const PetscReal dy = (PetscRealPart(coords[2]) - PetscRealPart(coords[0]));
1440       const PetscReal l  = 0.1/PetscSqrtReal(dx*dx + dy*dy);
1441 
1442       ierr = PetscDrawLine(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1443       ierr = PetscDrawLine(draw, PetscRealPart(coords[0])+l*dx, PetscRealPart(coords[1])+l*dy, PetscRealPart(coords[0])-l*dx, PetscRealPart(coords[1])-l*dy, PETSC_DRAW_BLACK);CHKERRQ(ierr);
1444       ierr = PetscDrawLine(draw, PetscRealPart(coords[2])+l*dx, PetscRealPart(coords[3])+l*dy, PetscRealPart(coords[2])-l*dx, PetscRealPart(coords[3])-l*dy, PETSC_DRAW_BLACK);CHKERRQ(ierr);
1445     }
1446     break;
1447     default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_SUP, "Cannot draw cells of dimension %D", cdim);
1448     }
1449     break;
1450   case DM_POLYTOPE_TRIANGLE:
1451     ierr = PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]),
1452                              PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1453                              PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1454                              PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2);CHKERRQ(ierr);
1455     ierr = PetscDrawLine(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1456     ierr = PetscDrawLine(draw, PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1457     ierr = PetscDrawLine(draw, PetscRealPart(coords[4]), PetscRealPart(coords[5]), PetscRealPart(coords[0]), PetscRealPart(coords[1]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1458     break;
1459   case DM_POLYTOPE_QUADRILATERAL:
1460     ierr = PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]),
1461                               PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1462                               PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1463                               PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2);CHKERRQ(ierr);
1464     ierr = PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), PetscRealPart(coords[6]), PetscRealPart(coords[7]),
1465                               PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1466                               PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1467                               PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2);CHKERRQ(ierr);
1468     ierr = PetscDrawLine(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1469     ierr = PetscDrawLine(draw, PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1470     ierr = PetscDrawLine(draw, PetscRealPart(coords[4]), PetscRealPart(coords[5]), PetscRealPart(coords[6]), PetscRealPart(coords[7]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1471     ierr = PetscDrawLine(draw, PetscRealPart(coords[6]), PetscRealPart(coords[7]), PetscRealPart(coords[0]), PetscRealPart(coords[1]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1472     break;
1473   default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_SUP, "Cannot draw cells of type %s", DMPolytopeTypes[ct]);
1474   }
1475   PetscFunctionReturn(0);
1476 }
1477 
1478 static PetscErrorCode DMPlexDrawCellHighOrder(DM dm, PetscDraw draw, PetscInt cell, const PetscScalar coords[], PetscInt edgeDiv, PetscReal refCoords[], PetscReal edgeCoords[])
1479 {
1480   DMPolytopeType ct;
1481   PetscReal      centroid[2] = {0., 0.};
1482   PetscMPIInt    rank;
1483   PetscInt       fillColor, v, e, d;
1484   PetscErrorCode ierr;
1485 
1486   PetscFunctionBegin;
1487   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRMPI(ierr);
1488   ierr = DMPlexGetCellType(dm, cell, &ct);CHKERRQ(ierr);
1489   fillColor = PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2;
1490   switch (ct) {
1491   case DM_POLYTOPE_TRIANGLE:
1492     {
1493       PetscReal refVertices[6] = {-1., -1., 1., -1., -1., 1.};
1494 
1495       for (v = 0; v < 3; ++v) {centroid[0] += PetscRealPart(coords[v*2+0])/3.;centroid[1] += PetscRealPart(coords[v*2+1])/3.;}
1496       for (e = 0; e < 3; ++e) {
1497         refCoords[0] = refVertices[e*2+0];
1498         refCoords[1] = refVertices[e*2+1];
1499         for (d = 1; d <= edgeDiv; ++d) {
1500           refCoords[d*2+0] = refCoords[0] + (refVertices[(e+1)%3 * 2 + 0] - refCoords[0])*d/edgeDiv;
1501           refCoords[d*2+1] = refCoords[1] + (refVertices[(e+1)%3 * 2 + 1] - refCoords[1])*d/edgeDiv;
1502         }
1503         ierr = DMPlexReferenceToCoordinates(dm, cell, edgeDiv+1, refCoords, edgeCoords);CHKERRQ(ierr);
1504         for (d = 0; d < edgeDiv; ++d) {
1505           ierr = PetscDrawTriangle(draw, centroid[0], centroid[1], edgeCoords[d*2+0], edgeCoords[d*2+1], edgeCoords[(d+1)*2+0], edgeCoords[(d+1)*2+1], fillColor, fillColor, fillColor);CHKERRQ(ierr);
1506           ierr = PetscDrawLine(draw, edgeCoords[d*2+0], edgeCoords[d*2+1], edgeCoords[(d+1)*2+0], edgeCoords[(d+1)*2+1], PETSC_DRAW_BLACK);CHKERRQ(ierr);
1507         }
1508       }
1509     }
1510     break;
1511   default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_SUP, "Cannot draw cells of type %s", DMPolytopeTypes[ct]);
1512   }
1513   PetscFunctionReturn(0);
1514 }
1515 
1516 static PetscErrorCode DMPlexView_Draw(DM dm, PetscViewer viewer)
1517 {
1518   PetscDraw          draw;
1519   DM                 cdm;
1520   PetscSection       coordSection;
1521   Vec                coordinates;
1522   const PetscScalar *coords;
1523   PetscReal          xyl[2],xyr[2],bound[4] = {PETSC_MAX_REAL, PETSC_MAX_REAL, PETSC_MIN_REAL, PETSC_MIN_REAL};
1524   PetscReal         *refCoords, *edgeCoords;
1525   PetscBool          isnull, drawAffine = PETSC_TRUE;
1526   PetscInt           dim, vStart, vEnd, cStart, cEnd, c, N, edgeDiv = 4;
1527   PetscErrorCode     ierr;
1528 
1529   PetscFunctionBegin;
1530   ierr = DMGetCoordinateDim(dm, &dim);CHKERRQ(ierr);
1531   if (dim > 2) SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Cannot draw meshes of dimension %D", dim);
1532   ierr = PetscOptionsGetBool(((PetscObject) dm)->options, ((PetscObject) dm)->prefix, "-dm_view_draw_affine", &drawAffine, NULL);CHKERRQ(ierr);
1533   if (!drawAffine) {ierr = PetscMalloc2((edgeDiv+1)*dim, &refCoords, (edgeDiv+1)*dim, &edgeCoords);CHKERRQ(ierr);}
1534   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
1535   ierr = DMGetLocalSection(cdm, &coordSection);CHKERRQ(ierr);
1536   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
1537   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
1538   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
1539 
1540   ierr = PetscViewerDrawGetDraw(viewer, 0, &draw);CHKERRQ(ierr);
1541   ierr = PetscDrawIsNull(draw, &isnull);CHKERRQ(ierr);
1542   if (isnull) PetscFunctionReturn(0);
1543   ierr = PetscDrawSetTitle(draw, "Mesh");CHKERRQ(ierr);
1544 
1545   ierr = VecGetLocalSize(coordinates, &N);CHKERRQ(ierr);
1546   ierr = VecGetArrayRead(coordinates, &coords);CHKERRQ(ierr);
1547   for (c = 0; c < N; c += dim) {
1548     bound[0] = PetscMin(bound[0], PetscRealPart(coords[c]));   bound[2] = PetscMax(bound[2], PetscRealPart(coords[c]));
1549     bound[1] = PetscMin(bound[1], PetscRealPart(coords[c+1])); bound[3] = PetscMax(bound[3], PetscRealPart(coords[c+1]));
1550   }
1551   ierr = VecRestoreArrayRead(coordinates, &coords);CHKERRQ(ierr);
1552   ierr = MPIU_Allreduce(&bound[0],xyl,2,MPIU_REAL,MPIU_MIN,PetscObjectComm((PetscObject)dm));CHKERRMPI(ierr);
1553   ierr = MPIU_Allreduce(&bound[2],xyr,2,MPIU_REAL,MPIU_MAX,PetscObjectComm((PetscObject)dm));CHKERRMPI(ierr);
1554   ierr = PetscDrawSetCoordinates(draw, xyl[0], xyl[1], xyr[0], xyr[1]);CHKERRQ(ierr);
1555   ierr = PetscDrawClear(draw);CHKERRQ(ierr);
1556 
1557   for (c = cStart; c < cEnd; ++c) {
1558     PetscScalar *coords = NULL;
1559     PetscInt     numCoords;
1560 
1561     ierr = DMPlexVecGetClosureAtDepth_Internal(dm, coordSection, coordinates, c, 0, &numCoords, &coords);CHKERRQ(ierr);
1562     if (drawAffine) {
1563       ierr = DMPlexDrawCell(dm, draw, c, coords);CHKERRQ(ierr);
1564     } else {
1565       ierr = DMPlexDrawCellHighOrder(dm, draw, c, coords, edgeDiv, refCoords, edgeCoords);CHKERRQ(ierr);
1566     }
1567     ierr = DMPlexVecRestoreClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
1568   }
1569   if (!drawAffine) {ierr = PetscFree2(refCoords, edgeCoords);CHKERRQ(ierr);}
1570   ierr = PetscDrawFlush(draw);CHKERRQ(ierr);
1571   ierr = PetscDrawPause(draw);CHKERRQ(ierr);
1572   ierr = PetscDrawSave(draw);CHKERRQ(ierr);
1573   PetscFunctionReturn(0);
1574 }
1575 
1576 #if defined(PETSC_HAVE_EXODUSII)
1577 #include <exodusII.h>
1578 #include <petscviewerexodusii.h>
1579 #endif
1580 
1581 PetscErrorCode DMView_Plex(DM dm, PetscViewer viewer)
1582 {
1583   PetscBool      iascii, ishdf5, isvtk, isdraw, flg, isglvis, isexodus;
1584   char           name[PETSC_MAX_PATH_LEN];
1585   PetscErrorCode ierr;
1586 
1587   PetscFunctionBegin;
1588   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1589   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1590   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERASCII,    &iascii);CHKERRQ(ierr);
1591   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,      &isvtk);CHKERRQ(ierr);
1592   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,     &ishdf5);CHKERRQ(ierr);
1593   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERDRAW,     &isdraw);CHKERRQ(ierr);
1594   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERGLVIS,    &isglvis);CHKERRQ(ierr);
1595   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWEREXODUSII, &isexodus);CHKERRQ(ierr);
1596   if (iascii) {
1597     PetscViewerFormat format;
1598     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
1599     if (format == PETSC_VIEWER_ASCII_GLVIS) {
1600       ierr = DMPlexView_GLVis(dm, viewer);CHKERRQ(ierr);
1601     } else {
1602       ierr = DMPlexView_Ascii(dm, viewer);CHKERRQ(ierr);
1603     }
1604   } else if (ishdf5) {
1605 #if defined(PETSC_HAVE_HDF5)
1606     ierr = DMPlexView_HDF5_Internal(dm, viewer);CHKERRQ(ierr);
1607 #else
1608     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1609 #endif
1610   } else if (isvtk) {
1611     ierr = DMPlexVTKWriteAll((PetscObject) dm,viewer);CHKERRQ(ierr);
1612   } else if (isdraw) {
1613     ierr = DMPlexView_Draw(dm, viewer);CHKERRQ(ierr);
1614   } else if (isglvis) {
1615     ierr = DMPlexView_GLVis(dm, viewer);CHKERRQ(ierr);
1616 #if defined(PETSC_HAVE_EXODUSII)
1617   } else if (isexodus) {
1618 /*
1619       exodusII requires that all sets be part of exactly one cell set.
1620       If the dm does not have a "Cell Sets" label defined, we create one
1621       with ID 1, containig all cells.
1622       Note that if the Cell Sets label is defined but does not cover all cells,
1623       we may still have a problem. This should probably be checked here or in the viewer;
1624     */
1625     PetscInt numCS;
1626     ierr = DMGetLabelSize(dm,"Cell Sets",&numCS);CHKERRQ(ierr);
1627     if (!numCS) {
1628       PetscInt cStart, cEnd, c;
1629       ierr = DMCreateLabel(dm, "Cell Sets");CHKERRQ(ierr);
1630       ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
1631       for (c = cStart; c < cEnd; ++c) {ierr = DMSetLabelValue(dm, "Cell Sets", c, 1);CHKERRQ(ierr);}
1632     }
1633     ierr = DMView_PlexExodusII(dm, viewer);CHKERRQ(ierr);
1634 #endif
1635   } else {
1636     SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Viewer type %s not yet supported for DMPlex writing", ((PetscObject)viewer)->type_name);
1637   }
1638   /* Optionally view the partition */
1639   ierr = PetscOptionsHasName(((PetscObject) dm)->options, ((PetscObject) dm)->prefix, "-dm_partition_view", &flg);CHKERRQ(ierr);
1640   if (flg) {
1641     Vec ranks;
1642     ierr = DMPlexCreateRankField(dm, &ranks);CHKERRQ(ierr);
1643     ierr = VecView(ranks, viewer);CHKERRQ(ierr);
1644     ierr = VecDestroy(&ranks);CHKERRQ(ierr);
1645   }
1646   /* Optionally view a label */
1647   ierr = PetscOptionsGetString(((PetscObject) dm)->options, ((PetscObject) dm)->prefix, "-dm_label_view", name, sizeof(name), &flg);CHKERRQ(ierr);
1648   if (flg) {
1649     DMLabel label;
1650     Vec     val;
1651 
1652     ierr = DMGetLabel(dm, name, &label);CHKERRQ(ierr);
1653     if (!label) SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Label %s provided to -dm_label_view does not exist in this DM", name);
1654     ierr = DMPlexCreateLabelField(dm, label, &val);CHKERRQ(ierr);
1655     ierr = VecView(val, viewer);CHKERRQ(ierr);
1656     ierr = VecDestroy(&val);CHKERRQ(ierr);
1657   }
1658   PetscFunctionReturn(0);
1659 }
1660 
1661 /*@
1662   DMPlexTopologyView - Saves a DMPlex topology into a file
1663 
1664   Collective on DM
1665 
1666   Input Parameters:
1667 + dm     - The DM whose topology is to be saved
1668 - viewer - The PetscViewer for saving
1669 
1670   Level: advanced
1671 
1672 .seealso: DMView(), DMPlexCoordinatesView(), DMPlexLabelsView(), DMPlexTopologyLoad()
1673 @*/
1674 PetscErrorCode DMPlexTopologyView(DM dm, PetscViewer viewer)
1675 {
1676   PetscBool      ishdf5;
1677   PetscErrorCode ierr;
1678 
1679   PetscFunctionBegin;
1680   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1681   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1682   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
1683   ierr = PetscLogEventBegin(DMPLEX_TopologyView,viewer,0,0,0);CHKERRQ(ierr);
1684   if (ishdf5) {
1685 #if defined(PETSC_HAVE_HDF5)
1686     PetscViewerFormat format;
1687     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
1688     if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
1689       IS globalPointNumbering;
1690 
1691       ierr = DMPlexCreatePointNumbering(dm, &globalPointNumbering);CHKERRQ(ierr);
1692       ierr = DMPlexTopologyView_HDF5_Internal(dm, globalPointNumbering, viewer);CHKERRQ(ierr);
1693       ierr = ISDestroy(&globalPointNumbering);CHKERRQ(ierr);
1694     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 output.", PetscViewerFormats[format]);
1695 #else
1696     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1697 #endif
1698   }
1699   ierr = PetscLogEventEnd(DMPLEX_TopologyView,viewer,0,0,0);CHKERRQ(ierr);
1700   PetscFunctionReturn(0);
1701 }
1702 
1703 /*@
1704   DMPlexCoordinatesView - Saves DMPlex coordinates into a file
1705 
1706   Collective on DM
1707 
1708   Input Parameters:
1709 + dm     - The DM whose coordinates are to be saved
1710 - viewer - The PetscViewer for saving
1711 
1712   Level: advanced
1713 
1714 .seealso: DMView(), DMPlexTopologyView(), DMPlexLabelsView(), DMPlexCoordinatesLoad()
1715 @*/
1716 PetscErrorCode DMPlexCoordinatesView(DM dm, PetscViewer viewer)
1717 {
1718   PetscBool      ishdf5;
1719   PetscErrorCode ierr;
1720 
1721   PetscFunctionBegin;
1722   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1723   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1724   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
1725   ierr = PetscLogEventBegin(DMPLEX_CoordinatesView,viewer,0,0,0);CHKERRQ(ierr);
1726   if (ishdf5) {
1727 #if defined(PETSC_HAVE_HDF5)
1728     PetscViewerFormat format;
1729     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
1730     if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
1731       ierr = DMPlexCoordinatesView_HDF5_Internal(dm, viewer);CHKERRQ(ierr);
1732     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 input.", PetscViewerFormats[format]);
1733 #else
1734     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1735 #endif
1736   }
1737   ierr = PetscLogEventEnd(DMPLEX_CoordinatesView,viewer,0,0,0);CHKERRQ(ierr);
1738   PetscFunctionReturn(0);
1739 }
1740 
1741 /*@
1742   DMPlexLabelsView - Saves DMPlex labels into a file
1743 
1744   Collective on DM
1745 
1746   Input Parameters:
1747 + dm     - The DM whose labels are to be saved
1748 - viewer - The PetscViewer for saving
1749 
1750   Level: advanced
1751 
1752 .seealso: DMView(), DMPlexTopologyView(), DMPlexCoordinatesView(), DMPlexLabelsLoad()
1753 @*/
1754 PetscErrorCode DMPlexLabelsView(DM dm, PetscViewer viewer)
1755 {
1756   PetscBool      ishdf5;
1757   PetscErrorCode ierr;
1758 
1759   PetscFunctionBegin;
1760   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1761   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1762   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
1763   ierr = PetscLogEventBegin(DMPLEX_LabelsView,viewer,0,0,0);CHKERRQ(ierr);
1764   if (ishdf5) {
1765 #if defined(PETSC_HAVE_HDF5)
1766     IS                globalPointNumbering;
1767     PetscViewerFormat format;
1768 
1769     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
1770     if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
1771       ierr = DMPlexCreatePointNumbering(dm, &globalPointNumbering);CHKERRQ(ierr);
1772       ierr = DMPlexLabelsView_HDF5_Internal(dm, globalPointNumbering, viewer);CHKERRQ(ierr);
1773       ierr = ISDestroy(&globalPointNumbering);CHKERRQ(ierr);
1774     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 input.", PetscViewerFormats[format]);
1775 #else
1776     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1777 #endif
1778   }
1779   ierr = PetscLogEventEnd(DMPLEX_LabelsView,viewer,0,0,0);CHKERRQ(ierr);
1780   PetscFunctionReturn(0);
1781 }
1782 
1783 /*@
1784   DMPlexSectionView - Saves a section associated with a DMPlex
1785 
1786   Collective on DM
1787 
1788   Input Parameters:
1789 + dm         - The DM that contains the topology on which the section to be saved is defined
1790 . viewer     - The PetscViewer for saving
1791 - sectiondm  - The DM that contains the section to be saved
1792 
1793   Level: advanced
1794 
1795   Notes:
1796   This function is a wrapper around PetscSectionView(); in addition to the raw section, it saves information that associates the section points to the topology (dm) points. When the topology (dm) and the section are later loaded with DMPlexTopologyLoad() and DMPlexSectionLoad(), respectively, this information is used to match section points with topology points.
1797 
1798   In general dm and sectiondm are two different objects, the former carrying the topology and the latter carrying the section, and have been given a topology name and a section name, respectively, with PetscObjectSetName(). In practice, however, they can be the same object if it carries both topology and section; in that case the name of the object is used as both the topology name and the section name.
1799 
1800 .seealso: DMView(), DMPlexTopologyView(), DMPlexCoordinatesView(), DMPlexLabelsView(), DMPlexGlobalVectorView(), DMPlexLocalVectorView(), PetscSectionView(), DMPlexSectionLoad()
1801 @*/
1802 PetscErrorCode DMPlexSectionView(DM dm, PetscViewer viewer, DM sectiondm)
1803 {
1804   PetscBool      ishdf5;
1805   PetscErrorCode ierr;
1806 
1807   PetscFunctionBegin;
1808   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1809   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1810   PetscValidHeaderSpecific(sectiondm, DM_CLASSID, 3);
1811   ierr = PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERHDF5,&ishdf5);CHKERRQ(ierr);
1812   ierr = PetscLogEventBegin(DMPLEX_SectionView,viewer,0,0,0);CHKERRQ(ierr);
1813   if (ishdf5) {
1814 #if defined(PETSC_HAVE_HDF5)
1815     ierr = DMPlexSectionView_HDF5_Internal(dm, viewer, sectiondm);CHKERRQ(ierr);
1816 #else
1817     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1818 #endif
1819   }
1820   ierr = PetscLogEventEnd(DMPLEX_SectionView,viewer,0,0,0);CHKERRQ(ierr);
1821   PetscFunctionReturn(0);
1822 }
1823 
1824 /*@
1825   DMPlexGlobalVectorView - Saves a global vector
1826 
1827   Collective on DM
1828 
1829   Input Parameters:
1830 + dm        - The DM that represents the topology
1831 . viewer    - The PetscViewer to save data with
1832 . sectiondm - The DM that contains the global section on which vec is defined
1833 - vec       - The global vector to be saved
1834 
1835   Level: advanced
1836 
1837   Notes:
1838   In general dm and sectiondm are two different objects, the former carrying the topology and the latter carrying the section, and have been given a topology name and a section name, respectively, with PetscObjectSetName(). In practice, however, they can be the same object if it carries both topology and section; in that case the name of the object is used as both the topology name and the section name.
1839 
1840   Typical calling sequence
1841 $       DMCreate(PETSC_COMM_WORLD, &dm);
1842 $       DMSetType(dm, DMPLEX);
1843 $       PetscObjectSetName((PetscObject)dm, "topologydm_name");
1844 $       DMClone(dm, &sectiondm);
1845 $       PetscObjectSetName((PetscObject)sectiondm, "sectiondm_name");
1846 $       PetscSectionCreate(PETSC_COMM_WORLD, &section);
1847 $       DMPlexGetChart(sectiondm, &pStart, &pEnd);
1848 $       PetscSectionSetChart(section, pStart, pEnd);
1849 $       PetscSectionSetUp(section);
1850 $       DMSetLocalSection(sectiondm, section);
1851 $       PetscSectionDestroy(&section);
1852 $       DMGetGlobalVector(sectiondm, &vec);
1853 $       PetscObjectSetName((PetscObject)vec, "vec_name");
1854 $       DMPlexTopologyView(dm, viewer);
1855 $       DMPlexSectionView(dm, viewer, sectiondm);
1856 $       DMPlexGlobalVectorView(dm, viewer, sectiondm, vec);
1857 $       DMRestoreGlobalVector(sectiondm, &vec);
1858 $       DMDestroy(&sectiondm);
1859 $       DMDestroy(&dm);
1860 
1861 .seealso: DMPlexTopologyView(), DMPlexSectionView(), DMPlexLocalVectorView(), DMPlexGlobalVectorLoad(), DMPlexLocalVectorLoad()
1862 @*/
1863 PetscErrorCode DMPlexGlobalVectorView(DM dm, PetscViewer viewer, DM sectiondm, Vec vec)
1864 {
1865   PetscBool       ishdf5;
1866   PetscErrorCode  ierr;
1867 
1868   PetscFunctionBegin;
1869   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1870   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1871   PetscValidHeaderSpecific(sectiondm, DM_CLASSID, 3);
1872   PetscValidHeaderSpecific(vec, VEC_CLASSID, 4);
1873   /* Check consistency */
1874   {
1875     PetscSection  section;
1876     PetscBool     includesConstraints;
1877     PetscInt      m, m1;
1878 
1879     ierr = VecGetLocalSize(vec, &m1);CHKERRQ(ierr);
1880     ierr = DMGetGlobalSection(sectiondm, &section);CHKERRQ(ierr);
1881     ierr = PetscSectionGetIncludesConstraints(section, &includesConstraints);CHKERRQ(ierr);
1882     if (includesConstraints) {ierr = PetscSectionGetStorageSize(section, &m);CHKERRQ(ierr);}
1883     else {ierr = PetscSectionGetConstrainedStorageSize(section, &m);CHKERRQ(ierr);}
1884     if (m1 != m) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Global vector size (%D) != global section storage size (%D)", m1, m);
1885   }
1886   ierr = PetscObjectTypeCompare((PetscObject)viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
1887   ierr = PetscLogEventBegin(DMPLEX_GlobalVectorView,viewer,0,0,0);CHKERRQ(ierr);
1888   if (ishdf5) {
1889 #if defined(PETSC_HAVE_HDF5)
1890     ierr = DMPlexGlobalVectorView_HDF5_Internal(dm, viewer, sectiondm, vec);CHKERRQ(ierr);
1891 #else
1892     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1893 #endif
1894   }
1895   ierr = PetscLogEventEnd(DMPLEX_GlobalVectorView,viewer,0,0,0);CHKERRQ(ierr);
1896   PetscFunctionReturn(0);
1897 }
1898 
1899 /*@
1900   DMPlexLocalVectorView - Saves a local vector
1901 
1902   Collective on DM
1903 
1904   Input Parameters:
1905 + dm        - The DM that represents the topology
1906 . viewer    - The PetscViewer to save data with
1907 . sectiondm - The DM that contains the local section on which vec is defined; may be the same as dm
1908 - vec       - The local vector to be saved
1909 
1910   Level: advanced
1911 
1912   Notes:
1913   In general dm and sectiondm are two different objects, the former carrying the topology and the latter carrying the section, and have been given a topology name and a section name, respectively, with PetscObjectSetName(). In practice, however, they can be the same object if it carries both topology and section; in that case the name of the object is used as both the topology name and the section name.
1914 
1915   Typical calling sequence
1916 $       DMCreate(PETSC_COMM_WORLD, &dm);
1917 $       DMSetType(dm, DMPLEX);
1918 $       PetscObjectSetName((PetscObject)dm, "topologydm_name");
1919 $       DMClone(dm, &sectiondm);
1920 $       PetscObjectSetName((PetscObject)sectiondm, "sectiondm_name");
1921 $       PetscSectionCreate(PETSC_COMM_WORLD, &section);
1922 $       DMPlexGetChart(sectiondm, &pStart, &pEnd);
1923 $       PetscSectionSetChart(section, pStart, pEnd);
1924 $       PetscSectionSetUp(section);
1925 $       DMSetLocalSection(sectiondm, section);
1926 $       DMGetLocalVector(sectiondm, &vec);
1927 $       PetscObjectSetName((PetscObject)vec, "vec_name");
1928 $       DMPlexTopologyView(dm, viewer);
1929 $       DMPlexSectionView(dm, viewer, sectiondm);
1930 $       DMPlexLocalVectorView(dm, viewer, sectiondm, vec);
1931 $       DMRestoreLocalVector(sectiondm, &vec);
1932 $       DMDestroy(&sectiondm);
1933 $       DMDestroy(&dm);
1934 
1935 .seealso: DMPlexTopologyView(), DMPlexSectionView(), DMPlexGlobalVectorView(), DMPlexGlobalVectorLoad(), DMPlexLocalVectorLoad()
1936 @*/
1937 PetscErrorCode DMPlexLocalVectorView(DM dm, PetscViewer viewer, DM sectiondm, Vec vec)
1938 {
1939   PetscBool       ishdf5;
1940   PetscErrorCode  ierr;
1941 
1942   PetscFunctionBegin;
1943   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1944   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1945   PetscValidHeaderSpecific(sectiondm, DM_CLASSID, 3);
1946   PetscValidHeaderSpecific(vec, VEC_CLASSID, 4);
1947   /* Check consistency */
1948   {
1949     PetscSection  section;
1950     PetscBool     includesConstraints;
1951     PetscInt      m, m1;
1952 
1953     ierr = VecGetLocalSize(vec, &m1);CHKERRQ(ierr);
1954     ierr = DMGetLocalSection(sectiondm, &section);CHKERRQ(ierr);
1955     ierr = PetscSectionGetIncludesConstraints(section, &includesConstraints);CHKERRQ(ierr);
1956     if (includesConstraints) {ierr = PetscSectionGetStorageSize(section, &m);CHKERRQ(ierr);}
1957     else {ierr = PetscSectionGetConstrainedStorageSize(section, &m);CHKERRQ(ierr);}
1958     if (m1 != m) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Local vector size (%D) != local section storage size (%D)", m1, m);
1959   }
1960   ierr = PetscObjectTypeCompare((PetscObject)viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
1961   ierr = PetscLogEventBegin(DMPLEX_LocalVectorView,viewer,0,0,0);CHKERRQ(ierr);
1962   if (ishdf5) {
1963 #if defined(PETSC_HAVE_HDF5)
1964     ierr = DMPlexLocalVectorView_HDF5_Internal(dm, viewer, sectiondm, vec);CHKERRQ(ierr);
1965 #else
1966     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1967 #endif
1968   }
1969   ierr = PetscLogEventEnd(DMPLEX_LocalVectorView,viewer,0,0,0);CHKERRQ(ierr);
1970   PetscFunctionReturn(0);
1971 }
1972 
1973 PetscErrorCode DMLoad_Plex(DM dm, PetscViewer viewer)
1974 {
1975   PetscBool      ishdf5;
1976   PetscErrorCode ierr;
1977 
1978   PetscFunctionBegin;
1979   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1980   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1981   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,   &ishdf5);CHKERRQ(ierr);
1982   if (ishdf5) {
1983 #if defined(PETSC_HAVE_HDF5)
1984     PetscViewerFormat format;
1985     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
1986     if (format == PETSC_VIEWER_HDF5_XDMF || format == PETSC_VIEWER_HDF5_VIZ) {
1987       ierr = DMPlexLoad_HDF5_Xdmf_Internal(dm, viewer);CHKERRQ(ierr);
1988     } else if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
1989       ierr = DMPlexLoad_HDF5_Internal(dm, viewer);CHKERRQ(ierr);
1990     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 input.", PetscViewerFormats[format]);
1991     PetscFunctionReturn(0);
1992 #else
1993     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1994 #endif
1995   } else SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Viewer type %s not yet supported for DMPlex loading", ((PetscObject)viewer)->type_name);
1996 }
1997 
1998 /*@
1999   DMPlexTopologyLoad - Loads a topology into a DMPlex
2000 
2001   Collective on DM
2002 
2003   Input Parameters:
2004 + dm     - The DM into which the topology is loaded
2005 - viewer - The PetscViewer for the saved topology
2006 
2007   Output Parameters:
2008 . globalToLocalPointSF - The PetscSF that pushes points in [0, N) to the associated points in the loaded plex, where N is the global number of points; NULL if unneeded
2009 
2010   Level: advanced
2011 
2012 .seealso: DMLoad(), DMPlexCoordinatesLoad(), DMPlexLabelsLoad(), DMView(), PetscViewerHDF5Open(), PetscViewerPushFormat()
2013 @*/
2014 PetscErrorCode DMPlexTopologyLoad(DM dm, PetscViewer viewer, PetscSF *globalToLocalPointSF)
2015 {
2016   PetscBool      ishdf5;
2017   PetscErrorCode ierr;
2018 
2019   PetscFunctionBegin;
2020   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2021   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
2022   if (globalToLocalPointSF) PetscValidPointer(globalToLocalPointSF, 3);
2023   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
2024   ierr = PetscLogEventBegin(DMPLEX_TopologyLoad,viewer,0,0,0);CHKERRQ(ierr);
2025   if (ishdf5) {
2026 #if defined(PETSC_HAVE_HDF5)
2027     PetscViewerFormat format;
2028     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
2029     if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
2030       ierr = DMPlexTopologyLoad_HDF5_Internal(dm, viewer, globalToLocalPointSF);CHKERRQ(ierr);
2031     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 input.", PetscViewerFormats[format]);
2032 #else
2033     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
2034 #endif
2035   }
2036   ierr = PetscLogEventEnd(DMPLEX_TopologyLoad,viewer,0,0,0);CHKERRQ(ierr);
2037   PetscFunctionReturn(0);
2038 }
2039 
2040 /*@
2041   DMPlexCoordinatesLoad - Loads coordinates into a DMPlex
2042 
2043   Collective on DM
2044 
2045   Input Parameters:
2046 + dm     - The DM into which the coordinates are loaded
2047 . viewer - The PetscViewer for the saved coordinates
2048 - globalToLocalPointSF - The SF returned by DMPlexTopologyLoad() when loading dm from viewer
2049 
2050   Level: advanced
2051 
2052 .seealso: DMLoad(), DMPlexTopologyLoad(), DMPlexLabelsLoad(), DMView(), PetscViewerHDF5Open(), PetscViewerPushFormat()
2053 @*/
2054 PetscErrorCode DMPlexCoordinatesLoad(DM dm, PetscViewer viewer, PetscSF globalToLocalPointSF)
2055 {
2056   PetscBool      ishdf5;
2057   PetscErrorCode ierr;
2058 
2059   PetscFunctionBegin;
2060   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2061   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
2062   PetscValidHeaderSpecific(globalToLocalPointSF, PETSCSF_CLASSID, 3);
2063   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
2064   ierr = PetscLogEventBegin(DMPLEX_CoordinatesLoad,viewer,0,0,0);CHKERRQ(ierr);
2065   if (ishdf5) {
2066 #if defined(PETSC_HAVE_HDF5)
2067     PetscViewerFormat format;
2068     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
2069     if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
2070       ierr = DMPlexCoordinatesLoad_HDF5_Internal(dm, viewer, globalToLocalPointSF);CHKERRQ(ierr);
2071     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 input.", PetscViewerFormats[format]);
2072 #else
2073     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
2074 #endif
2075   }
2076   ierr = PetscLogEventEnd(DMPLEX_CoordinatesLoad,viewer,0,0,0);CHKERRQ(ierr);
2077   PetscFunctionReturn(0);
2078 }
2079 
2080 /*@
2081   DMPlexLabelsLoad - Loads labels into a DMPlex
2082 
2083   Collective on DM
2084 
2085   Input Parameters:
2086 + dm     - The DM into which the labels are loaded
2087 - viewer - The PetscViewer for the saved labels
2088 
2089   Level: advanced
2090 
2091 .seealso: DMLoad(), DMPlexTopologyLoad(), DMPlexCoordinatesLoad(), DMView(), PetscViewerHDF5Open(), PetscViewerPushFormat()
2092 @*/
2093 PetscErrorCode DMPlexLabelsLoad(DM dm, PetscViewer viewer)
2094 {
2095   PetscBool      ishdf5;
2096   PetscErrorCode ierr;
2097 
2098   PetscFunctionBegin;
2099   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2100   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
2101   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
2102   ierr = PetscLogEventBegin(DMPLEX_LabelsLoad,viewer,0,0,0);CHKERRQ(ierr);
2103   if (ishdf5) {
2104 #if defined(PETSC_HAVE_HDF5)
2105     PetscViewerFormat format;
2106 
2107     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
2108     if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
2109       ierr = DMPlexLabelsLoad_HDF5_Internal(dm, viewer);CHKERRQ(ierr);
2110     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 input.", PetscViewerFormats[format]);
2111 #else
2112     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
2113 #endif
2114   }
2115   ierr = PetscLogEventEnd(DMPLEX_LabelsLoad,viewer,0,0,0);CHKERRQ(ierr);
2116   PetscFunctionReturn(0);
2117 }
2118 
2119 /*@
2120   DMPlexSectionLoad - Loads section into a DMPlex
2121 
2122   Collective on DM
2123 
2124   Input Parameters:
2125 + dm          - The DM that represents the topology
2126 . viewer      - The PetscViewer that represents the on-disk section (sectionA)
2127 . sectiondm   - The DM into which the on-disk section (sectionA) is migrated
2128 - globalToLocalPointSF - The SF returned by DMPlexTopologyLoad() when loading dm from viewer
2129 
2130   Output Parameters
2131 + globalDofSF - The SF that migrates any on-disk Vec data associated with sectionA into a global Vec associated with the sectiondm's global section (NULL if not needed)
2132 - localDofSF  - The SF that migrates any on-disk Vec data associated with sectionA into a local Vec associated with the sectiondm's local section (NULL if not needed)
2133 
2134   Level: advanced
2135 
2136   Notes:
2137   This function is a wrapper around PetscSectionLoad(); it loads, in addition to the raw section, a list of global point numbers that associates each on-disk section point with a global point number in [0, NX), where NX is the number of topology points in dm. Noting that globalToLocalPointSF associates each topology point in dm with a global number in [0, NX), one can readily establish an association of the on-disk section points with the topology points.
2138 
2139   In general dm and sectiondm are two different objects, the former carrying the topology and the latter carrying the section, and have been given a topology name and a section name, respectively, with PetscObjectSetName(). In practice, however, they can be the same object if it carries both topology and section; in that case the name of the object is used as both the topology name and the section name.
2140 
2141   The output parameter, globalDofSF (localDofSF), can later be used with DMPlexGlobalVectorLoad() (DMPlexLocalVectorLoad()) to load on-disk vectors into global (local) vectors associated with sectiondm's global (local) section.
2142 
2143   Example using 2 processes:
2144 $  NX (number of points on dm): 4
2145 $  sectionA                   : the on-disk section
2146 $  vecA                       : a vector associated with sectionA
2147 $  sectionB                   : sectiondm's local section constructed in this function
2148 $  vecB (local)               : a vector associated with sectiondm's local section
2149 $  vecB (global)              : a vector associated with sectiondm's global section
2150 $
2151 $                                     rank 0    rank 1
2152 $  vecA (global)                  : [.0 .4 .1 | .2 .3]        <- to be loaded in DMPlexGlobalVectorLoad() or DMPlexLocalVectorLoad()
2153 $  sectionA->atlasOff             :       0 2 | 1             <- loaded in PetscSectionLoad()
2154 $  sectionA->atlasDof             :       1 3 | 1             <- loaded in PetscSectionLoad()
2155 $  sectionA's global point numbers:       0 2 | 3             <- loaded in DMPlexSectionLoad()
2156 $  [0, NX)                        :       0 1 | 2 3           <- conceptual partition used in globalToLocalPointSF
2157 $  sectionB's global point numbers:     0 1 3 | 3 2           <- associated with [0, NX) by globalToLocalPointSF
2158 $  sectionB->atlasDof             :     1 0 1 | 1 3
2159 $  sectionB->atlasOff (no perm)   :     0 1 1 | 0 1
2160 $  vecB (local)                   :   [.0 .4] | [.4 .1 .2 .3] <- to be constructed by calling DMPlexLocalVectorLoad() with localDofSF
2161 $  vecB (global)                  :    [.0 .4 | .1 .2 .3]     <- to be constructed by calling DMPlexGlobalVectorLoad() with globalDofSF
2162 $
2163 $  where "|" represents a partition of loaded data, and global point 3 is assumed to be owned by rank 0.
2164 
2165 .seealso: DMLoad(), DMPlexTopologyLoad(), DMPlexCoordinatesLoad(), DMPlexLabelsLoad(), DMPlexGlobalVectorLoad(), DMPlexLocalVectorLoad(), PetscSectionLoad(), DMPlexSectionView()
2166 @*/
2167 PetscErrorCode DMPlexSectionLoad(DM dm, PetscViewer viewer, DM sectiondm, PetscSF globalToLocalPointSF, PetscSF *globalDofSF, PetscSF *localDofSF)
2168 {
2169   PetscBool      ishdf5;
2170   PetscErrorCode ierr;
2171 
2172   PetscFunctionBegin;
2173   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2174   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
2175   PetscValidHeaderSpecific(sectiondm, DM_CLASSID, 3);
2176   PetscValidHeaderSpecific(globalToLocalPointSF, PETSCSF_CLASSID, 4);
2177   if (globalDofSF) PetscValidPointer(globalDofSF, 5);
2178   if (localDofSF) PetscValidPointer(localDofSF, 6);
2179   ierr = PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERHDF5,&ishdf5);CHKERRQ(ierr);
2180   ierr = PetscLogEventBegin(DMPLEX_SectionLoad,viewer,0,0,0);CHKERRQ(ierr);
2181   if (ishdf5) {
2182 #if defined(PETSC_HAVE_HDF5)
2183     ierr = DMPlexSectionLoad_HDF5_Internal(dm, viewer, sectiondm, globalToLocalPointSF, globalDofSF, localDofSF);CHKERRQ(ierr);
2184 #else
2185     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
2186 #endif
2187   }
2188   ierr = PetscLogEventEnd(DMPLEX_SectionLoad,viewer,0,0,0);CHKERRQ(ierr);
2189   PetscFunctionReturn(0);
2190 }
2191 
2192 /*@
2193   DMPlexGlobalVectorLoad - Loads on-disk vector data into a global vector
2194 
2195   Collective on DM
2196 
2197   Input Parameters:
2198 + dm        - The DM that represents the topology
2199 . viewer    - The PetscViewer that represents the on-disk vector data
2200 . sectiondm - The DM that contains the global section on which vec is defined
2201 . sf        - The SF that migrates the on-disk vector data into vec
2202 - vec       - The global vector to set values of
2203 
2204   Level: advanced
2205 
2206   Notes:
2207   In general dm and sectiondm are two different objects, the former carrying the topology and the latter carrying the section, and have been given a topology name and a section name, respectively, with PetscObjectSetName(). In practice, however, they can be the same object if it carries both topology and section; in that case the name of the object is used as both the topology name and the section name.
2208 
2209   Typical calling sequence
2210 $       DMCreate(PETSC_COMM_WORLD, &dm);
2211 $       DMSetType(dm, DMPLEX);
2212 $       PetscObjectSetName((PetscObject)dm, "topologydm_name");
2213 $       DMPlexTopologyLoad(dm, viewer, &sfX);
2214 $       DMClone(dm, &sectiondm);
2215 $       PetscObjectSetName((PetscObject)sectiondm, "sectiondm_name");
2216 $       DMPlexSectionLoad(dm, viewer, sectiondm, sfX, &gsf, NULL);
2217 $       DMGetGlobalVector(sectiondm, &vec);
2218 $       PetscObjectSetName((PetscObject)vec, "vec_name");
2219 $       DMPlexGlobalVectorLoad(dm, viewer, sectiondm, gsf, vec);
2220 $       DMRestoreGlobalVector(sectiondm, &vec);
2221 $       PetscSFDestroy(&gsf);
2222 $       PetscSFDestroy(&sfX);
2223 $       DMDestroy(&sectiondm);
2224 $       DMDestroy(&dm);
2225 
2226 .seealso: DMPlexTopologyLoad(), DMPlexSectionLoad(), DMPlexLocalVectorLoad(), DMPlexGlobalVectorView(), DMPlexLocalVectorView()
2227 @*/
2228 PetscErrorCode DMPlexGlobalVectorLoad(DM dm, PetscViewer viewer, DM sectiondm, PetscSF sf, Vec vec)
2229 {
2230   PetscBool       ishdf5;
2231   PetscErrorCode  ierr;
2232 
2233   PetscFunctionBegin;
2234   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2235   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
2236   PetscValidHeaderSpecific(sectiondm, DM_CLASSID, 3);
2237   PetscValidHeaderSpecific(sf, PETSCSF_CLASSID, 4);
2238   PetscValidHeaderSpecific(vec, VEC_CLASSID, 5);
2239   /* Check consistency */
2240   {
2241     PetscSection  section;
2242     PetscBool     includesConstraints;
2243     PetscInt      m, m1;
2244 
2245     ierr = VecGetLocalSize(vec, &m1);CHKERRQ(ierr);
2246     ierr = DMGetGlobalSection(sectiondm, &section);CHKERRQ(ierr);
2247     ierr = PetscSectionGetIncludesConstraints(section, &includesConstraints);CHKERRQ(ierr);
2248     if (includesConstraints) {ierr = PetscSectionGetStorageSize(section, &m);CHKERRQ(ierr);}
2249     else {ierr = PetscSectionGetConstrainedStorageSize(section, &m);CHKERRQ(ierr);}
2250     if (m1 != m) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Global vector size (%D) != global section storage size (%D)", m1, m);
2251   }
2252   ierr = PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERHDF5,&ishdf5);CHKERRQ(ierr);
2253   ierr = PetscLogEventBegin(DMPLEX_GlobalVectorLoad,viewer,0,0,0);CHKERRQ(ierr);
2254   if (ishdf5) {
2255 #if defined(PETSC_HAVE_HDF5)
2256     ierr = DMPlexVecLoad_HDF5_Internal(dm, viewer, sectiondm, sf, vec);CHKERRQ(ierr);
2257 #else
2258     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
2259 #endif
2260   }
2261   ierr = PetscLogEventEnd(DMPLEX_GlobalVectorLoad,viewer,0,0,0);CHKERRQ(ierr);
2262   PetscFunctionReturn(0);
2263 }
2264 
2265 /*@
2266   DMPlexLocalVectorLoad - Loads on-disk vector data into a local vector
2267 
2268   Collective on DM
2269 
2270   Input Parameters:
2271 + dm        - The DM that represents the topology
2272 . viewer    - The PetscViewer that represents the on-disk vector data
2273 . sectiondm - The DM that contains the local section on which vec is defined
2274 . sf        - The SF that migrates the on-disk vector data into vec
2275 - vec       - The local vector to set values of
2276 
2277   Level: advanced
2278 
2279   Notes:
2280   In general dm and sectiondm are two different objects, the former carrying the topology and the latter carrying the section, and have been given a topology name and a section name, respectively, with PetscObjectSetName(). In practice, however, they can be the same object if it carries both topology and section; in that case the name of the object is used as both the topology name and the section name.
2281 
2282   Typical calling sequence
2283 $       DMCreate(PETSC_COMM_WORLD, &dm);
2284 $       DMSetType(dm, DMPLEX);
2285 $       PetscObjectSetName((PetscObject)dm, "topologydm_name");
2286 $       DMPlexTopologyLoad(dm, viewer, &sfX);
2287 $       DMClone(dm, &sectiondm);
2288 $       PetscObjectSetName((PetscObject)sectiondm, "sectiondm_name");
2289 $       DMPlexSectionLoad(dm, viewer, sectiondm, sfX, NULL, &lsf);
2290 $       DMGetLocalVector(sectiondm, &vec);
2291 $       PetscObjectSetName((PetscObject)vec, "vec_name");
2292 $       DMPlexLocalVectorLoad(dm, viewer, sectiondm, lsf, vec);
2293 $       DMRestoreLocalVector(sectiondm, &vec);
2294 $       PetscSFDestroy(&lsf);
2295 $       PetscSFDestroy(&sfX);
2296 $       DMDestroy(&sectiondm);
2297 $       DMDestroy(&dm);
2298 
2299 .seealso: DMPlexTopologyLoad(), DMPlexSectionLoad(), DMPlexGlobalVectorLoad(), DMPlexGlobalVectorView(), DMPlexLocalVectorView()
2300 @*/
2301 PetscErrorCode DMPlexLocalVectorLoad(DM dm, PetscViewer viewer, DM sectiondm, PetscSF sf, Vec vec)
2302 {
2303   PetscBool       ishdf5;
2304   PetscErrorCode  ierr;
2305 
2306   PetscFunctionBegin;
2307   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2308   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
2309   PetscValidHeaderSpecific(sectiondm, DM_CLASSID, 3);
2310   PetscValidHeaderSpecific(sf, PETSCSF_CLASSID, 4);
2311   PetscValidHeaderSpecific(vec, VEC_CLASSID, 5);
2312   /* Check consistency */
2313   {
2314     PetscSection  section;
2315     PetscBool     includesConstraints;
2316     PetscInt      m, m1;
2317 
2318     ierr = VecGetLocalSize(vec, &m1);CHKERRQ(ierr);
2319     ierr = DMGetLocalSection(sectiondm, &section);CHKERRQ(ierr);
2320     ierr = PetscSectionGetIncludesConstraints(section, &includesConstraints);CHKERRQ(ierr);
2321     if (includesConstraints) {ierr = PetscSectionGetStorageSize(section, &m);CHKERRQ(ierr);}
2322     else {ierr = PetscSectionGetConstrainedStorageSize(section, &m);CHKERRQ(ierr);}
2323     if (m1 != m) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Local vector size (%D) != local section storage size (%D)", m1, m);
2324   }
2325   ierr = PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERHDF5,&ishdf5);CHKERRQ(ierr);
2326   ierr = PetscLogEventBegin(DMPLEX_LocalVectorLoad,viewer,0,0,0);CHKERRQ(ierr);
2327   if (ishdf5) {
2328 #if defined(PETSC_HAVE_HDF5)
2329     ierr = DMPlexVecLoad_HDF5_Internal(dm, viewer, sectiondm, sf, vec);CHKERRQ(ierr);
2330 #else
2331     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
2332 #endif
2333   }
2334   ierr = PetscLogEventEnd(DMPLEX_LocalVectorLoad,viewer,0,0,0);CHKERRQ(ierr);
2335   PetscFunctionReturn(0);
2336 }
2337 
2338 PetscErrorCode DMDestroy_Plex(DM dm)
2339 {
2340   DM_Plex       *mesh = (DM_Plex*) dm->data;
2341   PetscErrorCode ierr;
2342 
2343   PetscFunctionBegin;
2344   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMSetUpGLVisViewer_C",NULL);CHKERRQ(ierr);
2345   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMPlexInsertBoundaryValues_C", NULL);CHKERRQ(ierr);
2346   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMCreateNeumannOverlap_C", NULL);CHKERRQ(ierr);
2347   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMInterpolateSolution_C", NULL);CHKERRQ(ierr);
2348   if (--mesh->refct > 0) PetscFunctionReturn(0);
2349   ierr = PetscSectionDestroy(&mesh->coneSection);CHKERRQ(ierr);
2350   ierr = PetscFree(mesh->cones);CHKERRQ(ierr);
2351   ierr = PetscFree(mesh->coneOrientations);CHKERRQ(ierr);
2352   ierr = PetscSectionDestroy(&mesh->supportSection);CHKERRQ(ierr);
2353   ierr = PetscSectionDestroy(&mesh->subdomainSection);CHKERRQ(ierr);
2354   ierr = PetscFree(mesh->supports);CHKERRQ(ierr);
2355   ierr = PetscFree(mesh->facesTmp);CHKERRQ(ierr);
2356   ierr = PetscFree(mesh->tetgenOpts);CHKERRQ(ierr);
2357   ierr = PetscFree(mesh->triangleOpts);CHKERRQ(ierr);
2358   ierr = PetscFree(mesh->transformType);CHKERRQ(ierr);
2359   ierr = PetscPartitionerDestroy(&mesh->partitioner);CHKERRQ(ierr);
2360   ierr = DMLabelDestroy(&mesh->subpointMap);CHKERRQ(ierr);
2361   ierr = ISDestroy(&mesh->subpointIS);CHKERRQ(ierr);
2362   ierr = ISDestroy(&mesh->globalVertexNumbers);CHKERRQ(ierr);
2363   ierr = ISDestroy(&mesh->globalCellNumbers);CHKERRQ(ierr);
2364   ierr = PetscSectionDestroy(&mesh->anchorSection);CHKERRQ(ierr);
2365   ierr = ISDestroy(&mesh->anchorIS);CHKERRQ(ierr);
2366   ierr = PetscSectionDestroy(&mesh->parentSection);CHKERRQ(ierr);
2367   ierr = PetscFree(mesh->parents);CHKERRQ(ierr);
2368   ierr = PetscFree(mesh->childIDs);CHKERRQ(ierr);
2369   ierr = PetscSectionDestroy(&mesh->childSection);CHKERRQ(ierr);
2370   ierr = PetscFree(mesh->children);CHKERRQ(ierr);
2371   ierr = DMDestroy(&mesh->referenceTree);CHKERRQ(ierr);
2372   ierr = PetscGridHashDestroy(&mesh->lbox);CHKERRQ(ierr);
2373   ierr = PetscFree(mesh->neighbors);CHKERRQ(ierr);
2374   if (mesh->metricCtx) { ierr = PetscFree(mesh->metricCtx);CHKERRQ(ierr); }
2375   /* This was originally freed in DMDestroy(), but that prevents reference counting of backend objects */
2376   ierr = PetscFree(mesh);CHKERRQ(ierr);
2377   PetscFunctionReturn(0);
2378 }
2379 
2380 PetscErrorCode DMCreateMatrix_Plex(DM dm, Mat *J)
2381 {
2382   PetscSection           sectionGlobal;
2383   PetscInt               bs = -1, mbs;
2384   PetscInt               localSize;
2385   PetscBool              isShell, isBlock, isSeqBlock, isMPIBlock, isSymBlock, isSymSeqBlock, isSymMPIBlock, isMatIS;
2386   PetscErrorCode         ierr;
2387   MatType                mtype;
2388   ISLocalToGlobalMapping ltog;
2389 
2390   PetscFunctionBegin;
2391   ierr = MatInitializePackage();CHKERRQ(ierr);
2392   mtype = dm->mattype;
2393   ierr = DMGetGlobalSection(dm, &sectionGlobal);CHKERRQ(ierr);
2394   /* ierr = PetscSectionGetStorageSize(sectionGlobal, &localSize);CHKERRQ(ierr); */
2395   ierr = PetscSectionGetConstrainedStorageSize(sectionGlobal, &localSize);CHKERRQ(ierr);
2396   ierr = MatCreate(PetscObjectComm((PetscObject)dm), J);CHKERRQ(ierr);
2397   ierr = MatSetSizes(*J, localSize, localSize, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
2398   ierr = MatSetType(*J, mtype);CHKERRQ(ierr);
2399   ierr = MatSetFromOptions(*J);CHKERRQ(ierr);
2400   ierr = MatGetBlockSize(*J, &mbs);CHKERRQ(ierr);
2401   if (mbs > 1) bs = mbs;
2402   ierr = PetscStrcmp(mtype, MATSHELL, &isShell);CHKERRQ(ierr);
2403   ierr = PetscStrcmp(mtype, MATBAIJ, &isBlock);CHKERRQ(ierr);
2404   ierr = PetscStrcmp(mtype, MATSEQBAIJ, &isSeqBlock);CHKERRQ(ierr);
2405   ierr = PetscStrcmp(mtype, MATMPIBAIJ, &isMPIBlock);CHKERRQ(ierr);
2406   ierr = PetscStrcmp(mtype, MATSBAIJ, &isSymBlock);CHKERRQ(ierr);
2407   ierr = PetscStrcmp(mtype, MATSEQSBAIJ, &isSymSeqBlock);CHKERRQ(ierr);
2408   ierr = PetscStrcmp(mtype, MATMPISBAIJ, &isSymMPIBlock);CHKERRQ(ierr);
2409   ierr = PetscStrcmp(mtype, MATIS, &isMatIS);CHKERRQ(ierr);
2410   if (!isShell) {
2411     PetscSection subSection;
2412     PetscBool    fillMatrix = (PetscBool)(!dm->prealloc_only && !isMatIS);
2413     PetscInt    *dnz, *onz, *dnzu, *onzu, bsLocal[2], bsMinMax[2], *ltogidx, lsize;
2414     PetscInt     pStart, pEnd, p, dof, cdof;
2415 
2416     /* Set localtoglobalmapping on the matrix for MatSetValuesLocal() to work (it also creates the local matrices in case of MATIS) */
2417     if (isMatIS) { /* need a different l2g map than the one computed by DMGetLocalToGlobalMapping */
2418       PetscSection section;
2419       PetscInt     size;
2420 
2421       ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);
2422       ierr = PetscSectionGetStorageSize(section, &size);CHKERRQ(ierr);
2423       ierr = PetscMalloc1(size,&ltogidx);CHKERRQ(ierr);
2424       ierr = DMPlexGetSubdomainSection(dm, &subSection);CHKERRQ(ierr);
2425     } else {
2426       ierr = DMGetLocalToGlobalMapping(dm,&ltog);CHKERRQ(ierr);
2427     }
2428     ierr = PetscSectionGetChart(sectionGlobal, &pStart, &pEnd);CHKERRQ(ierr);
2429     for (p = pStart, lsize = 0; p < pEnd; ++p) {
2430       PetscInt bdof;
2431 
2432       ierr = PetscSectionGetDof(sectionGlobal, p, &dof);CHKERRQ(ierr);
2433       ierr = PetscSectionGetConstraintDof(sectionGlobal, p, &cdof);CHKERRQ(ierr);
2434       dof  = dof < 0 ? -(dof+1) : dof;
2435       bdof = cdof && (dof-cdof) ? 1 : dof;
2436       if (dof) {
2437         if (bs < 0)          {bs = bdof;}
2438         else if (bs != bdof) {bs = 1; if (!isMatIS) break;}
2439       }
2440       if (isMatIS) {
2441         PetscInt loff,c,off;
2442         ierr = PetscSectionGetOffset(subSection, p, &loff);CHKERRQ(ierr);
2443         ierr = PetscSectionGetOffset(sectionGlobal, p, &off);CHKERRQ(ierr);
2444         for (c = 0; c < dof-cdof; ++c, ++lsize) ltogidx[loff+c] = off > -1 ? off+c : -(off+1)+c;
2445       }
2446     }
2447     /* Must have same blocksize on all procs (some might have no points) */
2448     bsLocal[0] = bs < 0 ? PETSC_MAX_INT : bs; bsLocal[1] = bs;
2449     ierr = PetscGlobalMinMaxInt(PetscObjectComm((PetscObject) dm), bsLocal, bsMinMax);CHKERRQ(ierr);
2450     if (bsMinMax[0] != bsMinMax[1]) {bs = 1;}
2451     else                            {bs = bsMinMax[0];}
2452     bs = PetscMax(1,bs);
2453     if (isMatIS) { /* Must reduce indices by blocksize */
2454       PetscInt l;
2455 
2456       lsize = lsize/bs;
2457       if (bs > 1) for (l = 0; l < lsize; ++l) ltogidx[l] = ltogidx[l*bs]/bs;
2458       ierr = ISLocalToGlobalMappingCreate(PetscObjectComm((PetscObject)dm), bs, lsize, ltogidx, PETSC_OWN_POINTER, &ltog);CHKERRQ(ierr);
2459     }
2460     ierr = MatSetLocalToGlobalMapping(*J,ltog,ltog);CHKERRQ(ierr);
2461     if (isMatIS) {
2462       ierr = ISLocalToGlobalMappingDestroy(&ltog);CHKERRQ(ierr);
2463     }
2464     ierr = PetscCalloc4(localSize/bs, &dnz, localSize/bs, &onz, localSize/bs, &dnzu, localSize/bs, &onzu);CHKERRQ(ierr);
2465     ierr = DMPlexPreallocateOperator(dm, bs, dnz, onz, dnzu, onzu, *J, fillMatrix);CHKERRQ(ierr);
2466     ierr = PetscFree4(dnz, onz, dnzu, onzu);CHKERRQ(ierr);
2467   }
2468   ierr = MatSetDM(*J, dm);CHKERRQ(ierr);
2469   PetscFunctionReturn(0);
2470 }
2471 
2472 /*@
2473   DMPlexGetSubdomainSection - Returns the section associated with the subdomain
2474 
2475   Not collective
2476 
2477   Input Parameter:
2478 . mesh - The DMPlex
2479 
2480   Output Parameters:
2481 . subsection - The subdomain section
2482 
2483   Level: developer
2484 
2485 .seealso:
2486 @*/
2487 PetscErrorCode DMPlexGetSubdomainSection(DM dm, PetscSection *subsection)
2488 {
2489   DM_Plex       *mesh = (DM_Plex*) dm->data;
2490   PetscErrorCode ierr;
2491 
2492   PetscFunctionBegin;
2493   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2494   if (!mesh->subdomainSection) {
2495     PetscSection section;
2496     PetscSF      sf;
2497 
2498     ierr = PetscSFCreate(PETSC_COMM_SELF,&sf);CHKERRQ(ierr);
2499     ierr = DMGetLocalSection(dm,&section);CHKERRQ(ierr);
2500     ierr = PetscSectionCreateGlobalSection(section,sf,PETSC_FALSE,PETSC_TRUE,&mesh->subdomainSection);CHKERRQ(ierr);
2501     ierr = PetscSFDestroy(&sf);CHKERRQ(ierr);
2502   }
2503   *subsection = mesh->subdomainSection;
2504   PetscFunctionReturn(0);
2505 }
2506 
2507 /*@
2508   DMPlexGetChart - Return the interval for all mesh points [pStart, pEnd)
2509 
2510   Not collective
2511 
2512   Input Parameter:
2513 . mesh - The DMPlex
2514 
2515   Output Parameters:
2516 + pStart - The first mesh point
2517 - pEnd   - The upper bound for mesh points
2518 
2519   Level: beginner
2520 
2521 .seealso: DMPlexCreate(), DMPlexSetChart()
2522 @*/
2523 PetscErrorCode DMPlexGetChart(DM dm, PetscInt *pStart, PetscInt *pEnd)
2524 {
2525   DM_Plex       *mesh = (DM_Plex*) dm->data;
2526   PetscErrorCode ierr;
2527 
2528   PetscFunctionBegin;
2529   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2530   ierr = PetscSectionGetChart(mesh->coneSection, pStart, pEnd);CHKERRQ(ierr);
2531   PetscFunctionReturn(0);
2532 }
2533 
2534 /*@
2535   DMPlexSetChart - Set the interval for all mesh points [pStart, pEnd)
2536 
2537   Not collective
2538 
2539   Input Parameters:
2540 + mesh - The DMPlex
2541 . pStart - The first mesh point
2542 - pEnd   - The upper bound for mesh points
2543 
2544   Output Parameters:
2545 
2546   Level: beginner
2547 
2548 .seealso: DMPlexCreate(), DMPlexGetChart()
2549 @*/
2550 PetscErrorCode DMPlexSetChart(DM dm, PetscInt pStart, PetscInt pEnd)
2551 {
2552   DM_Plex       *mesh = (DM_Plex*) dm->data;
2553   PetscErrorCode ierr;
2554 
2555   PetscFunctionBegin;
2556   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2557   ierr = PetscSectionSetChart(mesh->coneSection, pStart, pEnd);CHKERRQ(ierr);
2558   ierr = PetscSectionSetChart(mesh->supportSection, pStart, pEnd);CHKERRQ(ierr);
2559   PetscFunctionReturn(0);
2560 }
2561 
2562 /*@
2563   DMPlexGetConeSize - Return the number of in-edges for this point in the DAG
2564 
2565   Not collective
2566 
2567   Input Parameters:
2568 + mesh - The DMPlex
2569 - p - The point, which must lie in the chart set with DMPlexSetChart()
2570 
2571   Output Parameter:
2572 . size - The cone size for point p
2573 
2574   Level: beginner
2575 
2576 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart()
2577 @*/
2578 PetscErrorCode DMPlexGetConeSize(DM dm, PetscInt p, PetscInt *size)
2579 {
2580   DM_Plex       *mesh = (DM_Plex*) dm->data;
2581   PetscErrorCode ierr;
2582 
2583   PetscFunctionBegin;
2584   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2585   PetscValidPointer(size, 3);
2586   ierr = PetscSectionGetDof(mesh->coneSection, p, size);CHKERRQ(ierr);
2587   PetscFunctionReturn(0);
2588 }
2589 
2590 /*@
2591   DMPlexSetConeSize - Set the number of in-edges for this point in the DAG
2592 
2593   Not collective
2594 
2595   Input Parameters:
2596 + mesh - The DMPlex
2597 . p - The point, which must lie in the chart set with DMPlexSetChart()
2598 - size - The cone size for point p
2599 
2600   Output Parameter:
2601 
2602   Note:
2603   This should be called after DMPlexSetChart().
2604 
2605   Level: beginner
2606 
2607 .seealso: DMPlexCreate(), DMPlexGetConeSize(), DMPlexSetChart()
2608 @*/
2609 PetscErrorCode DMPlexSetConeSize(DM dm, PetscInt p, PetscInt size)
2610 {
2611   DM_Plex       *mesh = (DM_Plex*) dm->data;
2612   PetscErrorCode ierr;
2613 
2614   PetscFunctionBegin;
2615   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2616   ierr = PetscSectionSetDof(mesh->coneSection, p, size);CHKERRQ(ierr);
2617 
2618   mesh->maxConeSize = PetscMax(mesh->maxConeSize, size);
2619   PetscFunctionReturn(0);
2620 }
2621 
2622 /*@
2623   DMPlexAddConeSize - Add the given number of in-edges to this point in the DAG
2624 
2625   Not collective
2626 
2627   Input Parameters:
2628 + mesh - The DMPlex
2629 . p - The point, which must lie in the chart set with DMPlexSetChart()
2630 - size - The additional cone size for point p
2631 
2632   Output Parameter:
2633 
2634   Note:
2635   This should be called after DMPlexSetChart().
2636 
2637   Level: beginner
2638 
2639 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexGetConeSize(), DMPlexSetChart()
2640 @*/
2641 PetscErrorCode DMPlexAddConeSize(DM dm, PetscInt p, PetscInt size)
2642 {
2643   DM_Plex       *mesh = (DM_Plex*) dm->data;
2644   PetscInt       csize;
2645   PetscErrorCode ierr;
2646 
2647   PetscFunctionBegin;
2648   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2649   ierr = PetscSectionAddDof(mesh->coneSection, p, size);CHKERRQ(ierr);
2650   ierr = PetscSectionGetDof(mesh->coneSection, p, &csize);CHKERRQ(ierr);
2651 
2652   mesh->maxConeSize = PetscMax(mesh->maxConeSize, csize);
2653   PetscFunctionReturn(0);
2654 }
2655 
2656 /*@C
2657   DMPlexGetCone - Return the points on the in-edges for this point in the DAG
2658 
2659   Not collective
2660 
2661   Input Parameters:
2662 + dm - The DMPlex
2663 - p - The point, which must lie in the chart set with DMPlexSetChart()
2664 
2665   Output Parameter:
2666 . cone - An array of points which are on the in-edges for point p
2667 
2668   Level: beginner
2669 
2670   Fortran Notes:
2671   Since it returns an array, this routine is only available in Fortran 90, and you must
2672   include petsc.h90 in your code.
2673   You must also call DMPlexRestoreCone() after you finish using the returned array.
2674   DMPlexRestoreCone() is not needed/available in C.
2675 
2676 .seealso: DMPlexGetConeSize(), DMPlexSetCone(), DMPlexGetConeTuple(), DMPlexSetChart()
2677 @*/
2678 PetscErrorCode DMPlexGetCone(DM dm, PetscInt p, const PetscInt *cone[])
2679 {
2680   DM_Plex       *mesh = (DM_Plex*) dm->data;
2681   PetscInt       off;
2682   PetscErrorCode ierr;
2683 
2684   PetscFunctionBegin;
2685   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2686   PetscValidPointer(cone, 3);
2687   ierr  = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2688   *cone = &mesh->cones[off];
2689   PetscFunctionReturn(0);
2690 }
2691 
2692 /*@C
2693   DMPlexGetConeTuple - Return the points on the in-edges of several points in the DAG
2694 
2695   Not collective
2696 
2697   Input Parameters:
2698 + dm - The DMPlex
2699 - p - The IS of points, which must lie in the chart set with DMPlexSetChart()
2700 
2701   Output Parameters:
2702 + pConesSection - PetscSection describing the layout of pCones
2703 - pCones - An array of points which are on the in-edges for the point set p
2704 
2705   Level: intermediate
2706 
2707 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexGetConeRecursive(), DMPlexSetChart()
2708 @*/
2709 PetscErrorCode DMPlexGetConeTuple(DM dm, IS p, PetscSection *pConesSection, IS *pCones)
2710 {
2711   PetscSection        cs, newcs;
2712   PetscInt            *cones;
2713   PetscInt            *newarr=NULL;
2714   PetscInt            n;
2715   PetscErrorCode      ierr;
2716 
2717   PetscFunctionBegin;
2718   ierr = DMPlexGetCones(dm, &cones);CHKERRQ(ierr);
2719   ierr = DMPlexGetConeSection(dm, &cs);CHKERRQ(ierr);
2720   ierr = PetscSectionExtractDofsFromArray(cs, MPIU_INT, cones, p, &newcs, pCones ? ((void**)&newarr) : NULL);CHKERRQ(ierr);
2721   if (pConesSection) *pConesSection = newcs;
2722   if (pCones) {
2723     ierr = PetscSectionGetStorageSize(newcs, &n);CHKERRQ(ierr);
2724     ierr = ISCreateGeneral(PetscObjectComm((PetscObject)p), n, newarr, PETSC_OWN_POINTER, pCones);CHKERRQ(ierr);
2725   }
2726   PetscFunctionReturn(0);
2727 }
2728 
2729 /*@
2730   DMPlexGetConeRecursiveVertices - Expand each given point into its cone points and do that recursively until we end up just with vertices.
2731 
2732   Not collective
2733 
2734   Input Parameters:
2735 + dm - The DMPlex
2736 - points - The IS of points, which must lie in the chart set with DMPlexSetChart()
2737 
2738   Output Parameter:
2739 . expandedPoints - An array of vertices recursively expanded from input points
2740 
2741   Level: advanced
2742 
2743   Notes:
2744   Like DMPlexGetConeRecursive but returns only the 0-depth IS (i.e. vertices only) and no sections.
2745   There is no corresponding Restore function, just call ISDestroy() on the returned IS to deallocate.
2746 
2747 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexGetConeTuple(), DMPlexGetConeRecursive(), DMPlexRestoreConeRecursive(), DMPlexGetDepth()
2748 @*/
2749 PetscErrorCode DMPlexGetConeRecursiveVertices(DM dm, IS points, IS *expandedPoints)
2750 {
2751   IS                  *expandedPointsAll;
2752   PetscInt            depth;
2753   PetscErrorCode      ierr;
2754 
2755   PetscFunctionBegin;
2756   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2757   PetscValidHeaderSpecific(points, IS_CLASSID, 2);
2758   PetscValidPointer(expandedPoints, 3);
2759   ierr = DMPlexGetConeRecursive(dm, points, &depth, &expandedPointsAll, NULL);CHKERRQ(ierr);
2760   *expandedPoints = expandedPointsAll[0];
2761   ierr = PetscObjectReference((PetscObject)expandedPointsAll[0]);CHKERRQ(ierr);
2762   ierr = DMPlexRestoreConeRecursive(dm, points, &depth, &expandedPointsAll, NULL);CHKERRQ(ierr);
2763   PetscFunctionReturn(0);
2764 }
2765 
2766 /*@
2767   DMPlexGetConeRecursive - Expand each given point into its cone points and do that recursively until we end up just with vertices (DAG points of depth 0, i.e. without cones).
2768 
2769   Not collective
2770 
2771   Input Parameters:
2772 + dm - The DMPlex
2773 - points - The IS of points, which must lie in the chart set with DMPlexSetChart()
2774 
2775   Output Parameters:
2776 + depth - (optional) Size of the output arrays, equal to DMPlex depth, returned by DMPlexGetDepth()
2777 . expandedPoints - (optional) An array of index sets with recursively expanded cones
2778 - sections - (optional) An array of sections which describe mappings from points to their cone points
2779 
2780   Level: advanced
2781 
2782   Notes:
2783   Like DMPlexGetConeTuple() but recursive.
2784 
2785   Array expandedPoints has size equal to depth. Each expandedPoints[d] contains DAG points with maximum depth d, recursively cone-wise expanded from the input points.
2786   For example, for d=0 it contains only vertices, for d=1 it can contain vertices and edges, etc.
2787 
2788   Array section has size equal to depth.  Each PetscSection sections[d] realizes mapping from expandedPoints[d+1] (section points) to expandedPoints[d] (section dofs) as follows:
2789   (1) DAG points in expandedPoints[d+1] with depth d+1 to their cone points in expandedPoints[d];
2790   (2) DAG points in expandedPoints[d+1] with depth in [0,d] to the same points in expandedPoints[d].
2791 
2792 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexGetConeTuple(), DMPlexRestoreConeRecursive(), DMPlexGetConeRecursiveVertices(), DMPlexGetDepth()
2793 @*/
2794 PetscErrorCode DMPlexGetConeRecursive(DM dm, IS points, PetscInt *depth, IS *expandedPoints[], PetscSection *sections[])
2795 {
2796   const PetscInt      *arr0=NULL, *cone=NULL;
2797   PetscInt            *arr=NULL, *newarr=NULL;
2798   PetscInt            d, depth_, i, n, newn, cn, co, start, end;
2799   IS                  *expandedPoints_;
2800   PetscSection        *sections_;
2801   PetscErrorCode      ierr;
2802 
2803   PetscFunctionBegin;
2804   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2805   PetscValidHeaderSpecific(points, IS_CLASSID, 2);
2806   if (depth) PetscValidIntPointer(depth, 3);
2807   if (expandedPoints) PetscValidPointer(expandedPoints, 4);
2808   if (sections) PetscValidPointer(sections, 5);
2809   ierr = ISGetLocalSize(points, &n);CHKERRQ(ierr);
2810   ierr = ISGetIndices(points, &arr0);CHKERRQ(ierr);
2811   ierr = DMPlexGetDepth(dm, &depth_);CHKERRQ(ierr);
2812   ierr = PetscCalloc1(depth_, &expandedPoints_);CHKERRQ(ierr);
2813   ierr = PetscCalloc1(depth_, &sections_);CHKERRQ(ierr);
2814   arr = (PetscInt*) arr0; /* this is ok because first generation of arr is not modified */
2815   for (d=depth_-1; d>=0; d--) {
2816     ierr = PetscSectionCreate(PETSC_COMM_SELF, &sections_[d]);CHKERRQ(ierr);
2817     ierr = PetscSectionSetChart(sections_[d], 0, n);CHKERRQ(ierr);
2818     for (i=0; i<n; i++) {
2819       ierr = DMPlexGetDepthStratum(dm, d+1, &start, &end);CHKERRQ(ierr);
2820       if (arr[i] >= start && arr[i] < end) {
2821         ierr = DMPlexGetConeSize(dm, arr[i], &cn);CHKERRQ(ierr);
2822         ierr = PetscSectionSetDof(sections_[d], i, cn);CHKERRQ(ierr);
2823       } else {
2824         ierr = PetscSectionSetDof(sections_[d], i, 1);CHKERRQ(ierr);
2825       }
2826     }
2827     ierr = PetscSectionSetUp(sections_[d]);CHKERRQ(ierr);
2828     ierr = PetscSectionGetStorageSize(sections_[d], &newn);CHKERRQ(ierr);
2829     ierr = PetscMalloc1(newn, &newarr);CHKERRQ(ierr);
2830     for (i=0; i<n; i++) {
2831       ierr = PetscSectionGetDof(sections_[d], i, &cn);CHKERRQ(ierr);
2832       ierr = PetscSectionGetOffset(sections_[d], i, &co);CHKERRQ(ierr);
2833       if (cn > 1) {
2834         ierr = DMPlexGetCone(dm, arr[i], &cone);CHKERRQ(ierr);
2835         ierr = PetscMemcpy(&newarr[co], cone, cn*sizeof(PetscInt));CHKERRQ(ierr);
2836       } else {
2837         newarr[co] = arr[i];
2838       }
2839     }
2840     ierr = ISCreateGeneral(PETSC_COMM_SELF, newn, newarr, PETSC_OWN_POINTER, &expandedPoints_[d]);CHKERRQ(ierr);
2841     arr = newarr;
2842     n = newn;
2843   }
2844   ierr = ISRestoreIndices(points, &arr0);CHKERRQ(ierr);
2845   *depth = depth_;
2846   if (expandedPoints) *expandedPoints = expandedPoints_;
2847   else {
2848     for (d=0; d<depth_; d++) {ierr = ISDestroy(&expandedPoints_[d]);CHKERRQ(ierr);}
2849     ierr = PetscFree(expandedPoints_);CHKERRQ(ierr);
2850   }
2851   if (sections) *sections = sections_;
2852   else {
2853     for (d=0; d<depth_; d++) {ierr = PetscSectionDestroy(&sections_[d]);CHKERRQ(ierr);}
2854     ierr = PetscFree(sections_);CHKERRQ(ierr);
2855   }
2856   PetscFunctionReturn(0);
2857 }
2858 
2859 /*@
2860   DMPlexRestoreConeRecursive - Deallocates arrays created by DMPlexGetConeRecursive
2861 
2862   Not collective
2863 
2864   Input Parameters:
2865 + dm - The DMPlex
2866 - points - The IS of points, which must lie in the chart set with DMPlexSetChart()
2867 
2868   Output Parameters:
2869 + depth - (optional) Size of the output arrays, equal to DMPlex depth, returned by DMPlexGetDepth()
2870 . expandedPoints - (optional) An array of recursively expanded cones
2871 - sections - (optional) An array of sections which describe mappings from points to their cone points
2872 
2873   Level: advanced
2874 
2875   Notes:
2876   See DMPlexGetConeRecursive() for details.
2877 
2878 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexGetConeTuple(), DMPlexGetConeRecursive(), DMPlexGetConeRecursiveVertices(), DMPlexGetDepth()
2879 @*/
2880 PetscErrorCode DMPlexRestoreConeRecursive(DM dm, IS points, PetscInt *depth, IS *expandedPoints[], PetscSection *sections[])
2881 {
2882   PetscInt            d, depth_;
2883   PetscErrorCode      ierr;
2884 
2885   PetscFunctionBegin;
2886   ierr = DMPlexGetDepth(dm, &depth_);CHKERRQ(ierr);
2887   if (depth && *depth != depth_) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "depth changed since last call to DMPlexGetConeRecursive");
2888   if (depth) *depth = 0;
2889   if (expandedPoints) {
2890     for (d=0; d<depth_; d++) {ierr = ISDestroy(&((*expandedPoints)[d]));CHKERRQ(ierr);}
2891     ierr = PetscFree(*expandedPoints);CHKERRQ(ierr);
2892   }
2893   if (sections)  {
2894     for (d=0; d<depth_; d++) {ierr = PetscSectionDestroy(&((*sections)[d]));CHKERRQ(ierr);}
2895     ierr = PetscFree(*sections);CHKERRQ(ierr);
2896   }
2897   PetscFunctionReturn(0);
2898 }
2899 
2900 /*@
2901   DMPlexSetCone - Set the points on the in-edges for this point in the DAG; that is these are the points that cover the specific point
2902 
2903   Not collective
2904 
2905   Input Parameters:
2906 + mesh - The DMPlex
2907 . p - The point, which must lie in the chart set with DMPlexSetChart()
2908 - cone - An array of points which are on the in-edges for point p
2909 
2910   Output Parameter:
2911 
2912   Note:
2913   This should be called after all calls to DMPlexSetConeSize() and DMSetUp().
2914 
2915   Level: beginner
2916 
2917 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp(), DMPlexSetSupport(), DMPlexSetSupportSize()
2918 @*/
2919 PetscErrorCode DMPlexSetCone(DM dm, PetscInt p, const PetscInt cone[])
2920 {
2921   DM_Plex       *mesh = (DM_Plex*) dm->data;
2922   PetscInt       pStart, pEnd;
2923   PetscInt       dof, off, c;
2924   PetscErrorCode ierr;
2925 
2926   PetscFunctionBegin;
2927   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2928   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
2929   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2930   if (dof) PetscValidPointer(cone, 3);
2931   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2932   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
2933   for (c = 0; c < dof; ++c) {
2934     if ((cone[c] < pStart) || (cone[c] >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone point %D is not in the valid range [%D, %D)", cone[c], pStart, pEnd);
2935     mesh->cones[off+c] = cone[c];
2936   }
2937   PetscFunctionReturn(0);
2938 }
2939 
2940 /*@C
2941   DMPlexGetConeOrientation - Return the orientations on the in-edges for this point in the DAG
2942 
2943   Not collective
2944 
2945   Input Parameters:
2946 + mesh - The DMPlex
2947 - p - The point, which must lie in the chart set with DMPlexSetChart()
2948 
2949   Output Parameter:
2950 . coneOrientation - An array of orientations which are on the in-edges for point p. An orientation is an
2951                     integer giving the prescription for cone traversal.
2952 
2953   Level: beginner
2954 
2955   Notes:
2956   The number indexes the symmetry transformations for the cell type (see manual). Orientation 0 is always
2957   the identity transformation. Negative orientation indicates reflection so that -(o+1) is the reflection
2958   of o, however it is not necessarily the inverse. To get the inverse, use DMPolytopeTypeComposeOrientationInv()
2959   with the identity.
2960 
2961   Fortran Notes:
2962   Since it returns an array, this routine is only available in Fortran 90, and you must
2963   include petsc.h90 in your code.
2964   You must also call DMPlexRestoreConeOrientation() after you finish using the returned array.
2965   DMPlexRestoreConeOrientation() is not needed/available in C.
2966 
2967 .seealso: DMPolytopeTypeComposeOrientation(), DMPolytopeTypeComposeOrientationInv(), DMPlexCreate(), DMPlexGetCone(), DMPlexSetCone(), DMPlexSetChart()
2968 @*/
2969 PetscErrorCode DMPlexGetConeOrientation(DM dm, PetscInt p, const PetscInt *coneOrientation[])
2970 {
2971   DM_Plex       *mesh = (DM_Plex*) dm->data;
2972   PetscInt       off;
2973   PetscErrorCode ierr;
2974 
2975   PetscFunctionBegin;
2976   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2977   if (PetscDefined(USE_DEBUG)) {
2978     PetscInt dof;
2979     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2980     if (dof) PetscValidPointer(coneOrientation, 3);
2981   }
2982   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2983 
2984   *coneOrientation = &mesh->coneOrientations[off];
2985   PetscFunctionReturn(0);
2986 }
2987 
2988 /*@
2989   DMPlexSetConeOrientation - Set the orientations on the in-edges for this point in the DAG
2990 
2991   Not collective
2992 
2993   Input Parameters:
2994 + mesh - The DMPlex
2995 . p - The point, which must lie in the chart set with DMPlexSetChart()
2996 - coneOrientation - An array of orientations
2997   Output Parameter:
2998 
2999   Notes:
3000   This should be called after all calls to DMPlexSetConeSize() and DMSetUp().
3001 
3002   The meaning of coneOrientation is detailed in DMPlexGetConeOrientation().
3003 
3004   Level: beginner
3005 
3006 .seealso: DMPlexCreate(), DMPlexGetConeOrientation(), DMPlexSetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
3007 @*/
3008 PetscErrorCode DMPlexSetConeOrientation(DM dm, PetscInt p, const PetscInt coneOrientation[])
3009 {
3010   DM_Plex       *mesh = (DM_Plex*) dm->data;
3011   PetscInt       pStart, pEnd;
3012   PetscInt       dof, off, c;
3013   PetscErrorCode ierr;
3014 
3015   PetscFunctionBegin;
3016   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3017   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
3018   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
3019   if (dof) PetscValidPointer(coneOrientation, 3);
3020   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
3021   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
3022   for (c = 0; c < dof; ++c) {
3023     PetscInt cdof, o = coneOrientation[c];
3024 
3025     ierr = PetscSectionGetDof(mesh->coneSection, mesh->cones[off+c], &cdof);CHKERRQ(ierr);
3026     if (o && ((o < -(cdof+1)) || (o >= cdof))) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone orientation %D is not in the valid range [%D. %D)", o, -(cdof+1), cdof);
3027     mesh->coneOrientations[off+c] = o;
3028   }
3029   PetscFunctionReturn(0);
3030 }
3031 
3032 /*@
3033   DMPlexInsertCone - Insert a point into the in-edges for the point p in the DAG
3034 
3035   Not collective
3036 
3037   Input Parameters:
3038 + mesh - The DMPlex
3039 . p - The point, which must lie in the chart set with DMPlexSetChart()
3040 . conePos - The local index in the cone where the point should be put
3041 - conePoint - The mesh point to insert
3042 
3043   Level: beginner
3044 
3045 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
3046 @*/
3047 PetscErrorCode DMPlexInsertCone(DM dm, PetscInt p, PetscInt conePos, PetscInt conePoint)
3048 {
3049   DM_Plex       *mesh = (DM_Plex*) dm->data;
3050   PetscInt       pStart, pEnd;
3051   PetscInt       dof, off;
3052   PetscErrorCode ierr;
3053 
3054   PetscFunctionBegin;
3055   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3056   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
3057   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
3058   if ((conePoint < pStart) || (conePoint >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone point %D is not in the valid range [%D, %D)", conePoint, pStart, pEnd);
3059   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
3060   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
3061   if ((conePos < 0) || (conePos >= dof)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone position %D of point %D is not in the valid range [0, %D)", conePos, p, dof);
3062   mesh->cones[off+conePos] = conePoint;
3063   PetscFunctionReturn(0);
3064 }
3065 
3066 /*@
3067   DMPlexInsertConeOrientation - Insert a point orientation for the in-edge for the point p in the DAG
3068 
3069   Not collective
3070 
3071   Input Parameters:
3072 + mesh - The DMPlex
3073 . p - The point, which must lie in the chart set with DMPlexSetChart()
3074 . conePos - The local index in the cone where the point should be put
3075 - coneOrientation - The point orientation to insert
3076 
3077   Level: beginner
3078 
3079   Notes:
3080   The meaning of coneOrientation values is detailed in DMPlexGetConeOrientation().
3081 
3082 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
3083 @*/
3084 PetscErrorCode DMPlexInsertConeOrientation(DM dm, PetscInt p, PetscInt conePos, PetscInt coneOrientation)
3085 {
3086   DM_Plex       *mesh = (DM_Plex*) dm->data;
3087   PetscInt       pStart, pEnd;
3088   PetscInt       dof, off;
3089   PetscErrorCode ierr;
3090 
3091   PetscFunctionBegin;
3092   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3093   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
3094   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
3095   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
3096   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
3097   if ((conePos < 0) || (conePos >= dof)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone position %D of point %D is not in the valid range [0, %D)", conePos, p, dof);
3098   mesh->coneOrientations[off+conePos] = coneOrientation;
3099   PetscFunctionReturn(0);
3100 }
3101 
3102 /*@
3103   DMPlexGetSupportSize - Return the number of out-edges for this point in the DAG
3104 
3105   Not collective
3106 
3107   Input Parameters:
3108 + mesh - The DMPlex
3109 - p - The point, which must lie in the chart set with DMPlexSetChart()
3110 
3111   Output Parameter:
3112 . size - The support size for point p
3113 
3114   Level: beginner
3115 
3116 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart(), DMPlexGetConeSize()
3117 @*/
3118 PetscErrorCode DMPlexGetSupportSize(DM dm, PetscInt p, PetscInt *size)
3119 {
3120   DM_Plex       *mesh = (DM_Plex*) dm->data;
3121   PetscErrorCode ierr;
3122 
3123   PetscFunctionBegin;
3124   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3125   PetscValidPointer(size, 3);
3126   ierr = PetscSectionGetDof(mesh->supportSection, p, size);CHKERRQ(ierr);
3127   PetscFunctionReturn(0);
3128 }
3129 
3130 /*@
3131   DMPlexSetSupportSize - Set the number of out-edges for this point in the DAG
3132 
3133   Not collective
3134 
3135   Input Parameters:
3136 + mesh - The DMPlex
3137 . p - The point, which must lie in the chart set with DMPlexSetChart()
3138 - size - The support size for point p
3139 
3140   Output Parameter:
3141 
3142   Note:
3143   This should be called after DMPlexSetChart().
3144 
3145   Level: beginner
3146 
3147 .seealso: DMPlexCreate(), DMPlexGetSupportSize(), DMPlexSetChart()
3148 @*/
3149 PetscErrorCode DMPlexSetSupportSize(DM dm, PetscInt p, PetscInt size)
3150 {
3151   DM_Plex       *mesh = (DM_Plex*) dm->data;
3152   PetscErrorCode ierr;
3153 
3154   PetscFunctionBegin;
3155   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3156   ierr = PetscSectionSetDof(mesh->supportSection, p, size);CHKERRQ(ierr);
3157 
3158   mesh->maxSupportSize = PetscMax(mesh->maxSupportSize, size);
3159   PetscFunctionReturn(0);
3160 }
3161 
3162 /*@C
3163   DMPlexGetSupport - Return the points on the out-edges for this point in the DAG
3164 
3165   Not collective
3166 
3167   Input Parameters:
3168 + mesh - The DMPlex
3169 - p - The point, which must lie in the chart set with DMPlexSetChart()
3170 
3171   Output Parameter:
3172 . support - An array of points which are on the out-edges for point p
3173 
3174   Level: beginner
3175 
3176   Fortran Notes:
3177   Since it returns an array, this routine is only available in Fortran 90, and you must
3178   include petsc.h90 in your code.
3179   You must also call DMPlexRestoreSupport() after you finish using the returned array.
3180   DMPlexRestoreSupport() is not needed/available in C.
3181 
3182 .seealso: DMPlexGetSupportSize(), DMPlexSetSupport(), DMPlexGetCone(), DMPlexSetChart()
3183 @*/
3184 PetscErrorCode DMPlexGetSupport(DM dm, PetscInt p, const PetscInt *support[])
3185 {
3186   DM_Plex       *mesh = (DM_Plex*) dm->data;
3187   PetscInt       off;
3188   PetscErrorCode ierr;
3189 
3190   PetscFunctionBegin;
3191   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3192   PetscValidPointer(support, 3);
3193   ierr     = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
3194   *support = &mesh->supports[off];
3195   PetscFunctionReturn(0);
3196 }
3197 
3198 /*@
3199   DMPlexSetSupport - Set the points on the out-edges for this point in the DAG, that is the list of points that this point covers
3200 
3201   Not collective
3202 
3203   Input Parameters:
3204 + mesh - The DMPlex
3205 . p - The point, which must lie in the chart set with DMPlexSetChart()
3206 - support - An array of points which are on the out-edges for point p
3207 
3208   Output Parameter:
3209 
3210   Note:
3211   This should be called after all calls to DMPlexSetSupportSize() and DMSetUp().
3212 
3213   Level: beginner
3214 
3215 .seealso: DMPlexSetCone(), DMPlexSetConeSize(), DMPlexCreate(), DMPlexGetSupport(), DMPlexSetChart(), DMPlexSetSupportSize(), DMSetUp()
3216 @*/
3217 PetscErrorCode DMPlexSetSupport(DM dm, PetscInt p, const PetscInt support[])
3218 {
3219   DM_Plex       *mesh = (DM_Plex*) dm->data;
3220   PetscInt       pStart, pEnd;
3221   PetscInt       dof, off, c;
3222   PetscErrorCode ierr;
3223 
3224   PetscFunctionBegin;
3225   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3226   ierr = PetscSectionGetChart(mesh->supportSection, &pStart, &pEnd);CHKERRQ(ierr);
3227   ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
3228   if (dof) PetscValidPointer(support, 3);
3229   ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
3230   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
3231   for (c = 0; c < dof; ++c) {
3232     if ((support[c] < pStart) || (support[c] >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support point %D is not in the valid range [%D, %D)", support[c], pStart, pEnd);
3233     mesh->supports[off+c] = support[c];
3234   }
3235   PetscFunctionReturn(0);
3236 }
3237 
3238 /*@
3239   DMPlexInsertSupport - Insert a point into the out-edges for the point p in the DAG
3240 
3241   Not collective
3242 
3243   Input Parameters:
3244 + mesh - The DMPlex
3245 . p - The point, which must lie in the chart set with DMPlexSetChart()
3246 . supportPos - The local index in the cone where the point should be put
3247 - supportPoint - The mesh point to insert
3248 
3249   Level: beginner
3250 
3251 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
3252 @*/
3253 PetscErrorCode DMPlexInsertSupport(DM dm, PetscInt p, PetscInt supportPos, PetscInt supportPoint)
3254 {
3255   DM_Plex       *mesh = (DM_Plex*) dm->data;
3256   PetscInt       pStart, pEnd;
3257   PetscInt       dof, off;
3258   PetscErrorCode ierr;
3259 
3260   PetscFunctionBegin;
3261   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3262   ierr = PetscSectionGetChart(mesh->supportSection, &pStart, &pEnd);CHKERRQ(ierr);
3263   ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
3264   ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
3265   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
3266   if ((supportPoint < pStart) || (supportPoint >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support point %D is not in the valid range [%D, %D)", supportPoint, pStart, pEnd);
3267   if (supportPos >= dof) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support position %D of point %D is not in the valid range [0, %D)", supportPos, p, dof);
3268   mesh->supports[off+supportPos] = supportPoint;
3269   PetscFunctionReturn(0);
3270 }
3271 
3272 /* Converts an orientation o in the current numbering to the previous scheme used in Plex */
3273 PetscInt DMPolytopeConvertNewOrientation_Internal(DMPolytopeType ct, PetscInt o)
3274 {
3275   switch (ct) {
3276     case DM_POLYTOPE_SEGMENT:
3277       if (o == -1) return -2;
3278       break;
3279     case DM_POLYTOPE_TRIANGLE:
3280       if (o == -3) return -1;
3281       if (o == -2) return -3;
3282       if (o == -1) return -2;
3283       break;
3284     case DM_POLYTOPE_QUADRILATERAL:
3285       if (o == -4) return -2;
3286       if (o == -3) return -1;
3287       if (o == -2) return -4;
3288       if (o == -1) return -3;
3289       break;
3290     default: return o;
3291   }
3292   return o;
3293 }
3294 
3295 /* Converts an orientation o in the previous scheme used in Plex to the current numbering */
3296 PetscInt DMPolytopeConvertOldOrientation_Internal(DMPolytopeType ct, PetscInt o)
3297 {
3298   switch (ct) {
3299     case DM_POLYTOPE_SEGMENT:
3300       if ((o == -2) || (o == 1)) return -1;
3301       if (o == -1) return 0;
3302       break;
3303     case DM_POLYTOPE_TRIANGLE:
3304       if (o == -3) return -2;
3305       if (o == -2) return -1;
3306       if (o == -1) return -3;
3307       break;
3308     case DM_POLYTOPE_QUADRILATERAL:
3309       if (o == -4) return -2;
3310       if (o == -3) return -1;
3311       if (o == -2) return -4;
3312       if (o == -1) return -3;
3313       break;
3314     default: return o;
3315   }
3316   return o;
3317 }
3318 
3319 /* Takes in a mesh whose orientations are in the previous scheme and converts them all to the current numbering */
3320 PetscErrorCode DMPlexConvertOldOrientations_Internal(DM dm)
3321 {
3322   PetscInt       pStart, pEnd, p;
3323   PetscErrorCode ierr;
3324 
3325   PetscFunctionBegin;
3326   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
3327   for (p = pStart; p < pEnd; ++p) {
3328     const PetscInt *cone, *ornt;
3329     PetscInt        coneSize, c;
3330 
3331     ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
3332     ierr = DMPlexGetCone(dm, p, &cone);CHKERRQ(ierr);
3333     ierr = DMPlexGetConeOrientation(dm, p, &ornt);CHKERRQ(ierr);
3334     for (c = 0; c < coneSize; ++c) {
3335       DMPolytopeType ct;
3336       const PetscInt o = ornt[c];
3337 
3338       ierr = DMPlexGetCellType(dm, cone[c], &ct);CHKERRQ(ierr);
3339       switch (ct) {
3340         case DM_POLYTOPE_SEGMENT:
3341           if ((o == -2) || (o == 1)) {ierr = DMPlexInsertConeOrientation(dm, p, c, -1);CHKERRQ(ierr);}
3342           if (o == -1) {ierr = DMPlexInsertConeOrientation(dm, p, c, 0);CHKERRQ(ierr);}
3343           break;
3344         case DM_POLYTOPE_TRIANGLE:
3345           if (o == -3) {ierr = DMPlexInsertConeOrientation(dm, p, c, -2);CHKERRQ(ierr);}
3346           if (o == -2) {ierr = DMPlexInsertConeOrientation(dm, p, c, -1);CHKERRQ(ierr);}
3347           if (o == -1) {ierr = DMPlexInsertConeOrientation(dm, p, c, -3);CHKERRQ(ierr);}
3348           break;
3349         case DM_POLYTOPE_QUADRILATERAL:
3350           if (o == -4) {ierr = DMPlexInsertConeOrientation(dm, p, c, -2);CHKERRQ(ierr);}
3351           if (o == -3) {ierr = DMPlexInsertConeOrientation(dm, p, c, -1);CHKERRQ(ierr);}
3352           if (o == -2) {ierr = DMPlexInsertConeOrientation(dm, p, c, -4);CHKERRQ(ierr);}
3353           if (o == -1) {ierr = DMPlexInsertConeOrientation(dm, p, c, -3);CHKERRQ(ierr);}
3354           break;
3355         default: break;
3356       }
3357     }
3358   }
3359   PetscFunctionReturn(0);
3360 }
3361 
3362 static PetscErrorCode DMPlexGetTransitiveClosure_Depth1_Private(DM dm, PetscInt p, PetscInt ornt, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
3363 {
3364   DMPolytopeType  ct = DM_POLYTOPE_UNKNOWN;
3365   PetscInt       *closure;
3366   const PetscInt *tmp = NULL, *tmpO = NULL;
3367   PetscInt        off = 0, tmpSize, t;
3368   PetscErrorCode  ierr;
3369 
3370   PetscFunctionBeginHot;
3371   if (ornt) {
3372     ierr = DMPlexGetCellType(dm, p, &ct);CHKERRQ(ierr);
3373     if (ct == DM_POLYTOPE_FV_GHOST || ct == DM_POLYTOPE_INTERIOR_GHOST || ct == DM_POLYTOPE_UNKNOWN) ct = DM_POLYTOPE_UNKNOWN;
3374   }
3375   if (*points) {
3376     closure = *points;
3377   } else {
3378     PetscInt maxConeSize, maxSupportSize;
3379     ierr = DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize);CHKERRQ(ierr);
3380     ierr = DMGetWorkArray(dm, 2*(PetscMax(maxConeSize, maxSupportSize)+1), MPIU_INT, &closure);CHKERRQ(ierr);
3381   }
3382   if (useCone) {
3383     ierr = DMPlexGetConeSize(dm, p, &tmpSize);CHKERRQ(ierr);
3384     ierr = DMPlexGetCone(dm, p, &tmp);CHKERRQ(ierr);
3385     ierr = DMPlexGetConeOrientation(dm, p, &tmpO);CHKERRQ(ierr);
3386   } else {
3387     ierr = DMPlexGetSupportSize(dm, p, &tmpSize);CHKERRQ(ierr);
3388     ierr = DMPlexGetSupport(dm, p, &tmp);CHKERRQ(ierr);
3389   }
3390   if (ct == DM_POLYTOPE_UNKNOWN) {
3391     closure[off++] = p;
3392     closure[off++] = 0;
3393     for (t = 0; t < tmpSize; ++t) {
3394       closure[off++] = tmp[t];
3395       closure[off++] = tmpO ? tmpO[t] : 0;
3396     }
3397   } else {
3398     const PetscInt *arr = DMPolytopeTypeGetArrangment(ct, ornt);CHKERRQ(ierr);
3399 
3400     /* We assume that cells with a valid type have faces with a valid type */
3401     closure[off++] = p;
3402     closure[off++] = ornt;
3403     for (t = 0; t < tmpSize; ++t) {
3404       DMPolytopeType ft;
3405 
3406       ierr = DMPlexGetCellType(dm, tmp[t], &ft);CHKERRQ(ierr);
3407       closure[off++] = tmp[arr[t]];
3408       closure[off++] = tmpO ? DMPolytopeTypeComposeOrientation(ft, ornt, tmpO[t]) : 0;
3409     }
3410   }
3411   if (numPoints) *numPoints = tmpSize+1;
3412   if (points)    *points    = closure;
3413   PetscFunctionReturn(0);
3414 }
3415 
3416 /* We need a special tensor verison becasue we want to allow duplicate points in the endcaps for hybrid cells */
3417 static PetscErrorCode DMPlexTransitiveClosure_Tensor_Internal(DM dm, PetscInt point, DMPolytopeType ct, PetscInt o, PetscBool useCone, PetscInt *numPoints, PetscInt **points)
3418 {
3419   const PetscInt *arr = DMPolytopeTypeGetArrangment(ct, o);
3420   const PetscInt *cone, *ornt;
3421   PetscInt       *pts,  *closure = NULL;
3422   DMPolytopeType  ft;
3423   PetscInt        maxConeSize, maxSupportSize, coneSeries, supportSeries, maxSize;
3424   PetscInt        dim, coneSize, c, d, clSize, cl;
3425   PetscErrorCode  ierr;
3426 
3427   PetscFunctionBeginHot;
3428   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
3429   ierr = DMPlexGetConeSize(dm, point, &coneSize);CHKERRQ(ierr);
3430   ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
3431   ierr = DMPlexGetConeOrientation(dm, point, &ornt);CHKERRQ(ierr);
3432   ierr = DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize);CHKERRQ(ierr);
3433   coneSeries    = (maxConeSize    > 1) ? ((PetscPowInt(maxConeSize,    dim+1)-1)/(maxConeSize-1))    : dim+1;
3434   supportSeries = (maxSupportSize > 1) ? ((PetscPowInt(maxSupportSize, dim+1)-1)/(maxSupportSize-1)) : dim+1;
3435   maxSize       = PetscMax(coneSeries, supportSeries);
3436   if (*points) {pts  = *points;}
3437   else         {ierr = DMGetWorkArray(dm, 2*maxSize, MPIU_INT, &pts);CHKERRQ(ierr);}
3438   c    = 0;
3439   pts[c++] = point;
3440   pts[c++] = o;
3441   ierr = DMPlexGetCellType(dm, cone[arr[0*2+0]], &ft);CHKERRQ(ierr);
3442   ierr = DMPlexGetTransitiveClosure_Internal(dm, cone[arr[0*2+0]], DMPolytopeTypeComposeOrientation(ft, arr[0*2+1], ornt[0]), useCone, &clSize, &closure);CHKERRQ(ierr);
3443   for (cl = 0; cl < clSize*2; cl += 2) {pts[c++] = closure[cl]; pts[c++] = closure[cl+1];}
3444   ierr = DMPlexGetTransitiveClosure_Internal(dm, cone[arr[1*2+0]], DMPolytopeTypeComposeOrientation(ft, arr[1*2+1], ornt[1]), useCone, &clSize, &closure);CHKERRQ(ierr);
3445   for (cl = 0; cl < clSize*2; cl += 2) {pts[c++] = closure[cl]; pts[c++] = closure[cl+1];}
3446   ierr = DMPlexRestoreTransitiveClosure(dm, cone[0], useCone, &clSize, &closure);CHKERRQ(ierr);
3447   for (d = 2; d < coneSize; ++d) {
3448     ierr = DMPlexGetCellType(dm, cone[arr[d*2+0]], &ft);CHKERRQ(ierr);
3449     pts[c++] = cone[arr[d*2+0]];
3450     pts[c++] = DMPolytopeTypeComposeOrientation(ft, arr[d*2+1], ornt[d]);
3451   }
3452   if (dim >= 3) {
3453     for (d = 2; d < coneSize; ++d) {
3454       const PetscInt  fpoint = cone[arr[d*2+0]];
3455       const PetscInt *fcone, *fornt;
3456       PetscInt        fconeSize, fc, i;
3457 
3458       ierr = DMPlexGetCellType(dm, fpoint, &ft);CHKERRQ(ierr);
3459       const PetscInt *farr = DMPolytopeTypeGetArrangment(ft, DMPolytopeTypeComposeOrientation(ft, arr[d*2+1], ornt[d]));
3460       ierr = DMPlexGetConeSize(dm, fpoint, &fconeSize);CHKERRQ(ierr);
3461       ierr = DMPlexGetCone(dm, fpoint, &fcone);CHKERRQ(ierr);
3462       ierr = DMPlexGetConeOrientation(dm, fpoint, &fornt);CHKERRQ(ierr);
3463       for (fc = 0; fc < fconeSize; ++fc) {
3464         const PetscInt cp = fcone[farr[fc*2+0]];
3465         const PetscInt co = farr[fc*2+1];
3466 
3467         for (i = 0; i < c; i += 2) if (pts[i] == cp) break;
3468         if (i == c) {
3469           ierr = DMPlexGetCellType(dm, cp, &ft);CHKERRQ(ierr);
3470           pts[c++] = cp;
3471           pts[c++] = DMPolytopeTypeComposeOrientation(ft, co, fornt[farr[fc*2+0]]);
3472         }
3473       }
3474     }
3475   }
3476   *numPoints = c/2;
3477   *points    = pts;
3478   PetscFunctionReturn(0);
3479 }
3480 
3481 PetscErrorCode DMPlexGetTransitiveClosure_Internal(DM dm, PetscInt p, PetscInt ornt, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
3482 {
3483   DMPolytopeType ct;
3484   PetscInt      *closure, *fifo;
3485   PetscInt       closureSize = 0, fifoStart = 0, fifoSize = 0;
3486   PetscInt       maxConeSize, maxSupportSize, coneSeries, supportSeries;
3487   PetscInt       depth, maxSize;
3488   PetscErrorCode ierr;
3489 
3490   PetscFunctionBeginHot;
3491   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3492   if (depth == 1) {
3493     ierr = DMPlexGetTransitiveClosure_Depth1_Private(dm, p, ornt, useCone, numPoints, points);CHKERRQ(ierr);
3494     PetscFunctionReturn(0);
3495   }
3496   ierr = DMPlexGetCellType(dm, p, &ct);CHKERRQ(ierr);
3497   if (ct == DM_POLYTOPE_FV_GHOST || ct == DM_POLYTOPE_INTERIOR_GHOST || ct == DM_POLYTOPE_UNKNOWN) ct = DM_POLYTOPE_UNKNOWN;
3498   if (ct == DM_POLYTOPE_SEG_PRISM_TENSOR || ct == DM_POLYTOPE_TRI_PRISM_TENSOR || ct == DM_POLYTOPE_QUAD_PRISM_TENSOR) {
3499     ierr = DMPlexTransitiveClosure_Tensor_Internal(dm, p, ct, ornt, useCone, numPoints, points);CHKERRQ(ierr);
3500     PetscFunctionReturn(0);
3501   }
3502   ierr = DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize);CHKERRQ(ierr);
3503   coneSeries    = (maxConeSize    > 1) ? ((PetscPowInt(maxConeSize,    depth+1)-1)/(maxConeSize-1))    : depth+1;
3504   supportSeries = (maxSupportSize > 1) ? ((PetscPowInt(maxSupportSize, depth+1)-1)/(maxSupportSize-1)) : depth+1;
3505   maxSize       = PetscMax(coneSeries, supportSeries);
3506   ierr = DMGetWorkArray(dm, 3*maxSize, MPIU_INT, &fifo);CHKERRQ(ierr);
3507   if (*points) {closure = *points;}
3508   else         {ierr = DMGetWorkArray(dm, 2*maxSize, MPIU_INT, &closure);CHKERRQ(ierr);}
3509   closure[closureSize++] = p;
3510   closure[closureSize++] = ornt;
3511   fifo[fifoSize++]       = p;
3512   fifo[fifoSize++]       = ornt;
3513   fifo[fifoSize++]       = ct;
3514   /* Should kick out early when depth is reached, rather than checking all vertices for empty cones */
3515   while (fifoSize - fifoStart) {
3516     const PetscInt       q    = fifo[fifoStart++];
3517     const PetscInt       o    = fifo[fifoStart++];
3518     const DMPolytopeType qt   = (DMPolytopeType) fifo[fifoStart++];
3519     const PetscInt      *qarr = DMPolytopeTypeGetArrangment(qt, o);
3520     const PetscInt      *tmp, *tmpO;
3521     PetscInt             tmpSize, t;
3522 
3523     if (PetscDefined(USE_DEBUG)) {
3524       PetscInt nO = DMPolytopeTypeGetNumArrangments(qt)/2;
3525       if (o && (o >= nO || o < -nO)) SETERRQ5(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid orientation %D not in [%D,%D) for %s %D", o, -nO, nO, DMPolytopeTypes[qt], q);
3526     }
3527     if (useCone) {
3528       ierr = DMPlexGetConeSize(dm, q, &tmpSize);CHKERRQ(ierr);
3529       ierr = DMPlexGetCone(dm, q, &tmp);CHKERRQ(ierr);
3530       ierr = DMPlexGetConeOrientation(dm, q, &tmpO);CHKERRQ(ierr);
3531     } else {
3532       ierr = DMPlexGetSupportSize(dm, q, &tmpSize);CHKERRQ(ierr);
3533       ierr = DMPlexGetSupport(dm, q, &tmp);CHKERRQ(ierr);
3534       tmpO = NULL;
3535     }
3536     for (t = 0; t < tmpSize; ++t) {
3537       const PetscInt ip = useCone && qarr ? qarr[t*2]   : t;
3538       const PetscInt io = useCone && qarr ? qarr[t*2+1] : 0;
3539       const PetscInt cp = tmp[ip];
3540       ierr = DMPlexGetCellType(dm, cp, &ct);CHKERRQ(ierr);
3541       const PetscInt co = tmpO ? DMPolytopeTypeComposeOrientation(ct, io, tmpO[ip]) : 0;
3542       PetscInt       c;
3543 
3544       /* Check for duplicate */
3545       for (c = 0; c < closureSize; c += 2) {
3546         if (closure[c] == cp) break;
3547       }
3548       if (c == closureSize) {
3549         closure[closureSize++] = cp;
3550         closure[closureSize++] = co;
3551         fifo[fifoSize++]       = cp;
3552         fifo[fifoSize++]       = co;
3553         fifo[fifoSize++]       = ct;
3554       }
3555     }
3556   }
3557   ierr = DMRestoreWorkArray(dm, 3*maxSize, MPIU_INT, &fifo);CHKERRQ(ierr);
3558   if (numPoints) *numPoints = closureSize/2;
3559   if (points)    *points    = closure;
3560   PetscFunctionReturn(0);
3561 }
3562 
3563 /*@C
3564   DMPlexGetTransitiveClosure - Return the points on the transitive closure of the in-edges or out-edges for this point in the DAG
3565 
3566   Not collective
3567 
3568   Input Parameters:
3569 + dm      - The DMPlex
3570 . p       - The mesh point
3571 - useCone - PETSC_TRUE for the closure, otherwise return the star
3572 
3573   Input/Output Parameter:
3574 . points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...];
3575            if NULL on input, internal storage will be returned, otherwise the provided array is used
3576 
3577   Output Parameter:
3578 . numPoints - The number of points in the closure, so points[] is of size 2*numPoints
3579 
3580   Note:
3581   If using internal storage (points is NULL on input), each call overwrites the last output.
3582 
3583   Fortran Notes:
3584   Since it returns an array, this routine is only available in Fortran 90, and you must include petsc.h90 in your code.
3585 
3586   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3587 
3588   Level: beginner
3589 
3590 .seealso: DMPlexRestoreTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
3591 @*/
3592 PetscErrorCode DMPlexGetTransitiveClosure(DM dm, PetscInt p, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
3593 {
3594   PetscErrorCode ierr;
3595 
3596   PetscFunctionBeginHot;
3597   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3598   if (numPoints) PetscValidIntPointer(numPoints, 4);
3599   if (points)    PetscValidPointer(points, 5);
3600   ierr = DMPlexGetTransitiveClosure_Internal(dm, p, 0, useCone, numPoints, points);CHKERRQ(ierr);
3601   PetscFunctionReturn(0);
3602 }
3603 
3604 /*@C
3605   DMPlexRestoreTransitiveClosure - Restore the array of points on the transitive closure of the in-edges or out-edges for this point in the DAG
3606 
3607   Not collective
3608 
3609   Input Parameters:
3610 + dm        - The DMPlex
3611 . p         - The mesh point
3612 . useCone   - PETSC_TRUE for the closure, otherwise return the star
3613 . numPoints - The number of points in the closure, so points[] is of size 2*numPoints
3614 - points    - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...]
3615 
3616   Note:
3617   If not using internal storage (points is not NULL on input), this call is unnecessary
3618 
3619   Fortran Notes:
3620   Since it returns an array, this routine is only available in Fortran 90, and you must include petsc.h90 in your code.
3621 
3622   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3623 
3624   Level: beginner
3625 
3626 .seealso: DMPlexGetTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
3627 @*/
3628 PetscErrorCode DMPlexRestoreTransitiveClosure(DM dm, PetscInt p, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
3629 {
3630   PetscErrorCode ierr;
3631 
3632   PetscFunctionBeginHot;
3633   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3634   if (numPoints) *numPoints = 0;
3635   ierr = DMRestoreWorkArray(dm, 0, MPIU_INT, points);CHKERRQ(ierr);
3636   PetscFunctionReturn(0);
3637 }
3638 
3639 /*@
3640   DMPlexGetMaxSizes - Return the maximum number of in-edges (cone) and out-edges (support) for any point in the DAG
3641 
3642   Not collective
3643 
3644   Input Parameter:
3645 . mesh - The DMPlex
3646 
3647   Output Parameters:
3648 + maxConeSize - The maximum number of in-edges
3649 - maxSupportSize - The maximum number of out-edges
3650 
3651   Level: beginner
3652 
3653 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart()
3654 @*/
3655 PetscErrorCode DMPlexGetMaxSizes(DM dm, PetscInt *maxConeSize, PetscInt *maxSupportSize)
3656 {
3657   DM_Plex *mesh = (DM_Plex*) dm->data;
3658 
3659   PetscFunctionBegin;
3660   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3661   if (maxConeSize)    *maxConeSize    = mesh->maxConeSize;
3662   if (maxSupportSize) *maxSupportSize = mesh->maxSupportSize;
3663   PetscFunctionReturn(0);
3664 }
3665 
3666 PetscErrorCode DMSetUp_Plex(DM dm)
3667 {
3668   DM_Plex       *mesh = (DM_Plex*) dm->data;
3669   PetscInt       size;
3670   PetscErrorCode ierr;
3671 
3672   PetscFunctionBegin;
3673   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3674   ierr = PetscSectionSetUp(mesh->coneSection);CHKERRQ(ierr);
3675   ierr = PetscSectionGetStorageSize(mesh->coneSection, &size);CHKERRQ(ierr);
3676   ierr = PetscMalloc1(size, &mesh->cones);CHKERRQ(ierr);
3677   ierr = PetscCalloc1(size, &mesh->coneOrientations);CHKERRQ(ierr);
3678   ierr = PetscLogObjectMemory((PetscObject) dm, size*2*sizeof(PetscInt));CHKERRQ(ierr);
3679   if (mesh->maxSupportSize) {
3680     ierr = PetscSectionSetUp(mesh->supportSection);CHKERRQ(ierr);
3681     ierr = PetscSectionGetStorageSize(mesh->supportSection, &size);CHKERRQ(ierr);
3682     ierr = PetscMalloc1(size, &mesh->supports);CHKERRQ(ierr);
3683     ierr = PetscLogObjectMemory((PetscObject) dm, size*sizeof(PetscInt));CHKERRQ(ierr);
3684   }
3685   PetscFunctionReturn(0);
3686 }
3687 
3688 PetscErrorCode DMCreateSubDM_Plex(DM dm, PetscInt numFields, const PetscInt fields[], IS *is, DM *subdm)
3689 {
3690   PetscErrorCode ierr;
3691 
3692   PetscFunctionBegin;
3693   if (subdm) {ierr = DMClone(dm, subdm);CHKERRQ(ierr);}
3694   ierr = DMCreateSectionSubDM(dm, numFields, fields, is, subdm);CHKERRQ(ierr);
3695   if (subdm) {(*subdm)->useNatural = dm->useNatural;}
3696   if (dm->useNatural && dm->sfMigration) {
3697     PetscSF        sfMigrationInv,sfNatural;
3698     PetscSection   section, sectionSeq;
3699 
3700     (*subdm)->sfMigration = dm->sfMigration;
3701     ierr = PetscObjectReference((PetscObject) dm->sfMigration);CHKERRQ(ierr);
3702     ierr = DMGetLocalSection((*subdm), &section);CHKERRQ(ierr);
3703     ierr = PetscSFCreateInverseSF((*subdm)->sfMigration, &sfMigrationInv);CHKERRQ(ierr);
3704     ierr = PetscSectionCreate(PetscObjectComm((PetscObject) (*subdm)), &sectionSeq);CHKERRQ(ierr);
3705     ierr = PetscSFDistributeSection(sfMigrationInv, section, NULL, sectionSeq);CHKERRQ(ierr);
3706 
3707     ierr = DMPlexCreateGlobalToNaturalSF(*subdm, sectionSeq, (*subdm)->sfMigration, &sfNatural);CHKERRQ(ierr);
3708     (*subdm)->sfNatural = sfNatural;
3709     ierr = PetscSectionDestroy(&sectionSeq);CHKERRQ(ierr);
3710     ierr = PetscSFDestroy(&sfMigrationInv);CHKERRQ(ierr);
3711   }
3712   PetscFunctionReturn(0);
3713 }
3714 
3715 PetscErrorCode DMCreateSuperDM_Plex(DM dms[], PetscInt len, IS **is, DM *superdm)
3716 {
3717   PetscErrorCode ierr;
3718   PetscInt       i = 0;
3719 
3720   PetscFunctionBegin;
3721   ierr = DMClone(dms[0], superdm);CHKERRQ(ierr);
3722   ierr = DMCreateSectionSuperDM(dms, len, is, superdm);CHKERRQ(ierr);
3723   (*superdm)->useNatural = PETSC_FALSE;
3724   for (i = 0; i < len; i++) {
3725     if (dms[i]->useNatural && dms[i]->sfMigration) {
3726       PetscSF        sfMigrationInv,sfNatural;
3727       PetscSection   section, sectionSeq;
3728 
3729       (*superdm)->sfMigration = dms[i]->sfMigration;
3730       ierr = PetscObjectReference((PetscObject) dms[i]->sfMigration);CHKERRQ(ierr);
3731       (*superdm)->useNatural = PETSC_TRUE;
3732       ierr = DMGetLocalSection((*superdm), &section);CHKERRQ(ierr);
3733       ierr = PetscSFCreateInverseSF((*superdm)->sfMigration, &sfMigrationInv);CHKERRQ(ierr);
3734       ierr = PetscSectionCreate(PetscObjectComm((PetscObject) (*superdm)), &sectionSeq);CHKERRQ(ierr);
3735       ierr = PetscSFDistributeSection(sfMigrationInv, section, NULL, sectionSeq);CHKERRQ(ierr);
3736 
3737       ierr = DMPlexCreateGlobalToNaturalSF(*superdm, sectionSeq, (*superdm)->sfMigration, &sfNatural);CHKERRQ(ierr);
3738       (*superdm)->sfNatural = sfNatural;
3739       ierr = PetscSectionDestroy(&sectionSeq);CHKERRQ(ierr);
3740       ierr = PetscSFDestroy(&sfMigrationInv);CHKERRQ(ierr);
3741       break;
3742     }
3743   }
3744   PetscFunctionReturn(0);
3745 }
3746 
3747 /*@
3748   DMPlexSymmetrize - Create support (out-edge) information from cone (in-edge) information
3749 
3750   Not collective
3751 
3752   Input Parameter:
3753 . mesh - The DMPlex
3754 
3755   Output Parameter:
3756 
3757   Note:
3758   This should be called after all calls to DMPlexSetCone()
3759 
3760   Level: beginner
3761 
3762 .seealso: DMPlexCreate(), DMPlexSetChart(), DMPlexSetConeSize(), DMPlexSetCone()
3763 @*/
3764 PetscErrorCode DMPlexSymmetrize(DM dm)
3765 {
3766   DM_Plex       *mesh = (DM_Plex*) dm->data;
3767   PetscInt      *offsets;
3768   PetscInt       supportSize;
3769   PetscInt       pStart, pEnd, p;
3770   PetscErrorCode ierr;
3771 
3772   PetscFunctionBegin;
3773   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3774   if (mesh->supports) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONGSTATE, "Supports were already setup in this DMPlex");
3775   ierr = PetscLogEventBegin(DMPLEX_Symmetrize,dm,0,0,0);CHKERRQ(ierr);
3776   /* Calculate support sizes */
3777   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
3778   for (p = pStart; p < pEnd; ++p) {
3779     PetscInt dof, off, c;
3780 
3781     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
3782     ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
3783     for (c = off; c < off+dof; ++c) {
3784       ierr = PetscSectionAddDof(mesh->supportSection, mesh->cones[c], 1);CHKERRQ(ierr);
3785     }
3786   }
3787   for (p = pStart; p < pEnd; ++p) {
3788     PetscInt dof;
3789 
3790     ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
3791 
3792     mesh->maxSupportSize = PetscMax(mesh->maxSupportSize, dof);
3793   }
3794   ierr = PetscSectionSetUp(mesh->supportSection);CHKERRQ(ierr);
3795   /* Calculate supports */
3796   ierr = PetscSectionGetStorageSize(mesh->supportSection, &supportSize);CHKERRQ(ierr);
3797   ierr = PetscMalloc1(supportSize, &mesh->supports);CHKERRQ(ierr);
3798   ierr = PetscCalloc1(pEnd - pStart, &offsets);CHKERRQ(ierr);
3799   for (p = pStart; p < pEnd; ++p) {
3800     PetscInt dof, off, c;
3801 
3802     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
3803     ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
3804     for (c = off; c < off+dof; ++c) {
3805       const PetscInt q = mesh->cones[c];
3806       PetscInt       offS;
3807 
3808       ierr = PetscSectionGetOffset(mesh->supportSection, q, &offS);CHKERRQ(ierr);
3809 
3810       mesh->supports[offS+offsets[q]] = p;
3811       ++offsets[q];
3812     }
3813   }
3814   ierr = PetscFree(offsets);CHKERRQ(ierr);
3815   ierr = PetscLogEventEnd(DMPLEX_Symmetrize,dm,0,0,0);CHKERRQ(ierr);
3816   PetscFunctionReturn(0);
3817 }
3818 
3819 static PetscErrorCode DMPlexCreateDepthStratum(DM dm, DMLabel label, PetscInt depth, PetscInt pStart, PetscInt pEnd)
3820 {
3821   IS             stratumIS;
3822   PetscErrorCode ierr;
3823 
3824   PetscFunctionBegin;
3825   if (pStart >= pEnd) PetscFunctionReturn(0);
3826   if (PetscDefined(USE_DEBUG)) {
3827     PetscInt  qStart, qEnd, numLevels, level;
3828     PetscBool overlap = PETSC_FALSE;
3829     ierr = DMLabelGetNumValues(label, &numLevels);CHKERRQ(ierr);
3830     for (level = 0; level < numLevels; level++) {
3831       ierr = DMLabelGetStratumBounds(label, level, &qStart, &qEnd);CHKERRQ(ierr);
3832       if ((pStart >= qStart && pStart < qEnd) || (pEnd > qStart && pEnd <= qEnd)) {overlap = PETSC_TRUE; break;}
3833     }
3834     if (overlap) SETERRQ6(PETSC_COMM_SELF, PETSC_ERR_PLIB, "New depth %D range [%D,%D) overlaps with depth %D range [%D,%D)", depth, pStart, pEnd, level, qStart, qEnd);
3835   }
3836   ierr = ISCreateStride(PETSC_COMM_SELF, pEnd-pStart, pStart, 1, &stratumIS);CHKERRQ(ierr);
3837   ierr = DMLabelSetStratumIS(label, depth, stratumIS);CHKERRQ(ierr);
3838   ierr = ISDestroy(&stratumIS);CHKERRQ(ierr);
3839   PetscFunctionReturn(0);
3840 }
3841 
3842 /*@
3843   DMPlexStratify - The DAG for most topologies is a graded poset (https://en.wikipedia.org/wiki/Graded_poset), and
3844   can be illustrated by a Hasse Diagram (https://en.wikipedia.org/wiki/Hasse_diagram). The strata group all points of the
3845   same grade, and this function calculates the strata. This grade can be seen as the height (or depth) of the point in
3846   the DAG.
3847 
3848   Collective on dm
3849 
3850   Input Parameter:
3851 . mesh - The DMPlex
3852 
3853   Output Parameter:
3854 
3855   Notes:
3856   Concretely, DMPlexStratify() creates a new label named "depth" containing the depth in the DAG of each point. For cell-vertex
3857   meshes, vertices are depth 0 and cells are depth 1. For fully interpolated meshes, depth 0 for vertices, 1 for edges, and so on
3858   until cells have depth equal to the dimension of the mesh. The depth label can be accessed through DMPlexGetDepthLabel() or DMPlexGetDepthStratum(), or
3859   manually via DMGetLabel().  The height is defined implicitly by height = maxDimension - depth, and can be accessed
3860   via DMPlexGetHeightStratum().  For example, cells have height 0 and faces have height 1.
3861 
3862   The depth of a point is calculated by executing a breadth-first search (BFS) on the DAG. This could produce surprising results
3863   if run on a partially interpolated mesh, meaning one that had some edges and faces, but not others. For example, suppose that
3864   we had a mesh consisting of one triangle (c0) and three vertices (v0, v1, v2), and only one edge is on the boundary so we choose
3865   to interpolate only that one (e0), so that
3866 $  cone(c0) = {e0, v2}
3867 $  cone(e0) = {v0, v1}
3868   If DMPlexStratify() is run on this mesh, it will give depths
3869 $  depth 0 = {v0, v1, v2}
3870 $  depth 1 = {e0, c0}
3871   where the triangle has been given depth 1, instead of 2, because it is reachable from vertex v2.
3872 
3873   DMPlexStratify() should be called after all calls to DMPlexSymmetrize()
3874 
3875   Level: beginner
3876 
3877 .seealso: DMPlexCreate(), DMPlexSymmetrize(), DMPlexComputeCellTypes()
3878 @*/
3879 PetscErrorCode DMPlexStratify(DM dm)
3880 {
3881   DM_Plex       *mesh = (DM_Plex*) dm->data;
3882   DMLabel        label;
3883   PetscInt       pStart, pEnd, p;
3884   PetscInt       numRoots = 0, numLeaves = 0;
3885   PetscErrorCode ierr;
3886 
3887   PetscFunctionBegin;
3888   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3889   ierr = PetscLogEventBegin(DMPLEX_Stratify,dm,0,0,0);CHKERRQ(ierr);
3890 
3891   /* Create depth label */
3892   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
3893   ierr = DMCreateLabel(dm, "depth");CHKERRQ(ierr);
3894   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
3895 
3896   {
3897     /* Initialize roots and count leaves */
3898     PetscInt sMin = PETSC_MAX_INT;
3899     PetscInt sMax = PETSC_MIN_INT;
3900     PetscInt coneSize, supportSize;
3901 
3902     for (p = pStart; p < pEnd; ++p) {
3903       ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
3904       ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
3905       if (!coneSize && supportSize) {
3906         sMin = PetscMin(p, sMin);
3907         sMax = PetscMax(p, sMax);
3908         ++numRoots;
3909       } else if (!supportSize && coneSize) {
3910         ++numLeaves;
3911       } else if (!supportSize && !coneSize) {
3912         /* Isolated points */
3913         sMin = PetscMin(p, sMin);
3914         sMax = PetscMax(p, sMax);
3915       }
3916     }
3917     ierr = DMPlexCreateDepthStratum(dm, label, 0, sMin, sMax+1);CHKERRQ(ierr);
3918   }
3919 
3920   if (numRoots + numLeaves == (pEnd - pStart)) {
3921     PetscInt sMin = PETSC_MAX_INT;
3922     PetscInt sMax = PETSC_MIN_INT;
3923     PetscInt coneSize, supportSize;
3924 
3925     for (p = pStart; p < pEnd; ++p) {
3926       ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
3927       ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
3928       if (!supportSize && coneSize) {
3929         sMin = PetscMin(p, sMin);
3930         sMax = PetscMax(p, sMax);
3931       }
3932     }
3933     ierr = DMPlexCreateDepthStratum(dm, label, 1, sMin, sMax+1);CHKERRQ(ierr);
3934   } else {
3935     PetscInt level = 0;
3936     PetscInt qStart, qEnd, q;
3937 
3938     ierr = DMLabelGetStratumBounds(label, level, &qStart, &qEnd);CHKERRQ(ierr);
3939     while (qEnd > qStart) {
3940       PetscInt sMin = PETSC_MAX_INT;
3941       PetscInt sMax = PETSC_MIN_INT;
3942 
3943       for (q = qStart; q < qEnd; ++q) {
3944         const PetscInt *support;
3945         PetscInt        supportSize, s;
3946 
3947         ierr = DMPlexGetSupportSize(dm, q, &supportSize);CHKERRQ(ierr);
3948         ierr = DMPlexGetSupport(dm, q, &support);CHKERRQ(ierr);
3949         for (s = 0; s < supportSize; ++s) {
3950           sMin = PetscMin(support[s], sMin);
3951           sMax = PetscMax(support[s], sMax);
3952         }
3953       }
3954       ierr = DMLabelGetNumValues(label, &level);CHKERRQ(ierr);
3955       ierr = DMPlexCreateDepthStratum(dm, label, level, sMin, sMax+1);CHKERRQ(ierr);
3956       ierr = DMLabelGetStratumBounds(label, level, &qStart, &qEnd);CHKERRQ(ierr);
3957     }
3958   }
3959   { /* just in case there is an empty process */
3960     PetscInt numValues, maxValues = 0, v;
3961 
3962     ierr = DMLabelGetNumValues(label, &numValues);CHKERRQ(ierr);
3963     ierr = MPI_Allreduce(&numValues,&maxValues,1,MPIU_INT,MPI_MAX,PetscObjectComm((PetscObject)dm));CHKERRMPI(ierr);
3964     for (v = numValues; v < maxValues; v++) {
3965       ierr = DMLabelAddStratum(label, v);CHKERRQ(ierr);
3966     }
3967   }
3968   ierr = PetscObjectStateGet((PetscObject) label, &mesh->depthState);CHKERRQ(ierr);
3969   ierr = PetscLogEventEnd(DMPLEX_Stratify,dm,0,0,0);CHKERRQ(ierr);
3970   PetscFunctionReturn(0);
3971 }
3972 
3973 PetscErrorCode DMPlexComputeCellType_Internal(DM dm, PetscInt p, PetscInt pdepth, DMPolytopeType *pt)
3974 {
3975   DMPolytopeType ct = DM_POLYTOPE_UNKNOWN;
3976   PetscInt       dim, depth, pheight, coneSize;
3977   PetscErrorCode ierr;
3978 
3979   PetscFunctionBeginHot;
3980   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
3981   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3982   ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
3983   pheight = depth - pdepth;
3984   if (depth <= 1) {
3985     switch (pdepth) {
3986       case 0: ct = DM_POLYTOPE_POINT;break;
3987       case 1:
3988         switch (coneSize) {
3989           case 2: ct = DM_POLYTOPE_SEGMENT;break;
3990           case 3: ct = DM_POLYTOPE_TRIANGLE;break;
3991           case 4:
3992           switch (dim) {
3993             case 2: ct = DM_POLYTOPE_QUADRILATERAL;break;
3994             case 3: ct = DM_POLYTOPE_TETRAHEDRON;break;
3995             default: break;
3996           }
3997           break;
3998         case 5: ct = DM_POLYTOPE_PYRAMID;break;
3999         case 6: ct = DM_POLYTOPE_TRI_PRISM_TENSOR;break;
4000         case 8: ct = DM_POLYTOPE_HEXAHEDRON;break;
4001         default: break;
4002       }
4003     }
4004   } else {
4005     if (pdepth == 0) {
4006       ct = DM_POLYTOPE_POINT;
4007     } else if (pheight == 0) {
4008       switch (dim) {
4009         case 1:
4010           switch (coneSize) {
4011             case 2: ct = DM_POLYTOPE_SEGMENT;break;
4012             default: break;
4013           }
4014           break;
4015         case 2:
4016           switch (coneSize) {
4017             case 3: ct = DM_POLYTOPE_TRIANGLE;break;
4018             case 4: ct = DM_POLYTOPE_QUADRILATERAL;break;
4019             default: break;
4020           }
4021           break;
4022         case 3:
4023           switch (coneSize) {
4024             case 4: ct = DM_POLYTOPE_TETRAHEDRON;break;
4025             case 5:
4026             {
4027               const PetscInt *cone;
4028               PetscInt        faceConeSize;
4029 
4030               ierr = DMPlexGetCone(dm, p, &cone);CHKERRQ(ierr);
4031               ierr = DMPlexGetConeSize(dm, cone[0], &faceConeSize);CHKERRQ(ierr);
4032               switch (faceConeSize) {
4033                 case 3: ct = DM_POLYTOPE_TRI_PRISM_TENSOR;break;
4034                 case 4: ct = DM_POLYTOPE_PYRAMID;break;
4035               }
4036             }
4037             break;
4038             case 6: ct = DM_POLYTOPE_HEXAHEDRON;break;
4039             default: break;
4040           }
4041           break;
4042         default: break;
4043       }
4044     } else if (pheight > 0) {
4045       switch (coneSize) {
4046         case 2: ct = DM_POLYTOPE_SEGMENT;break;
4047         case 3: ct = DM_POLYTOPE_TRIANGLE;break;
4048         case 4: ct = DM_POLYTOPE_QUADRILATERAL;break;
4049         default: break;
4050       }
4051     }
4052   }
4053   *pt = ct;
4054   PetscFunctionReturn(0);
4055 }
4056 
4057 /*@
4058   DMPlexComputeCellTypes - Infer the polytope type of every cell using its dimension and cone size.
4059 
4060   Collective on dm
4061 
4062   Input Parameter:
4063 . mesh - The DMPlex
4064 
4065   DMPlexComputeCellTypes() should be called after all calls to DMPlexSymmetrize() and DMPlexStratify()
4066 
4067   Level: developer
4068 
4069   Note: This function is normally called automatically by Plex when a cell type is requested. It creates an
4070   internal DMLabel named "celltype" which can be directly accessed using DMGetLabel(). A user may disable
4071   automatic creation by creating the label manually, using DMCreateLabel(dm, "celltype").
4072 
4073 .seealso: DMPlexCreate(), DMPlexSymmetrize(), DMPlexStratify(), DMGetLabel(), DMCreateLabel()
4074 @*/
4075 PetscErrorCode DMPlexComputeCellTypes(DM dm)
4076 {
4077   DM_Plex       *mesh;
4078   DMLabel        ctLabel;
4079   PetscInt       pStart, pEnd, p;
4080   PetscErrorCode ierr;
4081 
4082   PetscFunctionBegin;
4083   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4084   mesh = (DM_Plex *) dm->data;
4085   ierr = DMCreateLabel(dm, "celltype");CHKERRQ(ierr);
4086   ierr = DMPlexGetCellTypeLabel(dm, &ctLabel);CHKERRQ(ierr);
4087   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
4088   for (p = pStart; p < pEnd; ++p) {
4089     DMPolytopeType ct = DM_POLYTOPE_UNKNOWN;
4090     PetscInt       pdepth;
4091 
4092     ierr = DMPlexGetPointDepth(dm, p, &pdepth);CHKERRQ(ierr);
4093     ierr = DMPlexComputeCellType_Internal(dm, p, pdepth, &ct);CHKERRQ(ierr);
4094     if (ct == DM_POLYTOPE_UNKNOWN) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_SUP, "Point %D is screwed up", p);
4095     ierr = DMLabelSetValue(ctLabel, p, ct);CHKERRQ(ierr);
4096   }
4097   ierr = PetscObjectStateGet((PetscObject) ctLabel, &mesh->celltypeState);CHKERRQ(ierr);
4098   ierr = PetscObjectViewFromOptions((PetscObject) ctLabel, NULL, "-dm_plex_celltypes_view");CHKERRQ(ierr);
4099   PetscFunctionReturn(0);
4100 }
4101 
4102 /*@C
4103   DMPlexGetJoin - Get an array for the join of the set of points
4104 
4105   Not Collective
4106 
4107   Input Parameters:
4108 + dm - The DMPlex object
4109 . numPoints - The number of input points for the join
4110 - points - The input points
4111 
4112   Output Parameters:
4113 + numCoveredPoints - The number of points in the join
4114 - coveredPoints - The points in the join
4115 
4116   Level: intermediate
4117 
4118   Note: Currently, this is restricted to a single level join
4119 
4120   Fortran Notes:
4121   Since it returns an array, this routine is only available in Fortran 90, and you must
4122   include petsc.h90 in your code.
4123 
4124   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
4125 
4126 .seealso: DMPlexRestoreJoin(), DMPlexGetMeet()
4127 @*/
4128 PetscErrorCode DMPlexGetJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
4129 {
4130   DM_Plex       *mesh = (DM_Plex*) dm->data;
4131   PetscInt      *join[2];
4132   PetscInt       joinSize, i = 0;
4133   PetscInt       dof, off, p, c, m;
4134   PetscErrorCode ierr;
4135 
4136   PetscFunctionBegin;
4137   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4138   PetscValidIntPointer(points, 3);
4139   PetscValidIntPointer(numCoveredPoints, 4);
4140   PetscValidPointer(coveredPoints, 5);
4141   ierr = DMGetWorkArray(dm, mesh->maxSupportSize, MPIU_INT, &join[0]);CHKERRQ(ierr);
4142   ierr = DMGetWorkArray(dm, mesh->maxSupportSize, MPIU_INT, &join[1]);CHKERRQ(ierr);
4143   /* Copy in support of first point */
4144   ierr = PetscSectionGetDof(mesh->supportSection, points[0], &dof);CHKERRQ(ierr);
4145   ierr = PetscSectionGetOffset(mesh->supportSection, points[0], &off);CHKERRQ(ierr);
4146   for (joinSize = 0; joinSize < dof; ++joinSize) {
4147     join[i][joinSize] = mesh->supports[off+joinSize];
4148   }
4149   /* Check each successive support */
4150   for (p = 1; p < numPoints; ++p) {
4151     PetscInt newJoinSize = 0;
4152 
4153     ierr = PetscSectionGetDof(mesh->supportSection, points[p], &dof);CHKERRQ(ierr);
4154     ierr = PetscSectionGetOffset(mesh->supportSection, points[p], &off);CHKERRQ(ierr);
4155     for (c = 0; c < dof; ++c) {
4156       const PetscInt point = mesh->supports[off+c];
4157 
4158       for (m = 0; m < joinSize; ++m) {
4159         if (point == join[i][m]) {
4160           join[1-i][newJoinSize++] = point;
4161           break;
4162         }
4163       }
4164     }
4165     joinSize = newJoinSize;
4166     i        = 1-i;
4167   }
4168   *numCoveredPoints = joinSize;
4169   *coveredPoints    = join[i];
4170   ierr              = DMRestoreWorkArray(dm, mesh->maxSupportSize, MPIU_INT, &join[1-i]);CHKERRQ(ierr);
4171   PetscFunctionReturn(0);
4172 }
4173 
4174 /*@C
4175   DMPlexRestoreJoin - Restore an array for the join of the set of points
4176 
4177   Not Collective
4178 
4179   Input Parameters:
4180 + dm - The DMPlex object
4181 . numPoints - The number of input points for the join
4182 - points - The input points
4183 
4184   Output Parameters:
4185 + numCoveredPoints - The number of points in the join
4186 - coveredPoints - The points in the join
4187 
4188   Fortran Notes:
4189   Since it returns an array, this routine is only available in Fortran 90, and you must
4190   include petsc.h90 in your code.
4191 
4192   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
4193 
4194   Level: intermediate
4195 
4196 .seealso: DMPlexGetJoin(), DMPlexGetFullJoin(), DMPlexGetMeet()
4197 @*/
4198 PetscErrorCode DMPlexRestoreJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
4199 {
4200   PetscErrorCode ierr;
4201 
4202   PetscFunctionBegin;
4203   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4204   if (points) PetscValidIntPointer(points,3);
4205   if (numCoveredPoints) PetscValidIntPointer(numCoveredPoints,4);
4206   PetscValidPointer(coveredPoints, 5);
4207   ierr = DMRestoreWorkArray(dm, 0, MPIU_INT, (void*) coveredPoints);CHKERRQ(ierr);
4208   if (numCoveredPoints) *numCoveredPoints = 0;
4209   PetscFunctionReturn(0);
4210 }
4211 
4212 /*@C
4213   DMPlexGetFullJoin - Get an array for the join of the set of points
4214 
4215   Not Collective
4216 
4217   Input Parameters:
4218 + dm - The DMPlex object
4219 . numPoints - The number of input points for the join
4220 - points - The input points
4221 
4222   Output Parameters:
4223 + numCoveredPoints - The number of points in the join
4224 - coveredPoints - The points in the join
4225 
4226   Fortran Notes:
4227   Since it returns an array, this routine is only available in Fortran 90, and you must
4228   include petsc.h90 in your code.
4229 
4230   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
4231 
4232   Level: intermediate
4233 
4234 .seealso: DMPlexGetJoin(), DMPlexRestoreJoin(), DMPlexGetMeet()
4235 @*/
4236 PetscErrorCode DMPlexGetFullJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
4237 {
4238   DM_Plex       *mesh = (DM_Plex*) dm->data;
4239   PetscInt      *offsets, **closures;
4240   PetscInt      *join[2];
4241   PetscInt       depth = 0, maxSize, joinSize = 0, i = 0;
4242   PetscInt       p, d, c, m, ms;
4243   PetscErrorCode ierr;
4244 
4245   PetscFunctionBegin;
4246   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4247   PetscValidIntPointer(points, 3);
4248   PetscValidIntPointer(numCoveredPoints, 4);
4249   PetscValidPointer(coveredPoints, 5);
4250 
4251   ierr    = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
4252   ierr    = PetscCalloc1(numPoints, &closures);CHKERRQ(ierr);
4253   ierr    = DMGetWorkArray(dm, numPoints*(depth+2), MPIU_INT, &offsets);CHKERRQ(ierr);
4254   ms      = mesh->maxSupportSize;
4255   maxSize = (ms > 1) ? ((PetscPowInt(ms,depth+1)-1)/(ms-1)) : depth + 1;
4256   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &join[0]);CHKERRQ(ierr);
4257   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &join[1]);CHKERRQ(ierr);
4258 
4259   for (p = 0; p < numPoints; ++p) {
4260     PetscInt closureSize;
4261 
4262     ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_FALSE, &closureSize, &closures[p]);CHKERRQ(ierr);
4263 
4264     offsets[p*(depth+2)+0] = 0;
4265     for (d = 0; d < depth+1; ++d) {
4266       PetscInt pStart, pEnd, i;
4267 
4268       ierr = DMPlexGetDepthStratum(dm, d, &pStart, &pEnd);CHKERRQ(ierr);
4269       for (i = offsets[p*(depth+2)+d]; i < closureSize; ++i) {
4270         if ((pStart > closures[p][i*2]) || (pEnd <= closures[p][i*2])) {
4271           offsets[p*(depth+2)+d+1] = i;
4272           break;
4273         }
4274       }
4275       if (i == closureSize) offsets[p*(depth+2)+d+1] = i;
4276     }
4277     if (offsets[p*(depth+2)+depth+1] != closureSize) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Total size of closure %D should be %D", offsets[p*(depth+2)+depth+1], closureSize);
4278   }
4279   for (d = 0; d < depth+1; ++d) {
4280     PetscInt dof;
4281 
4282     /* Copy in support of first point */
4283     dof = offsets[d+1] - offsets[d];
4284     for (joinSize = 0; joinSize < dof; ++joinSize) {
4285       join[i][joinSize] = closures[0][(offsets[d]+joinSize)*2];
4286     }
4287     /* Check each successive cone */
4288     for (p = 1; p < numPoints && joinSize; ++p) {
4289       PetscInt newJoinSize = 0;
4290 
4291       dof = offsets[p*(depth+2)+d+1] - offsets[p*(depth+2)+d];
4292       for (c = 0; c < dof; ++c) {
4293         const PetscInt point = closures[p][(offsets[p*(depth+2)+d]+c)*2];
4294 
4295         for (m = 0; m < joinSize; ++m) {
4296           if (point == join[i][m]) {
4297             join[1-i][newJoinSize++] = point;
4298             break;
4299           }
4300         }
4301       }
4302       joinSize = newJoinSize;
4303       i        = 1-i;
4304     }
4305     if (joinSize) break;
4306   }
4307   *numCoveredPoints = joinSize;
4308   *coveredPoints    = join[i];
4309   for (p = 0; p < numPoints; ++p) {
4310     ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_FALSE, NULL, &closures[p]);CHKERRQ(ierr);
4311   }
4312   ierr = PetscFree(closures);CHKERRQ(ierr);
4313   ierr = DMRestoreWorkArray(dm, numPoints*(depth+2), MPIU_INT, &offsets);CHKERRQ(ierr);
4314   ierr = DMRestoreWorkArray(dm, mesh->maxSupportSize, MPIU_INT, &join[1-i]);CHKERRQ(ierr);
4315   PetscFunctionReturn(0);
4316 }
4317 
4318 /*@C
4319   DMPlexGetMeet - Get an array for the meet of the set of points
4320 
4321   Not Collective
4322 
4323   Input Parameters:
4324 + dm - The DMPlex object
4325 . numPoints - The number of input points for the meet
4326 - points - The input points
4327 
4328   Output Parameters:
4329 + numCoveredPoints - The number of points in the meet
4330 - coveredPoints - The points in the meet
4331 
4332   Level: intermediate
4333 
4334   Note: Currently, this is restricted to a single level meet
4335 
4336   Fortran Notes:
4337   Since it returns an array, this routine is only available in Fortran 90, and you must
4338   include petsc.h90 in your code.
4339 
4340   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
4341 
4342 .seealso: DMPlexRestoreMeet(), DMPlexGetJoin()
4343 @*/
4344 PetscErrorCode DMPlexGetMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveringPoints, const PetscInt **coveringPoints)
4345 {
4346   DM_Plex       *mesh = (DM_Plex*) dm->data;
4347   PetscInt      *meet[2];
4348   PetscInt       meetSize, i = 0;
4349   PetscInt       dof, off, p, c, m;
4350   PetscErrorCode ierr;
4351 
4352   PetscFunctionBegin;
4353   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4354   PetscValidPointer(points, 3);
4355   PetscValidPointer(numCoveringPoints, 4);
4356   PetscValidPointer(coveringPoints, 5);
4357   ierr = DMGetWorkArray(dm, mesh->maxConeSize, MPIU_INT, &meet[0]);CHKERRQ(ierr);
4358   ierr = DMGetWorkArray(dm, mesh->maxConeSize, MPIU_INT, &meet[1]);CHKERRQ(ierr);
4359   /* Copy in cone of first point */
4360   ierr = PetscSectionGetDof(mesh->coneSection, points[0], &dof);CHKERRQ(ierr);
4361   ierr = PetscSectionGetOffset(mesh->coneSection, points[0], &off);CHKERRQ(ierr);
4362   for (meetSize = 0; meetSize < dof; ++meetSize) {
4363     meet[i][meetSize] = mesh->cones[off+meetSize];
4364   }
4365   /* Check each successive cone */
4366   for (p = 1; p < numPoints; ++p) {
4367     PetscInt newMeetSize = 0;
4368 
4369     ierr = PetscSectionGetDof(mesh->coneSection, points[p], &dof);CHKERRQ(ierr);
4370     ierr = PetscSectionGetOffset(mesh->coneSection, points[p], &off);CHKERRQ(ierr);
4371     for (c = 0; c < dof; ++c) {
4372       const PetscInt point = mesh->cones[off+c];
4373 
4374       for (m = 0; m < meetSize; ++m) {
4375         if (point == meet[i][m]) {
4376           meet[1-i][newMeetSize++] = point;
4377           break;
4378         }
4379       }
4380     }
4381     meetSize = newMeetSize;
4382     i        = 1-i;
4383   }
4384   *numCoveringPoints = meetSize;
4385   *coveringPoints    = meet[i];
4386   ierr               = DMRestoreWorkArray(dm, mesh->maxConeSize, MPIU_INT, &meet[1-i]);CHKERRQ(ierr);
4387   PetscFunctionReturn(0);
4388 }
4389 
4390 /*@C
4391   DMPlexRestoreMeet - Restore an array for the meet of the set of points
4392 
4393   Not Collective
4394 
4395   Input Parameters:
4396 + dm - The DMPlex object
4397 . numPoints - The number of input points for the meet
4398 - points - The input points
4399 
4400   Output Parameters:
4401 + numCoveredPoints - The number of points in the meet
4402 - coveredPoints - The points in the meet
4403 
4404   Level: intermediate
4405 
4406   Fortran Notes:
4407   Since it returns an array, this routine is only available in Fortran 90, and you must
4408   include petsc.h90 in your code.
4409 
4410   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
4411 
4412 .seealso: DMPlexGetMeet(), DMPlexGetFullMeet(), DMPlexGetJoin()
4413 @*/
4414 PetscErrorCode DMPlexRestoreMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
4415 {
4416   PetscErrorCode ierr;
4417 
4418   PetscFunctionBegin;
4419   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4420   if (points) PetscValidIntPointer(points,3);
4421   if (numCoveredPoints) PetscValidIntPointer(numCoveredPoints,4);
4422   PetscValidPointer(coveredPoints,5);
4423   ierr = DMRestoreWorkArray(dm, 0, MPIU_INT, (void*) coveredPoints);CHKERRQ(ierr);
4424   if (numCoveredPoints) *numCoveredPoints = 0;
4425   PetscFunctionReturn(0);
4426 }
4427 
4428 /*@C
4429   DMPlexGetFullMeet - Get an array for the meet of the set of points
4430 
4431   Not Collective
4432 
4433   Input Parameters:
4434 + dm - The DMPlex object
4435 . numPoints - The number of input points for the meet
4436 - points - The input points
4437 
4438   Output Parameters:
4439 + numCoveredPoints - The number of points in the meet
4440 - coveredPoints - The points in the meet
4441 
4442   Level: intermediate
4443 
4444   Fortran Notes:
4445   Since it returns an array, this routine is only available in Fortran 90, and you must
4446   include petsc.h90 in your code.
4447 
4448   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
4449 
4450 .seealso: DMPlexGetMeet(), DMPlexRestoreMeet(), DMPlexGetJoin()
4451 @*/
4452 PetscErrorCode DMPlexGetFullMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
4453 {
4454   DM_Plex       *mesh = (DM_Plex*) dm->data;
4455   PetscInt      *offsets, **closures;
4456   PetscInt      *meet[2];
4457   PetscInt       height = 0, maxSize, meetSize = 0, i = 0;
4458   PetscInt       p, h, c, m, mc;
4459   PetscErrorCode ierr;
4460 
4461   PetscFunctionBegin;
4462   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4463   PetscValidPointer(points, 3);
4464   PetscValidPointer(numCoveredPoints, 4);
4465   PetscValidPointer(coveredPoints, 5);
4466 
4467   ierr    = DMPlexGetDepth(dm, &height);CHKERRQ(ierr);
4468   ierr    = PetscMalloc1(numPoints, &closures);CHKERRQ(ierr);
4469   ierr    = DMGetWorkArray(dm, numPoints*(height+2), MPIU_INT, &offsets);CHKERRQ(ierr);
4470   mc      = mesh->maxConeSize;
4471   maxSize = (mc > 1) ? ((PetscPowInt(mc,height+1)-1)/(mc-1)) : height + 1;
4472   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &meet[0]);CHKERRQ(ierr);
4473   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &meet[1]);CHKERRQ(ierr);
4474 
4475   for (p = 0; p < numPoints; ++p) {
4476     PetscInt closureSize;
4477 
4478     ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_TRUE, &closureSize, &closures[p]);CHKERRQ(ierr);
4479 
4480     offsets[p*(height+2)+0] = 0;
4481     for (h = 0; h < height+1; ++h) {
4482       PetscInt pStart, pEnd, i;
4483 
4484       ierr = DMPlexGetHeightStratum(dm, h, &pStart, &pEnd);CHKERRQ(ierr);
4485       for (i = offsets[p*(height+2)+h]; i < closureSize; ++i) {
4486         if ((pStart > closures[p][i*2]) || (pEnd <= closures[p][i*2])) {
4487           offsets[p*(height+2)+h+1] = i;
4488           break;
4489         }
4490       }
4491       if (i == closureSize) offsets[p*(height+2)+h+1] = i;
4492     }
4493     if (offsets[p*(height+2)+height+1] != closureSize) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Total size of closure %D should be %D", offsets[p*(height+2)+height+1], closureSize);
4494   }
4495   for (h = 0; h < height+1; ++h) {
4496     PetscInt dof;
4497 
4498     /* Copy in cone of first point */
4499     dof = offsets[h+1] - offsets[h];
4500     for (meetSize = 0; meetSize < dof; ++meetSize) {
4501       meet[i][meetSize] = closures[0][(offsets[h]+meetSize)*2];
4502     }
4503     /* Check each successive cone */
4504     for (p = 1; p < numPoints && meetSize; ++p) {
4505       PetscInt newMeetSize = 0;
4506 
4507       dof = offsets[p*(height+2)+h+1] - offsets[p*(height+2)+h];
4508       for (c = 0; c < dof; ++c) {
4509         const PetscInt point = closures[p][(offsets[p*(height+2)+h]+c)*2];
4510 
4511         for (m = 0; m < meetSize; ++m) {
4512           if (point == meet[i][m]) {
4513             meet[1-i][newMeetSize++] = point;
4514             break;
4515           }
4516         }
4517       }
4518       meetSize = newMeetSize;
4519       i        = 1-i;
4520     }
4521     if (meetSize) break;
4522   }
4523   *numCoveredPoints = meetSize;
4524   *coveredPoints    = meet[i];
4525   for (p = 0; p < numPoints; ++p) {
4526     ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_TRUE, NULL, &closures[p]);CHKERRQ(ierr);
4527   }
4528   ierr = PetscFree(closures);CHKERRQ(ierr);
4529   ierr = DMRestoreWorkArray(dm, numPoints*(height+2), MPIU_INT, &offsets);CHKERRQ(ierr);
4530   ierr = DMRestoreWorkArray(dm, mesh->maxConeSize, MPIU_INT, &meet[1-i]);CHKERRQ(ierr);
4531   PetscFunctionReturn(0);
4532 }
4533 
4534 /*@C
4535   DMPlexEqual - Determine if two DMs have the same topology
4536 
4537   Not Collective
4538 
4539   Input Parameters:
4540 + dmA - A DMPlex object
4541 - dmB - A DMPlex object
4542 
4543   Output Parameters:
4544 . equal - PETSC_TRUE if the topologies are identical
4545 
4546   Level: intermediate
4547 
4548   Notes:
4549   We are not solving graph isomorphism, so we do not permutation.
4550 
4551 .seealso: DMPlexGetCone()
4552 @*/
4553 PetscErrorCode DMPlexEqual(DM dmA, DM dmB, PetscBool *equal)
4554 {
4555   PetscInt       depth, depthB, pStart, pEnd, pStartB, pEndB, p;
4556   PetscErrorCode ierr;
4557 
4558   PetscFunctionBegin;
4559   PetscValidHeaderSpecific(dmA, DM_CLASSID, 1);
4560   PetscValidHeaderSpecific(dmB, DM_CLASSID, 2);
4561   PetscValidPointer(equal, 3);
4562 
4563   *equal = PETSC_FALSE;
4564   ierr = DMPlexGetDepth(dmA, &depth);CHKERRQ(ierr);
4565   ierr = DMPlexGetDepth(dmB, &depthB);CHKERRQ(ierr);
4566   if (depth != depthB) PetscFunctionReturn(0);
4567   ierr = DMPlexGetChart(dmA, &pStart,  &pEnd);CHKERRQ(ierr);
4568   ierr = DMPlexGetChart(dmB, &pStartB, &pEndB);CHKERRQ(ierr);
4569   if ((pStart != pStartB) || (pEnd != pEndB)) PetscFunctionReturn(0);
4570   for (p = pStart; p < pEnd; ++p) {
4571     const PetscInt *cone, *coneB, *ornt, *orntB, *support, *supportB;
4572     PetscInt        coneSize, coneSizeB, c, supportSize, supportSizeB, s;
4573 
4574     ierr = DMPlexGetConeSize(dmA, p, &coneSize);CHKERRQ(ierr);
4575     ierr = DMPlexGetCone(dmA, p, &cone);CHKERRQ(ierr);
4576     ierr = DMPlexGetConeOrientation(dmA, p, &ornt);CHKERRQ(ierr);
4577     ierr = DMPlexGetConeSize(dmB, p, &coneSizeB);CHKERRQ(ierr);
4578     ierr = DMPlexGetCone(dmB, p, &coneB);CHKERRQ(ierr);
4579     ierr = DMPlexGetConeOrientation(dmB, p, &orntB);CHKERRQ(ierr);
4580     if (coneSize != coneSizeB) PetscFunctionReturn(0);
4581     for (c = 0; c < coneSize; ++c) {
4582       if (cone[c] != coneB[c]) PetscFunctionReturn(0);
4583       if (ornt[c] != orntB[c]) PetscFunctionReturn(0);
4584     }
4585     ierr = DMPlexGetSupportSize(dmA, p, &supportSize);CHKERRQ(ierr);
4586     ierr = DMPlexGetSupport(dmA, p, &support);CHKERRQ(ierr);
4587     ierr = DMPlexGetSupportSize(dmB, p, &supportSizeB);CHKERRQ(ierr);
4588     ierr = DMPlexGetSupport(dmB, p, &supportB);CHKERRQ(ierr);
4589     if (supportSize != supportSizeB) PetscFunctionReturn(0);
4590     for (s = 0; s < supportSize; ++s) {
4591       if (support[s] != supportB[s]) PetscFunctionReturn(0);
4592     }
4593   }
4594   *equal = PETSC_TRUE;
4595   PetscFunctionReturn(0);
4596 }
4597 
4598 /*@C
4599   DMPlexGetNumFaceVertices - Returns the number of vertices on a face
4600 
4601   Not Collective
4602 
4603   Input Parameters:
4604 + dm         - The DMPlex
4605 . cellDim    - The cell dimension
4606 - numCorners - The number of vertices on a cell
4607 
4608   Output Parameters:
4609 . numFaceVertices - The number of vertices on a face
4610 
4611   Level: developer
4612 
4613   Notes:
4614   Of course this can only work for a restricted set of symmetric shapes
4615 
4616 .seealso: DMPlexGetCone()
4617 @*/
4618 PetscErrorCode DMPlexGetNumFaceVertices(DM dm, PetscInt cellDim, PetscInt numCorners, PetscInt *numFaceVertices)
4619 {
4620   MPI_Comm       comm;
4621   PetscErrorCode ierr;
4622 
4623   PetscFunctionBegin;
4624   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
4625   PetscValidPointer(numFaceVertices,4);
4626   switch (cellDim) {
4627   case 0:
4628     *numFaceVertices = 0;
4629     break;
4630   case 1:
4631     *numFaceVertices = 1;
4632     break;
4633   case 2:
4634     switch (numCorners) {
4635     case 3: /* triangle */
4636       *numFaceVertices = 2; /* Edge has 2 vertices */
4637       break;
4638     case 4: /* quadrilateral */
4639       *numFaceVertices = 2; /* Edge has 2 vertices */
4640       break;
4641     case 6: /* quadratic triangle, tri and quad cohesive Lagrange cells */
4642       *numFaceVertices = 3; /* Edge has 3 vertices */
4643       break;
4644     case 9: /* quadratic quadrilateral, quadratic quad cohesive Lagrange cells */
4645       *numFaceVertices = 3; /* Edge has 3 vertices */
4646       break;
4647     default:
4648       SETERRQ2(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid number of face corners %D for dimension %D", numCorners, cellDim);
4649     }
4650     break;
4651   case 3:
4652     switch (numCorners) {
4653     case 4: /* tetradehdron */
4654       *numFaceVertices = 3; /* Face has 3 vertices */
4655       break;
4656     case 6: /* tet cohesive cells */
4657       *numFaceVertices = 4; /* Face has 4 vertices */
4658       break;
4659     case 8: /* hexahedron */
4660       *numFaceVertices = 4; /* Face has 4 vertices */
4661       break;
4662     case 9: /* tet cohesive Lagrange cells */
4663       *numFaceVertices = 6; /* Face has 6 vertices */
4664       break;
4665     case 10: /* quadratic tetrahedron */
4666       *numFaceVertices = 6; /* Face has 6 vertices */
4667       break;
4668     case 12: /* hex cohesive Lagrange cells */
4669       *numFaceVertices = 6; /* Face has 6 vertices */
4670       break;
4671     case 18: /* quadratic tet cohesive Lagrange cells */
4672       *numFaceVertices = 6; /* Face has 6 vertices */
4673       break;
4674     case 27: /* quadratic hexahedron, quadratic hex cohesive Lagrange cells */
4675       *numFaceVertices = 9; /* Face has 9 vertices */
4676       break;
4677     default:
4678       SETERRQ2(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid number of face corners %D for dimension %D", numCorners, cellDim);
4679     }
4680     break;
4681   default:
4682     SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid cell dimension %D", cellDim);
4683   }
4684   PetscFunctionReturn(0);
4685 }
4686 
4687 /*@
4688   DMPlexGetDepthLabel - Get the DMLabel recording the depth of each point
4689 
4690   Not Collective
4691 
4692   Input Parameter:
4693 . dm    - The DMPlex object
4694 
4695   Output Parameter:
4696 . depthLabel - The DMLabel recording point depth
4697 
4698   Level: developer
4699 
4700 .seealso: DMPlexGetDepth(), DMPlexGetHeightStratum(), DMPlexGetDepthStratum(), DMPlexGetPointDepth(),
4701 @*/
4702 PetscErrorCode DMPlexGetDepthLabel(DM dm, DMLabel *depthLabel)
4703 {
4704   PetscFunctionBegin;
4705   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4706   PetscValidPointer(depthLabel, 2);
4707   *depthLabel = dm->depthLabel;
4708   PetscFunctionReturn(0);
4709 }
4710 
4711 /*@
4712   DMPlexGetDepth - Get the depth of the DAG representing this mesh
4713 
4714   Not Collective
4715 
4716   Input Parameter:
4717 . dm    - The DMPlex object
4718 
4719   Output Parameter:
4720 . depth - The number of strata (breadth first levels) in the DAG
4721 
4722   Level: developer
4723 
4724   Notes:
4725   This returns maximum of point depths over all points, i.e. maximum value of the label returned by DMPlexGetDepthLabel().
4726   The point depth is described more in detail in DMPlexGetDepthStratum().
4727   An empty mesh gives -1.
4728 
4729 .seealso: DMPlexGetDepthLabel(), DMPlexGetDepthStratum(), DMPlexGetPointDepth(), DMPlexSymmetrize()
4730 @*/
4731 PetscErrorCode DMPlexGetDepth(DM dm, PetscInt *depth)
4732 {
4733   DMLabel        label;
4734   PetscInt       d = 0;
4735   PetscErrorCode ierr;
4736 
4737   PetscFunctionBegin;
4738   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4739   PetscValidPointer(depth, 2);
4740   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
4741   if (label) {ierr = DMLabelGetNumValues(label, &d);CHKERRQ(ierr);}
4742   *depth = d-1;
4743   PetscFunctionReturn(0);
4744 }
4745 
4746 /*@
4747   DMPlexGetDepthStratum - Get the bounds [start, end) for all points at a certain depth.
4748 
4749   Not Collective
4750 
4751   Input Parameters:
4752 + dm           - The DMPlex object
4753 - stratumValue - The requested depth
4754 
4755   Output Parameters:
4756 + start - The first point at this depth
4757 - end   - One beyond the last point at this depth
4758 
4759   Notes:
4760   Depth indexing is related to topological dimension.  Depth stratum 0 contains the lowest topological dimension points,
4761   often "vertices".  If the mesh is "interpolated" (see DMPlexInterpolate()), then depth stratum 1 contains the next
4762   higher dimension, e.g., "edges".
4763 
4764   Level: developer
4765 
4766 .seealso: DMPlexGetHeightStratum(), DMPlexGetDepth(), DMPlexGetDepthLabel(), DMPlexGetPointDepth(), DMPlexSymmetrize(), DMPlexInterpolate()
4767 @*/
4768 PetscErrorCode DMPlexGetDepthStratum(DM dm, PetscInt stratumValue, PetscInt *start, PetscInt *end)
4769 {
4770   DMLabel        label;
4771   PetscInt       pStart, pEnd;
4772   PetscErrorCode ierr;
4773 
4774   PetscFunctionBegin;
4775   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4776   if (start) {PetscValidPointer(start, 3); *start = 0;}
4777   if (end)   {PetscValidPointer(end,   4); *end   = 0;}
4778   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
4779   if (pStart == pEnd) PetscFunctionReturn(0);
4780   if (stratumValue < 0) {
4781     if (start) *start = pStart;
4782     if (end)   *end   = pEnd;
4783     PetscFunctionReturn(0);
4784   }
4785   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
4786   if (!label) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "No label named depth was found");
4787   ierr = DMLabelGetStratumBounds(label, stratumValue, start, end);CHKERRQ(ierr);
4788   PetscFunctionReturn(0);
4789 }
4790 
4791 /*@
4792   DMPlexGetHeightStratum - Get the bounds [start, end) for all points at a certain height.
4793 
4794   Not Collective
4795 
4796   Input Parameters:
4797 + dm           - The DMPlex object
4798 - stratumValue - The requested height
4799 
4800   Output Parameters:
4801 + start - The first point at this height
4802 - end   - One beyond the last point at this height
4803 
4804   Notes:
4805   Height indexing is related to topological codimension.  Height stratum 0 contains the highest topological dimension
4806   points, often called "cells" or "elements".  If the mesh is "interpolated" (see DMPlexInterpolate()), then height
4807   stratum 1 contains the boundary of these "cells", often called "faces" or "facets".
4808 
4809   Level: developer
4810 
4811 .seealso: DMPlexGetDepthStratum(), DMPlexGetDepth(), DMPlexGetPointHeight()
4812 @*/
4813 PetscErrorCode DMPlexGetHeightStratum(DM dm, PetscInt stratumValue, PetscInt *start, PetscInt *end)
4814 {
4815   DMLabel        label;
4816   PetscInt       depth, pStart, pEnd;
4817   PetscErrorCode ierr;
4818 
4819   PetscFunctionBegin;
4820   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4821   if (start) {PetscValidPointer(start, 3); *start = 0;}
4822   if (end)   {PetscValidPointer(end,   4); *end   = 0;}
4823   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
4824   if (pStart == pEnd) PetscFunctionReturn(0);
4825   if (stratumValue < 0) {
4826     if (start) *start = pStart;
4827     if (end)   *end   = pEnd;
4828     PetscFunctionReturn(0);
4829   }
4830   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
4831   if (!label) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "No label named depth was found");
4832   ierr = DMLabelGetNumValues(label, &depth);CHKERRQ(ierr);
4833   ierr = DMLabelGetStratumBounds(label, depth-1-stratumValue, start, end);CHKERRQ(ierr);
4834   PetscFunctionReturn(0);
4835 }
4836 
4837 /*@
4838   DMPlexGetPointDepth - Get the depth of a given point
4839 
4840   Not Collective
4841 
4842   Input Parameters:
4843 + dm    - The DMPlex object
4844 - point - The point
4845 
4846   Output Parameter:
4847 . depth - The depth of the point
4848 
4849   Level: intermediate
4850 
4851 .seealso: DMPlexGetCellType(), DMPlexGetDepthLabel(), DMPlexGetDepth(), DMPlexGetPointHeight()
4852 @*/
4853 PetscErrorCode DMPlexGetPointDepth(DM dm, PetscInt point, PetscInt *depth)
4854 {
4855   PetscErrorCode ierr;
4856 
4857   PetscFunctionBegin;
4858   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4859   PetscValidIntPointer(depth, 3);
4860   ierr = DMLabelGetValue(dm->depthLabel, point, depth);CHKERRQ(ierr);
4861   PetscFunctionReturn(0);
4862 }
4863 
4864 /*@
4865   DMPlexGetPointHeight - Get the height of a given point
4866 
4867   Not Collective
4868 
4869   Input Parameters:
4870 + dm    - The DMPlex object
4871 - point - The point
4872 
4873   Output Parameter:
4874 . height - The height of the point
4875 
4876   Level: intermediate
4877 
4878 .seealso: DMPlexGetCellType(), DMPlexGetDepthLabel(), DMPlexGetDepth(), DMPlexGetPointDepth()
4879 @*/
4880 PetscErrorCode DMPlexGetPointHeight(DM dm, PetscInt point, PetscInt *height)
4881 {
4882   PetscInt       n, pDepth;
4883   PetscErrorCode ierr;
4884 
4885   PetscFunctionBegin;
4886   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4887   PetscValidIntPointer(height, 3);
4888   ierr = DMLabelGetNumValues(dm->depthLabel, &n);CHKERRQ(ierr);
4889   ierr = DMLabelGetValue(dm->depthLabel, point, &pDepth);CHKERRQ(ierr);
4890   *height = n - 1 - pDepth;  /* DAG depth is n-1 */
4891   PetscFunctionReturn(0);
4892 }
4893 
4894 /*@
4895   DMPlexGetCellTypeLabel - Get the DMLabel recording the polytope type of each cell
4896 
4897   Not Collective
4898 
4899   Input Parameter:
4900 . dm - The DMPlex object
4901 
4902   Output Parameter:
4903 . celltypeLabel - The DMLabel recording cell polytope type
4904 
4905   Note: This function will trigger automatica computation of cell types. This can be disabled by calling
4906   DMCreateLabel(dm, "celltype") beforehand.
4907 
4908   Level: developer
4909 
4910 .seealso: DMPlexGetCellType(), DMPlexGetDepthLabel(), DMCreateLabel()
4911 @*/
4912 PetscErrorCode DMPlexGetCellTypeLabel(DM dm, DMLabel *celltypeLabel)
4913 {
4914   PetscErrorCode ierr;
4915 
4916   PetscFunctionBegin;
4917   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4918   PetscValidPointer(celltypeLabel, 2);
4919   if (!dm->celltypeLabel) {ierr = DMPlexComputeCellTypes(dm);CHKERRQ(ierr);}
4920   *celltypeLabel = dm->celltypeLabel;
4921   PetscFunctionReturn(0);
4922 }
4923 
4924 /*@
4925   DMPlexGetCellType - Get the polytope type of a given cell
4926 
4927   Not Collective
4928 
4929   Input Parameters:
4930 + dm   - The DMPlex object
4931 - cell - The cell
4932 
4933   Output Parameter:
4934 . celltype - The polytope type of the cell
4935 
4936   Level: intermediate
4937 
4938 .seealso: DMPlexGetCellTypeLabel(), DMPlexGetDepthLabel(), DMPlexGetDepth()
4939 @*/
4940 PetscErrorCode DMPlexGetCellType(DM dm, PetscInt cell, DMPolytopeType *celltype)
4941 {
4942   DMLabel        label;
4943   PetscInt       ct;
4944   PetscErrorCode ierr;
4945 
4946   PetscFunctionBegin;
4947   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4948   PetscValidPointer(celltype, 3);
4949   ierr = DMPlexGetCellTypeLabel(dm, &label);CHKERRQ(ierr);
4950   ierr = DMLabelGetValue(label, cell, &ct);CHKERRQ(ierr);
4951   if (ct < 0) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Cell %D has not been assigned a cell type", cell);
4952   *celltype = (DMPolytopeType) ct;
4953   PetscFunctionReturn(0);
4954 }
4955 
4956 /*@
4957   DMPlexSetCellType - Set the polytope type of a given cell
4958 
4959   Not Collective
4960 
4961   Input Parameters:
4962 + dm   - The DMPlex object
4963 . cell - The cell
4964 - celltype - The polytope type of the cell
4965 
4966   Note: By default, cell types will be automatically computed using DMPlexComputeCellTypes() before this function
4967   is executed. This function will override the computed type. However, if automatic classification will not succeed
4968   and a user wants to manually specify all types, the classification must be disabled by calling
4969   DMCreaateLabel(dm, "celltype") before getting or setting any cell types.
4970 
4971   Level: advanced
4972 
4973 .seealso: DMPlexGetCellTypeLabel(), DMPlexGetDepthLabel(), DMPlexGetDepth(), DMPlexComputeCellTypes(), DMCreateLabel()
4974 @*/
4975 PetscErrorCode DMPlexSetCellType(DM dm, PetscInt cell, DMPolytopeType celltype)
4976 {
4977   DMLabel        label;
4978   PetscErrorCode ierr;
4979 
4980   PetscFunctionBegin;
4981   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4982   ierr = DMPlexGetCellTypeLabel(dm, &label);CHKERRQ(ierr);
4983   ierr = DMLabelSetValue(label, cell, celltype);CHKERRQ(ierr);
4984   PetscFunctionReturn(0);
4985 }
4986 
4987 PetscErrorCode DMCreateCoordinateDM_Plex(DM dm, DM *cdm)
4988 {
4989   PetscSection   section, s;
4990   Mat            m;
4991   PetscInt       maxHeight;
4992   PetscErrorCode ierr;
4993 
4994   PetscFunctionBegin;
4995   ierr = DMClone(dm, cdm);CHKERRQ(ierr);
4996   ierr = DMPlexGetMaxProjectionHeight(dm, &maxHeight);CHKERRQ(ierr);
4997   ierr = DMPlexSetMaxProjectionHeight(*cdm, maxHeight);CHKERRQ(ierr);
4998   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &section);CHKERRQ(ierr);
4999   ierr = DMSetLocalSection(*cdm, section);CHKERRQ(ierr);
5000   ierr = PetscSectionDestroy(&section);CHKERRQ(ierr);
5001   ierr = PetscSectionCreate(PETSC_COMM_SELF, &s);CHKERRQ(ierr);
5002   ierr = MatCreate(PETSC_COMM_SELF, &m);CHKERRQ(ierr);
5003   ierr = DMSetDefaultConstraints(*cdm, s, m);CHKERRQ(ierr);
5004   ierr = PetscSectionDestroy(&s);CHKERRQ(ierr);
5005   ierr = MatDestroy(&m);CHKERRQ(ierr);
5006 
5007   ierr = DMSetNumFields(*cdm, 1);CHKERRQ(ierr);
5008   ierr = DMCreateDS(*cdm);CHKERRQ(ierr);
5009   PetscFunctionReturn(0);
5010 }
5011 
5012 PetscErrorCode DMCreateCoordinateField_Plex(DM dm, DMField *field)
5013 {
5014   Vec            coordsLocal;
5015   DM             coordsDM;
5016   PetscErrorCode ierr;
5017 
5018   PetscFunctionBegin;
5019   *field = NULL;
5020   ierr = DMGetCoordinatesLocal(dm,&coordsLocal);CHKERRQ(ierr);
5021   ierr = DMGetCoordinateDM(dm,&coordsDM);CHKERRQ(ierr);
5022   if (coordsLocal && coordsDM) {
5023     ierr = DMFieldCreateDS(coordsDM, 0, coordsLocal, field);CHKERRQ(ierr);
5024   }
5025   PetscFunctionReturn(0);
5026 }
5027 
5028 /*@C
5029   DMPlexGetConeSection - Return a section which describes the layout of cone data
5030 
5031   Not Collective
5032 
5033   Input Parameters:
5034 . dm        - The DMPlex object
5035 
5036   Output Parameter:
5037 . section - The PetscSection object
5038 
5039   Level: developer
5040 
5041 .seealso: DMPlexGetSupportSection(), DMPlexGetCones(), DMPlexGetConeOrientations()
5042 @*/
5043 PetscErrorCode DMPlexGetConeSection(DM dm, PetscSection *section)
5044 {
5045   DM_Plex *mesh = (DM_Plex*) dm->data;
5046 
5047   PetscFunctionBegin;
5048   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5049   if (section) *section = mesh->coneSection;
5050   PetscFunctionReturn(0);
5051 }
5052 
5053 /*@C
5054   DMPlexGetSupportSection - Return a section which describes the layout of support data
5055 
5056   Not Collective
5057 
5058   Input Parameters:
5059 . dm        - The DMPlex object
5060 
5061   Output Parameter:
5062 . section - The PetscSection object
5063 
5064   Level: developer
5065 
5066 .seealso: DMPlexGetConeSection()
5067 @*/
5068 PetscErrorCode DMPlexGetSupportSection(DM dm, PetscSection *section)
5069 {
5070   DM_Plex *mesh = (DM_Plex*) dm->data;
5071 
5072   PetscFunctionBegin;
5073   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5074   if (section) *section = mesh->supportSection;
5075   PetscFunctionReturn(0);
5076 }
5077 
5078 /*@C
5079   DMPlexGetCones - Return cone data
5080 
5081   Not Collective
5082 
5083   Input Parameters:
5084 . dm        - The DMPlex object
5085 
5086   Output Parameter:
5087 . cones - The cone for each point
5088 
5089   Level: developer
5090 
5091 .seealso: DMPlexGetConeSection()
5092 @*/
5093 PetscErrorCode DMPlexGetCones(DM dm, PetscInt *cones[])
5094 {
5095   DM_Plex *mesh = (DM_Plex*) dm->data;
5096 
5097   PetscFunctionBegin;
5098   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5099   if (cones) *cones = mesh->cones;
5100   PetscFunctionReturn(0);
5101 }
5102 
5103 /*@C
5104   DMPlexGetConeOrientations - Return cone orientation data
5105 
5106   Not Collective
5107 
5108   Input Parameters:
5109 . dm        - The DMPlex object
5110 
5111   Output Parameter:
5112 . coneOrientations - The array of cone orientations for all points
5113 
5114   Level: developer
5115 
5116   Notes:
5117   The PetscSection returned by DMPlexGetConeSection() partitions coneOrientations into cone orientations of particular points as returned by DMPlexGetConeOrientation().
5118 
5119   The meaning of coneOrientations values is detailed in DMPlexGetConeOrientation().
5120 
5121 .seealso: DMPlexGetConeSection(), DMPlexGetConeOrientation()
5122 @*/
5123 PetscErrorCode DMPlexGetConeOrientations(DM dm, PetscInt *coneOrientations[])
5124 {
5125   DM_Plex *mesh = (DM_Plex*) dm->data;
5126 
5127   PetscFunctionBegin;
5128   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5129   if (coneOrientations) *coneOrientations = mesh->coneOrientations;
5130   PetscFunctionReturn(0);
5131 }
5132 
5133 /******************************** FEM Support **********************************/
5134 
5135 /*
5136  Returns number of components and tensor degree for the field.  For interpolated meshes, line should be a point
5137  representing a line in the section.
5138 */
5139 static PetscErrorCode PetscSectionFieldGetTensorDegree_Private(PetscSection section,PetscInt field,PetscInt line,PetscBool vertexchart,PetscInt *Nc,PetscInt *k)
5140 {
5141   PetscErrorCode ierr;
5142 
5143   PetscFunctionBeginHot;
5144   ierr = PetscSectionGetFieldComponents(section, field, Nc);CHKERRQ(ierr);
5145   if (line < 0) {
5146     *k = 0;
5147     *Nc = 0;
5148   } else if (vertexchart) {            /* If we only have a vertex chart, we must have degree k=1 */
5149     *k = 1;
5150   } else {                      /* Assume the full interpolated mesh is in the chart; lines in particular */
5151     /* An order k SEM disc has k-1 dofs on an edge */
5152     ierr = PetscSectionGetFieldDof(section, line, field, k);CHKERRQ(ierr);
5153     *k = *k / *Nc + 1;
5154   }
5155   PetscFunctionReturn(0);
5156 }
5157 
5158 /*@
5159 
5160   DMPlexSetClosurePermutationTensor - Create a permutation from the default (BFS) point ordering in the closure, to a
5161   lexicographic ordering over the tensor product cell (i.e., line, quad, hex, etc.), and set this permutation in the
5162   section provided (or the section of the DM).
5163 
5164   Input Parameters:
5165 + dm      - The DM
5166 . point   - Either a cell (highest dim point) or an edge (dim 1 point), or PETSC_DETERMINE
5167 - section - The PetscSection to reorder, or NULL for the default section
5168 
5169   Note: The point is used to determine the number of dofs/field on an edge. For SEM, this is related to the polynomial
5170   degree of the basis.
5171 
5172   Example:
5173   A typical interpolated single-quad mesh might order points as
5174 .vb
5175   [c0, v1, v2, v3, v4, e5, e6, e7, e8]
5176 
5177   v4 -- e6 -- v3
5178   |           |
5179   e7    c0    e8
5180   |           |
5181   v1 -- e5 -- v2
5182 .ve
5183 
5184   (There is no significance to the ordering described here.)  The default section for a Q3 quad might typically assign
5185   dofs in the order of points, e.g.,
5186 .vb
5187     c0 -> [0,1,2,3]
5188     v1 -> [4]
5189     ...
5190     e5 -> [8, 9]
5191 .ve
5192 
5193   which corresponds to the dofs
5194 .vb
5195     6   10  11  7
5196     13  2   3   15
5197     12  0   1   14
5198     4   8   9   5
5199 .ve
5200 
5201   The closure in BFS ordering works through height strata (cells, edges, vertices) to produce the ordering
5202 .vb
5203   0 1 2 3 8 9 14 15 11 10 13 12 4 5 7 6
5204 .ve
5205 
5206   After calling DMPlexSetClosurePermutationTensor(), the closure will be ordered lexicographically,
5207 .vb
5208    4 8 9 5 12 0 1 14 13 2 3 15 6 10 11 7
5209 .ve
5210 
5211   Level: developer
5212 
5213 .seealso: DMGetLocalSection(), PetscSectionSetClosurePermutation(), DMSetGlobalSection()
5214 @*/
5215 PetscErrorCode DMPlexSetClosurePermutationTensor(DM dm, PetscInt point, PetscSection section)
5216 {
5217   DMLabel        label;
5218   PetscInt       dim, depth = -1, eStart = -1, Nf;
5219   PetscBool      vertexchart;
5220   PetscErrorCode ierr;
5221 
5222   PetscFunctionBegin;
5223   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
5224   if (dim < 1) PetscFunctionReturn(0);
5225   if (point < 0) {
5226     PetscInt sStart,sEnd;
5227 
5228     ierr = DMPlexGetDepthStratum(dm, 1, &sStart, &sEnd);CHKERRQ(ierr);
5229     point = sEnd-sStart ? sStart : point;
5230   }
5231   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
5232   if (point >= 0) { ierr = DMLabelGetValue(label, point, &depth);CHKERRQ(ierr); }
5233   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
5234   if (depth == 1) {eStart = point;}
5235   else if  (depth == dim) {
5236     const PetscInt *cone;
5237 
5238     ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
5239     if (dim == 2) eStart = cone[0];
5240     else if (dim == 3) {
5241       const PetscInt *cone2;
5242       ierr = DMPlexGetCone(dm, cone[0], &cone2);CHKERRQ(ierr);
5243       eStart = cone2[0];
5244     } else SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Point %D of depth %D cannot be used to bootstrap spectral ordering for dim %D", point, depth, dim);
5245   } else if (depth >= 0) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Point %D of depth %D cannot be used to bootstrap spectral ordering for dim %D", point, depth, dim);
5246   {                             /* Determine whether the chart covers all points or just vertices. */
5247     PetscInt pStart,pEnd,cStart,cEnd;
5248     ierr = DMPlexGetDepthStratum(dm,0,&pStart,&pEnd);CHKERRQ(ierr);
5249     ierr = PetscSectionGetChart(section,&cStart,&cEnd);CHKERRQ(ierr);
5250     if (pStart == cStart && pEnd == cEnd) vertexchart = PETSC_TRUE; /* Just vertices */
5251     else vertexchart = PETSC_FALSE;                                 /* Assume all interpolated points are in chart */
5252   }
5253   ierr = PetscSectionGetNumFields(section, &Nf);CHKERRQ(ierr);
5254   for (PetscInt d=1; d<=dim; d++) {
5255     PetscInt k, f, Nc, c, i, j, size = 0, offset = 0, foffset = 0;
5256     PetscInt *perm;
5257 
5258     for (f = 0; f < Nf; ++f) {
5259       ierr = PetscSectionFieldGetTensorDegree_Private(section,f,eStart,vertexchart,&Nc,&k);CHKERRQ(ierr);
5260       size += PetscPowInt(k+1, d)*Nc;
5261     }
5262     ierr = PetscMalloc1(size, &perm);CHKERRQ(ierr);
5263     for (f = 0; f < Nf; ++f) {
5264       switch (d) {
5265       case 1:
5266         ierr = PetscSectionFieldGetTensorDegree_Private(section,f,eStart,vertexchart,&Nc,&k);CHKERRQ(ierr);
5267         /*
5268          Original ordering is [ edge of length k-1; vtx0; vtx1 ]
5269          We want              [ vtx0; edge of length k-1; vtx1 ]
5270          */
5271         for (c=0; c<Nc; c++,offset++) perm[offset] = (k-1)*Nc + c + foffset;
5272         for (i=0; i<k-1; i++) for (c=0; c<Nc; c++,offset++) perm[offset] = i*Nc + c + foffset;
5273         for (c=0; c<Nc; c++,offset++) perm[offset] = k*Nc + c + foffset;
5274         foffset = offset;
5275         break;
5276       case 2:
5277         /* The original quad closure is oriented clockwise, {f, e_b, e_r, e_t, e_l, v_lb, v_rb, v_tr, v_tl} */
5278         ierr = PetscSectionFieldGetTensorDegree_Private(section,f,eStart,vertexchart,&Nc,&k);CHKERRQ(ierr);
5279         /* The SEM order is
5280 
5281          v_lb, {e_b}, v_rb,
5282          e^{(k-1)-i}_l, {f^{i*(k-1)}}, e^i_r,
5283          v_lt, reverse {e_t}, v_rt
5284          */
5285         {
5286           const PetscInt of   = 0;
5287           const PetscInt oeb  = of   + PetscSqr(k-1);
5288           const PetscInt oer  = oeb  + (k-1);
5289           const PetscInt oet  = oer  + (k-1);
5290           const PetscInt oel  = oet  + (k-1);
5291           const PetscInt ovlb = oel  + (k-1);
5292           const PetscInt ovrb = ovlb + 1;
5293           const PetscInt ovrt = ovrb + 1;
5294           const PetscInt ovlt = ovrt + 1;
5295           PetscInt       o;
5296 
5297           /* bottom */
5298           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovlb*Nc + c + foffset;
5299           for (o = oeb; o < oer; ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5300           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovrb*Nc + c + foffset;
5301           /* middle */
5302           for (i = 0; i < k-1; ++i) {
5303             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oel+(k-2)-i)*Nc + c + foffset;
5304             for (o = of+(k-1)*i; o < of+(k-1)*(i+1); ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5305             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oer+i)*Nc + c + foffset;
5306           }
5307           /* top */
5308           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovlt*Nc + c + foffset;
5309           for (o = oel-1; o >= oet; --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5310           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovrt*Nc + c + foffset;
5311           foffset = offset;
5312         }
5313         break;
5314       case 3:
5315         /* The original hex closure is
5316 
5317          {c,
5318          f_b, f_t, f_f, f_b, f_r, f_l,
5319          e_bl, e_bb, e_br, e_bf,  e_tf, e_tr, e_tb, e_tl,  e_rf, e_lf, e_lb, e_rb,
5320          v_blf, v_blb, v_brb, v_brf, v_tlf, v_trf, v_trb, v_tlb}
5321          */
5322         ierr = PetscSectionFieldGetTensorDegree_Private(section,f,eStart,vertexchart,&Nc,&k);CHKERRQ(ierr);
5323         /* The SEM order is
5324          Bottom Slice
5325          v_blf, {e^{(k-1)-n}_bf}, v_brf,
5326          e^{i}_bl, f^{n*(k-1)+(k-1)-i}_b, e^{(k-1)-i}_br,
5327          v_blb, {e_bb}, v_brb,
5328 
5329          Middle Slice (j)
5330          {e^{(k-1)-j}_lf}, {f^{j*(k-1)+n}_f}, e^j_rf,
5331          f^{i*(k-1)+j}_l, {c^{(j*(k-1) + i)*(k-1)+n}_t}, f^{j*(k-1)+i}_r,
5332          e^j_lb, {f^{j*(k-1)+(k-1)-n}_b}, e^{(k-1)-j}_rb,
5333 
5334          Top Slice
5335          v_tlf, {e_tf}, v_trf,
5336          e^{(k-1)-i}_tl, {f^{i*(k-1)}_t}, e^{i}_tr,
5337          v_tlb, {e^{(k-1)-n}_tb}, v_trb,
5338          */
5339         {
5340           const PetscInt oc    = 0;
5341           const PetscInt ofb   = oc    + PetscSqr(k-1)*(k-1);
5342           const PetscInt oft   = ofb   + PetscSqr(k-1);
5343           const PetscInt off   = oft   + PetscSqr(k-1);
5344           const PetscInt ofk   = off   + PetscSqr(k-1);
5345           const PetscInt ofr   = ofk   + PetscSqr(k-1);
5346           const PetscInt ofl   = ofr   + PetscSqr(k-1);
5347           const PetscInt oebl  = ofl   + PetscSqr(k-1);
5348           const PetscInt oebb  = oebl  + (k-1);
5349           const PetscInt oebr  = oebb  + (k-1);
5350           const PetscInt oebf  = oebr  + (k-1);
5351           const PetscInt oetf  = oebf  + (k-1);
5352           const PetscInt oetr  = oetf  + (k-1);
5353           const PetscInt oetb  = oetr  + (k-1);
5354           const PetscInt oetl  = oetb  + (k-1);
5355           const PetscInt oerf  = oetl  + (k-1);
5356           const PetscInt oelf  = oerf  + (k-1);
5357           const PetscInt oelb  = oelf  + (k-1);
5358           const PetscInt oerb  = oelb  + (k-1);
5359           const PetscInt ovblf = oerb  + (k-1);
5360           const PetscInt ovblb = ovblf + 1;
5361           const PetscInt ovbrb = ovblb + 1;
5362           const PetscInt ovbrf = ovbrb + 1;
5363           const PetscInt ovtlf = ovbrf + 1;
5364           const PetscInt ovtrf = ovtlf + 1;
5365           const PetscInt ovtrb = ovtrf + 1;
5366           const PetscInt ovtlb = ovtrb + 1;
5367           PetscInt       o, n;
5368 
5369           /* Bottom Slice */
5370           /*   bottom */
5371           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovblf*Nc + c + foffset;
5372           for (o = oetf-1; o >= oebf; --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5373           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovbrf*Nc + c + foffset;
5374           /*   middle */
5375           for (i = 0; i < k-1; ++i) {
5376             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oebl+i)*Nc + c + foffset;
5377             for (n = 0; n < k-1; ++n) {o = ofb+n*(k-1)+i; for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;}
5378             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oebr+(k-2)-i)*Nc + c + foffset;
5379           }
5380           /*   top */
5381           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovblb*Nc + c + foffset;
5382           for (o = oebb; o < oebr; ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5383           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovbrb*Nc + c + foffset;
5384 
5385           /* Middle Slice */
5386           for (j = 0; j < k-1; ++j) {
5387             /*   bottom */
5388             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oelf+(k-2)-j)*Nc + c + foffset;
5389             for (o = off+j*(k-1); o < off+(j+1)*(k-1); ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5390             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oerf+j)*Nc + c + foffset;
5391             /*   middle */
5392             for (i = 0; i < k-1; ++i) {
5393               for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (ofl+i*(k-1)+j)*Nc + c + foffset;
5394               for (n = 0; n < k-1; ++n) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oc+(j*(k-1)+i)*(k-1)+n)*Nc + c + foffset;
5395               for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (ofr+j*(k-1)+i)*Nc + c + foffset;
5396             }
5397             /*   top */
5398             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oelb+j)*Nc + c + foffset;
5399             for (o = ofk+j*(k-1)+(k-2); o >= ofk+j*(k-1); --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5400             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oerb+(k-2)-j)*Nc + c + foffset;
5401           }
5402 
5403           /* Top Slice */
5404           /*   bottom */
5405           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtlf*Nc + c + foffset;
5406           for (o = oetf; o < oetr; ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5407           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtrf*Nc + c + foffset;
5408           /*   middle */
5409           for (i = 0; i < k-1; ++i) {
5410             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oetl+(k-2)-i)*Nc + c + foffset;
5411             for (n = 0; n < k-1; ++n) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oft+i*(k-1)+n)*Nc + c + foffset;
5412             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oetr+i)*Nc + c + foffset;
5413           }
5414           /*   top */
5415           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtlb*Nc + c + foffset;
5416           for (o = oetl-1; o >= oetb; --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5417           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtrb*Nc + c + foffset;
5418 
5419           foffset = offset;
5420         }
5421         break;
5422       default: SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_OUTOFRANGE, "No spectral ordering for dimension %D", d);
5423       }
5424     }
5425     if (offset != size) SETERRQ2(PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Number of permutation entries %D != %D", offset, size);
5426     /* Check permutation */
5427     {
5428       PetscInt *check;
5429 
5430       ierr = PetscMalloc1(size, &check);CHKERRQ(ierr);
5431       for (i = 0; i < size; ++i) {check[i] = -1; if (perm[i] < 0 || perm[i] >= size) SETERRQ2(PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Invalid permutation index p[%D] = %D", i, perm[i]);}
5432       for (i = 0; i < size; ++i) check[perm[i]] = i;
5433       for (i = 0; i < size; ++i) {if (check[i] < 0) SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Missing permutation index %D", i);}
5434       ierr = PetscFree(check);CHKERRQ(ierr);
5435     }
5436     ierr = PetscSectionSetClosurePermutation_Internal(section, (PetscObject) dm, d, size, PETSC_OWN_POINTER, perm);CHKERRQ(ierr);
5437   }
5438   PetscFunctionReturn(0);
5439 }
5440 
5441 PetscErrorCode DMPlexGetPointDualSpaceFEM(DM dm, PetscInt point, PetscInt field, PetscDualSpace *dspace)
5442 {
5443   PetscDS        prob;
5444   PetscInt       depth, Nf, h;
5445   DMLabel        label;
5446   PetscErrorCode ierr;
5447 
5448   PetscFunctionBeginHot;
5449   ierr = DMGetDS(dm, &prob);CHKERRQ(ierr);
5450   Nf      = prob->Nf;
5451   label   = dm->depthLabel;
5452   *dspace = NULL;
5453   if (field < Nf) {
5454     PetscObject disc = prob->disc[field];
5455 
5456     if (disc->classid == PETSCFE_CLASSID) {
5457       PetscDualSpace dsp;
5458 
5459       ierr = PetscFEGetDualSpace((PetscFE)disc,&dsp);CHKERRQ(ierr);
5460       ierr = DMLabelGetNumValues(label,&depth);CHKERRQ(ierr);
5461       ierr = DMLabelGetValue(label,point,&h);CHKERRQ(ierr);
5462       h    = depth - 1 - h;
5463       if (h) {
5464         ierr = PetscDualSpaceGetHeightSubspace(dsp,h,dspace);CHKERRQ(ierr);
5465       } else {
5466         *dspace = dsp;
5467       }
5468     }
5469   }
5470   PetscFunctionReturn(0);
5471 }
5472 
5473 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecGetClosure_Depth1_Static(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
5474 {
5475   PetscScalar    *array, *vArray;
5476   const PetscInt *cone, *coneO;
5477   PetscInt        pStart, pEnd, p, numPoints, size = 0, offset = 0;
5478   PetscErrorCode  ierr;
5479 
5480   PetscFunctionBeginHot;
5481   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
5482   ierr = DMPlexGetConeSize(dm, point, &numPoints);CHKERRQ(ierr);
5483   ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
5484   ierr = DMPlexGetConeOrientation(dm, point, &coneO);CHKERRQ(ierr);
5485   if (!values || !*values) {
5486     if ((point >= pStart) && (point < pEnd)) {
5487       PetscInt dof;
5488 
5489       ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
5490       size += dof;
5491     }
5492     for (p = 0; p < numPoints; ++p) {
5493       const PetscInt cp = cone[p];
5494       PetscInt       dof;
5495 
5496       if ((cp < pStart) || (cp >= pEnd)) continue;
5497       ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
5498       size += dof;
5499     }
5500     if (!values) {
5501       if (csize) *csize = size;
5502       PetscFunctionReturn(0);
5503     }
5504     ierr = DMGetWorkArray(dm, size, MPIU_SCALAR, &array);CHKERRQ(ierr);
5505   } else {
5506     array = *values;
5507   }
5508   size = 0;
5509   ierr = VecGetArray(v, &vArray);CHKERRQ(ierr);
5510   if ((point >= pStart) && (point < pEnd)) {
5511     PetscInt     dof, off, d;
5512     PetscScalar *varr;
5513 
5514     ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
5515     ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
5516     varr = &vArray[off];
5517     for (d = 0; d < dof; ++d, ++offset) {
5518       array[offset] = varr[d];
5519     }
5520     size += dof;
5521   }
5522   for (p = 0; p < numPoints; ++p) {
5523     const PetscInt cp = cone[p];
5524     PetscInt       o  = coneO[p];
5525     PetscInt       dof, off, d;
5526     PetscScalar   *varr;
5527 
5528     if ((cp < pStart) || (cp >= pEnd)) continue;
5529     ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
5530     ierr = PetscSectionGetOffset(section, cp, &off);CHKERRQ(ierr);
5531     varr = &vArray[off];
5532     if (o >= 0) {
5533       for (d = 0; d < dof; ++d, ++offset) {
5534         array[offset] = varr[d];
5535       }
5536     } else {
5537       for (d = dof-1; d >= 0; --d, ++offset) {
5538         array[offset] = varr[d];
5539       }
5540     }
5541     size += dof;
5542   }
5543   ierr = VecRestoreArray(v, &vArray);CHKERRQ(ierr);
5544   if (!*values) {
5545     if (csize) *csize = size;
5546     *values = array;
5547   } else {
5548     if (size > *csize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Size of input array %D < actual size %D", *csize, size);
5549     *csize = size;
5550   }
5551   PetscFunctionReturn(0);
5552 }
5553 
5554 /* Compress out points not in the section */
5555 PETSC_STATIC_INLINE PetscErrorCode CompressPoints_Private(PetscSection section, PetscInt *numPoints, PetscInt points[])
5556 {
5557   const PetscInt np = *numPoints;
5558   PetscInt       pStart, pEnd, p, q;
5559   PetscErrorCode ierr;
5560 
5561   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
5562   for (p = 0, q = 0; p < np; ++p) {
5563     const PetscInt r = points[p*2];
5564     if ((r >= pStart) && (r < pEnd)) {
5565       points[q*2]   = r;
5566       points[q*2+1] = points[p*2+1];
5567       ++q;
5568     }
5569   }
5570   *numPoints = q;
5571   return 0;
5572 }
5573 
5574 /* Compressed closure does not apply closure permutation */
5575 PetscErrorCode DMPlexGetCompressedClosure(DM dm, PetscSection section, PetscInt point, PetscInt *numPoints, PetscInt **points, PetscSection *clSec, IS *clPoints, const PetscInt **clp)
5576 {
5577   const PetscInt *cla = NULL;
5578   PetscInt       np, *pts = NULL;
5579   PetscErrorCode ierr;
5580 
5581   PetscFunctionBeginHot;
5582   ierr = PetscSectionGetClosureIndex(section, (PetscObject) dm, clSec, clPoints);CHKERRQ(ierr);
5583   if (*clPoints) {
5584     PetscInt dof, off;
5585 
5586     ierr = PetscSectionGetDof(*clSec, point, &dof);CHKERRQ(ierr);
5587     ierr = PetscSectionGetOffset(*clSec, point, &off);CHKERRQ(ierr);
5588     ierr = ISGetIndices(*clPoints, &cla);CHKERRQ(ierr);
5589     np   = dof/2;
5590     pts  = (PetscInt *) &cla[off];
5591   } else {
5592     ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &np, &pts);CHKERRQ(ierr);
5593     ierr = CompressPoints_Private(section, &np, pts);CHKERRQ(ierr);
5594   }
5595   *numPoints = np;
5596   *points    = pts;
5597   *clp       = cla;
5598   PetscFunctionReturn(0);
5599 }
5600 
5601 PetscErrorCode DMPlexRestoreCompressedClosure(DM dm, PetscSection section, PetscInt point, PetscInt *numPoints, PetscInt **points, PetscSection *clSec, IS *clPoints, const PetscInt **clp)
5602 {
5603   PetscErrorCode ierr;
5604 
5605   PetscFunctionBeginHot;
5606   if (!*clPoints) {
5607     ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, numPoints, points);CHKERRQ(ierr);
5608   } else {
5609     ierr = ISRestoreIndices(*clPoints, clp);CHKERRQ(ierr);
5610   }
5611   *numPoints = 0;
5612   *points    = NULL;
5613   *clSec     = NULL;
5614   *clPoints  = NULL;
5615   *clp       = NULL;
5616   PetscFunctionReturn(0);
5617 }
5618 
5619 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecGetClosure_Static(DM dm, PetscSection section, PetscInt numPoints, const PetscInt points[], const PetscInt clperm[], const PetscScalar vArray[], PetscInt *size, PetscScalar array[])
5620 {
5621   PetscInt          offset = 0, p;
5622   const PetscInt    **perms = NULL;
5623   const PetscScalar **flips = NULL;
5624   PetscErrorCode    ierr;
5625 
5626   PetscFunctionBeginHot;
5627   *size = 0;
5628   ierr = PetscSectionGetPointSyms(section,numPoints,points,&perms,&flips);CHKERRQ(ierr);
5629   for (p = 0; p < numPoints; p++) {
5630     const PetscInt    point = points[2*p];
5631     const PetscInt    *perm = perms ? perms[p] : NULL;
5632     const PetscScalar *flip = flips ? flips[p] : NULL;
5633     PetscInt          dof, off, d;
5634     const PetscScalar *varr;
5635 
5636     ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
5637     ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
5638     varr = &vArray[off];
5639     if (clperm) {
5640       if (perm) {
5641         for (d = 0; d < dof; d++) array[clperm[offset + perm[d]]]  = varr[d];
5642       } else {
5643         for (d = 0; d < dof; d++) array[clperm[offset +      d ]]  = varr[d];
5644       }
5645       if (flip) {
5646         for (d = 0; d < dof; d++) array[clperm[offset +      d ]] *= flip[d];
5647       }
5648     } else {
5649       if (perm) {
5650         for (d = 0; d < dof; d++) array[offset + perm[d]]  = varr[d];
5651       } else {
5652         for (d = 0; d < dof; d++) array[offset +      d ]  = varr[d];
5653       }
5654       if (flip) {
5655         for (d = 0; d < dof; d++) array[offset +      d ] *= flip[d];
5656       }
5657     }
5658     offset += dof;
5659   }
5660   ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms,&flips);CHKERRQ(ierr);
5661   *size = offset;
5662   PetscFunctionReturn(0);
5663 }
5664 
5665 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecGetClosure_Fields_Static(DM dm, PetscSection section, PetscInt numPoints, const PetscInt points[], PetscInt numFields, const PetscInt clperm[], const PetscScalar vArray[], PetscInt *size, PetscScalar array[])
5666 {
5667   PetscInt          offset = 0, f;
5668   PetscErrorCode    ierr;
5669 
5670   PetscFunctionBeginHot;
5671   *size = 0;
5672   for (f = 0; f < numFields; ++f) {
5673     PetscInt          p;
5674     const PetscInt    **perms = NULL;
5675     const PetscScalar **flips = NULL;
5676 
5677     ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
5678     for (p = 0; p < numPoints; p++) {
5679       const PetscInt    point = points[2*p];
5680       PetscInt          fdof, foff, b;
5681       const PetscScalar *varr;
5682       const PetscInt    *perm = perms ? perms[p] : NULL;
5683       const PetscScalar *flip = flips ? flips[p] : NULL;
5684 
5685       ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
5686       ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
5687       varr = &vArray[foff];
5688       if (clperm) {
5689         if (perm) {for (b = 0; b < fdof; b++) {array[clperm[offset + perm[b]]]  = varr[b];}}
5690         else      {for (b = 0; b < fdof; b++) {array[clperm[offset +      b ]]  = varr[b];}}
5691         if (flip) {for (b = 0; b < fdof; b++) {array[clperm[offset +      b ]] *= flip[b];}}
5692       } else {
5693         if (perm) {for (b = 0; b < fdof; b++) {array[offset + perm[b]]  = varr[b];}}
5694         else      {for (b = 0; b < fdof; b++) {array[offset +      b ]  = varr[b];}}
5695         if (flip) {for (b = 0; b < fdof; b++) {array[offset +      b ] *= flip[b];}}
5696       }
5697       offset += fdof;
5698     }
5699     ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
5700   }
5701   *size = offset;
5702   PetscFunctionReturn(0);
5703 }
5704 
5705 /*@C
5706   DMPlexVecGetClosure - Get an array of the values on the closure of 'point'
5707 
5708   Not collective
5709 
5710   Input Parameters:
5711 + dm - The DM
5712 . section - The section describing the layout in v, or NULL to use the default section
5713 . v - The local vector
5714 - point - The point in the DM
5715 
5716   Input/Output Parameters:
5717 + csize  - The size of the input values array, or NULL; on output the number of values in the closure
5718 - values - An array to use for the values, or NULL to have it allocated automatically;
5719            if the user provided NULL, it is a borrowed array and should not be freed
5720 
5721 $ Note that DMPlexVecGetClosure/DMPlexVecRestoreClosure only allocates the values array if it set to NULL in the
5722 $ calling function. This is because DMPlexVecGetClosure() is typically called in the inner loop of a Vec or Mat
5723 $ assembly function, and a user may already have allocated storage for this operation.
5724 $
5725 $ A typical use could be
5726 $
5727 $  values = NULL;
5728 $  ierr = DMPlexVecGetClosure(dm, NULL, v, p, &clSize, &values);CHKERRQ(ierr);
5729 $  for (cl = 0; cl < clSize; ++cl) {
5730 $    <Compute on closure>
5731 $  }
5732 $  ierr = DMPlexVecRestoreClosure(dm, NULL, v, p, &clSize, &values);CHKERRQ(ierr);
5733 $
5734 $ or
5735 $
5736 $  PetscMalloc1(clMaxSize, &values);
5737 $  for (p = pStart; p < pEnd; ++p) {
5738 $    clSize = clMaxSize;
5739 $    ierr = DMPlexVecGetClosure(dm, NULL, v, p, &clSize, &values);CHKERRQ(ierr);
5740 $    for (cl = 0; cl < clSize; ++cl) {
5741 $      <Compute on closure>
5742 $    }
5743 $  }
5744 $  PetscFree(values);
5745 
5746   Fortran Notes:
5747   Since it returns an array, this routine is only available in Fortran 90, and you must
5748   include petsc.h90 in your code.
5749 
5750   The csize argument is not present in the Fortran 90 binding since it is internal to the array.
5751 
5752   Level: intermediate
5753 
5754 .seealso DMPlexVecRestoreClosure(), DMPlexVecSetClosure(), DMPlexMatSetClosure()
5755 @*/
5756 PetscErrorCode DMPlexVecGetClosure(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
5757 {
5758   PetscSection       clSection;
5759   IS                 clPoints;
5760   PetscInt          *points = NULL;
5761   const PetscInt    *clp, *perm;
5762   PetscInt           depth, numFields, numPoints, asize;
5763   PetscErrorCode     ierr;
5764 
5765   PetscFunctionBeginHot;
5766   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5767   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
5768   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
5769   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
5770   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
5771   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5772   if (depth == 1 && numFields < 2) {
5773     ierr = DMPlexVecGetClosure_Depth1_Static(dm, section, v, point, csize, values);CHKERRQ(ierr);
5774     PetscFunctionReturn(0);
5775   }
5776   /* Get points */
5777   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5778   /* Get sizes */
5779   asize = 0;
5780   for (PetscInt p = 0; p < numPoints*2; p += 2) {
5781     PetscInt dof;
5782     ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5783     asize += dof;
5784   }
5785   if (values) {
5786     const PetscScalar *vArray;
5787     PetscInt          size;
5788 
5789     if (*values) {
5790       if (PetscUnlikely(*csize < asize)) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Provided array size %D not sufficient to hold closure size %D", *csize, asize);
5791     } else {ierr = DMGetWorkArray(dm, asize, MPIU_SCALAR, values);CHKERRQ(ierr);}
5792     ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, depth, asize, &perm);CHKERRQ(ierr);
5793     ierr = VecGetArrayRead(v, &vArray);CHKERRQ(ierr);
5794     /* Get values */
5795     if (numFields > 0) {ierr = DMPlexVecGetClosure_Fields_Static(dm, section, numPoints, points, numFields, perm, vArray, &size, *values);CHKERRQ(ierr);}
5796     else               {ierr = DMPlexVecGetClosure_Static(dm, section, numPoints, points, perm, vArray, &size, *values);CHKERRQ(ierr);}
5797     if (PetscUnlikely(asize != size)) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Section size %D does not match Vec closure size %D", asize, size);
5798     /* Cleanup array */
5799     ierr = VecRestoreArrayRead(v, &vArray);CHKERRQ(ierr);
5800   }
5801   if (csize) *csize = asize;
5802   /* Cleanup points */
5803   ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5804   PetscFunctionReturn(0);
5805 }
5806 
5807 PetscErrorCode DMPlexVecGetClosureAtDepth_Internal(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt depth, PetscInt *csize, PetscScalar *values[])
5808 {
5809   DMLabel            depthLabel;
5810   PetscSection       clSection;
5811   IS                 clPoints;
5812   PetscScalar       *array;
5813   const PetscScalar *vArray;
5814   PetscInt          *points = NULL;
5815   const PetscInt    *clp, *perm = NULL;
5816   PetscInt           mdepth, numFields, numPoints, Np = 0, p, clsize, size;
5817   PetscErrorCode     ierr;
5818 
5819   PetscFunctionBeginHot;
5820   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5821   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
5822   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
5823   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
5824   ierr = DMPlexGetDepth(dm, &mdepth);CHKERRQ(ierr);
5825   ierr = DMPlexGetDepthLabel(dm, &depthLabel);CHKERRQ(ierr);
5826   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5827   if (mdepth == 1 && numFields < 2) {
5828     ierr = DMPlexVecGetClosure_Depth1_Static(dm, section, v, point, csize, values);CHKERRQ(ierr);
5829     PetscFunctionReturn(0);
5830   }
5831   /* Get points */
5832   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5833   for (clsize=0,p=0; p<Np; p++) {
5834     PetscInt dof;
5835     ierr = PetscSectionGetDof(section, points[2*p], &dof);CHKERRQ(ierr);
5836     clsize += dof;
5837   }
5838   ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, depth, clsize, &perm);CHKERRQ(ierr);
5839   /* Filter points */
5840   for (p = 0; p < numPoints*2; p += 2) {
5841     PetscInt dep;
5842 
5843     ierr = DMLabelGetValue(depthLabel, points[p], &dep);CHKERRQ(ierr);
5844     if (dep != depth) continue;
5845     points[Np*2+0] = points[p];
5846     points[Np*2+1] = points[p+1];
5847     ++Np;
5848   }
5849   /* Get array */
5850   if (!values || !*values) {
5851     PetscInt asize = 0, dof;
5852 
5853     for (p = 0; p < Np*2; p += 2) {
5854       ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5855       asize += dof;
5856     }
5857     if (!values) {
5858       ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5859       if (csize) *csize = asize;
5860       PetscFunctionReturn(0);
5861     }
5862     ierr = DMGetWorkArray(dm, asize, MPIU_SCALAR, &array);CHKERRQ(ierr);
5863   } else {
5864     array = *values;
5865   }
5866   ierr = VecGetArrayRead(v, &vArray);CHKERRQ(ierr);
5867   /* Get values */
5868   if (numFields > 0) {ierr = DMPlexVecGetClosure_Fields_Static(dm, section, Np, points, numFields, perm, vArray, &size, array);CHKERRQ(ierr);}
5869   else               {ierr = DMPlexVecGetClosure_Static(dm, section, Np, points, perm, vArray, &size, array);CHKERRQ(ierr);}
5870   /* Cleanup points */
5871   ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5872   /* Cleanup array */
5873   ierr = VecRestoreArrayRead(v, &vArray);CHKERRQ(ierr);
5874   if (!*values) {
5875     if (csize) *csize = size;
5876     *values = array;
5877   } else {
5878     if (size > *csize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Size of input array %D < actual size %D", *csize, size);
5879     *csize = size;
5880   }
5881   PetscFunctionReturn(0);
5882 }
5883 
5884 /*@C
5885   DMPlexVecRestoreClosure - Restore the array of the values on the closure of 'point'
5886 
5887   Not collective
5888 
5889   Input Parameters:
5890 + dm - The DM
5891 . section - The section describing the layout in v, or NULL to use the default section
5892 . v - The local vector
5893 . point - The point in the DM
5894 . csize - The number of values in the closure, or NULL
5895 - values - The array of values, which is a borrowed array and should not be freed
5896 
5897   Note that the array values are discarded and not copied back into v. In order to copy values back to v, use DMPlexVecSetClosure()
5898 
5899   Fortran Notes:
5900   Since it returns an array, this routine is only available in Fortran 90, and you must
5901   include petsc.h90 in your code.
5902 
5903   The csize argument is not present in the Fortran 90 binding since it is internal to the array.
5904 
5905   Level: intermediate
5906 
5907 .seealso DMPlexVecGetClosure(), DMPlexVecSetClosure(), DMPlexMatSetClosure()
5908 @*/
5909 PetscErrorCode DMPlexVecRestoreClosure(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
5910 {
5911   PetscInt       size = 0;
5912   PetscErrorCode ierr;
5913 
5914   PetscFunctionBegin;
5915   /* Should work without recalculating size */
5916   ierr = DMRestoreWorkArray(dm, size, MPIU_SCALAR, (void*) values);CHKERRQ(ierr);
5917   *values = NULL;
5918   PetscFunctionReturn(0);
5919 }
5920 
5921 PETSC_STATIC_INLINE void add   (PetscScalar *x, PetscScalar y) {*x += y;}
5922 PETSC_STATIC_INLINE void insert(PetscScalar *x, PetscScalar y) {*x  = y;}
5923 
5924 PETSC_STATIC_INLINE PetscErrorCode updatePoint_private(PetscSection section, PetscInt point, PetscInt dof, void (*fuse)(PetscScalar*, PetscScalar), PetscBool setBC, const PetscInt perm[], const PetscScalar flip[], const PetscInt clperm[], const PetscScalar values[], PetscInt offset, PetscScalar array[])
5925 {
5926   PetscInt        cdof;   /* The number of constraints on this point */
5927   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
5928   PetscScalar    *a;
5929   PetscInt        off, cind = 0, k;
5930   PetscErrorCode  ierr;
5931 
5932   PetscFunctionBegin;
5933   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
5934   ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
5935   a    = &array[off];
5936   if (!cdof || setBC) {
5937     if (clperm) {
5938       if (perm) {for (k = 0; k < dof; ++k) {fuse(&a[k], values[clperm[offset+perm[k]]] * (flip ? flip[perm[k]] : 1.));}}
5939       else      {for (k = 0; k < dof; ++k) {fuse(&a[k], values[clperm[offset+     k ]] * (flip ? flip[     k ] : 1.));}}
5940     } else {
5941       if (perm) {for (k = 0; k < dof; ++k) {fuse(&a[k], values[offset+perm[k]] * (flip ? flip[perm[k]] : 1.));}}
5942       else      {for (k = 0; k < dof; ++k) {fuse(&a[k], values[offset+     k ] * (flip ? flip[     k ] : 1.));}}
5943     }
5944   } else {
5945     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
5946     if (clperm) {
5947       if (perm) {for (k = 0; k < dof; ++k) {
5948           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
5949           fuse(&a[k], values[clperm[offset+perm[k]]] * (flip ? flip[perm[k]] : 1.));
5950         }
5951       } else {
5952         for (k = 0; k < dof; ++k) {
5953           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
5954           fuse(&a[k], values[clperm[offset+     k ]] * (flip ? flip[     k ] : 1.));
5955         }
5956       }
5957     } else {
5958       if (perm) {
5959         for (k = 0; k < dof; ++k) {
5960           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
5961           fuse(&a[k], values[offset+perm[k]] * (flip ? flip[perm[k]] : 1.));
5962         }
5963       } else {
5964         for (k = 0; k < dof; ++k) {
5965           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
5966           fuse(&a[k], values[offset+     k ] * (flip ? flip[     k ] : 1.));
5967         }
5968       }
5969     }
5970   }
5971   PetscFunctionReturn(0);
5972 }
5973 
5974 PETSC_STATIC_INLINE PetscErrorCode updatePointBC_private(PetscSection section, PetscInt point, PetscInt dof, void (*fuse)(PetscScalar*, PetscScalar), const PetscInt perm[], const PetscScalar flip[], const PetscInt clperm[], const PetscScalar values[], PetscInt offset, PetscScalar array[])
5975 {
5976   PetscInt        cdof;   /* The number of constraints on this point */
5977   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
5978   PetscScalar    *a;
5979   PetscInt        off, cind = 0, k;
5980   PetscErrorCode  ierr;
5981 
5982   PetscFunctionBegin;
5983   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
5984   ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
5985   a    = &array[off];
5986   if (cdof) {
5987     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
5988     if (clperm) {
5989       if (perm) {
5990         for (k = 0; k < dof; ++k) {
5991           if ((cind < cdof) && (k == cdofs[cind])) {
5992             fuse(&a[k], values[clperm[offset+perm[k]]] * (flip ? flip[perm[k]] : 1.));
5993             cind++;
5994           }
5995         }
5996       } else {
5997         for (k = 0; k < dof; ++k) {
5998           if ((cind < cdof) && (k == cdofs[cind])) {
5999             fuse(&a[k], values[clperm[offset+     k ]] * (flip ? flip[     k ] : 1.));
6000             cind++;
6001           }
6002         }
6003       }
6004     } else {
6005       if (perm) {
6006         for (k = 0; k < dof; ++k) {
6007           if ((cind < cdof) && (k == cdofs[cind])) {
6008             fuse(&a[k], values[offset+perm[k]] * (flip ? flip[perm[k]] : 1.));
6009             cind++;
6010           }
6011         }
6012       } else {
6013         for (k = 0; k < dof; ++k) {
6014           if ((cind < cdof) && (k == cdofs[cind])) {
6015             fuse(&a[k], values[offset+     k ] * (flip ? flip[     k ] : 1.));
6016             cind++;
6017           }
6018         }
6019       }
6020     }
6021   }
6022   PetscFunctionReturn(0);
6023 }
6024 
6025 PETSC_STATIC_INLINE PetscErrorCode updatePointFields_private(PetscSection section, PetscInt point, const PetscInt *perm, const PetscScalar *flip, PetscInt f, void (*fuse)(PetscScalar*, PetscScalar), PetscBool setBC, const PetscInt clperm[], const PetscScalar values[], PetscInt *offset, PetscScalar array[])
6026 {
6027   PetscScalar    *a;
6028   PetscInt        fdof, foff, fcdof, foffset = *offset;
6029   const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
6030   PetscInt        cind = 0, b;
6031   PetscErrorCode  ierr;
6032 
6033   PetscFunctionBegin;
6034   ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
6035   ierr = PetscSectionGetFieldConstraintDof(section, point, f, &fcdof);CHKERRQ(ierr);
6036   ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
6037   a    = &array[foff];
6038   if (!fcdof || setBC) {
6039     if (clperm) {
6040       if (perm) {for (b = 0; b < fdof; b++) {fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));}}
6041       else      {for (b = 0; b < fdof; b++) {fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));}}
6042     } else {
6043       if (perm) {for (b = 0; b < fdof; b++) {fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));}}
6044       else      {for (b = 0; b < fdof; b++) {fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));}}
6045     }
6046   } else {
6047     ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
6048     if (clperm) {
6049       if (perm) {
6050         for (b = 0; b < fdof; b++) {
6051           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
6052           fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));
6053         }
6054       } else {
6055         for (b = 0; b < fdof; b++) {
6056           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
6057           fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));
6058         }
6059       }
6060     } else {
6061       if (perm) {
6062         for (b = 0; b < fdof; b++) {
6063           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
6064           fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));
6065         }
6066       } else {
6067         for (b = 0; b < fdof; b++) {
6068           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
6069           fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));
6070         }
6071       }
6072     }
6073   }
6074   *offset += fdof;
6075   PetscFunctionReturn(0);
6076 }
6077 
6078 PETSC_STATIC_INLINE PetscErrorCode updatePointFieldsBC_private(PetscSection section, PetscInt point, const PetscInt perm[], const PetscScalar flip[], PetscInt f, PetscInt Ncc, const PetscInt comps[], void (*fuse)(PetscScalar*, PetscScalar), const PetscInt clperm[], const PetscScalar values[], PetscInt *offset, PetscScalar array[])
6079 {
6080   PetscScalar    *a;
6081   PetscInt        fdof, foff, fcdof, foffset = *offset;
6082   const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
6083   PetscInt        Nc, cind = 0, ncind = 0, b;
6084   PetscBool       ncSet, fcSet;
6085   PetscErrorCode  ierr;
6086 
6087   PetscFunctionBegin;
6088   ierr = PetscSectionGetFieldComponents(section, f, &Nc);CHKERRQ(ierr);
6089   ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
6090   ierr = PetscSectionGetFieldConstraintDof(section, point, f, &fcdof);CHKERRQ(ierr);
6091   ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
6092   a    = &array[foff];
6093   if (fcdof) {
6094     /* We just override fcdof and fcdofs with Ncc and comps */
6095     ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
6096     if (clperm) {
6097       if (perm) {
6098         if (comps) {
6099           for (b = 0; b < fdof; b++) {
6100             ncSet = fcSet = PETSC_FALSE;
6101             if (b%Nc == comps[ncind]) {ncind = (ncind+1)%Ncc; ncSet = PETSC_TRUE;}
6102             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
6103             if (ncSet && fcSet) {fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));}
6104           }
6105         } else {
6106           for (b = 0; b < fdof; b++) {
6107             if ((cind < fcdof) && (b == fcdofs[cind])) {
6108               fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));
6109               ++cind;
6110             }
6111           }
6112         }
6113       } else {
6114         if (comps) {
6115           for (b = 0; b < fdof; b++) {
6116             ncSet = fcSet = PETSC_FALSE;
6117             if (b%Nc == comps[ncind]) {ncind = (ncind+1)%Ncc; ncSet = PETSC_TRUE;}
6118             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
6119             if (ncSet && fcSet) {fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));}
6120           }
6121         } else {
6122           for (b = 0; b < fdof; b++) {
6123             if ((cind < fcdof) && (b == fcdofs[cind])) {
6124               fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));
6125               ++cind;
6126             }
6127           }
6128         }
6129       }
6130     } else {
6131       if (perm) {
6132         if (comps) {
6133           for (b = 0; b < fdof; b++) {
6134             ncSet = fcSet = PETSC_FALSE;
6135             if (b%Nc == comps[ncind]) {ncind = (ncind+1)%Ncc; ncSet = PETSC_TRUE;}
6136             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
6137             if (ncSet && fcSet) {fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));}
6138           }
6139         } else {
6140           for (b = 0; b < fdof; b++) {
6141             if ((cind < fcdof) && (b == fcdofs[cind])) {
6142               fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));
6143               ++cind;
6144             }
6145           }
6146         }
6147       } else {
6148         if (comps) {
6149           for (b = 0; b < fdof; b++) {
6150             ncSet = fcSet = PETSC_FALSE;
6151             if (b%Nc == comps[ncind]) {ncind = (ncind+1)%Ncc; ncSet = PETSC_TRUE;}
6152             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
6153             if (ncSet && fcSet) {fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));}
6154           }
6155         } else {
6156           for (b = 0; b < fdof; b++) {
6157             if ((cind < fcdof) && (b == fcdofs[cind])) {
6158               fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));
6159               ++cind;
6160             }
6161           }
6162         }
6163       }
6164     }
6165   }
6166   *offset += fdof;
6167   PetscFunctionReturn(0);
6168 }
6169 
6170 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecSetClosure_Depth1_Static(DM dm, PetscSection section, Vec v, PetscInt point, const PetscScalar values[], InsertMode mode)
6171 {
6172   PetscScalar    *array;
6173   const PetscInt *cone, *coneO;
6174   PetscInt        pStart, pEnd, p, numPoints, off, dof;
6175   PetscErrorCode  ierr;
6176 
6177   PetscFunctionBeginHot;
6178   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
6179   ierr = DMPlexGetConeSize(dm, point, &numPoints);CHKERRQ(ierr);
6180   ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
6181   ierr = DMPlexGetConeOrientation(dm, point, &coneO);CHKERRQ(ierr);
6182   ierr = VecGetArray(v, &array);CHKERRQ(ierr);
6183   for (p = 0, off = 0; p <= numPoints; ++p, off += dof) {
6184     const PetscInt cp = !p ? point : cone[p-1];
6185     const PetscInt o  = !p ? 0     : coneO[p-1];
6186 
6187     if ((cp < pStart) || (cp >= pEnd)) {dof = 0; continue;}
6188     ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
6189     /* ADD_VALUES */
6190     {
6191       const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
6192       PetscScalar    *a;
6193       PetscInt        cdof, coff, cind = 0, k;
6194 
6195       ierr = PetscSectionGetConstraintDof(section, cp, &cdof);CHKERRQ(ierr);
6196       ierr = PetscSectionGetOffset(section, cp, &coff);CHKERRQ(ierr);
6197       a    = &array[coff];
6198       if (!cdof) {
6199         if (o >= 0) {
6200           for (k = 0; k < dof; ++k) {
6201             a[k] += values[off+k];
6202           }
6203         } else {
6204           for (k = 0; k < dof; ++k) {
6205             a[k] += values[off+dof-k-1];
6206           }
6207         }
6208       } else {
6209         ierr = PetscSectionGetConstraintIndices(section, cp, &cdofs);CHKERRQ(ierr);
6210         if (o >= 0) {
6211           for (k = 0; k < dof; ++k) {
6212             if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
6213             a[k] += values[off+k];
6214           }
6215         } else {
6216           for (k = 0; k < dof; ++k) {
6217             if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
6218             a[k] += values[off+dof-k-1];
6219           }
6220         }
6221       }
6222     }
6223   }
6224   ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
6225   PetscFunctionReturn(0);
6226 }
6227 
6228 /*@C
6229   DMPlexVecSetClosure - Set an array of the values on the closure of 'point'
6230 
6231   Not collective
6232 
6233   Input Parameters:
6234 + dm - The DM
6235 . section - The section describing the layout in v, or NULL to use the default section
6236 . v - The local vector
6237 . point - The point in the DM
6238 . values - The array of values
6239 - mode - The insert mode. One of INSERT_ALL_VALUES, ADD_ALL_VALUES, INSERT_VALUES, ADD_VALUES, INSERT_BC_VALUES, and ADD_BC_VALUES,
6240          where INSERT_ALL_VALUES and ADD_ALL_VALUES also overwrite boundary conditions.
6241 
6242   Fortran Notes:
6243   This routine is only available in Fortran 90, and you must include petsc.h90 in your code.
6244 
6245   Level: intermediate
6246 
6247 .seealso DMPlexVecGetClosure(), DMPlexMatSetClosure()
6248 @*/
6249 PetscErrorCode DMPlexVecSetClosure(DM dm, PetscSection section, Vec v, PetscInt point, const PetscScalar values[], InsertMode mode)
6250 {
6251   PetscSection    clSection;
6252   IS              clPoints;
6253   PetscScalar    *array;
6254   PetscInt       *points = NULL;
6255   const PetscInt *clp, *clperm = NULL;
6256   PetscInt        depth, numFields, numPoints, p, clsize;
6257   PetscErrorCode  ierr;
6258 
6259   PetscFunctionBeginHot;
6260   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6261   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
6262   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
6263   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
6264   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
6265   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
6266   if (depth == 1 && numFields < 2 && mode == ADD_VALUES) {
6267     ierr = DMPlexVecSetClosure_Depth1_Static(dm, section, v, point, values, mode);CHKERRQ(ierr);
6268     PetscFunctionReturn(0);
6269   }
6270   /* Get points */
6271   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
6272   for (clsize=0,p=0; p<numPoints; p++) {
6273     PetscInt dof;
6274     ierr = PetscSectionGetDof(section, points[2*p], &dof);CHKERRQ(ierr);
6275     clsize += dof;
6276   }
6277   ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, depth, clsize, &clperm);CHKERRQ(ierr);
6278   /* Get array */
6279   ierr = VecGetArray(v, &array);CHKERRQ(ierr);
6280   /* Get values */
6281   if (numFields > 0) {
6282     PetscInt offset = 0, f;
6283     for (f = 0; f < numFields; ++f) {
6284       const PetscInt    **perms = NULL;
6285       const PetscScalar **flips = NULL;
6286 
6287       ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
6288       switch (mode) {
6289       case INSERT_VALUES:
6290         for (p = 0; p < numPoints; p++) {
6291           const PetscInt    point = points[2*p];
6292           const PetscInt    *perm = perms ? perms[p] : NULL;
6293           const PetscScalar *flip = flips ? flips[p] : NULL;
6294           updatePointFields_private(section, point, perm, flip, f, insert, PETSC_FALSE, clperm, values, &offset, array);
6295         } break;
6296       case INSERT_ALL_VALUES:
6297         for (p = 0; p < numPoints; p++) {
6298           const PetscInt    point = points[2*p];
6299           const PetscInt    *perm = perms ? perms[p] : NULL;
6300           const PetscScalar *flip = flips ? flips[p] : NULL;
6301           updatePointFields_private(section, point, perm, flip, f, insert, PETSC_TRUE, clperm, values, &offset, array);
6302         } break;
6303       case INSERT_BC_VALUES:
6304         for (p = 0; p < numPoints; p++) {
6305           const PetscInt    point = points[2*p];
6306           const PetscInt    *perm = perms ? perms[p] : NULL;
6307           const PetscScalar *flip = flips ? flips[p] : NULL;
6308           updatePointFieldsBC_private(section, point, perm, flip, f, -1, NULL, insert, clperm, values, &offset, array);
6309         } break;
6310       case ADD_VALUES:
6311         for (p = 0; p < numPoints; p++) {
6312           const PetscInt    point = points[2*p];
6313           const PetscInt    *perm = perms ? perms[p] : NULL;
6314           const PetscScalar *flip = flips ? flips[p] : NULL;
6315           updatePointFields_private(section, point, perm, flip, f, add, PETSC_FALSE, clperm, values, &offset, array);
6316         } break;
6317       case ADD_ALL_VALUES:
6318         for (p = 0; p < numPoints; p++) {
6319           const PetscInt    point = points[2*p];
6320           const PetscInt    *perm = perms ? perms[p] : NULL;
6321           const PetscScalar *flip = flips ? flips[p] : NULL;
6322           updatePointFields_private(section, point, perm, flip, f, add, PETSC_TRUE, clperm, values, &offset, array);
6323         } break;
6324       case ADD_BC_VALUES:
6325         for (p = 0; p < numPoints; p++) {
6326           const PetscInt    point = points[2*p];
6327           const PetscInt    *perm = perms ? perms[p] : NULL;
6328           const PetscScalar *flip = flips ? flips[p] : NULL;
6329           updatePointFieldsBC_private(section, point, perm, flip, f, -1, NULL, add, clperm, values, &offset, array);
6330         } break;
6331       default:
6332         SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %d", mode);
6333       }
6334       ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
6335     }
6336   } else {
6337     PetscInt dof, off;
6338     const PetscInt    **perms = NULL;
6339     const PetscScalar **flips = NULL;
6340 
6341     ierr = PetscSectionGetPointSyms(section,numPoints,points,&perms,&flips);CHKERRQ(ierr);
6342     switch (mode) {
6343     case INSERT_VALUES:
6344       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
6345         const PetscInt    point = points[2*p];
6346         const PetscInt    *perm = perms ? perms[p] : NULL;
6347         const PetscScalar *flip = flips ? flips[p] : NULL;
6348         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
6349         updatePoint_private(section, point, dof, insert, PETSC_FALSE, perm, flip, clperm, values, off, array);
6350       } break;
6351     case INSERT_ALL_VALUES:
6352       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
6353         const PetscInt    point = points[2*p];
6354         const PetscInt    *perm = perms ? perms[p] : NULL;
6355         const PetscScalar *flip = flips ? flips[p] : NULL;
6356         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
6357         updatePoint_private(section, point, dof, insert, PETSC_TRUE,  perm, flip, clperm, values, off, array);
6358       } break;
6359     case INSERT_BC_VALUES:
6360       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
6361         const PetscInt    point = points[2*p];
6362         const PetscInt    *perm = perms ? perms[p] : NULL;
6363         const PetscScalar *flip = flips ? flips[p] : NULL;
6364         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
6365         updatePointBC_private(section, point, dof, insert,  perm, flip, clperm, values, off, array);
6366       } break;
6367     case ADD_VALUES:
6368       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
6369         const PetscInt    point = points[2*p];
6370         const PetscInt    *perm = perms ? perms[p] : NULL;
6371         const PetscScalar *flip = flips ? flips[p] : NULL;
6372         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
6373         updatePoint_private(section, point, dof, add,    PETSC_FALSE, perm, flip, clperm, values, off, array);
6374       } break;
6375     case ADD_ALL_VALUES:
6376       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
6377         const PetscInt    point = points[2*p];
6378         const PetscInt    *perm = perms ? perms[p] : NULL;
6379         const PetscScalar *flip = flips ? flips[p] : NULL;
6380         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
6381         updatePoint_private(section, point, dof, add,    PETSC_TRUE,  perm, flip, clperm, values, off, array);
6382       } break;
6383     case ADD_BC_VALUES:
6384       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
6385         const PetscInt    point = points[2*p];
6386         const PetscInt    *perm = perms ? perms[p] : NULL;
6387         const PetscScalar *flip = flips ? flips[p] : NULL;
6388         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
6389         updatePointBC_private(section, point, dof, add,  perm, flip, clperm, values, off, array);
6390       } break;
6391     default:
6392       SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %d", mode);
6393     }
6394     ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms,&flips);CHKERRQ(ierr);
6395   }
6396   /* Cleanup points */
6397   ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
6398   /* Cleanup array */
6399   ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
6400   PetscFunctionReturn(0);
6401 }
6402 
6403 /* Check whether the given point is in the label. If not, update the offset to skip this point */
6404 PETSC_STATIC_INLINE PetscErrorCode CheckPoint_Private(DMLabel label, PetscInt labelId, PetscSection section, PetscInt point, PetscInt f, PetscInt *offset)
6405 {
6406   PetscFunctionBegin;
6407   if (label) {
6408     PetscInt       val, fdof;
6409     PetscErrorCode ierr;
6410 
6411     /* There is a problem with this:
6412          Suppose we have two label values, defining surfaces, interecting along a line in 3D. When we add cells to the label, the cells that
6413        touch both surfaces must pick a label value. Thus we miss setting values for the surface with that other value intersecting that cell.
6414        Thus I am only going to check val != -1, not val != labelId
6415     */
6416     ierr = DMLabelGetValue(label, point, &val);CHKERRQ(ierr);
6417     if (val < 0) {
6418       ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
6419       *offset += fdof;
6420       PetscFunctionReturn(1);
6421     }
6422   }
6423   PetscFunctionReturn(0);
6424 }
6425 
6426 /* Unlike DMPlexVecSetClosure(), this uses plex-native closure permutation, not a user-specified permutation such as DMPlexSetClosurePermutationTensor(). */
6427 PetscErrorCode DMPlexVecSetFieldClosure_Internal(DM dm, PetscSection section, Vec v, PetscBool fieldActive[], PetscInt point, PetscInt Ncc, const PetscInt comps[], DMLabel label, PetscInt labelId, const PetscScalar values[], InsertMode mode)
6428 {
6429   PetscSection      clSection;
6430   IS                clPoints;
6431   PetscScalar       *array;
6432   PetscInt          *points = NULL;
6433   const PetscInt    *clp;
6434   PetscInt          numFields, numPoints, p;
6435   PetscInt          offset = 0, f;
6436   PetscErrorCode    ierr;
6437 
6438   PetscFunctionBeginHot;
6439   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6440   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
6441   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
6442   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
6443   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
6444   /* Get points */
6445   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
6446   /* Get array */
6447   ierr = VecGetArray(v, &array);CHKERRQ(ierr);
6448   /* Get values */
6449   for (f = 0; f < numFields; ++f) {
6450     const PetscInt    **perms = NULL;
6451     const PetscScalar **flips = NULL;
6452 
6453     if (!fieldActive[f]) {
6454       for (p = 0; p < numPoints*2; p += 2) {
6455         PetscInt fdof;
6456         ierr = PetscSectionGetFieldDof(section, points[p], f, &fdof);CHKERRQ(ierr);
6457         offset += fdof;
6458       }
6459       continue;
6460     }
6461     ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
6462     switch (mode) {
6463     case INSERT_VALUES:
6464       for (p = 0; p < numPoints; p++) {
6465         const PetscInt    point = points[2*p];
6466         const PetscInt    *perm = perms ? perms[p] : NULL;
6467         const PetscScalar *flip = flips ? flips[p] : NULL;
6468         ierr = CheckPoint_Private(label, labelId, section, point, f, &offset); if (ierr) continue;
6469         updatePointFields_private(section, point, perm, flip, f, insert, PETSC_FALSE, NULL, values, &offset, array);
6470       } break;
6471     case INSERT_ALL_VALUES:
6472       for (p = 0; p < numPoints; p++) {
6473         const PetscInt    point = points[2*p];
6474         const PetscInt    *perm = perms ? perms[p] : NULL;
6475         const PetscScalar *flip = flips ? flips[p] : NULL;
6476         ierr = CheckPoint_Private(label, labelId, section, point, f, &offset); if (ierr) continue;
6477         updatePointFields_private(section, point, perm, flip, f, insert, PETSC_TRUE, NULL, values, &offset, array);
6478       } break;
6479     case INSERT_BC_VALUES:
6480       for (p = 0; p < numPoints; p++) {
6481         const PetscInt    point = points[2*p];
6482         const PetscInt    *perm = perms ? perms[p] : NULL;
6483         const PetscScalar *flip = flips ? flips[p] : NULL;
6484         ierr = CheckPoint_Private(label, labelId, section, point, f, &offset); if (ierr) continue;
6485         updatePointFieldsBC_private(section, point, perm, flip, f, Ncc, comps, insert, NULL, values, &offset, array);
6486       } break;
6487     case ADD_VALUES:
6488       for (p = 0; p < numPoints; p++) {
6489         const PetscInt    point = points[2*p];
6490         const PetscInt    *perm = perms ? perms[p] : NULL;
6491         const PetscScalar *flip = flips ? flips[p] : NULL;
6492         ierr = CheckPoint_Private(label, labelId, section, point, f, &offset); if (ierr) continue;
6493         updatePointFields_private(section, point, perm, flip, f, add, PETSC_FALSE, NULL, values, &offset, array);
6494       } break;
6495     case ADD_ALL_VALUES:
6496       for (p = 0; p < numPoints; p++) {
6497         const PetscInt    point = points[2*p];
6498         const PetscInt    *perm = perms ? perms[p] : NULL;
6499         const PetscScalar *flip = flips ? flips[p] : NULL;
6500         ierr = CheckPoint_Private(label, labelId, section, point, f, &offset); if (ierr) continue;
6501         updatePointFields_private(section, point, perm, flip, f, add, PETSC_TRUE, NULL, values, &offset, array);
6502       } break;
6503     default:
6504       SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %d", mode);
6505     }
6506     ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
6507   }
6508   /* Cleanup points */
6509   ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
6510   /* Cleanup array */
6511   ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
6512   PetscFunctionReturn(0);
6513 }
6514 
6515 static PetscErrorCode DMPlexPrintMatSetValues(PetscViewer viewer, Mat A, PetscInt point, PetscInt numRIndices, const PetscInt rindices[], PetscInt numCIndices, const PetscInt cindices[], const PetscScalar values[])
6516 {
6517   PetscMPIInt    rank;
6518   PetscInt       i, j;
6519   PetscErrorCode ierr;
6520 
6521   PetscFunctionBegin;
6522   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRMPI(ierr);
6523   ierr = PetscViewerASCIIPrintf(viewer, "[%d]mat for point %D\n", rank, point);CHKERRQ(ierr);
6524   for (i = 0; i < numRIndices; i++) {ierr = PetscViewerASCIIPrintf(viewer, "[%d]mat row indices[%D] = %D\n", rank, i, rindices[i]);CHKERRQ(ierr);}
6525   for (i = 0; i < numCIndices; i++) {ierr = PetscViewerASCIIPrintf(viewer, "[%d]mat col indices[%D] = %D\n", rank, i, cindices[i]);CHKERRQ(ierr);}
6526   numCIndices = numCIndices ? numCIndices : numRIndices;
6527   if (!values) PetscFunctionReturn(0);
6528   for (i = 0; i < numRIndices; i++) {
6529     ierr = PetscViewerASCIIPrintf(viewer, "[%d]", rank);CHKERRQ(ierr);
6530     for (j = 0; j < numCIndices; j++) {
6531 #if defined(PETSC_USE_COMPLEX)
6532       ierr = PetscViewerASCIIPrintf(viewer, " (%g,%g)", (double)PetscRealPart(values[i*numCIndices+j]), (double)PetscImaginaryPart(values[i*numCIndices+j]));CHKERRQ(ierr);
6533 #else
6534       ierr = PetscViewerASCIIPrintf(viewer, " %g", (double)values[i*numCIndices+j]);CHKERRQ(ierr);
6535 #endif
6536     }
6537     ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
6538   }
6539   PetscFunctionReturn(0);
6540 }
6541 
6542 /*
6543   DMPlexGetIndicesPoint_Internal - Add the indices for dofs on a point to an index array
6544 
6545   Input Parameters:
6546 + section - The section for this data layout
6547 . islocal - Is the section (and thus indices being requested) local or global?
6548 . point   - The point contributing dofs with these indices
6549 . off     - The global offset of this point
6550 . loff    - The local offset of each field
6551 . setBC   - The flag determining whether to include indices of boundary values
6552 . perm    - A permutation of the dofs on this point, or NULL
6553 - indperm - A permutation of the entire indices array, or NULL
6554 
6555   Output Parameter:
6556 . indices - Indices for dofs on this point
6557 
6558   Level: developer
6559 
6560   Note: The indices could be local or global, depending on the value of 'off'.
6561 */
6562 PetscErrorCode DMPlexGetIndicesPoint_Internal(PetscSection section, PetscBool islocal,PetscInt point, PetscInt off, PetscInt *loff, PetscBool setBC, const PetscInt perm[], const PetscInt indperm[], PetscInt indices[])
6563 {
6564   PetscInt        dof;   /* The number of unknowns on this point */
6565   PetscInt        cdof;  /* The number of constraints on this point */
6566   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
6567   PetscInt        cind = 0, k;
6568   PetscErrorCode  ierr;
6569 
6570   PetscFunctionBegin;
6571   if (!islocal && setBC) SETERRQ(PetscObjectComm((PetscObject)section),PETSC_ERR_ARG_INCOMP,"setBC incompatible with global indices; use a local section or disable setBC");
6572   ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
6573   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
6574   if (!cdof || setBC) {
6575     for (k = 0; k < dof; ++k) {
6576       const PetscInt preind = perm ? *loff+perm[k] : *loff+k;
6577       const PetscInt ind    = indperm ? indperm[preind] : preind;
6578 
6579       indices[ind] = off + k;
6580     }
6581   } else {
6582     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
6583     for (k = 0; k < dof; ++k) {
6584       const PetscInt preind = perm ? *loff+perm[k] : *loff+k;
6585       const PetscInt ind    = indperm ? indperm[preind] : preind;
6586 
6587       if ((cind < cdof) && (k == cdofs[cind])) {
6588         /* Insert check for returning constrained indices */
6589         indices[ind] = -(off+k+1);
6590         ++cind;
6591       } else {
6592         indices[ind] = off + k - (islocal ? 0 : cind);
6593       }
6594     }
6595   }
6596   *loff += dof;
6597   PetscFunctionReturn(0);
6598 }
6599 
6600 /*
6601  DMPlexGetIndicesPointFields_Internal - gets section indices for a point in its canonical ordering.
6602 
6603  Input Parameters:
6604 + section - a section (global or local)
6605 - islocal - PETSC_TRUE if requesting local indices (i.e., section is local); PETSC_FALSE for global
6606 . point - point within section
6607 . off - The offset of this point in the (local or global) indexed space - should match islocal and (usually) the section
6608 . foffs - array of length numFields containing the offset in canonical point ordering (the location in indices) of each field
6609 . setBC - identify constrained (boundary condition) points via involution.
6610 . perms - perms[f][permsoff][:] is a permutation of dofs within each field
6611 . permsoff - offset
6612 - indperm - index permutation
6613 
6614  Output Parameter:
6615 . foffs - each entry is incremented by the number of (unconstrained if setBC=FALSE) dofs in that field
6616 . indices - array to hold indices (as defined by section) of each dof associated with point
6617 
6618  Notes:
6619  If section is local and setBC=true, there is no distinction between constrained and unconstrained dofs.
6620  If section is local and setBC=false, the indices for constrained points are the involution -(i+1) of their position
6621  in the local vector.
6622 
6623  If section is global and setBC=false, the indices for constrained points are negative (and their value is not
6624  significant).  It is invalid to call with a global section and setBC=true.
6625 
6626  Developer Note:
6627  The section is only used for field layout, so islocal is technically a statement about the offset (off).  At some point
6628  in the future, global sections may have fields set, in which case we could pass the global section and obtain the
6629  offset could be obtained from the section instead of passing it explicitly as we do now.
6630 
6631  Example:
6632  Suppose a point contains one field with three components, and for which the unconstrained indices are {10, 11, 12}.
6633  When the middle component is constrained, we get the array {10, -12, 12} for (islocal=TRUE, setBC=FALSE).
6634  Note that -12 is the involution of 11, so the user can involute negative indices to recover local indices.
6635  The global vector does not store constrained dofs, so when this function returns global indices, say {110, -112, 111}, the value of -112 is an arbitrary flag that should not be interpreted beyond its sign.
6636 
6637  Level: developer
6638 */
6639 PetscErrorCode DMPlexGetIndicesPointFields_Internal(PetscSection section, PetscBool islocal, PetscInt point, PetscInt off, PetscInt foffs[], PetscBool setBC, const PetscInt ***perms, PetscInt permsoff, const PetscInt indperm[], PetscInt indices[])
6640 {
6641   PetscInt       numFields, foff, f;
6642   PetscErrorCode ierr;
6643 
6644   PetscFunctionBegin;
6645   if (!islocal && setBC) SETERRQ(PetscObjectComm((PetscObject)section),PETSC_ERR_ARG_INCOMP,"setBC incompatible with global indices; use a local section or disable setBC");
6646   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
6647   for (f = 0, foff = 0; f < numFields; ++f) {
6648     PetscInt        fdof, cfdof;
6649     const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
6650     PetscInt        cind = 0, b;
6651     const PetscInt  *perm = (perms && perms[f]) ? perms[f][permsoff] : NULL;
6652 
6653     ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
6654     ierr = PetscSectionGetFieldConstraintDof(section, point, f, &cfdof);CHKERRQ(ierr);
6655     if (!cfdof || setBC) {
6656       for (b = 0; b < fdof; ++b) {
6657         const PetscInt preind = perm ? foffs[f]+perm[b] : foffs[f]+b;
6658         const PetscInt ind    = indperm ? indperm[preind] : preind;
6659 
6660         indices[ind] = off+foff+b;
6661       }
6662     } else {
6663       ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
6664       for (b = 0; b < fdof; ++b) {
6665         const PetscInt preind = perm ? foffs[f]+perm[b] : foffs[f]+b;
6666         const PetscInt ind    = indperm ? indperm[preind] : preind;
6667 
6668         if ((cind < cfdof) && (b == fcdofs[cind])) {
6669           indices[ind] = -(off+foff+b+1);
6670           ++cind;
6671         } else {
6672           indices[ind] = off + foff + b - (islocal ? 0 : cind);
6673         }
6674       }
6675     }
6676     foff     += (setBC || islocal ? fdof : (fdof - cfdof));
6677     foffs[f] += fdof;
6678   }
6679   PetscFunctionReturn(0);
6680 }
6681 
6682 /*
6683   This version believes the globalSection offsets for each field, rather than just the point offset
6684 
6685  . foffs - The offset into 'indices' for each field, since it is segregated by field
6686 
6687  Notes:
6688  The semantics of this function relate to that of setBC=FALSE in DMPlexGetIndicesPointFields_Internal.
6689  Since this function uses global indices, setBC=TRUE would be invalid, so no such argument exists.
6690 */
6691 static PetscErrorCode DMPlexGetIndicesPointFieldsSplit_Internal(PetscSection section, PetscSection globalSection, PetscInt point, PetscInt foffs[], const PetscInt ***perms, PetscInt permsoff, const PetscInt indperm[], PetscInt indices[])
6692 {
6693   PetscInt       numFields, foff, f;
6694   PetscErrorCode ierr;
6695 
6696   PetscFunctionBegin;
6697   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
6698   for (f = 0; f < numFields; ++f) {
6699     PetscInt        fdof, cfdof;
6700     const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
6701     PetscInt        cind = 0, b;
6702     const PetscInt  *perm = (perms && perms[f]) ? perms[f][permsoff] : NULL;
6703 
6704     ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
6705     ierr = PetscSectionGetFieldConstraintDof(section, point, f, &cfdof);CHKERRQ(ierr);
6706     ierr = PetscSectionGetFieldOffset(globalSection, point, f, &foff);CHKERRQ(ierr);
6707     if (!cfdof) {
6708       for (b = 0; b < fdof; ++b) {
6709         const PetscInt preind = perm ? foffs[f]+perm[b] : foffs[f]+b;
6710         const PetscInt ind    = indperm ? indperm[preind] : preind;
6711 
6712         indices[ind] = foff+b;
6713       }
6714     } else {
6715       ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
6716       for (b = 0; b < fdof; ++b) {
6717         const PetscInt preind = perm ? foffs[f]+perm[b] : foffs[f]+b;
6718         const PetscInt ind    = indperm ? indperm[preind] : preind;
6719 
6720         if ((cind < cfdof) && (b == fcdofs[cind])) {
6721           indices[ind] = -(foff+b+1);
6722           ++cind;
6723         } else {
6724           indices[ind] = foff+b-cind;
6725         }
6726       }
6727     }
6728     foffs[f] += fdof;
6729   }
6730   PetscFunctionReturn(0);
6731 }
6732 
6733 PetscErrorCode DMPlexAnchorsModifyMat(DM dm, PetscSection section, PetscInt numPoints, PetscInt numIndices, const PetscInt points[], const PetscInt ***perms, const PetscScalar values[], PetscInt *outNumPoints, PetscInt *outNumIndices, PetscInt *outPoints[], PetscScalar *outValues[], PetscInt offsets[], PetscBool multiplyLeft)
6734 {
6735   Mat             cMat;
6736   PetscSection    aSec, cSec;
6737   IS              aIS;
6738   PetscInt        aStart = -1, aEnd = -1;
6739   const PetscInt  *anchors;
6740   PetscInt        numFields, f, p, q, newP = 0;
6741   PetscInt        newNumPoints = 0, newNumIndices = 0;
6742   PetscInt        *newPoints, *indices, *newIndices;
6743   PetscInt        maxAnchor, maxDof;
6744   PetscInt        newOffsets[32];
6745   PetscInt        *pointMatOffsets[32];
6746   PetscInt        *newPointOffsets[32];
6747   PetscScalar     *pointMat[32];
6748   PetscScalar     *newValues=NULL,*tmpValues;
6749   PetscBool       anyConstrained = PETSC_FALSE;
6750   PetscErrorCode  ierr;
6751 
6752   PetscFunctionBegin;
6753   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6754   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
6755   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
6756 
6757   ierr = DMPlexGetAnchors(dm,&aSec,&aIS);CHKERRQ(ierr);
6758   /* if there are point-to-point constraints */
6759   if (aSec) {
6760     ierr = PetscArrayzero(newOffsets, 32);CHKERRQ(ierr);
6761     ierr = ISGetIndices(aIS,&anchors);CHKERRQ(ierr);
6762     ierr = PetscSectionGetChart(aSec,&aStart,&aEnd);CHKERRQ(ierr);
6763     /* figure out how many points are going to be in the new element matrix
6764      * (we allow double counting, because it's all just going to be summed
6765      * into the global matrix anyway) */
6766     for (p = 0; p < 2*numPoints; p+=2) {
6767       PetscInt b    = points[p];
6768       PetscInt bDof = 0, bSecDof;
6769 
6770       ierr = PetscSectionGetDof(section,b,&bSecDof);CHKERRQ(ierr);
6771       if (!bSecDof) {
6772         continue;
6773       }
6774       if (b >= aStart && b < aEnd) {
6775         ierr = PetscSectionGetDof(aSec,b,&bDof);CHKERRQ(ierr);
6776       }
6777       if (bDof) {
6778         /* this point is constrained */
6779         /* it is going to be replaced by its anchors */
6780         PetscInt bOff, q;
6781 
6782         anyConstrained = PETSC_TRUE;
6783         newNumPoints  += bDof;
6784         ierr = PetscSectionGetOffset(aSec,b,&bOff);CHKERRQ(ierr);
6785         for (q = 0; q < bDof; q++) {
6786           PetscInt a = anchors[bOff + q];
6787           PetscInt aDof;
6788 
6789           ierr           = PetscSectionGetDof(section,a,&aDof);CHKERRQ(ierr);
6790           newNumIndices += aDof;
6791           for (f = 0; f < numFields; ++f) {
6792             PetscInt fDof;
6793 
6794             ierr             = PetscSectionGetFieldDof(section, a, f, &fDof);CHKERRQ(ierr);
6795             newOffsets[f+1] += fDof;
6796           }
6797         }
6798       }
6799       else {
6800         /* this point is not constrained */
6801         newNumPoints++;
6802         newNumIndices += bSecDof;
6803         for (f = 0; f < numFields; ++f) {
6804           PetscInt fDof;
6805 
6806           ierr = PetscSectionGetFieldDof(section, b, f, &fDof);CHKERRQ(ierr);
6807           newOffsets[f+1] += fDof;
6808         }
6809       }
6810     }
6811   }
6812   if (!anyConstrained) {
6813     if (outNumPoints)  *outNumPoints  = 0;
6814     if (outNumIndices) *outNumIndices = 0;
6815     if (outPoints)     *outPoints     = NULL;
6816     if (outValues)     *outValues     = NULL;
6817     if (aSec) {ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);}
6818     PetscFunctionReturn(0);
6819   }
6820 
6821   if (outNumPoints)  *outNumPoints  = newNumPoints;
6822   if (outNumIndices) *outNumIndices = newNumIndices;
6823 
6824   for (f = 0; f < numFields; ++f) newOffsets[f+1] += newOffsets[f];
6825 
6826   if (!outPoints && !outValues) {
6827     if (offsets) {
6828       for (f = 0; f <= numFields; f++) {
6829         offsets[f] = newOffsets[f];
6830       }
6831     }
6832     if (aSec) {ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);}
6833     PetscFunctionReturn(0);
6834   }
6835 
6836   if (numFields && newOffsets[numFields] != newNumIndices) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", newOffsets[numFields], newNumIndices);
6837 
6838   ierr = DMGetDefaultConstraints(dm, &cSec, &cMat);CHKERRQ(ierr);
6839 
6840   /* workspaces */
6841   if (numFields) {
6842     for (f = 0; f < numFields; f++) {
6843       ierr = DMGetWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[f]);CHKERRQ(ierr);
6844       ierr = DMGetWorkArray(dm,numPoints+1,MPIU_INT,&newPointOffsets[f]);CHKERRQ(ierr);
6845     }
6846   }
6847   else {
6848     ierr = DMGetWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[0]);CHKERRQ(ierr);
6849     ierr = DMGetWorkArray(dm,numPoints,MPIU_INT,&newPointOffsets[0]);CHKERRQ(ierr);
6850   }
6851 
6852   /* get workspaces for the point-to-point matrices */
6853   if (numFields) {
6854     PetscInt totalOffset, totalMatOffset;
6855 
6856     for (p = 0; p < numPoints; p++) {
6857       PetscInt b    = points[2*p];
6858       PetscInt bDof = 0, bSecDof;
6859 
6860       ierr = PetscSectionGetDof(section,b,&bSecDof);CHKERRQ(ierr);
6861       if (!bSecDof) {
6862         for (f = 0; f < numFields; f++) {
6863           newPointOffsets[f][p + 1] = 0;
6864           pointMatOffsets[f][p + 1] = 0;
6865         }
6866         continue;
6867       }
6868       if (b >= aStart && b < aEnd) {
6869         ierr = PetscSectionGetDof(aSec, b, &bDof);CHKERRQ(ierr);
6870       }
6871       if (bDof) {
6872         for (f = 0; f < numFields; f++) {
6873           PetscInt fDof, q, bOff, allFDof = 0;
6874 
6875           ierr = PetscSectionGetFieldDof(section, b, f, &fDof);CHKERRQ(ierr);
6876           ierr = PetscSectionGetOffset(aSec, b, &bOff);CHKERRQ(ierr);
6877           for (q = 0; q < bDof; q++) {
6878             PetscInt a = anchors[bOff + q];
6879             PetscInt aFDof;
6880 
6881             ierr     = PetscSectionGetFieldDof(section, a, f, &aFDof);CHKERRQ(ierr);
6882             allFDof += aFDof;
6883           }
6884           newPointOffsets[f][p+1] = allFDof;
6885           pointMatOffsets[f][p+1] = fDof * allFDof;
6886         }
6887       }
6888       else {
6889         for (f = 0; f < numFields; f++) {
6890           PetscInt fDof;
6891 
6892           ierr = PetscSectionGetFieldDof(section, b, f, &fDof);CHKERRQ(ierr);
6893           newPointOffsets[f][p+1] = fDof;
6894           pointMatOffsets[f][p+1] = 0;
6895         }
6896       }
6897     }
6898     for (f = 0, totalOffset = 0, totalMatOffset = 0; f < numFields; f++) {
6899       newPointOffsets[f][0] = totalOffset;
6900       pointMatOffsets[f][0] = totalMatOffset;
6901       for (p = 0; p < numPoints; p++) {
6902         newPointOffsets[f][p+1] += newPointOffsets[f][p];
6903         pointMatOffsets[f][p+1] += pointMatOffsets[f][p];
6904       }
6905       totalOffset    = newPointOffsets[f][numPoints];
6906       totalMatOffset = pointMatOffsets[f][numPoints];
6907       ierr = DMGetWorkArray(dm,pointMatOffsets[f][numPoints],MPIU_SCALAR,&pointMat[f]);CHKERRQ(ierr);
6908     }
6909   }
6910   else {
6911     for (p = 0; p < numPoints; p++) {
6912       PetscInt b    = points[2*p];
6913       PetscInt bDof = 0, bSecDof;
6914 
6915       ierr = PetscSectionGetDof(section,b,&bSecDof);CHKERRQ(ierr);
6916       if (!bSecDof) {
6917         newPointOffsets[0][p + 1] = 0;
6918         pointMatOffsets[0][p + 1] = 0;
6919         continue;
6920       }
6921       if (b >= aStart && b < aEnd) {
6922         ierr = PetscSectionGetDof(aSec, b, &bDof);CHKERRQ(ierr);
6923       }
6924       if (bDof) {
6925         PetscInt bOff, q, allDof = 0;
6926 
6927         ierr = PetscSectionGetOffset(aSec, b, &bOff);CHKERRQ(ierr);
6928         for (q = 0; q < bDof; q++) {
6929           PetscInt a = anchors[bOff + q], aDof;
6930 
6931           ierr    = PetscSectionGetDof(section, a, &aDof);CHKERRQ(ierr);
6932           allDof += aDof;
6933         }
6934         newPointOffsets[0][p+1] = allDof;
6935         pointMatOffsets[0][p+1] = bSecDof * allDof;
6936       }
6937       else {
6938         newPointOffsets[0][p+1] = bSecDof;
6939         pointMatOffsets[0][p+1] = 0;
6940       }
6941     }
6942     newPointOffsets[0][0] = 0;
6943     pointMatOffsets[0][0] = 0;
6944     for (p = 0; p < numPoints; p++) {
6945       newPointOffsets[0][p+1] += newPointOffsets[0][p];
6946       pointMatOffsets[0][p+1] += pointMatOffsets[0][p];
6947     }
6948     ierr = DMGetWorkArray(dm,pointMatOffsets[0][numPoints],MPIU_SCALAR,&pointMat[0]);CHKERRQ(ierr);
6949   }
6950 
6951   /* output arrays */
6952   ierr = DMGetWorkArray(dm,2*newNumPoints,MPIU_INT,&newPoints);CHKERRQ(ierr);
6953 
6954   /* get the point-to-point matrices; construct newPoints */
6955   ierr = PetscSectionGetMaxDof(aSec, &maxAnchor);CHKERRQ(ierr);
6956   ierr = PetscSectionGetMaxDof(section, &maxDof);CHKERRQ(ierr);
6957   ierr = DMGetWorkArray(dm,maxDof,MPIU_INT,&indices);CHKERRQ(ierr);
6958   ierr = DMGetWorkArray(dm,maxAnchor*maxDof,MPIU_INT,&newIndices);CHKERRQ(ierr);
6959   if (numFields) {
6960     for (p = 0, newP = 0; p < numPoints; p++) {
6961       PetscInt b    = points[2*p];
6962       PetscInt o    = points[2*p+1];
6963       PetscInt bDof = 0, bSecDof;
6964 
6965       ierr = PetscSectionGetDof(section, b, &bSecDof);CHKERRQ(ierr);
6966       if (!bSecDof) {
6967         continue;
6968       }
6969       if (b >= aStart && b < aEnd) {
6970         ierr = PetscSectionGetDof(aSec, b, &bDof);CHKERRQ(ierr);
6971       }
6972       if (bDof) {
6973         PetscInt fStart[32], fEnd[32], fAnchorStart[32], fAnchorEnd[32], bOff, q;
6974 
6975         fStart[0] = 0;
6976         fEnd[0]   = 0;
6977         for (f = 0; f < numFields; f++) {
6978           PetscInt fDof;
6979 
6980           ierr        = PetscSectionGetFieldDof(cSec, b, f, &fDof);CHKERRQ(ierr);
6981           fStart[f+1] = fStart[f] + fDof;
6982           fEnd[f+1]   = fStart[f+1];
6983         }
6984         ierr = PetscSectionGetOffset(cSec, b, &bOff);CHKERRQ(ierr);
6985         ierr = DMPlexGetIndicesPointFields_Internal(cSec, PETSC_TRUE, b, bOff, fEnd, PETSC_TRUE, perms, p, NULL, indices);CHKERRQ(ierr);
6986 
6987         fAnchorStart[0] = 0;
6988         fAnchorEnd[0]   = 0;
6989         for (f = 0; f < numFields; f++) {
6990           PetscInt fDof = newPointOffsets[f][p + 1] - newPointOffsets[f][p];
6991 
6992           fAnchorStart[f+1] = fAnchorStart[f] + fDof;
6993           fAnchorEnd[f+1]   = fAnchorStart[f + 1];
6994         }
6995         ierr = PetscSectionGetOffset(aSec, b, &bOff);CHKERRQ(ierr);
6996         for (q = 0; q < bDof; q++) {
6997           PetscInt a = anchors[bOff + q], aOff;
6998 
6999           /* we take the orientation of ap into account in the order that we constructed the indices above: the newly added points have no orientation */
7000           newPoints[2*(newP + q)]     = a;
7001           newPoints[2*(newP + q) + 1] = 0;
7002           ierr = PetscSectionGetOffset(section, a, &aOff);CHKERRQ(ierr);
7003           ierr = DMPlexGetIndicesPointFields_Internal(section, PETSC_TRUE, a, aOff, fAnchorEnd, PETSC_TRUE, NULL, -1, NULL, newIndices);CHKERRQ(ierr);
7004         }
7005         newP += bDof;
7006 
7007         if (outValues) {
7008           /* get the point-to-point submatrix */
7009           for (f = 0; f < numFields; f++) {
7010             ierr = MatGetValues(cMat,fEnd[f]-fStart[f],indices + fStart[f],fAnchorEnd[f] - fAnchorStart[f],newIndices + fAnchorStart[f],pointMat[f] + pointMatOffsets[f][p]);CHKERRQ(ierr);
7011           }
7012         }
7013       }
7014       else {
7015         newPoints[2 * newP]     = b;
7016         newPoints[2 * newP + 1] = o;
7017         newP++;
7018       }
7019     }
7020   } else {
7021     for (p = 0; p < numPoints; p++) {
7022       PetscInt b    = points[2*p];
7023       PetscInt o    = points[2*p+1];
7024       PetscInt bDof = 0, bSecDof;
7025 
7026       ierr = PetscSectionGetDof(section, b, &bSecDof);CHKERRQ(ierr);
7027       if (!bSecDof) {
7028         continue;
7029       }
7030       if (b >= aStart && b < aEnd) {
7031         ierr = PetscSectionGetDof(aSec, b, &bDof);CHKERRQ(ierr);
7032       }
7033       if (bDof) {
7034         PetscInt bEnd = 0, bAnchorEnd = 0, bOff;
7035 
7036         ierr = PetscSectionGetOffset(cSec, b, &bOff);CHKERRQ(ierr);
7037         ierr = DMPlexGetIndicesPoint_Internal(cSec, PETSC_TRUE, b, bOff, &bEnd, PETSC_TRUE, (perms && perms[0]) ? perms[0][p] : NULL, NULL, indices);CHKERRQ(ierr);
7038 
7039         ierr = PetscSectionGetOffset (aSec, b, &bOff);CHKERRQ(ierr);
7040         for (q = 0; q < bDof; q++) {
7041           PetscInt a = anchors[bOff + q], aOff;
7042 
7043           /* we take the orientation of ap into account in the order that we constructed the indices above: the newly added points have no orientation */
7044 
7045           newPoints[2*(newP + q)]     = a;
7046           newPoints[2*(newP + q) + 1] = 0;
7047           ierr = PetscSectionGetOffset(section, a, &aOff);CHKERRQ(ierr);
7048           ierr = DMPlexGetIndicesPoint_Internal(section, PETSC_TRUE, a, aOff, &bAnchorEnd, PETSC_TRUE, NULL, NULL, newIndices);CHKERRQ(ierr);
7049         }
7050         newP += bDof;
7051 
7052         /* get the point-to-point submatrix */
7053         if (outValues) {
7054           ierr = MatGetValues(cMat,bEnd,indices,bAnchorEnd,newIndices,pointMat[0] + pointMatOffsets[0][p]);CHKERRQ(ierr);
7055         }
7056       }
7057       else {
7058         newPoints[2 * newP]     = b;
7059         newPoints[2 * newP + 1] = o;
7060         newP++;
7061       }
7062     }
7063   }
7064 
7065   if (outValues) {
7066     ierr = DMGetWorkArray(dm,newNumIndices*numIndices,MPIU_SCALAR,&tmpValues);CHKERRQ(ierr);
7067     ierr = PetscArrayzero(tmpValues,newNumIndices*numIndices);CHKERRQ(ierr);
7068     /* multiply constraints on the right */
7069     if (numFields) {
7070       for (f = 0; f < numFields; f++) {
7071         PetscInt oldOff = offsets[f];
7072 
7073         for (p = 0; p < numPoints; p++) {
7074           PetscInt cStart = newPointOffsets[f][p];
7075           PetscInt b      = points[2 * p];
7076           PetscInt c, r, k;
7077           PetscInt dof;
7078 
7079           ierr = PetscSectionGetFieldDof(section,b,f,&dof);CHKERRQ(ierr);
7080           if (!dof) {
7081             continue;
7082           }
7083           if (pointMatOffsets[f][p] < pointMatOffsets[f][p + 1]) {
7084             PetscInt nCols         = newPointOffsets[f][p+1]-cStart;
7085             const PetscScalar *mat = pointMat[f] + pointMatOffsets[f][p];
7086 
7087             for (r = 0; r < numIndices; r++) {
7088               for (c = 0; c < nCols; c++) {
7089                 for (k = 0; k < dof; k++) {
7090                   tmpValues[r * newNumIndices + cStart + c] += values[r * numIndices + oldOff + k] * mat[k * nCols + c];
7091                 }
7092               }
7093             }
7094           }
7095           else {
7096             /* copy this column as is */
7097             for (r = 0; r < numIndices; r++) {
7098               for (c = 0; c < dof; c++) {
7099                 tmpValues[r * newNumIndices + cStart + c] = values[r * numIndices + oldOff + c];
7100               }
7101             }
7102           }
7103           oldOff += dof;
7104         }
7105       }
7106     }
7107     else {
7108       PetscInt oldOff = 0;
7109       for (p = 0; p < numPoints; p++) {
7110         PetscInt cStart = newPointOffsets[0][p];
7111         PetscInt b      = points[2 * p];
7112         PetscInt c, r, k;
7113         PetscInt dof;
7114 
7115         ierr = PetscSectionGetDof(section,b,&dof);CHKERRQ(ierr);
7116         if (!dof) {
7117           continue;
7118         }
7119         if (pointMatOffsets[0][p] < pointMatOffsets[0][p + 1]) {
7120           PetscInt nCols         = newPointOffsets[0][p+1]-cStart;
7121           const PetscScalar *mat = pointMat[0] + pointMatOffsets[0][p];
7122 
7123           for (r = 0; r < numIndices; r++) {
7124             for (c = 0; c < nCols; c++) {
7125               for (k = 0; k < dof; k++) {
7126                 tmpValues[r * newNumIndices + cStart + c] += mat[k * nCols + c] * values[r * numIndices + oldOff + k];
7127               }
7128             }
7129           }
7130         }
7131         else {
7132           /* copy this column as is */
7133           for (r = 0; r < numIndices; r++) {
7134             for (c = 0; c < dof; c++) {
7135               tmpValues[r * newNumIndices + cStart + c] = values[r * numIndices + oldOff + c];
7136             }
7137           }
7138         }
7139         oldOff += dof;
7140       }
7141     }
7142 
7143     if (multiplyLeft) {
7144       ierr = DMGetWorkArray(dm,newNumIndices*newNumIndices,MPIU_SCALAR,&newValues);CHKERRQ(ierr);
7145       ierr = PetscArrayzero(newValues,newNumIndices*newNumIndices);CHKERRQ(ierr);
7146       /* multiply constraints transpose on the left */
7147       if (numFields) {
7148         for (f = 0; f < numFields; f++) {
7149           PetscInt oldOff = offsets[f];
7150 
7151           for (p = 0; p < numPoints; p++) {
7152             PetscInt rStart = newPointOffsets[f][p];
7153             PetscInt b      = points[2 * p];
7154             PetscInt c, r, k;
7155             PetscInt dof;
7156 
7157             ierr = PetscSectionGetFieldDof(section,b,f,&dof);CHKERRQ(ierr);
7158             if (pointMatOffsets[f][p] < pointMatOffsets[f][p + 1]) {
7159               PetscInt nRows                        = newPointOffsets[f][p+1]-rStart;
7160               const PetscScalar *PETSC_RESTRICT mat = pointMat[f] + pointMatOffsets[f][p];
7161 
7162               for (r = 0; r < nRows; r++) {
7163                 for (c = 0; c < newNumIndices; c++) {
7164                   for (k = 0; k < dof; k++) {
7165                     newValues[(rStart + r) * newNumIndices + c] += mat[k * nRows + r] * tmpValues[(oldOff + k) * newNumIndices + c];
7166                   }
7167                 }
7168               }
7169             }
7170             else {
7171               /* copy this row as is */
7172               for (r = 0; r < dof; r++) {
7173                 for (c = 0; c < newNumIndices; c++) {
7174                   newValues[(rStart + r) * newNumIndices + c] = tmpValues[(oldOff + r) * newNumIndices + c];
7175                 }
7176               }
7177             }
7178             oldOff += dof;
7179           }
7180         }
7181       }
7182       else {
7183         PetscInt oldOff = 0;
7184 
7185         for (p = 0; p < numPoints; p++) {
7186           PetscInt rStart = newPointOffsets[0][p];
7187           PetscInt b      = points[2 * p];
7188           PetscInt c, r, k;
7189           PetscInt dof;
7190 
7191           ierr = PetscSectionGetDof(section,b,&dof);CHKERRQ(ierr);
7192           if (pointMatOffsets[0][p] < pointMatOffsets[0][p + 1]) {
7193             PetscInt nRows                        = newPointOffsets[0][p+1]-rStart;
7194             const PetscScalar *PETSC_RESTRICT mat = pointMat[0] + pointMatOffsets[0][p];
7195 
7196             for (r = 0; r < nRows; r++) {
7197               for (c = 0; c < newNumIndices; c++) {
7198                 for (k = 0; k < dof; k++) {
7199                   newValues[(rStart + r) * newNumIndices + c] += mat[k * nRows + r] * tmpValues[(oldOff + k) * newNumIndices + c];
7200                 }
7201               }
7202             }
7203           }
7204           else {
7205             /* copy this row as is */
7206             for (r = 0; r < dof; r++) {
7207               for (c = 0; c < newNumIndices; c++) {
7208                 newValues[(rStart + r) * newNumIndices + c] = tmpValues[(oldOff + r) * newNumIndices + c];
7209               }
7210             }
7211           }
7212           oldOff += dof;
7213         }
7214       }
7215 
7216       ierr = DMRestoreWorkArray(dm,newNumIndices*numIndices,MPIU_SCALAR,&tmpValues);CHKERRQ(ierr);
7217     }
7218     else {
7219       newValues = tmpValues;
7220     }
7221   }
7222 
7223   /* clean up */
7224   ierr = DMRestoreWorkArray(dm,maxDof,MPIU_INT,&indices);CHKERRQ(ierr);
7225   ierr = DMRestoreWorkArray(dm,maxAnchor*maxDof,MPIU_INT,&newIndices);CHKERRQ(ierr);
7226 
7227   if (numFields) {
7228     for (f = 0; f < numFields; f++) {
7229       ierr = DMRestoreWorkArray(dm,pointMatOffsets[f][numPoints],MPIU_SCALAR,&pointMat[f]);CHKERRQ(ierr);
7230       ierr = DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[f]);CHKERRQ(ierr);
7231       ierr = DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&newPointOffsets[f]);CHKERRQ(ierr);
7232     }
7233   }
7234   else {
7235     ierr = DMRestoreWorkArray(dm,pointMatOffsets[0][numPoints],MPIU_SCALAR,&pointMat[0]);CHKERRQ(ierr);
7236     ierr = DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[0]);CHKERRQ(ierr);
7237     ierr = DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&newPointOffsets[0]);CHKERRQ(ierr);
7238   }
7239   ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);
7240 
7241   /* output */
7242   if (outPoints) {
7243     *outPoints = newPoints;
7244   }
7245   else {
7246     ierr = DMRestoreWorkArray(dm,2*newNumPoints,MPIU_INT,&newPoints);CHKERRQ(ierr);
7247   }
7248   if (outValues) {
7249     *outValues = newValues;
7250   }
7251   for (f = 0; f <= numFields; f++) {
7252     offsets[f] = newOffsets[f];
7253   }
7254   PetscFunctionReturn(0);
7255 }
7256 
7257 /*@C
7258   DMPlexGetClosureIndices - Gets the global dof indices associated with the closure of the given point within the provided sections.
7259 
7260   Not collective
7261 
7262   Input Parameters:
7263 + dm         - The DM
7264 . section    - The PetscSection describing the points (a local section)
7265 . idxSection - The PetscSection from which to obtain indices (may be local or global)
7266 . point      - The point defining the closure
7267 - useClPerm  - Use the closure point permutation if available
7268 
7269   Output Parameters:
7270 + numIndices - The number of dof indices in the closure of point with the input sections
7271 . indices    - The dof indices
7272 . outOffsets - Array to write the field offsets into, or NULL
7273 - values     - The input values, which may be modified if sign flips are induced by the point symmetries, or NULL
7274 
7275   Notes:
7276   Must call DMPlexRestoreClosureIndices() to free allocated memory
7277 
7278   If idxSection is global, any constrained dofs (see DMAddBoundary(), for example) will get negative indices.  The value
7279   of those indices is not significant.  If idxSection is local, the constrained dofs will yield the involution -(idx+1)
7280   of their index in a local vector.  A caller who does not wish to distinguish those points may recover the nonnegative
7281   indices via involution, -(-(idx+1)+1)==idx.  Local indices are provided when idxSection == section, otherwise global
7282   indices (with the above semantics) are implied.
7283 
7284   Level: advanced
7285 
7286 .seealso DMPlexRestoreClosureIndices(), DMPlexVecGetClosure(), DMPlexMatSetClosure(), DMGetLocalSection(), DMGetGlobalSection()
7287 @*/
7288 PetscErrorCode DMPlexGetClosureIndices(DM dm, PetscSection section, PetscSection idxSection, PetscInt point, PetscBool useClPerm,
7289                                        PetscInt *numIndices, PetscInt *indices[], PetscInt outOffsets[], PetscScalar *values[])
7290 {
7291   /* Closure ordering */
7292   PetscSection        clSection;
7293   IS                  clPoints;
7294   const PetscInt     *clp;
7295   PetscInt           *points;
7296   const PetscInt     *clperm = NULL;
7297   /* Dof permutation and sign flips */
7298   const PetscInt    **perms[32] = {NULL};
7299   const PetscScalar **flips[32] = {NULL};
7300   PetscScalar        *valCopy   = NULL;
7301   /* Hanging node constraints */
7302   PetscInt           *pointsC = NULL;
7303   PetscScalar        *valuesC = NULL;
7304   PetscInt            NclC, NiC;
7305 
7306   PetscInt           *idx;
7307   PetscInt            Nf, Ncl, Ni = 0, offsets[32], p, f;
7308   PetscBool           isLocal = (section == idxSection) ? PETSC_TRUE : PETSC_FALSE;
7309   PetscErrorCode      ierr;
7310 
7311   PetscFunctionBeginHot;
7312   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7313   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
7314   PetscValidHeaderSpecific(idxSection, PETSC_SECTION_CLASSID, 3);
7315   if (numIndices) PetscValidPointer(numIndices, 6);
7316   if (indices)    PetscValidPointer(indices, 7);
7317   if (outOffsets) PetscValidPointer(outOffsets, 8);
7318   if (values)     PetscValidPointer(values, 9);
7319   ierr = PetscSectionGetNumFields(section, &Nf);CHKERRQ(ierr);
7320   if (Nf > 31) SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", Nf);
7321   ierr = PetscArrayzero(offsets, 32);CHKERRQ(ierr);
7322   /* 1) Get points in closure */
7323   ierr = DMPlexGetCompressedClosure(dm, section, point, &Ncl, &points, &clSection, &clPoints, &clp);CHKERRQ(ierr);
7324   if (useClPerm) {
7325     PetscInt depth, clsize;
7326     ierr = DMPlexGetPointDepth(dm, point, &depth);CHKERRQ(ierr);
7327     for (clsize=0,p=0; p<Ncl; p++) {
7328       PetscInt dof;
7329       ierr = PetscSectionGetDof(section, points[2*p], &dof);CHKERRQ(ierr);
7330       clsize += dof;
7331     }
7332     ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, depth, clsize, &clperm);CHKERRQ(ierr);
7333   }
7334   /* 2) Get number of indices on these points and field offsets from section */
7335   for (p = 0; p < Ncl*2; p += 2) {
7336     PetscInt dof, fdof;
7337 
7338     ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
7339     for (f = 0; f < Nf; ++f) {
7340       ierr = PetscSectionGetFieldDof(section, points[p], f, &fdof);CHKERRQ(ierr);
7341       offsets[f+1] += fdof;
7342     }
7343     Ni += dof;
7344   }
7345   for (f = 1; f < Nf; ++f) offsets[f+1] += offsets[f];
7346   if (Nf && offsets[Nf] != Ni) SETERRQ2(PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", offsets[Nf], Ni);
7347   /* 3) Get symmetries and sign flips. Apply sign flips to values if passed in (only works for square values matrix) */
7348   for (f = 0; f < PetscMax(1, Nf); ++f) {
7349     if (Nf) {ierr = PetscSectionGetFieldPointSyms(section, f, Ncl, points, &perms[f], &flips[f]);CHKERRQ(ierr);}
7350     else    {ierr = PetscSectionGetPointSyms(section, Ncl, points, &perms[f], &flips[f]);CHKERRQ(ierr);}
7351     /* may need to apply sign changes to the element matrix */
7352     if (values && flips[f]) {
7353       PetscInt foffset = offsets[f];
7354 
7355       for (p = 0; p < Ncl; ++p) {
7356         PetscInt           pnt  = points[2*p], fdof;
7357         const PetscScalar *flip = flips[f] ? flips[f][p] : NULL;
7358 
7359         if (!Nf) {ierr = PetscSectionGetDof(section, pnt, &fdof);CHKERRQ(ierr);}
7360         else     {ierr = PetscSectionGetFieldDof(section, pnt, f, &fdof);CHKERRQ(ierr);}
7361         if (flip) {
7362           PetscInt i, j, k;
7363 
7364           if (!valCopy) {
7365             ierr = DMGetWorkArray(dm, Ni*Ni, MPIU_SCALAR, &valCopy);CHKERRQ(ierr);
7366             for (j = 0; j < Ni * Ni; ++j) valCopy[j] = (*values)[j];
7367             *values = valCopy;
7368           }
7369           for (i = 0; i < fdof; ++i) {
7370             PetscScalar fval = flip[i];
7371 
7372             for (k = 0; k < Ni; ++k) {
7373               valCopy[Ni * (foffset + i) + k] *= fval;
7374               valCopy[Ni * k + (foffset + i)] *= fval;
7375             }
7376           }
7377         }
7378         foffset += fdof;
7379       }
7380     }
7381   }
7382   /* 4) Apply hanging node constraints. Get new symmetries and replace all storage with constrained storage */
7383   ierr = DMPlexAnchorsModifyMat(dm, section, Ncl, Ni, points, perms, values ? *values : NULL, &NclC, &NiC, &pointsC, values ? &valuesC : NULL, offsets, PETSC_TRUE);CHKERRQ(ierr);
7384   if (NclC) {
7385     if (valCopy) {ierr = DMRestoreWorkArray(dm, Ni*Ni, MPIU_SCALAR, &valCopy);CHKERRQ(ierr);}
7386     for (f = 0; f < PetscMax(1, Nf); ++f) {
7387       if (Nf) {ierr = PetscSectionRestoreFieldPointSyms(section, f, Ncl, points, &perms[f], &flips[f]);CHKERRQ(ierr);}
7388       else    {ierr = PetscSectionRestorePointSyms(section, Ncl, points, &perms[f], &flips[f]);CHKERRQ(ierr);}
7389     }
7390     for (f = 0; f < PetscMax(1, Nf); ++f) {
7391       if (Nf) {ierr = PetscSectionGetFieldPointSyms(section, f, NclC, pointsC, &perms[f], &flips[f]);CHKERRQ(ierr);}
7392       else    {ierr = PetscSectionGetPointSyms(section, NclC, pointsC, &perms[f], &flips[f]);CHKERRQ(ierr);}
7393     }
7394     ierr = DMPlexRestoreCompressedClosure(dm, section, point, &Ncl, &points, &clSection, &clPoints, &clp);CHKERRQ(ierr);
7395     Ncl     = NclC;
7396     Ni      = NiC;
7397     points  = pointsC;
7398     if (values) *values = valuesC;
7399   }
7400   /* 5) Calculate indices */
7401   ierr = DMGetWorkArray(dm, Ni, MPIU_INT, &idx);CHKERRQ(ierr);
7402   if (Nf) {
7403     PetscInt  idxOff;
7404     PetscBool useFieldOffsets;
7405 
7406     if (outOffsets) {for (f = 0; f <= Nf; f++) outOffsets[f] = offsets[f];}
7407     ierr = PetscSectionGetUseFieldOffsets(idxSection, &useFieldOffsets);CHKERRQ(ierr);
7408     if (useFieldOffsets) {
7409       for (p = 0; p < Ncl; ++p) {
7410         const PetscInt pnt = points[p*2];
7411 
7412         ierr = DMPlexGetIndicesPointFieldsSplit_Internal(section, idxSection, pnt, offsets, perms, p, clperm, idx);CHKERRQ(ierr);
7413       }
7414     } else {
7415       for (p = 0; p < Ncl; ++p) {
7416         const PetscInt pnt = points[p*2];
7417 
7418         ierr = PetscSectionGetOffset(idxSection, pnt, &idxOff);CHKERRQ(ierr);
7419         /* Note that we pass a local section even though we're using global offsets.  This is because global sections do
7420          * not (at the time of this writing) have fields set. They probably should, in which case we would pass the
7421          * global section. */
7422         ierr = DMPlexGetIndicesPointFields_Internal(section, isLocal, pnt, idxOff < 0 ? -(idxOff+1) : idxOff, offsets, PETSC_FALSE, perms, p, clperm, idx);CHKERRQ(ierr);
7423       }
7424     }
7425   } else {
7426     PetscInt off = 0, idxOff;
7427 
7428     for (p = 0; p < Ncl; ++p) {
7429       const PetscInt  pnt  = points[p*2];
7430       const PetscInt *perm = perms[0] ? perms[0][p] : NULL;
7431 
7432       ierr = PetscSectionGetOffset(idxSection, pnt, &idxOff);CHKERRQ(ierr);
7433       /* Note that we pass a local section even though we're using global offsets.  This is because global sections do
7434        * not (at the time of this writing) have fields set. They probably should, in which case we would pass the global section. */
7435       ierr = DMPlexGetIndicesPoint_Internal(section, isLocal, pnt, idxOff < 0 ? -(idxOff+1) : idxOff, &off, PETSC_FALSE, perm, clperm, idx);CHKERRQ(ierr);
7436     }
7437   }
7438   /* 6) Cleanup */
7439   for (f = 0; f < PetscMax(1, Nf); ++f) {
7440     if (Nf) {ierr = PetscSectionRestoreFieldPointSyms(section, f, Ncl, points, &perms[f], &flips[f]);CHKERRQ(ierr);}
7441     else    {ierr = PetscSectionRestorePointSyms(section, Ncl, points, &perms[f], &flips[f]);CHKERRQ(ierr);}
7442   }
7443   if (NclC) {
7444     ierr = DMRestoreWorkArray(dm, NclC*2, MPIU_INT, &pointsC);CHKERRQ(ierr);
7445   } else {
7446     ierr = DMPlexRestoreCompressedClosure(dm, section, point, &Ncl, &points, &clSection, &clPoints, &clp);CHKERRQ(ierr);
7447   }
7448 
7449   if (numIndices) *numIndices = Ni;
7450   if (indices)    *indices    = idx;
7451   PetscFunctionReturn(0);
7452 }
7453 
7454 /*@C
7455   DMPlexRestoreClosureIndices - Restores the global dof indices associated with the closure of the given point within the provided sections.
7456 
7457   Not collective
7458 
7459   Input Parameters:
7460 + dm         - The DM
7461 . section    - The PetscSection describing the points (a local section)
7462 . idxSection - The PetscSection from which to obtain indices (may be local or global)
7463 . point      - The point defining the closure
7464 - useClPerm  - Use the closure point permutation if available
7465 
7466   Output Parameters:
7467 + numIndices - The number of dof indices in the closure of point with the input sections
7468 . indices    - The dof indices
7469 . outOffsets - Array to write the field offsets into, or NULL
7470 - values     - The input values, which may be modified if sign flips are induced by the point symmetries, or NULL
7471 
7472   Notes:
7473   If values were modified, the user is responsible for calling DMRestoreWorkArray(dm, 0, MPIU_SCALAR, &values).
7474 
7475   If idxSection is global, any constrained dofs (see DMAddBoundary(), for example) will get negative indices.  The value
7476   of those indices is not significant.  If idxSection is local, the constrained dofs will yield the involution -(idx+1)
7477   of their index in a local vector.  A caller who does not wish to distinguish those points may recover the nonnegative
7478   indices via involution, -(-(idx+1)+1)==idx.  Local indices are provided when idxSection == section, otherwise global
7479   indices (with the above semantics) are implied.
7480 
7481   Level: advanced
7482 
7483 .seealso DMPlexGetClosureIndices(), DMPlexVecGetClosure(), DMPlexMatSetClosure(), DMGetLocalSection(), DMGetGlobalSection()
7484 @*/
7485 PetscErrorCode DMPlexRestoreClosureIndices(DM dm, PetscSection section, PetscSection idxSection, PetscInt point, PetscBool useClPerm,
7486                                            PetscInt *numIndices, PetscInt *indices[], PetscInt outOffsets[], PetscScalar *values[])
7487 {
7488   PetscErrorCode ierr;
7489 
7490   PetscFunctionBegin;
7491   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7492   PetscValidPointer(indices, 7);
7493   ierr = DMRestoreWorkArray(dm, 0, MPIU_INT, indices);CHKERRQ(ierr);
7494   PetscFunctionReturn(0);
7495 }
7496 
7497 /*@C
7498   DMPlexMatSetClosure - Set an array of the values on the closure of 'point'
7499 
7500   Not collective
7501 
7502   Input Parameters:
7503 + dm - The DM
7504 . section - The section describing the layout in v, or NULL to use the default section
7505 . globalSection - The section describing the layout in v, or NULL to use the default global section
7506 . A - The matrix
7507 . point - The point in the DM
7508 . values - The array of values
7509 - mode - The insert mode, where INSERT_ALL_VALUES and ADD_ALL_VALUES also overwrite boundary conditions
7510 
7511   Fortran Notes:
7512   This routine is only available in Fortran 90, and you must include petsc.h90 in your code.
7513 
7514   Level: intermediate
7515 
7516 .seealso DMPlexMatSetClosureGeneral(), DMPlexVecGetClosure(), DMPlexVecSetClosure()
7517 @*/
7518 PetscErrorCode DMPlexMatSetClosure(DM dm, PetscSection section, PetscSection globalSection, Mat A, PetscInt point, const PetscScalar values[], InsertMode mode)
7519 {
7520   DM_Plex           *mesh = (DM_Plex*) dm->data;
7521   PetscInt          *indices;
7522   PetscInt           numIndices;
7523   const PetscScalar *valuesOrig = values;
7524   PetscErrorCode     ierr;
7525 
7526   PetscFunctionBegin;
7527   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7528   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
7529   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
7530   if (!globalSection) {ierr = DMGetGlobalSection(dm, &globalSection);CHKERRQ(ierr);}
7531   PetscValidHeaderSpecific(globalSection, PETSC_SECTION_CLASSID, 3);
7532   PetscValidHeaderSpecific(A, MAT_CLASSID, 4);
7533 
7534   ierr = DMPlexGetClosureIndices(dm, section, globalSection, point, PETSC_TRUE, &numIndices, &indices, NULL, (PetscScalar **) &values);CHKERRQ(ierr);
7535 
7536   if (mesh->printSetValues) {ierr = DMPlexPrintMatSetValues(PETSC_VIEWER_STDOUT_SELF, A, point, numIndices, indices, 0, NULL, values);CHKERRQ(ierr);}
7537   ierr = MatSetValues(A, numIndices, indices, numIndices, indices, values, mode);
7538   if (ierr) {
7539     PetscMPIInt    rank;
7540     PetscErrorCode ierr2;
7541 
7542     ierr2 = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRMPI(ierr2);
7543     ierr2 = (*PetscErrorPrintf)("[%d]ERROR in DMPlexMatSetClosure\n", rank);CHKERRQ(ierr2);
7544     ierr2 = DMPlexPrintMatSetValues(PETSC_VIEWER_STDERR_SELF, A, point, numIndices, indices, 0, NULL, values);CHKERRQ(ierr2);
7545     ierr2 = DMPlexRestoreClosureIndices(dm, section, globalSection, point, PETSC_TRUE, &numIndices, &indices, NULL, (PetscScalar **) &values);CHKERRQ(ierr2);
7546     if (values != valuesOrig) {ierr2 = DMRestoreWorkArray(dm, 0, MPIU_SCALAR, &values);CHKERRQ(ierr2);}
7547     SETERRQ(PetscObjectComm((PetscObject)dm),ierr,"Not possible to set matrix values");
7548   }
7549   if (mesh->printFEM > 1) {
7550     PetscInt i;
7551     ierr = PetscPrintf(PETSC_COMM_SELF, "  Indices:");CHKERRQ(ierr);
7552     for (i = 0; i < numIndices; ++i) {ierr = PetscPrintf(PETSC_COMM_SELF, " %D", indices[i]);CHKERRQ(ierr);}
7553     ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
7554   }
7555 
7556   ierr = DMPlexRestoreClosureIndices(dm, section, globalSection, point, PETSC_TRUE, &numIndices, &indices, NULL, (PetscScalar **) &values);CHKERRQ(ierr);
7557   if (values != valuesOrig) {ierr = DMRestoreWorkArray(dm, 0, MPIU_SCALAR, &values);CHKERRQ(ierr);}
7558   PetscFunctionReturn(0);
7559 }
7560 
7561 /*@C
7562   DMPlexMatSetClosure - Set an array of the values on the closure of 'point' using a different row and column section
7563 
7564   Not collective
7565 
7566   Input Parameters:
7567 + dmRow - The DM for the row fields
7568 . sectionRow - The section describing the layout, or NULL to use the default section in dmRow
7569 . globalSectionRow - The section describing the layout, or NULL to use the default global section in dmRow
7570 . dmCol - The DM for the column fields
7571 . sectionCol - The section describing the layout, or NULL to use the default section in dmCol
7572 . globalSectionCol - The section describing the layout, or NULL to use the default global section in dmCol
7573 . A - The matrix
7574 . point - The point in the DMs
7575 . values - The array of values
7576 - mode - The insert mode, where INSERT_ALL_VALUES and ADD_ALL_VALUES also overwrite boundary conditions
7577 
7578   Level: intermediate
7579 
7580 .seealso DMPlexMatSetClosure(), DMPlexVecGetClosure(), DMPlexVecSetClosure()
7581 @*/
7582 PetscErrorCode DMPlexMatSetClosureGeneral(DM dmRow, PetscSection sectionRow, PetscSection globalSectionRow, DM dmCol, PetscSection sectionCol, PetscSection globalSectionCol, Mat A, PetscInt point, const PetscScalar values[], InsertMode mode)
7583 {
7584   DM_Plex           *mesh = (DM_Plex*) dmRow->data;
7585   PetscInt          *indicesRow, *indicesCol;
7586   PetscInt           numIndicesRow, numIndicesCol;
7587   const PetscScalar *valuesOrig = values;
7588   PetscErrorCode     ierr;
7589 
7590   PetscFunctionBegin;
7591   PetscValidHeaderSpecific(dmRow, DM_CLASSID, 1);
7592   if (!sectionRow) {ierr = DMGetLocalSection(dmRow, &sectionRow);CHKERRQ(ierr);}
7593   PetscValidHeaderSpecific(sectionRow, PETSC_SECTION_CLASSID, 2);
7594   if (!globalSectionRow) {ierr = DMGetGlobalSection(dmRow, &globalSectionRow);CHKERRQ(ierr);}
7595   PetscValidHeaderSpecific(globalSectionRow, PETSC_SECTION_CLASSID, 3);
7596   PetscValidHeaderSpecific(dmCol, DM_CLASSID, 4);
7597   if (!sectionCol) {ierr = DMGetLocalSection(dmCol, &sectionCol);CHKERRQ(ierr);}
7598   PetscValidHeaderSpecific(sectionCol, PETSC_SECTION_CLASSID, 5);
7599   if (!globalSectionCol) {ierr = DMGetGlobalSection(dmCol, &globalSectionCol);CHKERRQ(ierr);}
7600   PetscValidHeaderSpecific(globalSectionCol, PETSC_SECTION_CLASSID, 6);
7601   PetscValidHeaderSpecific(A, MAT_CLASSID, 7);
7602 
7603   ierr = DMPlexGetClosureIndices(dmRow, sectionRow, globalSectionRow, point, PETSC_TRUE, &numIndicesRow, &indicesRow, NULL, (PetscScalar **) &values);CHKERRQ(ierr);
7604   ierr = DMPlexGetClosureIndices(dmCol, sectionCol, globalSectionCol, point, PETSC_TRUE, &numIndicesCol, &indicesCol, NULL, (PetscScalar **) &values);CHKERRQ(ierr);
7605 
7606   if (mesh->printSetValues) {ierr = DMPlexPrintMatSetValues(PETSC_VIEWER_STDOUT_SELF, A, point, numIndicesRow, indicesRow, numIndicesCol, indicesCol, values);CHKERRQ(ierr);}
7607   ierr = MatSetValues(A, numIndicesRow, indicesRow, numIndicesCol, indicesCol, values, mode);
7608   if (ierr) {
7609     PetscMPIInt    rank;
7610     PetscErrorCode ierr2;
7611 
7612     ierr2 = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRMPI(ierr2);
7613     ierr2 = (*PetscErrorPrintf)("[%d]ERROR in DMPlexMatSetClosure\n", rank);CHKERRQ(ierr2);
7614     ierr2 = DMPlexPrintMatSetValues(PETSC_VIEWER_STDERR_SELF, A, point, numIndicesRow, indicesRow, numIndicesCol, indicesCol, values);CHKERRQ(ierr2);
7615     ierr2 = DMPlexRestoreClosureIndices(dmRow, sectionRow, globalSectionRow, point, PETSC_TRUE, &numIndicesRow, &indicesRow, NULL, (PetscScalar **) &values);CHKERRQ(ierr2);
7616     ierr2 = DMPlexRestoreClosureIndices(dmCol, sectionCol, globalSectionCol, point, PETSC_TRUE, &numIndicesCol, &indicesRow, NULL, (PetscScalar **) &values);CHKERRQ(ierr2);
7617     if (values != valuesOrig) {ierr2 = DMRestoreWorkArray(dmRow, 0, MPIU_SCALAR, &values);CHKERRQ(ierr2);}
7618     CHKERRQ(ierr);
7619   }
7620 
7621   ierr = DMPlexRestoreClosureIndices(dmRow, sectionRow, globalSectionRow, point, PETSC_TRUE, &numIndicesRow, &indicesRow, NULL, (PetscScalar **) &values);CHKERRQ(ierr);
7622   ierr = DMPlexRestoreClosureIndices(dmCol, sectionCol, globalSectionCol, point, PETSC_TRUE, &numIndicesCol, &indicesCol, NULL, (PetscScalar **) &values);CHKERRQ(ierr);
7623   if (values != valuesOrig) {ierr = DMRestoreWorkArray(dmRow, 0, MPIU_SCALAR, &values);CHKERRQ(ierr);}
7624   PetscFunctionReturn(0);
7625 }
7626 
7627 PetscErrorCode DMPlexMatSetClosureRefined(DM dmf, PetscSection fsection, PetscSection globalFSection, DM dmc, PetscSection csection, PetscSection globalCSection, Mat A, PetscInt point, const PetscScalar values[], InsertMode mode)
7628 {
7629   DM_Plex        *mesh   = (DM_Plex*) dmf->data;
7630   PetscInt       *fpoints = NULL, *ftotpoints = NULL;
7631   PetscInt       *cpoints = NULL;
7632   PetscInt       *findices, *cindices;
7633   const PetscInt *fclperm = NULL, *cclperm = NULL; /* Closure permutations cannot work here */
7634   PetscInt        foffsets[32], coffsets[32];
7635   DMPolytopeType  ct;
7636   PetscInt        numFields, numSubcells, maxFPoints, numFPoints, numCPoints, numFIndices, numCIndices, dof, off, globalOff, pStart, pEnd, p, q, r, s, f;
7637   PetscErrorCode  ierr;
7638 
7639   PetscFunctionBegin;
7640   PetscValidHeaderSpecific(dmf, DM_CLASSID, 1);
7641   PetscValidHeaderSpecific(dmc, DM_CLASSID, 4);
7642   if (!fsection) {ierr = DMGetLocalSection(dmf, &fsection);CHKERRQ(ierr);}
7643   PetscValidHeaderSpecific(fsection, PETSC_SECTION_CLASSID, 2);
7644   if (!csection) {ierr = DMGetLocalSection(dmc, &csection);CHKERRQ(ierr);}
7645   PetscValidHeaderSpecific(csection, PETSC_SECTION_CLASSID, 5);
7646   if (!globalFSection) {ierr = DMGetGlobalSection(dmf, &globalFSection);CHKERRQ(ierr);}
7647   PetscValidHeaderSpecific(globalFSection, PETSC_SECTION_CLASSID, 3);
7648   if (!globalCSection) {ierr = DMGetGlobalSection(dmc, &globalCSection);CHKERRQ(ierr);}
7649   PetscValidHeaderSpecific(globalCSection, PETSC_SECTION_CLASSID, 6);
7650   PetscValidHeaderSpecific(A, MAT_CLASSID, 7);
7651   ierr = PetscSectionGetNumFields(fsection, &numFields);CHKERRQ(ierr);
7652   if (numFields > 31) SETERRQ1(PetscObjectComm((PetscObject)dmf), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", numFields);
7653   ierr = PetscArrayzero(foffsets, 32);CHKERRQ(ierr);
7654   ierr = PetscArrayzero(coffsets, 32);CHKERRQ(ierr);
7655   /* Column indices */
7656   ierr = DMPlexGetTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
7657   maxFPoints = numCPoints;
7658   /* Compress out points not in the section */
7659   /*   TODO: Squeeze out points with 0 dof as well */
7660   ierr = PetscSectionGetChart(csection, &pStart, &pEnd);CHKERRQ(ierr);
7661   for (p = 0, q = 0; p < numCPoints*2; p += 2) {
7662     if ((cpoints[p] >= pStart) && (cpoints[p] < pEnd)) {
7663       cpoints[q*2]   = cpoints[p];
7664       cpoints[q*2+1] = cpoints[p+1];
7665       ++q;
7666     }
7667   }
7668   numCPoints = q;
7669   for (p = 0, numCIndices = 0; p < numCPoints*2; p += 2) {
7670     PetscInt fdof;
7671 
7672     ierr = PetscSectionGetDof(csection, cpoints[p], &dof);CHKERRQ(ierr);
7673     if (!dof) continue;
7674     for (f = 0; f < numFields; ++f) {
7675       ierr           = PetscSectionGetFieldDof(csection, cpoints[p], f, &fdof);CHKERRQ(ierr);
7676       coffsets[f+1] += fdof;
7677     }
7678     numCIndices += dof;
7679   }
7680   for (f = 1; f < numFields; ++f) coffsets[f+1] += coffsets[f];
7681   /* Row indices */
7682   ierr = DMPlexGetCellType(dmc, point, &ct);CHKERRQ(ierr);
7683   {
7684     DMPlexTransform tr;
7685     DMPolytopeType *rct;
7686     PetscInt       *rsize, *rcone, *rornt, Nt;
7687 
7688     ierr = DMPlexTransformCreate(PETSC_COMM_SELF, &tr);CHKERRQ(ierr);
7689     ierr = DMPlexTransformSetType(tr, DMPLEXREFINEREGULAR);CHKERRQ(ierr);
7690     ierr = DMPlexTransformCellTransform(tr, ct, point, NULL, &Nt, &rct, &rsize, &rcone, &rornt);CHKERRQ(ierr);
7691     numSubcells = rsize[Nt-1];
7692     ierr = DMPlexTransformDestroy(&tr);CHKERRQ(ierr);
7693   }
7694   ierr = DMGetWorkArray(dmf, maxFPoints*2*numSubcells, MPIU_INT, &ftotpoints);CHKERRQ(ierr);
7695   for (r = 0, q = 0; r < numSubcells; ++r) {
7696     /* TODO Map from coarse to fine cells */
7697     ierr = DMPlexGetTransitiveClosure(dmf, point*numSubcells + r, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
7698     /* Compress out points not in the section */
7699     ierr = PetscSectionGetChart(fsection, &pStart, &pEnd);CHKERRQ(ierr);
7700     for (p = 0; p < numFPoints*2; p += 2) {
7701       if ((fpoints[p] >= pStart) && (fpoints[p] < pEnd)) {
7702         ierr = PetscSectionGetDof(fsection, fpoints[p], &dof);CHKERRQ(ierr);
7703         if (!dof) continue;
7704         for (s = 0; s < q; ++s) if (fpoints[p] == ftotpoints[s*2]) break;
7705         if (s < q) continue;
7706         ftotpoints[q*2]   = fpoints[p];
7707         ftotpoints[q*2+1] = fpoints[p+1];
7708         ++q;
7709       }
7710     }
7711     ierr = DMPlexRestoreTransitiveClosure(dmf, point, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
7712   }
7713   numFPoints = q;
7714   for (p = 0, numFIndices = 0; p < numFPoints*2; p += 2) {
7715     PetscInt fdof;
7716 
7717     ierr = PetscSectionGetDof(fsection, ftotpoints[p], &dof);CHKERRQ(ierr);
7718     if (!dof) continue;
7719     for (f = 0; f < numFields; ++f) {
7720       ierr           = PetscSectionGetFieldDof(fsection, ftotpoints[p], f, &fdof);CHKERRQ(ierr);
7721       foffsets[f+1] += fdof;
7722     }
7723     numFIndices += dof;
7724   }
7725   for (f = 1; f < numFields; ++f) foffsets[f+1] += foffsets[f];
7726 
7727   if (numFields && foffsets[numFields] != numFIndices) SETERRQ2(PetscObjectComm((PetscObject)dmf), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", foffsets[numFields], numFIndices);
7728   if (numFields && coffsets[numFields] != numCIndices) SETERRQ2(PetscObjectComm((PetscObject)dmc), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", coffsets[numFields], numCIndices);
7729   ierr = DMGetWorkArray(dmf, numFIndices, MPIU_INT, &findices);CHKERRQ(ierr);
7730   ierr = DMGetWorkArray(dmc, numCIndices, MPIU_INT, &cindices);CHKERRQ(ierr);
7731   if (numFields) {
7732     const PetscInt **permsF[32] = {NULL};
7733     const PetscInt **permsC[32] = {NULL};
7734 
7735     for (f = 0; f < numFields; f++) {
7736       ierr = PetscSectionGetFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL);CHKERRQ(ierr);
7737       ierr = PetscSectionGetFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL);CHKERRQ(ierr);
7738     }
7739     for (p = 0; p < numFPoints; p++) {
7740       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff);CHKERRQ(ierr);
7741       ierr = DMPlexGetIndicesPointFields_Internal(fsection, PETSC_FALSE, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, foffsets, PETSC_FALSE, permsF, p, fclperm, findices);CHKERRQ(ierr);
7742     }
7743     for (p = 0; p < numCPoints; p++) {
7744       ierr = PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff);CHKERRQ(ierr);
7745       ierr = DMPlexGetIndicesPointFields_Internal(csection, PETSC_FALSE, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, coffsets, PETSC_FALSE, permsC, p, cclperm, cindices);CHKERRQ(ierr);
7746     }
7747     for (f = 0; f < numFields; f++) {
7748       ierr = PetscSectionRestoreFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL);CHKERRQ(ierr);
7749       ierr = PetscSectionRestoreFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL);CHKERRQ(ierr);
7750     }
7751   } else {
7752     const PetscInt **permsF = NULL;
7753     const PetscInt **permsC = NULL;
7754 
7755     ierr = PetscSectionGetPointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL);CHKERRQ(ierr);
7756     ierr = PetscSectionGetPointSyms(csection,numCPoints,cpoints,&permsC,NULL);CHKERRQ(ierr);
7757     for (p = 0, off = 0; p < numFPoints; p++) {
7758       const PetscInt *perm = permsF ? permsF[p] : NULL;
7759 
7760       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff);CHKERRQ(ierr);
7761       ierr = DMPlexGetIndicesPoint_Internal(fsection, PETSC_FALSE, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, fclperm, findices);CHKERRQ(ierr);
7762     }
7763     for (p = 0, off = 0; p < numCPoints; p++) {
7764       const PetscInt *perm = permsC ? permsC[p] : NULL;
7765 
7766       ierr = PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff);CHKERRQ(ierr);
7767       ierr = DMPlexGetIndicesPoint_Internal(csection, PETSC_FALSE, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, cclperm, cindices);CHKERRQ(ierr);
7768     }
7769     ierr = PetscSectionRestorePointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL);CHKERRQ(ierr);
7770     ierr = PetscSectionRestorePointSyms(csection,numCPoints,cpoints,&permsC,NULL);CHKERRQ(ierr);
7771   }
7772   if (mesh->printSetValues) {ierr = DMPlexPrintMatSetValues(PETSC_VIEWER_STDOUT_SELF, A, point, numFIndices, findices, numCIndices, cindices, values);CHKERRQ(ierr);}
7773   /* TODO: flips */
7774   ierr = MatSetValues(A, numFIndices, findices, numCIndices, cindices, values, mode);
7775   if (ierr) {
7776     PetscMPIInt    rank;
7777     PetscErrorCode ierr2;
7778 
7779     ierr2 = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRMPI(ierr2);
7780     ierr2 = (*PetscErrorPrintf)("[%d]ERROR in DMPlexMatSetClosure\n", rank);CHKERRQ(ierr2);
7781     ierr2 = DMPlexPrintMatSetValues(PETSC_VIEWER_STDERR_SELF, A, point, numFIndices, findices, numCIndices, cindices, values);CHKERRQ(ierr2);
7782     ierr2 = DMRestoreWorkArray(dmf, numFIndices, MPIU_INT, &findices);CHKERRQ(ierr2);
7783     ierr2 = DMRestoreWorkArray(dmc, numCIndices, MPIU_INT, &cindices);CHKERRQ(ierr2);
7784     CHKERRQ(ierr);
7785   }
7786   ierr = DMRestoreWorkArray(dmf, numCPoints*2*4, MPIU_INT, &ftotpoints);CHKERRQ(ierr);
7787   ierr = DMPlexRestoreTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
7788   ierr = DMRestoreWorkArray(dmf, numFIndices, MPIU_INT, &findices);CHKERRQ(ierr);
7789   ierr = DMRestoreWorkArray(dmc, numCIndices, MPIU_INT, &cindices);CHKERRQ(ierr);
7790   PetscFunctionReturn(0);
7791 }
7792 
7793 PetscErrorCode DMPlexMatGetClosureIndicesRefined(DM dmf, PetscSection fsection, PetscSection globalFSection, DM dmc, PetscSection csection, PetscSection globalCSection, PetscInt point, PetscInt cindices[], PetscInt findices[])
7794 {
7795   PetscInt      *fpoints = NULL, *ftotpoints = NULL;
7796   PetscInt      *cpoints = NULL;
7797   PetscInt       foffsets[32], coffsets[32];
7798   const PetscInt *fclperm = NULL, *cclperm = NULL; /* Closure permutations cannot work here */
7799   DMPolytopeType ct;
7800   PetscInt       numFields, numSubcells, maxFPoints, numFPoints, numCPoints, numFIndices, numCIndices, dof, off, globalOff, pStart, pEnd, p, q, r, s, f;
7801   PetscErrorCode ierr;
7802 
7803   PetscFunctionBegin;
7804   PetscValidHeaderSpecific(dmf, DM_CLASSID, 1);
7805   PetscValidHeaderSpecific(dmc, DM_CLASSID, 4);
7806   if (!fsection) {ierr = DMGetLocalSection(dmf, &fsection);CHKERRQ(ierr);}
7807   PetscValidHeaderSpecific(fsection, PETSC_SECTION_CLASSID, 2);
7808   if (!csection) {ierr = DMGetLocalSection(dmc, &csection);CHKERRQ(ierr);}
7809   PetscValidHeaderSpecific(csection, PETSC_SECTION_CLASSID, 5);
7810   if (!globalFSection) {ierr = DMGetGlobalSection(dmf, &globalFSection);CHKERRQ(ierr);}
7811   PetscValidHeaderSpecific(globalFSection, PETSC_SECTION_CLASSID, 3);
7812   if (!globalCSection) {ierr = DMGetGlobalSection(dmc, &globalCSection);CHKERRQ(ierr);}
7813   PetscValidHeaderSpecific(globalCSection, PETSC_SECTION_CLASSID, 6);
7814   ierr = PetscSectionGetNumFields(fsection, &numFields);CHKERRQ(ierr);
7815   if (numFields > 31) SETERRQ1(PetscObjectComm((PetscObject)dmf), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", numFields);
7816   ierr = PetscArrayzero(foffsets, 32);CHKERRQ(ierr);
7817   ierr = PetscArrayzero(coffsets, 32);CHKERRQ(ierr);
7818   /* Column indices */
7819   ierr = DMPlexGetTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
7820   maxFPoints = numCPoints;
7821   /* Compress out points not in the section */
7822   /*   TODO: Squeeze out points with 0 dof as well */
7823   ierr = PetscSectionGetChart(csection, &pStart, &pEnd);CHKERRQ(ierr);
7824   for (p = 0, q = 0; p < numCPoints*2; p += 2) {
7825     if ((cpoints[p] >= pStart) && (cpoints[p] < pEnd)) {
7826       cpoints[q*2]   = cpoints[p];
7827       cpoints[q*2+1] = cpoints[p+1];
7828       ++q;
7829     }
7830   }
7831   numCPoints = q;
7832   for (p = 0, numCIndices = 0; p < numCPoints*2; p += 2) {
7833     PetscInt fdof;
7834 
7835     ierr = PetscSectionGetDof(csection, cpoints[p], &dof);CHKERRQ(ierr);
7836     if (!dof) continue;
7837     for (f = 0; f < numFields; ++f) {
7838       ierr           = PetscSectionGetFieldDof(csection, cpoints[p], f, &fdof);CHKERRQ(ierr);
7839       coffsets[f+1] += fdof;
7840     }
7841     numCIndices += dof;
7842   }
7843   for (f = 1; f < numFields; ++f) coffsets[f+1] += coffsets[f];
7844   /* Row indices */
7845   ierr = DMPlexGetCellType(dmc, point, &ct);CHKERRQ(ierr);
7846   {
7847     DMPlexTransform tr;
7848     DMPolytopeType *rct;
7849     PetscInt       *rsize, *rcone, *rornt, Nt;
7850 
7851     ierr = DMPlexTransformCreate(PETSC_COMM_SELF, &tr);CHKERRQ(ierr);
7852     ierr = DMPlexTransformSetType(tr, DMPLEXREFINEREGULAR);CHKERRQ(ierr);
7853     ierr = DMPlexTransformCellTransform(tr, ct, point, NULL, &Nt, &rct, &rsize, &rcone, &rornt);CHKERRQ(ierr);
7854     numSubcells = rsize[Nt-1];
7855     ierr = DMPlexTransformDestroy(&tr);CHKERRQ(ierr);
7856   }
7857   ierr = DMGetWorkArray(dmf, maxFPoints*2*numSubcells, MPIU_INT, &ftotpoints);CHKERRQ(ierr);
7858   for (r = 0, q = 0; r < numSubcells; ++r) {
7859     /* TODO Map from coarse to fine cells */
7860     ierr = DMPlexGetTransitiveClosure(dmf, point*numSubcells + r, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
7861     /* Compress out points not in the section */
7862     ierr = PetscSectionGetChart(fsection, &pStart, &pEnd);CHKERRQ(ierr);
7863     for (p = 0; p < numFPoints*2; p += 2) {
7864       if ((fpoints[p] >= pStart) && (fpoints[p] < pEnd)) {
7865         ierr = PetscSectionGetDof(fsection, fpoints[p], &dof);CHKERRQ(ierr);
7866         if (!dof) continue;
7867         for (s = 0; s < q; ++s) if (fpoints[p] == ftotpoints[s*2]) break;
7868         if (s < q) continue;
7869         ftotpoints[q*2]   = fpoints[p];
7870         ftotpoints[q*2+1] = fpoints[p+1];
7871         ++q;
7872       }
7873     }
7874     ierr = DMPlexRestoreTransitiveClosure(dmf, point, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
7875   }
7876   numFPoints = q;
7877   for (p = 0, numFIndices = 0; p < numFPoints*2; p += 2) {
7878     PetscInt fdof;
7879 
7880     ierr = PetscSectionGetDof(fsection, ftotpoints[p], &dof);CHKERRQ(ierr);
7881     if (!dof) continue;
7882     for (f = 0; f < numFields; ++f) {
7883       ierr           = PetscSectionGetFieldDof(fsection, ftotpoints[p], f, &fdof);CHKERRQ(ierr);
7884       foffsets[f+1] += fdof;
7885     }
7886     numFIndices += dof;
7887   }
7888   for (f = 1; f < numFields; ++f) foffsets[f+1] += foffsets[f];
7889 
7890   if (numFields && foffsets[numFields] != numFIndices) SETERRQ2(PetscObjectComm((PetscObject)dmf), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", foffsets[numFields], numFIndices);
7891   if (numFields && coffsets[numFields] != numCIndices) SETERRQ2(PetscObjectComm((PetscObject)dmc), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", coffsets[numFields], numCIndices);
7892   if (numFields) {
7893     const PetscInt **permsF[32] = {NULL};
7894     const PetscInt **permsC[32] = {NULL};
7895 
7896     for (f = 0; f < numFields; f++) {
7897       ierr = PetscSectionGetFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL);CHKERRQ(ierr);
7898       ierr = PetscSectionGetFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL);CHKERRQ(ierr);
7899     }
7900     for (p = 0; p < numFPoints; p++) {
7901       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff);CHKERRQ(ierr);
7902       ierr = DMPlexGetIndicesPointFields_Internal(fsection, PETSC_FALSE, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, foffsets, PETSC_FALSE, permsF, p, fclperm, findices);CHKERRQ(ierr);
7903     }
7904     for (p = 0; p < numCPoints; p++) {
7905       ierr = PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff);CHKERRQ(ierr);
7906       ierr = DMPlexGetIndicesPointFields_Internal(csection, PETSC_FALSE, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, coffsets, PETSC_FALSE, permsC, p, cclperm, cindices);CHKERRQ(ierr);
7907     }
7908     for (f = 0; f < numFields; f++) {
7909       ierr = PetscSectionRestoreFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL);CHKERRQ(ierr);
7910       ierr = PetscSectionRestoreFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL);CHKERRQ(ierr);
7911     }
7912   } else {
7913     const PetscInt **permsF = NULL;
7914     const PetscInt **permsC = NULL;
7915 
7916     ierr = PetscSectionGetPointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL);CHKERRQ(ierr);
7917     ierr = PetscSectionGetPointSyms(csection,numCPoints,cpoints,&permsC,NULL);CHKERRQ(ierr);
7918     for (p = 0, off = 0; p < numFPoints; p++) {
7919       const PetscInt *perm = permsF ? permsF[p] : NULL;
7920 
7921       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff);CHKERRQ(ierr);
7922       ierr = DMPlexGetIndicesPoint_Internal(fsection, PETSC_FALSE, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, fclperm, findices);CHKERRQ(ierr);
7923     }
7924     for (p = 0, off = 0; p < numCPoints; p++) {
7925       const PetscInt *perm = permsC ? permsC[p] : NULL;
7926 
7927       ierr = PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff);CHKERRQ(ierr);
7928       ierr = DMPlexGetIndicesPoint_Internal(csection, PETSC_FALSE, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, cclperm, cindices);CHKERRQ(ierr);
7929     }
7930     ierr = PetscSectionRestorePointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL);CHKERRQ(ierr);
7931     ierr = PetscSectionRestorePointSyms(csection,numCPoints,cpoints,&permsC,NULL);CHKERRQ(ierr);
7932   }
7933   ierr = DMRestoreWorkArray(dmf, numCPoints*2*4, MPIU_INT, &ftotpoints);CHKERRQ(ierr);
7934   ierr = DMPlexRestoreTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
7935   PetscFunctionReturn(0);
7936 }
7937 
7938 /*@C
7939   DMPlexGetVTKCellHeight - Returns the height in the DAG used to determine which points are cells (normally 0)
7940 
7941   Input Parameter:
7942 . dm   - The DMPlex object
7943 
7944   Output Parameter:
7945 . cellHeight - The height of a cell
7946 
7947   Level: developer
7948 
7949 .seealso DMPlexSetVTKCellHeight()
7950 @*/
7951 PetscErrorCode DMPlexGetVTKCellHeight(DM dm, PetscInt *cellHeight)
7952 {
7953   DM_Plex *mesh = (DM_Plex*) dm->data;
7954 
7955   PetscFunctionBegin;
7956   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7957   PetscValidPointer(cellHeight, 2);
7958   *cellHeight = mesh->vtkCellHeight;
7959   PetscFunctionReturn(0);
7960 }
7961 
7962 /*@C
7963   DMPlexSetVTKCellHeight - Sets the height in the DAG used to determine which points are cells (normally 0)
7964 
7965   Input Parameters:
7966 + dm   - The DMPlex object
7967 - cellHeight - The height of a cell
7968 
7969   Level: developer
7970 
7971 .seealso DMPlexGetVTKCellHeight()
7972 @*/
7973 PetscErrorCode DMPlexSetVTKCellHeight(DM dm, PetscInt cellHeight)
7974 {
7975   DM_Plex *mesh = (DM_Plex*) dm->data;
7976 
7977   PetscFunctionBegin;
7978   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7979   mesh->vtkCellHeight = cellHeight;
7980   PetscFunctionReturn(0);
7981 }
7982 
7983 /*@
7984   DMPlexGetGhostCellStratum - Get the range of cells which are used to enforce FV boundary conditions
7985 
7986   Input Parameter:
7987 . dm - The DMPlex object
7988 
7989   Output Parameters:
7990 + gcStart - The first ghost cell, or NULL
7991 - gcEnd   - The upper bound on ghost cells, or NULL
7992 
7993   Level: advanced
7994 
7995 .seealso DMPlexConstructGhostCells(), DMPlexGetGhostCellStratum()
7996 @*/
7997 PetscErrorCode DMPlexGetGhostCellStratum(DM dm, PetscInt *gcStart, PetscInt *gcEnd)
7998 {
7999   DMLabel        ctLabel;
8000   PetscErrorCode ierr;
8001 
8002   PetscFunctionBegin;
8003   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8004   ierr = DMPlexGetCellTypeLabel(dm, &ctLabel);CHKERRQ(ierr);
8005   ierr = DMLabelGetStratumBounds(ctLabel, DM_POLYTOPE_FV_GHOST, gcStart, gcEnd);CHKERRQ(ierr);
8006   PetscFunctionReturn(0);
8007 }
8008 
8009 /* We can easily have a form that takes an IS instead */
8010 PetscErrorCode DMPlexCreateNumbering_Plex(DM dm, PetscInt pStart, PetscInt pEnd, PetscInt shift, PetscInt *globalSize, PetscSF sf, IS *numbering)
8011 {
8012   PetscSection   section, globalSection;
8013   PetscInt      *numbers, p;
8014   PetscErrorCode ierr;
8015 
8016   PetscFunctionBegin;
8017   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &section);CHKERRQ(ierr);
8018   ierr = PetscSectionSetChart(section, pStart, pEnd);CHKERRQ(ierr);
8019   for (p = pStart; p < pEnd; ++p) {
8020     ierr = PetscSectionSetDof(section, p, 1);CHKERRQ(ierr);
8021   }
8022   ierr = PetscSectionSetUp(section);CHKERRQ(ierr);
8023   ierr = PetscSectionCreateGlobalSection(section, sf, PETSC_FALSE, PETSC_FALSE, &globalSection);CHKERRQ(ierr);
8024   ierr = PetscMalloc1(pEnd - pStart, &numbers);CHKERRQ(ierr);
8025   for (p = pStart; p < pEnd; ++p) {
8026     ierr = PetscSectionGetOffset(globalSection, p, &numbers[p-pStart]);CHKERRQ(ierr);
8027     if (numbers[p-pStart] < 0) numbers[p-pStart] -= shift;
8028     else                       numbers[p-pStart] += shift;
8029   }
8030   ierr = ISCreateGeneral(PetscObjectComm((PetscObject) dm), pEnd - pStart, numbers, PETSC_OWN_POINTER, numbering);CHKERRQ(ierr);
8031   if (globalSize) {
8032     PetscLayout layout;
8033     ierr = PetscSectionGetPointLayout(PetscObjectComm((PetscObject) dm), globalSection, &layout);CHKERRQ(ierr);
8034     ierr = PetscLayoutGetSize(layout, globalSize);CHKERRQ(ierr);
8035     ierr = PetscLayoutDestroy(&layout);CHKERRQ(ierr);
8036   }
8037   ierr = PetscSectionDestroy(&section);CHKERRQ(ierr);
8038   ierr = PetscSectionDestroy(&globalSection);CHKERRQ(ierr);
8039   PetscFunctionReturn(0);
8040 }
8041 
8042 PetscErrorCode DMPlexCreateCellNumbering_Internal(DM dm, PetscBool includeHybrid, IS *globalCellNumbers)
8043 {
8044   PetscInt       cellHeight, cStart, cEnd;
8045   PetscErrorCode ierr;
8046 
8047   PetscFunctionBegin;
8048   ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
8049   if (includeHybrid) {ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);}
8050   else               {ierr = DMPlexGetSimplexOrBoxCells(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);}
8051   ierr = DMPlexCreateNumbering_Plex(dm, cStart, cEnd, 0, NULL, dm->sf, globalCellNumbers);CHKERRQ(ierr);
8052   PetscFunctionReturn(0);
8053 }
8054 
8055 /*@
8056   DMPlexGetCellNumbering - Get a global cell numbering for all cells on this process
8057 
8058   Input Parameter:
8059 . dm   - The DMPlex object
8060 
8061   Output Parameter:
8062 . globalCellNumbers - Global cell numbers for all cells on this process
8063 
8064   Level: developer
8065 
8066 .seealso DMPlexGetVertexNumbering()
8067 @*/
8068 PetscErrorCode DMPlexGetCellNumbering(DM dm, IS *globalCellNumbers)
8069 {
8070   DM_Plex       *mesh = (DM_Plex*) dm->data;
8071   PetscErrorCode ierr;
8072 
8073   PetscFunctionBegin;
8074   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8075   if (!mesh->globalCellNumbers) {ierr = DMPlexCreateCellNumbering_Internal(dm, PETSC_FALSE, &mesh->globalCellNumbers);CHKERRQ(ierr);}
8076   *globalCellNumbers = mesh->globalCellNumbers;
8077   PetscFunctionReturn(0);
8078 }
8079 
8080 PetscErrorCode DMPlexCreateVertexNumbering_Internal(DM dm, PetscBool includeHybrid, IS *globalVertexNumbers)
8081 {
8082   PetscInt       vStart, vEnd;
8083   PetscErrorCode ierr;
8084 
8085   PetscFunctionBegin;
8086   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8087   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
8088   ierr = DMPlexCreateNumbering_Plex(dm, vStart, vEnd, 0, NULL, dm->sf, globalVertexNumbers);CHKERRQ(ierr);
8089   PetscFunctionReturn(0);
8090 }
8091 
8092 /*@
8093   DMPlexGetVertexNumbering - Get a global vertex numbering for all vertices on this process
8094 
8095   Input Parameter:
8096 . dm   - The DMPlex object
8097 
8098   Output Parameter:
8099 . globalVertexNumbers - Global vertex numbers for all vertices on this process
8100 
8101   Level: developer
8102 
8103 .seealso DMPlexGetCellNumbering()
8104 @*/
8105 PetscErrorCode DMPlexGetVertexNumbering(DM dm, IS *globalVertexNumbers)
8106 {
8107   DM_Plex       *mesh = (DM_Plex*) dm->data;
8108   PetscErrorCode ierr;
8109 
8110   PetscFunctionBegin;
8111   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8112   if (!mesh->globalVertexNumbers) {ierr = DMPlexCreateVertexNumbering_Internal(dm, PETSC_FALSE, &mesh->globalVertexNumbers);CHKERRQ(ierr);}
8113   *globalVertexNumbers = mesh->globalVertexNumbers;
8114   PetscFunctionReturn(0);
8115 }
8116 
8117 /*@
8118   DMPlexCreatePointNumbering - Create a global numbering for all points on this process
8119 
8120   Input Parameter:
8121 . dm   - The DMPlex object
8122 
8123   Output Parameter:
8124 . globalPointNumbers - Global numbers for all points on this process
8125 
8126   Level: developer
8127 
8128 .seealso DMPlexGetCellNumbering()
8129 @*/
8130 PetscErrorCode DMPlexCreatePointNumbering(DM dm, IS *globalPointNumbers)
8131 {
8132   IS             nums[4];
8133   PetscInt       depths[4], gdepths[4], starts[4];
8134   PetscInt       depth, d, shift = 0;
8135   PetscErrorCode ierr;
8136 
8137   PetscFunctionBegin;
8138   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8139   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
8140   /* For unstratified meshes use dim instead of depth */
8141   if (depth < 0) {ierr = DMGetDimension(dm, &depth);CHKERRQ(ierr);}
8142   for (d = 0; d <= depth; ++d) {
8143     PetscInt end;
8144 
8145     depths[d] = depth-d;
8146     ierr = DMPlexGetDepthStratum(dm, depths[d], &starts[d], &end);CHKERRQ(ierr);
8147     if (!(starts[d]-end)) { starts[d] = depths[d] = -1; }
8148   }
8149   ierr = PetscSortIntWithArray(depth+1, starts, depths);CHKERRQ(ierr);
8150   ierr = MPIU_Allreduce(depths, gdepths, depth+1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject) dm));CHKERRMPI(ierr);
8151   for (d = 0; d <= depth; ++d) {
8152     if (starts[d] >= 0 && depths[d] != gdepths[d]) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Expected depth %D, found %D",depths[d],gdepths[d]);
8153   }
8154   for (d = 0; d <= depth; ++d) {
8155     PetscInt pStart, pEnd, gsize;
8156 
8157     ierr = DMPlexGetDepthStratum(dm, gdepths[d], &pStart, &pEnd);CHKERRQ(ierr);
8158     ierr = DMPlexCreateNumbering_Plex(dm, pStart, pEnd, shift, &gsize, dm->sf, &nums[d]);CHKERRQ(ierr);
8159     shift += gsize;
8160   }
8161   ierr = ISConcatenate(PetscObjectComm((PetscObject) dm), depth+1, nums, globalPointNumbers);CHKERRQ(ierr);
8162   for (d = 0; d <= depth; ++d) {ierr = ISDestroy(&nums[d]);CHKERRQ(ierr);}
8163   PetscFunctionReturn(0);
8164 }
8165 
8166 /*@
8167   DMPlexCreateRankField - Create a cell field whose value is the rank of the owner
8168 
8169   Input Parameter:
8170 . dm - The DMPlex object
8171 
8172   Output Parameter:
8173 . ranks - The rank field
8174 
8175   Options Database Keys:
8176 . -dm_partition_view - Adds the rank field into the DM output from -dm_view using the same viewer
8177 
8178   Level: intermediate
8179 
8180 .seealso: DMView()
8181 @*/
8182 PetscErrorCode DMPlexCreateRankField(DM dm, Vec *ranks)
8183 {
8184   DM             rdm;
8185   PetscFE        fe;
8186   PetscScalar   *r;
8187   PetscMPIInt    rank;
8188   DMPolytopeType ct;
8189   PetscInt       dim, cStart, cEnd, c;
8190   PetscBool      simplex;
8191   PetscErrorCode ierr;
8192 
8193   PetscFunctionBeginUser;
8194   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8195   PetscValidPointer(ranks, 2);
8196   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRMPI(ierr);
8197   ierr = DMClone(dm, &rdm);CHKERRQ(ierr);
8198   ierr = DMGetDimension(rdm, &dim);CHKERRQ(ierr);
8199   ierr = DMPlexGetHeightStratum(rdm, 0, &cStart, &cEnd);CHKERRQ(ierr);
8200   ierr = DMPlexGetCellType(dm, cStart, &ct);CHKERRQ(ierr);
8201   simplex = DMPolytopeTypeGetNumVertices(ct) == DMPolytopeTypeGetDim(ct)+1 ? PETSC_TRUE : PETSC_FALSE;
8202   ierr = PetscFECreateDefault(PETSC_COMM_SELF, dim, 1, simplex, "PETSc___rank_", -1, &fe);CHKERRQ(ierr);
8203   ierr = PetscObjectSetName((PetscObject) fe, "rank");CHKERRQ(ierr);
8204   ierr = DMSetField(rdm, 0, NULL, (PetscObject) fe);CHKERRQ(ierr);
8205   ierr = PetscFEDestroy(&fe);CHKERRQ(ierr);
8206   ierr = DMCreateDS(rdm);CHKERRQ(ierr);
8207   ierr = DMCreateGlobalVector(rdm, ranks);CHKERRQ(ierr);
8208   ierr = PetscObjectSetName((PetscObject) *ranks, "partition");CHKERRQ(ierr);
8209   ierr = VecGetArray(*ranks, &r);CHKERRQ(ierr);
8210   for (c = cStart; c < cEnd; ++c) {
8211     PetscScalar *lr;
8212 
8213     ierr = DMPlexPointGlobalRef(rdm, c, r, &lr);CHKERRQ(ierr);
8214     if (lr) *lr = rank;
8215   }
8216   ierr = VecRestoreArray(*ranks, &r);CHKERRQ(ierr);
8217   ierr = DMDestroy(&rdm);CHKERRQ(ierr);
8218   PetscFunctionReturn(0);
8219 }
8220 
8221 /*@
8222   DMPlexCreateLabelField - Create a cell field whose value is the label value for that cell
8223 
8224   Input Parameters:
8225 + dm    - The DMPlex
8226 - label - The DMLabel
8227 
8228   Output Parameter:
8229 . val - The label value field
8230 
8231   Options Database Keys:
8232 . -dm_label_view - Adds the label value field into the DM output from -dm_view using the same viewer
8233 
8234   Level: intermediate
8235 
8236 .seealso: DMView()
8237 @*/
8238 PetscErrorCode DMPlexCreateLabelField(DM dm, DMLabel label, Vec *val)
8239 {
8240   DM             rdm;
8241   PetscFE        fe;
8242   PetscScalar   *v;
8243   PetscInt       dim, cStart, cEnd, c;
8244   PetscErrorCode ierr;
8245 
8246   PetscFunctionBeginUser;
8247   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8248   PetscValidPointer(label, 2);
8249   PetscValidPointer(val, 3);
8250   ierr = DMClone(dm, &rdm);CHKERRQ(ierr);
8251   ierr = DMGetDimension(rdm, &dim);CHKERRQ(ierr);
8252   ierr = PetscFECreateDefault(PetscObjectComm((PetscObject) rdm), dim, 1, PETSC_TRUE, "PETSc___label_value_", -1, &fe);CHKERRQ(ierr);
8253   ierr = PetscObjectSetName((PetscObject) fe, "label_value");CHKERRQ(ierr);
8254   ierr = DMSetField(rdm, 0, NULL, (PetscObject) fe);CHKERRQ(ierr);
8255   ierr = PetscFEDestroy(&fe);CHKERRQ(ierr);
8256   ierr = DMCreateDS(rdm);CHKERRQ(ierr);
8257   ierr = DMPlexGetHeightStratum(rdm, 0, &cStart, &cEnd);CHKERRQ(ierr);
8258   ierr = DMCreateGlobalVector(rdm, val);CHKERRQ(ierr);
8259   ierr = PetscObjectSetName((PetscObject) *val, "label_value");CHKERRQ(ierr);
8260   ierr = VecGetArray(*val, &v);CHKERRQ(ierr);
8261   for (c = cStart; c < cEnd; ++c) {
8262     PetscScalar *lv;
8263     PetscInt     cval;
8264 
8265     ierr = DMPlexPointGlobalRef(rdm, c, v, &lv);CHKERRQ(ierr);
8266     ierr = DMLabelGetValue(label, c, &cval);CHKERRQ(ierr);
8267     *lv = cval;
8268   }
8269   ierr = VecRestoreArray(*val, &v);CHKERRQ(ierr);
8270   ierr = DMDestroy(&rdm);CHKERRQ(ierr);
8271   PetscFunctionReturn(0);
8272 }
8273 
8274 /*@
8275   DMPlexCheckSymmetry - Check that the adjacency information in the mesh is symmetric.
8276 
8277   Input Parameter:
8278 . dm - The DMPlex object
8279 
8280   Notes:
8281   This is a useful diagnostic when creating meshes programmatically.
8282 
8283   For the complete list of DMPlexCheck* functions, see DMSetFromOptions().
8284 
8285   Level: developer
8286 
8287 .seealso: DMCreate(), DMSetFromOptions()
8288 @*/
8289 PetscErrorCode DMPlexCheckSymmetry(DM dm)
8290 {
8291   PetscSection    coneSection, supportSection;
8292   const PetscInt *cone, *support;
8293   PetscInt        coneSize, c, supportSize, s;
8294   PetscInt        pStart, pEnd, p, pp, csize, ssize;
8295   PetscBool       storagecheck = PETSC_TRUE;
8296   PetscErrorCode  ierr;
8297 
8298   PetscFunctionBegin;
8299   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8300   ierr = DMViewFromOptions(dm, NULL, "-sym_dm_view");CHKERRQ(ierr);
8301   ierr = DMPlexGetConeSection(dm, &coneSection);CHKERRQ(ierr);
8302   ierr = DMPlexGetSupportSection(dm, &supportSection);CHKERRQ(ierr);
8303   /* Check that point p is found in the support of its cone points, and vice versa */
8304   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
8305   for (p = pStart; p < pEnd; ++p) {
8306     ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
8307     ierr = DMPlexGetCone(dm, p, &cone);CHKERRQ(ierr);
8308     for (c = 0; c < coneSize; ++c) {
8309       PetscBool dup = PETSC_FALSE;
8310       PetscInt  d;
8311       for (d = c-1; d >= 0; --d) {
8312         if (cone[c] == cone[d]) {dup = PETSC_TRUE; break;}
8313       }
8314       ierr = DMPlexGetSupportSize(dm, cone[c], &supportSize);CHKERRQ(ierr);
8315       ierr = DMPlexGetSupport(dm, cone[c], &support);CHKERRQ(ierr);
8316       for (s = 0; s < supportSize; ++s) {
8317         if (support[s] == p) break;
8318       }
8319       if ((s >= supportSize) || (dup && (support[s+1] != p))) {
8320         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %D cone: ", p);CHKERRQ(ierr);
8321         for (s = 0; s < coneSize; ++s) {
8322           ierr = PetscPrintf(PETSC_COMM_SELF, "%D, ", cone[s]);CHKERRQ(ierr);
8323         }
8324         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
8325         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %D support: ", cone[c]);CHKERRQ(ierr);
8326         for (s = 0; s < supportSize; ++s) {
8327           ierr = PetscPrintf(PETSC_COMM_SELF, "%D, ", support[s]);CHKERRQ(ierr);
8328         }
8329         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
8330         if (dup) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %D not repeatedly found in support of repeated cone point %D", p, cone[c]);
8331         else SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %D not found in support of cone point %D", p, cone[c]);
8332       }
8333     }
8334     ierr = DMPlexGetTreeParent(dm, p, &pp, NULL);CHKERRQ(ierr);
8335     if (p != pp) { storagecheck = PETSC_FALSE; continue; }
8336     ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
8337     ierr = DMPlexGetSupport(dm, p, &support);CHKERRQ(ierr);
8338     for (s = 0; s < supportSize; ++s) {
8339       ierr = DMPlexGetConeSize(dm, support[s], &coneSize);CHKERRQ(ierr);
8340       ierr = DMPlexGetCone(dm, support[s], &cone);CHKERRQ(ierr);
8341       for (c = 0; c < coneSize; ++c) {
8342         ierr = DMPlexGetTreeParent(dm, cone[c], &pp, NULL);CHKERRQ(ierr);
8343         if (cone[c] != pp) { c = 0; break; }
8344         if (cone[c] == p) break;
8345       }
8346       if (c >= coneSize) {
8347         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %D support: ", p);CHKERRQ(ierr);
8348         for (c = 0; c < supportSize; ++c) {
8349           ierr = PetscPrintf(PETSC_COMM_SELF, "%D, ", support[c]);CHKERRQ(ierr);
8350         }
8351         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
8352         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %D cone: ", support[s]);CHKERRQ(ierr);
8353         for (c = 0; c < coneSize; ++c) {
8354           ierr = PetscPrintf(PETSC_COMM_SELF, "%D, ", cone[c]);CHKERRQ(ierr);
8355         }
8356         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
8357         SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %D not found in cone of support point %D", p, support[s]);
8358       }
8359     }
8360   }
8361   if (storagecheck) {
8362     ierr = PetscSectionGetStorageSize(coneSection, &csize);CHKERRQ(ierr);
8363     ierr = PetscSectionGetStorageSize(supportSection, &ssize);CHKERRQ(ierr);
8364     if (csize != ssize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "Total cone size %D != Total support size %D", csize, ssize);
8365   }
8366   PetscFunctionReturn(0);
8367 }
8368 
8369 /*
8370   For submeshes with cohesive cells (see DMPlexConstructCohesiveCells()), we allow a special case where some of the boundary of a face (edges and vertices) are not duplicated. We call these special boundary points "unsplit", since the same edge or vertex appears in both copies of the face. These unsplit points throw off our counting, so we have to explicitly account for them here.
8371 */
8372 static PetscErrorCode DMPlexCellUnsplitVertices_Private(DM dm, PetscInt c, DMPolytopeType ct, PetscInt *unsplit)
8373 {
8374   DMPolytopeType  cct;
8375   PetscInt        ptpoints[4];
8376   const PetscInt *cone, *ccone, *ptcone;
8377   PetscInt        coneSize, cp, cconeSize, ccp, npt = 0, pt;
8378   PetscErrorCode  ierr;
8379 
8380   PetscFunctionBegin;
8381   *unsplit = 0;
8382   switch (ct) {
8383     case DM_POLYTOPE_POINT_PRISM_TENSOR:
8384       ptpoints[npt++] = c;
8385       break;
8386     case DM_POLYTOPE_SEG_PRISM_TENSOR:
8387       ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
8388       ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
8389       for (cp = 0; cp < coneSize; ++cp) {
8390         ierr = DMPlexGetCellType(dm, cone[cp], &cct);CHKERRQ(ierr);
8391         if (cct == DM_POLYTOPE_POINT_PRISM_TENSOR) ptpoints[npt++] = cone[cp];
8392       }
8393       break;
8394     case DM_POLYTOPE_TRI_PRISM_TENSOR:
8395     case DM_POLYTOPE_QUAD_PRISM_TENSOR:
8396       ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
8397       ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
8398       for (cp = 0; cp < coneSize; ++cp) {
8399         ierr = DMPlexGetCone(dm, cone[cp], &ccone);CHKERRQ(ierr);
8400         ierr = DMPlexGetConeSize(dm, cone[cp], &cconeSize);CHKERRQ(ierr);
8401         for (ccp = 0; ccp < cconeSize; ++ccp) {
8402           ierr = DMPlexGetCellType(dm, ccone[ccp], &cct);CHKERRQ(ierr);
8403           if (cct == DM_POLYTOPE_POINT_PRISM_TENSOR) {
8404             PetscInt p;
8405             for (p = 0; p < npt; ++p) if (ptpoints[p] == ccone[ccp]) break;
8406             if (p == npt) ptpoints[npt++] = ccone[ccp];
8407           }
8408         }
8409       }
8410       break;
8411     default: break;
8412   }
8413   for (pt = 0; pt < npt; ++pt) {
8414     ierr = DMPlexGetCone(dm, ptpoints[pt], &ptcone);CHKERRQ(ierr);
8415     if (ptcone[0] == ptcone[1]) ++(*unsplit);
8416   }
8417   PetscFunctionReturn(0);
8418 }
8419 
8420 /*@
8421   DMPlexCheckSkeleton - Check that each cell has the correct number of vertices
8422 
8423   Input Parameters:
8424 + dm - The DMPlex object
8425 - cellHeight - Normally 0
8426 
8427   Notes:
8428   This is a useful diagnostic when creating meshes programmatically.
8429   Currently applicable only to homogeneous simplex or tensor meshes.
8430 
8431   For the complete list of DMPlexCheck* functions, see DMSetFromOptions().
8432 
8433   Level: developer
8434 
8435 .seealso: DMCreate(), DMSetFromOptions()
8436 @*/
8437 PetscErrorCode DMPlexCheckSkeleton(DM dm, PetscInt cellHeight)
8438 {
8439   DMPlexInterpolatedFlag interp;
8440   DMPolytopeType         ct;
8441   PetscInt               vStart, vEnd, cStart, cEnd, c;
8442   PetscErrorCode         ierr;
8443 
8444   PetscFunctionBegin;
8445   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8446   ierr = DMPlexIsInterpolated(dm, &interp);CHKERRQ(ierr);
8447   ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
8448   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
8449   for (c = cStart; c < cEnd; ++c) {
8450     PetscInt *closure = NULL;
8451     PetscInt  coneSize, closureSize, cl, Nv = 0;
8452 
8453     ierr = DMPlexGetCellType(dm, c, &ct);CHKERRQ(ierr);
8454     if ((PetscInt) ct < 0) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %D has no cell type", c);
8455     if (ct == DM_POLYTOPE_UNKNOWN) continue;
8456     if (interp == DMPLEX_INTERPOLATED_FULL) {
8457       ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
8458       if (coneSize != DMPolytopeTypeGetConeSize(ct)) SETERRQ4(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %D of type %s has cone size %D != %D", c, DMPolytopeTypes[ct], coneSize, DMPolytopeTypeGetConeSize(ct));
8459     }
8460     ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
8461     for (cl = 0; cl < closureSize*2; cl += 2) {
8462       const PetscInt p = closure[cl];
8463       if ((p >= vStart) && (p < vEnd)) ++Nv;
8464     }
8465     ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
8466     /* Special Case: Tensor faces with identified vertices */
8467     if (Nv < DMPolytopeTypeGetNumVertices(ct)) {
8468       PetscInt unsplit;
8469 
8470       ierr = DMPlexCellUnsplitVertices_Private(dm, c, ct, &unsplit);CHKERRQ(ierr);
8471       if (Nv + unsplit == DMPolytopeTypeGetNumVertices(ct)) continue;
8472     }
8473     if (Nv != DMPolytopeTypeGetNumVertices(ct)) SETERRQ4(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %D of type %s has %D vertices != %D", c, DMPolytopeTypes[ct], Nv, DMPolytopeTypeGetNumVertices(ct));
8474   }
8475   PetscFunctionReturn(0);
8476 }
8477 
8478 /*@
8479   DMPlexCheckFaces - Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type
8480 
8481   Not Collective
8482 
8483   Input Parameters:
8484 + dm - The DMPlex object
8485 - cellHeight - Normally 0
8486 
8487   Notes:
8488   This is a useful diagnostic when creating meshes programmatically.
8489   This routine is only relevant for meshes that are fully interpolated across all ranks.
8490   It will error out if a partially interpolated mesh is given on some rank.
8491   It will do nothing for locally uninterpolated mesh (as there is nothing to check).
8492 
8493   For the complete list of DMPlexCheck* functions, see DMSetFromOptions().
8494 
8495   Level: developer
8496 
8497 .seealso: DMCreate(), DMPlexGetVTKCellHeight(), DMSetFromOptions()
8498 @*/
8499 PetscErrorCode DMPlexCheckFaces(DM dm, PetscInt cellHeight)
8500 {
8501   PetscInt       dim, depth, vStart, vEnd, cStart, cEnd, c, h;
8502   PetscErrorCode ierr;
8503   DMPlexInterpolatedFlag interpEnum;
8504 
8505   PetscFunctionBegin;
8506   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8507   ierr = DMPlexIsInterpolated(dm, &interpEnum);CHKERRQ(ierr);
8508   if (interpEnum == DMPLEX_INTERPOLATED_NONE) PetscFunctionReturn(0);
8509   if (interpEnum == DMPLEX_INTERPOLATED_PARTIAL) {
8510     PetscMPIInt rank;
8511     MPI_Comm    comm;
8512 
8513     ierr = PetscObjectGetComm((PetscObject) dm, &comm);CHKERRQ(ierr);
8514     ierr = MPI_Comm_rank(comm, &rank);CHKERRMPI(ierr);
8515     SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_SUP, "Mesh is only partially interpolated on rank %d, this is currently not supported", rank);
8516   }
8517 
8518   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
8519   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
8520   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
8521   for (h = cellHeight; h < PetscMin(depth, dim); ++h) {
8522     ierr = DMPlexGetHeightStratum(dm, h, &cStart, &cEnd);CHKERRQ(ierr);
8523     for (c = cStart; c < cEnd; ++c) {
8524       const PetscInt      *cone, *ornt, *faceSizes, *faces;
8525       const DMPolytopeType *faceTypes;
8526       DMPolytopeType        ct;
8527       PetscInt              numFaces, coneSize, f;
8528       PetscInt             *closure = NULL, closureSize, cl, numCorners = 0, fOff = 0, unsplit;
8529 
8530       ierr = DMPlexGetCellType(dm, c, &ct);CHKERRQ(ierr);
8531       ierr = DMPlexCellUnsplitVertices_Private(dm, c, ct, &unsplit);CHKERRQ(ierr);
8532       if (unsplit) continue;
8533       ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
8534       ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
8535       ierr = DMPlexGetConeOrientation(dm, c, &ornt);CHKERRQ(ierr);
8536       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
8537       for (cl = 0; cl < closureSize*2; cl += 2) {
8538         const PetscInt p = closure[cl];
8539         if ((p >= vStart) && (p < vEnd)) closure[numCorners++] = p;
8540       }
8541       ierr = DMPlexGetRawFaces_Internal(dm, ct, closure, &numFaces, &faceTypes, &faceSizes, &faces);CHKERRQ(ierr);
8542       if (coneSize != numFaces) SETERRQ4(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %D of type %s has %D faces but should have %D", c, DMPolytopeTypes[ct], coneSize, numFaces);
8543       for (f = 0; f < numFaces; ++f) {
8544         DMPolytopeType fct;
8545         PetscInt       *fclosure = NULL, fclosureSize, cl, fnumCorners = 0, v;
8546 
8547         ierr = DMPlexGetCellType(dm, cone[f], &fct);CHKERRQ(ierr);
8548         ierr = DMPlexGetTransitiveClosure_Internal(dm, cone[f], ornt[f], PETSC_TRUE, &fclosureSize, &fclosure);CHKERRQ(ierr);
8549         for (cl = 0; cl < fclosureSize*2; cl += 2) {
8550           const PetscInt p = fclosure[cl];
8551           if ((p >= vStart) && (p < vEnd)) fclosure[fnumCorners++] = p;
8552         }
8553         if (fnumCorners != faceSizes[f]) SETERRQ7(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %D of type %s (cone idx %D) of cell %D of type %s has %D vertices but should have %D", cone[f], DMPolytopeTypes[fct], f, c, DMPolytopeTypes[ct], fnumCorners, faceSizes[f]);
8554         for (v = 0; v < fnumCorners; ++v) {
8555           if (fclosure[v] != faces[fOff+v]) {
8556             PetscInt v1;
8557 
8558             ierr = PetscPrintf(PETSC_COMM_SELF, "face closure:");CHKERRQ(ierr);
8559             for (v1 = 0; v1 < fnumCorners; ++v1) {ierr = PetscPrintf(PETSC_COMM_SELF, " %D", fclosure[v1]);CHKERRQ(ierr);}
8560             ierr = PetscPrintf(PETSC_COMM_SELF, "\ncell face:");CHKERRQ(ierr);
8561             for (v1 = 0; v1 < fnumCorners; ++v1) {ierr = PetscPrintf(PETSC_COMM_SELF, " %D", faces[fOff+v1]);CHKERRQ(ierr);}
8562             ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
8563             SETERRQ9(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %D of type %s (cone idx %d, ornt %D) of cell %D of type %s vertex %D, %D != %D", cone[f], DMPolytopeTypes[fct], f, ornt[f], c, DMPolytopeTypes[ct], v, fclosure[v], faces[fOff+v]);
8564           }
8565         }
8566         ierr = DMPlexRestoreTransitiveClosure(dm, cone[f], PETSC_TRUE, &fclosureSize, &fclosure);CHKERRQ(ierr);
8567         fOff += faceSizes[f];
8568       }
8569       ierr = DMPlexRestoreRawFaces_Internal(dm, ct, closure, &numFaces, &faceTypes, &faceSizes, &faces);CHKERRQ(ierr);
8570       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
8571     }
8572   }
8573   PetscFunctionReturn(0);
8574 }
8575 
8576 /*@
8577   DMPlexCheckGeometry - Check the geometry of mesh cells
8578 
8579   Input Parameter:
8580 . dm - The DMPlex object
8581 
8582   Notes:
8583   This is a useful diagnostic when creating meshes programmatically.
8584 
8585   For the complete list of DMPlexCheck* functions, see DMSetFromOptions().
8586 
8587   Level: developer
8588 
8589 .seealso: DMCreate(), DMSetFromOptions()
8590 @*/
8591 PetscErrorCode DMPlexCheckGeometry(DM dm)
8592 {
8593   Vec            coordinates;
8594   PetscReal      detJ, J[9], refVol = 1.0;
8595   PetscReal      vol;
8596   PetscBool      periodic;
8597   PetscInt       dim, depth, dE, d, cStart, cEnd, c;
8598   PetscErrorCode ierr;
8599 
8600   PetscFunctionBegin;
8601   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
8602   ierr = DMGetCoordinateDim(dm, &dE);CHKERRQ(ierr);
8603   if (dim != dE) PetscFunctionReturn(0);
8604   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
8605   ierr = DMGetPeriodicity(dm, &periodic, NULL, NULL, NULL);CHKERRQ(ierr);
8606   for (d = 0; d < dim; ++d) refVol *= 2.0;
8607   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
8608   /* Make sure local coordinates are created, because that step is collective */
8609   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
8610   for (c = cStart; c < cEnd; ++c) {
8611     DMPolytopeType ct;
8612     PetscInt       unsplit;
8613     PetscBool      ignoreZeroVol = PETSC_FALSE;
8614 
8615     ierr = DMPlexGetCellType(dm, c, &ct);CHKERRQ(ierr);
8616     switch (ct) {
8617       case DM_POLYTOPE_SEG_PRISM_TENSOR:
8618       case DM_POLYTOPE_TRI_PRISM_TENSOR:
8619       case DM_POLYTOPE_QUAD_PRISM_TENSOR:
8620         ignoreZeroVol = PETSC_TRUE; break;
8621       default: break;
8622     }
8623     switch (ct) {
8624       case DM_POLYTOPE_TRI_PRISM:
8625       case DM_POLYTOPE_TRI_PRISM_TENSOR:
8626       case DM_POLYTOPE_QUAD_PRISM_TENSOR:
8627       case DM_POLYTOPE_PYRAMID:
8628         continue;
8629       default: break;
8630     }
8631     ierr = DMPlexCellUnsplitVertices_Private(dm, c, ct, &unsplit);CHKERRQ(ierr);
8632     if (unsplit) continue;
8633     ierr = DMPlexComputeCellGeometryFEM(dm, c, NULL, NULL, J, NULL, &detJ);CHKERRQ(ierr);
8634     if (detJ < -PETSC_SMALL || (detJ <= 0.0 && !ignoreZeroVol)) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Mesh cell %D of type %s is inverted, |J| = %g", c, DMPolytopeTypes[ct], (double) detJ);
8635     ierr = PetscInfo2(dm, "Cell %D FEM Volume %g\n", c, (double) detJ*refVol);CHKERRQ(ierr);
8636     if (depth > 1 && !periodic) {
8637       ierr = DMPlexComputeCellGeometryFVM(dm, c, &vol, NULL, NULL);CHKERRQ(ierr);
8638       if (vol < -PETSC_SMALL || (vol <= 0.0 && !ignoreZeroVol)) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Mesh cell %D of type %s is inverted, vol = %g", c, DMPolytopeTypes[ct], (double) vol);
8639       ierr = PetscInfo2(dm, "Cell %D FVM Volume %g\n", c, (double) vol);CHKERRQ(ierr);
8640     }
8641   }
8642   PetscFunctionReturn(0);
8643 }
8644 
8645 /*@
8646   DMPlexCheckPointSF - Check that several necessary conditions are met for the point SF of this plex.
8647 
8648   Input Parameters:
8649 . dm - The DMPlex object
8650 
8651   Notes:
8652   This is mainly intended for debugging/testing purposes.
8653   It currently checks only meshes with no partition overlapping.
8654 
8655   For the complete list of DMPlexCheck* functions, see DMSetFromOptions().
8656 
8657   Level: developer
8658 
8659 .seealso: DMGetPointSF(), DMSetFromOptions()
8660 @*/
8661 PetscErrorCode DMPlexCheckPointSF(DM dm)
8662 {
8663   PetscSF         pointSF;
8664   PetscInt        cellHeight, cStart, cEnd, l, nleaves, nroots, overlap;
8665   const PetscInt *locals, *rootdegree;
8666   PetscBool       distributed;
8667   PetscErrorCode  ierr;
8668 
8669   PetscFunctionBegin;
8670   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8671   ierr = DMGetPointSF(dm, &pointSF);CHKERRQ(ierr);
8672   ierr = DMPlexIsDistributed(dm, &distributed);CHKERRQ(ierr);
8673   if (!distributed) PetscFunctionReturn(0);
8674   ierr = DMPlexGetOverlap(dm, &overlap);CHKERRQ(ierr);
8675   if (overlap) {
8676     ierr = PetscPrintf(PetscObjectComm((PetscObject)dm), "Warning: DMPlexCheckPointSF() is currently not implemented for meshes with partition overlapping");CHKERRQ(ierr);
8677     PetscFunctionReturn(0);
8678   }
8679   if (!pointSF) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONGSTATE, "This DMPlex is distributed but does not have PointSF attached");
8680   ierr = PetscSFGetGraph(pointSF, &nroots, &nleaves, &locals, NULL);CHKERRQ(ierr);
8681   if (nroots < 0) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONGSTATE, "This DMPlex is distributed but its PointSF has no graph set");
8682   ierr = PetscSFComputeDegreeBegin(pointSF, &rootdegree);CHKERRQ(ierr);
8683   ierr = PetscSFComputeDegreeEnd(pointSF, &rootdegree);CHKERRQ(ierr);
8684 
8685   /* 1) check there are no faces in 2D, cells in 3D, in interface */
8686   ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
8687   ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
8688   for (l = 0; l < nleaves; ++l) {
8689     const PetscInt point = locals[l];
8690 
8691     if (point >= cStart && point < cEnd) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point SF contains %D which is a cell", point);
8692   }
8693 
8694   /* 2) if some point is in interface, then all its cone points must be also in interface (either as leaves or roots) */
8695   for (l = 0; l < nleaves; ++l) {
8696     const PetscInt  point = locals[l];
8697     const PetscInt *cone;
8698     PetscInt        coneSize, c, idx;
8699 
8700     ierr = DMPlexGetConeSize(dm, point, &coneSize);CHKERRQ(ierr);
8701     ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
8702     for (c = 0; c < coneSize; ++c) {
8703       if (!rootdegree[cone[c]]) {
8704         ierr = PetscFindInt(cone[c], nleaves, locals, &idx);CHKERRQ(ierr);
8705         if (idx < 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point SF contains %D but not %D from its cone", point, cone[c]);
8706       }
8707     }
8708   }
8709   PetscFunctionReturn(0);
8710 }
8711 
8712 PetscErrorCode DMPlexCheckAll_Internal(DM dm, PetscInt cellHeight)
8713 {
8714   PetscErrorCode ierr;
8715 
8716   PetscFunctionBegin;
8717   ierr = DMPlexCheckSymmetry(dm);CHKERRQ(ierr);
8718   ierr = DMPlexCheckSkeleton(dm, cellHeight);CHKERRQ(ierr);
8719   ierr = DMPlexCheckFaces(dm, cellHeight);CHKERRQ(ierr);
8720   ierr = DMPlexCheckGeometry(dm);CHKERRQ(ierr);
8721   ierr = DMPlexCheckPointSF(dm);CHKERRQ(ierr);
8722   ierr = DMPlexCheckInterfaceCones(dm);CHKERRQ(ierr);
8723   PetscFunctionReturn(0);
8724 }
8725 
8726 typedef struct cell_stats
8727 {
8728   PetscReal min, max, sum, squaresum;
8729   PetscInt  count;
8730 } cell_stats_t;
8731 
8732 static void MPIAPI cell_stats_reduce(void *a, void *b, int * len, MPI_Datatype *datatype)
8733 {
8734   PetscInt i, N = *len;
8735 
8736   for (i = 0; i < N; i++) {
8737     cell_stats_t *A = (cell_stats_t *) a;
8738     cell_stats_t *B = (cell_stats_t *) b;
8739 
8740     B->min = PetscMin(A->min,B->min);
8741     B->max = PetscMax(A->max,B->max);
8742     B->sum += A->sum;
8743     B->squaresum += A->squaresum;
8744     B->count += A->count;
8745   }
8746 }
8747 
8748 /*@
8749   DMPlexCheckCellShape - Checks the Jacobian of the mapping from reference to real cells and computes some minimal statistics.
8750 
8751   Collective on dm
8752 
8753   Input Parameters:
8754 + dm        - The DMPlex object
8755 . output    - If true, statistics will be displayed on stdout
8756 - condLimit - Display all cells above this condition number, or PETSC_DETERMINE for no cell output
8757 
8758   Notes:
8759   This is mainly intended for debugging/testing purposes.
8760 
8761   For the complete list of DMPlexCheck* functions, see DMSetFromOptions().
8762 
8763   Level: developer
8764 
8765 .seealso: DMSetFromOptions(), DMPlexComputeOrthogonalQuality()
8766 @*/
8767 PetscErrorCode DMPlexCheckCellShape(DM dm, PetscBool output, PetscReal condLimit)
8768 {
8769   DM             dmCoarse;
8770   cell_stats_t   stats, globalStats;
8771   MPI_Comm       comm = PetscObjectComm((PetscObject)dm);
8772   PetscReal      *J, *invJ, min = 0, max = 0, mean = 0, stdev = 0;
8773   PetscReal      limit = condLimit > 0 ? condLimit : PETSC_MAX_REAL;
8774   PetscInt       cdim, cStart, cEnd, c, eStart, eEnd, count = 0;
8775   PetscMPIInt    rank,size;
8776   PetscErrorCode ierr;
8777 
8778   PetscFunctionBegin;
8779   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8780   stats.min   = PETSC_MAX_REAL;
8781   stats.max   = PETSC_MIN_REAL;
8782   stats.sum   = stats.squaresum = 0.;
8783   stats.count = 0;
8784 
8785   ierr = MPI_Comm_size(comm, &size);CHKERRMPI(ierr);
8786   ierr = MPI_Comm_rank(comm, &rank);CHKERRMPI(ierr);
8787   ierr = DMGetCoordinateDim(dm,&cdim);CHKERRQ(ierr);
8788   ierr = PetscMalloc2(PetscSqr(cdim), &J, PetscSqr(cdim), &invJ);CHKERRQ(ierr);
8789   ierr = DMPlexGetSimplexOrBoxCells(dm,0,&cStart,&cEnd);CHKERRQ(ierr);
8790   ierr = DMPlexGetDepthStratum(dm,1,&eStart,&eEnd);CHKERRQ(ierr);
8791   for (c = cStart; c < cEnd; c++) {
8792     PetscInt  i;
8793     PetscReal frobJ = 0., frobInvJ = 0., cond2, cond, detJ;
8794 
8795     ierr = DMPlexComputeCellGeometryAffineFEM(dm,c,NULL,J,invJ,&detJ);CHKERRQ(ierr);
8796     if (detJ < 0.0) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Mesh cell %D is inverted", c);
8797     for (i = 0; i < PetscSqr(cdim); ++i) {
8798       frobJ    += J[i] * J[i];
8799       frobInvJ += invJ[i] * invJ[i];
8800     }
8801     cond2 = frobJ * frobInvJ;
8802     cond  = PetscSqrtReal(cond2);
8803 
8804     stats.min        = PetscMin(stats.min,cond);
8805     stats.max        = PetscMax(stats.max,cond);
8806     stats.sum       += cond;
8807     stats.squaresum += cond2;
8808     stats.count++;
8809     if (output && cond > limit) {
8810       PetscSection coordSection;
8811       Vec          coordsLocal;
8812       PetscScalar *coords = NULL;
8813       PetscInt     Nv, d, clSize, cl, *closure = NULL;
8814 
8815       ierr = DMGetCoordinatesLocal(dm, &coordsLocal);CHKERRQ(ierr);
8816       ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
8817       ierr = DMPlexVecGetClosure(dm, coordSection, coordsLocal, c, &Nv, &coords);CHKERRQ(ierr);
8818       ierr = PetscSynchronizedPrintf(comm, "[%d] Cell %D cond %g\n", rank, c, (double) cond);CHKERRQ(ierr);
8819       for (i = 0; i < Nv/cdim; ++i) {
8820         ierr = PetscSynchronizedPrintf(comm, "  Vertex %D: (", i);CHKERRQ(ierr);
8821         for (d = 0; d < cdim; ++d) {
8822           if (d > 0) {ierr = PetscSynchronizedPrintf(comm, ", ");CHKERRQ(ierr);}
8823           ierr = PetscSynchronizedPrintf(comm, "%g", (double) PetscRealPart(coords[i*cdim+d]));CHKERRQ(ierr);
8824         }
8825         ierr = PetscSynchronizedPrintf(comm, ")\n");CHKERRQ(ierr);
8826       }
8827       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &clSize, &closure);CHKERRQ(ierr);
8828       for (cl = 0; cl < clSize*2; cl += 2) {
8829         const PetscInt edge = closure[cl];
8830 
8831         if ((edge >= eStart) && (edge < eEnd)) {
8832           PetscReal len;
8833 
8834           ierr = DMPlexComputeCellGeometryFVM(dm, edge, &len, NULL, NULL);CHKERRQ(ierr);
8835           ierr = PetscSynchronizedPrintf(comm, "  Edge %D: length %g\n", edge, (double) len);CHKERRQ(ierr);
8836         }
8837       }
8838       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &clSize, &closure);CHKERRQ(ierr);
8839       ierr = DMPlexVecRestoreClosure(dm, coordSection, coordsLocal, c, &Nv, &coords);CHKERRQ(ierr);
8840     }
8841   }
8842   if (output) {ierr = PetscSynchronizedFlush(comm, NULL);CHKERRQ(ierr);}
8843 
8844   if (size > 1) {
8845     PetscMPIInt   blockLengths[2] = {4,1};
8846     MPI_Aint      blockOffsets[2] = {offsetof(cell_stats_t,min),offsetof(cell_stats_t,count)};
8847     MPI_Datatype  blockTypes[2]   = {MPIU_REAL,MPIU_INT}, statType;
8848     MPI_Op        statReduce;
8849 
8850     ierr = MPI_Type_create_struct(2,blockLengths,blockOffsets,blockTypes,&statType);CHKERRMPI(ierr);
8851     ierr = MPI_Type_commit(&statType);CHKERRMPI(ierr);
8852     ierr = MPI_Op_create(cell_stats_reduce, PETSC_TRUE, &statReduce);CHKERRMPI(ierr);
8853     ierr = MPI_Reduce(&stats,&globalStats,1,statType,statReduce,0,comm);CHKERRMPI(ierr);
8854     ierr = MPI_Op_free(&statReduce);CHKERRMPI(ierr);
8855     ierr = MPI_Type_free(&statType);CHKERRMPI(ierr);
8856   } else {
8857     ierr = PetscArraycpy(&globalStats,&stats,1);CHKERRQ(ierr);
8858   }
8859   if (rank == 0) {
8860     count = globalStats.count;
8861     min   = globalStats.min;
8862     max   = globalStats.max;
8863     mean  = globalStats.sum / globalStats.count;
8864     stdev = globalStats.count > 1 ? PetscSqrtReal(PetscMax((globalStats.squaresum - globalStats.count * mean * mean) / (globalStats.count - 1),0)) : 0.0;
8865   }
8866 
8867   if (output) {
8868     ierr = PetscPrintf(comm,"Mesh with %D cells, shape condition numbers: min = %g, max = %g, mean = %g, stddev = %g\n", count, (double) min, (double) max, (double) mean, (double) stdev);CHKERRQ(ierr);
8869   }
8870   ierr = PetscFree2(J,invJ);CHKERRQ(ierr);
8871 
8872   ierr = DMGetCoarseDM(dm,&dmCoarse);CHKERRQ(ierr);
8873   if (dmCoarse) {
8874     PetscBool isplex;
8875 
8876     ierr = PetscObjectTypeCompare((PetscObject)dmCoarse,DMPLEX,&isplex);CHKERRQ(ierr);
8877     if (isplex) {
8878       ierr = DMPlexCheckCellShape(dmCoarse,output,condLimit);CHKERRQ(ierr);
8879     }
8880   }
8881   PetscFunctionReturn(0);
8882 }
8883 
8884 /*@
8885   DMPlexComputeOrthogonalQuality - Compute cell-wise orthogonal quality mesh statistic. Optionally tags all cells with
8886   orthogonal quality below given tolerance.
8887 
8888   Collective on dm
8889 
8890   Input Parameters:
8891 + dm   - The DMPlex object
8892 . fv   - Optional PetscFV object for pre-computed cell/face centroid information
8893 - atol - [0, 1] Absolute tolerance for tagging cells.
8894 
8895   Output Parameters:
8896 + OrthQual      - Vec containing orthogonal quality per cell
8897 - OrthQualLabel - DMLabel tagging cells below atol with DM_ADAPT_REFINE
8898 
8899   Options Database Keys:
8900 + -dm_plex_orthogonal_quality_label_view - view OrthQualLabel if label is requested. Currently only PETSCVIEWERASCII is
8901 supported.
8902 - -dm_plex_orthogonal_quality_vec_view - view OrthQual vector.
8903 
8904   Notes:
8905   Orthogonal quality is given by the following formula:
8906 
8907   \min \left[ \frac{A_i \cdot f_i}{\|A_i\| \|f_i\|} , \frac{A_i \cdot c_i}{\|A_i\| \|c_i\|} \right]
8908 
8909   Where A_i is the i'th face-normal vector, f_i is the vector from the cell centroid to the i'th face centroid, and c_i
8910   is the vector from the current cells centroid to the centroid of its i'th neighbor (which shares a face with the
8911   current cell). This computes the vector similarity between each cell face and its corresponding neighbor centroid by
8912   calculating the cosine of the angle between these vectors.
8913 
8914   Orthogonal quality ranges from 1 (best) to 0 (worst).
8915 
8916   This routine is mainly useful for FVM, however is not restricted to only FVM. The PetscFV object is optionally used to check for
8917   pre-computed FVM cell data, but if it is not passed in then this data will be computed.
8918 
8919   Cells are tagged if they have an orthogonal quality less than or equal to the absolute tolerance.
8920 
8921   Level: intermediate
8922 
8923 .seealso: DMPlexCheckCellShape(), DMCreateLabel()
8924 @*/
8925 PetscErrorCode DMPlexComputeOrthogonalQuality(DM dm, PetscFV fv, PetscReal atol, Vec *OrthQual, DMLabel *OrthQualLabel)
8926 {
8927   PetscInt                nc, cellHeight, cStart, cEnd, cell, cellIter = 0;
8928   PetscInt                *idx;
8929   PetscScalar             *oqVals;
8930   const PetscScalar       *cellGeomArr, *faceGeomArr;
8931   PetscReal               *ci, *fi, *Ai;
8932   MPI_Comm                comm;
8933   Vec                     cellgeom, facegeom;
8934   DM                      dmFace, dmCell;
8935   IS                      glob;
8936   ISLocalToGlobalMapping  ltog;
8937   PetscViewer             vwr;
8938   PetscErrorCode          ierr;
8939 
8940   PetscFunctionBegin;
8941   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8942   if (fv) {PetscValidHeaderSpecific(fv, PETSCFV_CLASSID, 2);}
8943   PetscValidPointer(OrthQual, 4);
8944   if (PetscUnlikelyDebug(atol < 0.0 || atol > 1.0)) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"Absolute tolerance %g not in [0,1]",(double)atol);
8945   ierr = PetscObjectGetComm((PetscObject) dm, &comm);CHKERRQ(ierr);
8946   ierr = DMGetDimension(dm, &nc);CHKERRQ(ierr);
8947   if (nc < 2) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONGSTATE, "DM must have dimension >= 2 (current %D)", nc);
8948   {
8949     DMPlexInterpolatedFlag interpFlag;
8950 
8951     ierr = DMPlexIsInterpolated(dm, &interpFlag);CHKERRQ(ierr);
8952     if (interpFlag != DMPLEX_INTERPOLATED_FULL) {
8953       PetscMPIInt rank;
8954 
8955       ierr = MPI_Comm_rank(comm, &rank);CHKERRMPI(ierr);
8956       SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONGSTATE, "DM must be fully interpolated, DM on rank %d is not fully interpolated", rank);
8957     }
8958   }
8959   if (OrthQualLabel) {
8960     PetscValidPointer(OrthQualLabel, 5);
8961     ierr = DMCreateLabel(dm, "Orthogonal_Quality");CHKERRQ(ierr);
8962     ierr = DMGetLabel(dm, "Orthogonal_Quality", OrthQualLabel);CHKERRQ(ierr);
8963   } else {*OrthQualLabel = NULL;}
8964   ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
8965   ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
8966   ierr = DMPlexCreateCellNumbering_Internal(dm, PETSC_TRUE, &glob);CHKERRQ(ierr);
8967   ierr = ISLocalToGlobalMappingCreateIS(glob, &ltog);CHKERRQ(ierr);
8968   ierr = ISLocalToGlobalMappingSetType(ltog, ISLOCALTOGLOBALMAPPINGHASH);CHKERRQ(ierr);
8969   ierr = VecCreate(comm, OrthQual);CHKERRQ(ierr);
8970   ierr = VecSetType(*OrthQual, VECSTANDARD);CHKERRQ(ierr);
8971   ierr = VecSetSizes(*OrthQual, cEnd-cStart, PETSC_DETERMINE);CHKERRQ(ierr);
8972   ierr = VecSetLocalToGlobalMapping(*OrthQual, ltog);CHKERRQ(ierr);
8973   ierr = VecSetUp(*OrthQual);CHKERRQ(ierr);
8974   ierr = ISDestroy(&glob);CHKERRQ(ierr);
8975   ierr = ISLocalToGlobalMappingDestroy(&ltog);CHKERRQ(ierr);
8976   ierr = DMPlexGetDataFVM(dm, fv, &cellgeom, &facegeom, NULL);CHKERRQ(ierr);
8977   ierr = VecGetArrayRead(cellgeom, &cellGeomArr);CHKERRQ(ierr);
8978   ierr = VecGetArrayRead(facegeom, &faceGeomArr);CHKERRQ(ierr);
8979   ierr = VecGetDM(cellgeom, &dmCell);CHKERRQ(ierr);
8980   ierr = VecGetDM(facegeom, &dmFace);CHKERRQ(ierr);
8981   ierr = PetscMalloc5(cEnd-cStart, &idx, cEnd-cStart, &oqVals, nc, &ci, nc, &fi, nc, &Ai);CHKERRQ(ierr);
8982   for (cell = cStart; cell < cEnd; cellIter++,cell++) {
8983     PetscInt           cellneigh, cellneighiter = 0, adjSize = PETSC_DETERMINE;
8984     PetscInt           cellarr[2], *adj = NULL;
8985     PetscScalar        *cArr, *fArr;
8986     PetscReal          minvalc = 1.0, minvalf = 1.0;
8987     PetscFVCellGeom    *cg;
8988 
8989     idx[cellIter] = cell-cStart;
8990     cellarr[0] = cell;
8991     /* Make indexing into cellGeom easier */
8992     ierr = DMPlexPointLocalRead(dmCell, cell, cellGeomArr, &cg);CHKERRQ(ierr);
8993     ierr = DMPlexGetAdjacency_Internal(dm, cell, PETSC_TRUE, PETSC_FALSE, PETSC_FALSE, &adjSize, &adj);CHKERRQ(ierr);
8994     /* Technically 1 too big, but easier than fiddling with empty adjacency array */
8995     ierr = PetscCalloc2(adjSize, &cArr, adjSize, &fArr);CHKERRQ(ierr);
8996     for (cellneigh = 0; cellneigh < adjSize; cellneighiter++,cellneigh++) {
8997       PetscInt         i;
8998       const PetscInt   neigh = adj[cellneigh];
8999       PetscReal        normci = 0, normfi = 0, normai = 0;
9000       PetscFVCellGeom  *cgneigh;
9001       PetscFVFaceGeom  *fg;
9002 
9003       /* Don't count ourselves in the neighbor list */
9004       if (neigh == cell) continue;
9005       ierr = DMPlexPointLocalRead(dmCell, neigh, cellGeomArr, &cgneigh);CHKERRQ(ierr);
9006       cellarr[1] = neigh;
9007       {
9008         PetscInt       numcovpts;
9009         const PetscInt *covpts;
9010 
9011         ierr = DMPlexGetMeet(dm, 2, cellarr, &numcovpts, &covpts);CHKERRQ(ierr);
9012         ierr = DMPlexPointLocalRead(dmFace, covpts[0], faceGeomArr, &fg);CHKERRQ(ierr);
9013         ierr = DMPlexRestoreMeet(dm, 2, cellarr, &numcovpts, &covpts);CHKERRQ(ierr);
9014       }
9015 
9016       /* Compute c_i, f_i and their norms */
9017       for (i = 0; i < nc; i++) {
9018         ci[i] = cgneigh->centroid[i] - cg->centroid[i];
9019         fi[i] = fg->centroid[i] - cg->centroid[i];
9020         Ai[i] = fg->normal[i];
9021         normci += PetscPowReal(ci[i], 2);
9022         normfi += PetscPowReal(fi[i], 2);
9023         normai += PetscPowReal(Ai[i], 2);
9024       }
9025       normci = PetscSqrtReal(normci);
9026       normfi = PetscSqrtReal(normfi);
9027       normai = PetscSqrtReal(normai);
9028 
9029       /* Normalize and compute for each face-cell-normal pair */
9030       for (i = 0; i < nc; i++) {
9031         ci[i] = ci[i]/normci;
9032         fi[i] = fi[i]/normfi;
9033         Ai[i] = Ai[i]/normai;
9034         /* PetscAbs because I don't know if normals are guaranteed to point out */
9035         cArr[cellneighiter] += PetscAbs(Ai[i]*ci[i]);
9036         fArr[cellneighiter] += PetscAbs(Ai[i]*fi[i]);
9037       }
9038       if (PetscRealPart(cArr[cellneighiter]) < minvalc) {
9039         minvalc = PetscRealPart(cArr[cellneighiter]);
9040       }
9041       if (PetscRealPart(fArr[cellneighiter]) < minvalf) {
9042         minvalf = PetscRealPart(fArr[cellneighiter]);
9043       }
9044     }
9045     ierr = PetscFree(adj);CHKERRQ(ierr);
9046     ierr = PetscFree2(cArr, fArr);CHKERRQ(ierr);
9047     /* Defer to cell if they're equal */
9048     oqVals[cellIter] = PetscMin(minvalf, minvalc);
9049     if (OrthQualLabel) {
9050       if (PetscRealPart(oqVals[cellIter]) <= atol) {ierr = DMLabelSetValue(*OrthQualLabel, cell, DM_ADAPT_REFINE);CHKERRQ(ierr);}
9051     }
9052   }
9053   ierr = VecSetValuesLocal(*OrthQual, cEnd-cStart, idx, oqVals, INSERT_VALUES);CHKERRQ(ierr);
9054   ierr = VecAssemblyBegin(*OrthQual);CHKERRQ(ierr);
9055   ierr = VecAssemblyEnd(*OrthQual);CHKERRQ(ierr);
9056   ierr = VecRestoreArrayRead(cellgeom, &cellGeomArr);CHKERRQ(ierr);
9057   ierr = VecRestoreArrayRead(facegeom, &faceGeomArr);CHKERRQ(ierr);
9058   ierr = PetscOptionsGetViewer(comm, NULL, NULL, "-dm_plex_orthogonal_quality_label_view", &vwr, NULL, NULL);CHKERRQ(ierr);
9059   if (OrthQualLabel) {
9060     if (vwr) {ierr = DMLabelView(*OrthQualLabel, vwr);CHKERRQ(ierr);}
9061   }
9062   ierr = PetscFree5(idx, oqVals, ci, fi, Ai);CHKERRQ(ierr);
9063   ierr = PetscViewerDestroy(&vwr);CHKERRQ(ierr);
9064   ierr = VecViewFromOptions(*OrthQual, NULL, "-dm_plex_orthogonal_quality_vec_view");CHKERRQ(ierr);
9065   PetscFunctionReturn(0);
9066 }
9067 
9068 /* this is here insead of DMGetOutputDM because output DM still has constraints in the local indices that affect
9069  * interpolator construction */
9070 static PetscErrorCode DMGetFullDM(DM dm, DM *odm)
9071 {
9072   PetscSection   section, newSection, gsection;
9073   PetscSF        sf;
9074   PetscBool      hasConstraints, ghasConstraints;
9075   PetscErrorCode ierr;
9076 
9077   PetscFunctionBegin;
9078   PetscValidHeaderSpecific(dm,DM_CLASSID,1);
9079   PetscValidPointer(odm,2);
9080   ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);
9081   ierr = PetscSectionHasConstraints(section, &hasConstraints);CHKERRQ(ierr);
9082   ierr = MPI_Allreduce(&hasConstraints, &ghasConstraints, 1, MPIU_BOOL, MPI_LOR, PetscObjectComm((PetscObject) dm));CHKERRMPI(ierr);
9083   if (!ghasConstraints) {
9084     ierr = PetscObjectReference((PetscObject)dm);CHKERRQ(ierr);
9085     *odm = dm;
9086     PetscFunctionReturn(0);
9087   }
9088   ierr = DMClone(dm, odm);CHKERRQ(ierr);
9089   ierr = DMCopyFields(dm, *odm);CHKERRQ(ierr);
9090   ierr = DMGetLocalSection(*odm, &newSection);CHKERRQ(ierr);
9091   ierr = DMGetPointSF(*odm, &sf);CHKERRQ(ierr);
9092   ierr = PetscSectionCreateGlobalSection(newSection, sf, PETSC_TRUE, PETSC_FALSE, &gsection);CHKERRQ(ierr);
9093   ierr = DMSetGlobalSection(*odm, gsection);CHKERRQ(ierr);
9094   ierr = PetscSectionDestroy(&gsection);CHKERRQ(ierr);
9095   PetscFunctionReturn(0);
9096 }
9097 
9098 static PetscErrorCode DMCreateAffineInterpolationCorrection_Plex(DM dmc, DM dmf, Vec *shift)
9099 {
9100   DM             dmco, dmfo;
9101   Mat            interpo;
9102   Vec            rscale;
9103   Vec            cglobalo, clocal;
9104   Vec            fglobal, fglobalo, flocal;
9105   PetscBool      regular;
9106   PetscErrorCode ierr;
9107 
9108   PetscFunctionBegin;
9109   ierr = DMGetFullDM(dmc, &dmco);CHKERRQ(ierr);
9110   ierr = DMGetFullDM(dmf, &dmfo);CHKERRQ(ierr);
9111   ierr = DMSetCoarseDM(dmfo, dmco);CHKERRQ(ierr);
9112   ierr = DMPlexGetRegularRefinement(dmf, &regular);CHKERRQ(ierr);
9113   ierr = DMPlexSetRegularRefinement(dmfo, regular);CHKERRQ(ierr);
9114   ierr = DMCreateInterpolation(dmco, dmfo, &interpo, &rscale);CHKERRQ(ierr);
9115   ierr = DMCreateGlobalVector(dmco, &cglobalo);CHKERRQ(ierr);
9116   ierr = DMCreateLocalVector(dmc, &clocal);CHKERRQ(ierr);
9117   ierr = VecSet(cglobalo, 0.);CHKERRQ(ierr);
9118   ierr = VecSet(clocal, 0.);CHKERRQ(ierr);
9119   ierr = DMCreateGlobalVector(dmf, &fglobal);CHKERRQ(ierr);
9120   ierr = DMCreateGlobalVector(dmfo, &fglobalo);CHKERRQ(ierr);
9121   ierr = DMCreateLocalVector(dmf, &flocal);CHKERRQ(ierr);
9122   ierr = VecSet(fglobal, 0.);CHKERRQ(ierr);
9123   ierr = VecSet(fglobalo, 0.);CHKERRQ(ierr);
9124   ierr = VecSet(flocal, 0.);CHKERRQ(ierr);
9125   ierr = DMPlexInsertBoundaryValues(dmc, PETSC_TRUE, clocal, 0., NULL, NULL, NULL);CHKERRQ(ierr);
9126   ierr = DMLocalToGlobalBegin(dmco, clocal, INSERT_VALUES, cglobalo);CHKERRQ(ierr);
9127   ierr = DMLocalToGlobalEnd(dmco, clocal, INSERT_VALUES, cglobalo);CHKERRQ(ierr);
9128   ierr = MatMult(interpo, cglobalo, fglobalo);CHKERRQ(ierr);
9129   ierr = DMGlobalToLocalBegin(dmfo, fglobalo, INSERT_VALUES, flocal);CHKERRQ(ierr);
9130   ierr = DMGlobalToLocalEnd(dmfo, fglobalo, INSERT_VALUES, flocal);CHKERRQ(ierr);
9131   ierr = DMLocalToGlobalBegin(dmf, flocal, INSERT_VALUES, fglobal);CHKERRQ(ierr);
9132   ierr = DMLocalToGlobalEnd(dmf, flocal, INSERT_VALUES, fglobal);CHKERRQ(ierr);
9133   *shift = fglobal;
9134   ierr = VecDestroy(&flocal);CHKERRQ(ierr);
9135   ierr = VecDestroy(&fglobalo);CHKERRQ(ierr);
9136   ierr = VecDestroy(&clocal);CHKERRQ(ierr);
9137   ierr = VecDestroy(&cglobalo);CHKERRQ(ierr);
9138   ierr = VecDestroy(&rscale);CHKERRQ(ierr);
9139   ierr = MatDestroy(&interpo);CHKERRQ(ierr);
9140   ierr = DMDestroy(&dmfo);CHKERRQ(ierr);
9141   ierr = DMDestroy(&dmco);CHKERRQ(ierr);
9142   PetscFunctionReturn(0);
9143 }
9144 
9145 PETSC_INTERN PetscErrorCode DMInterpolateSolution_Plex(DM coarse, DM fine, Mat interp, Vec coarseSol, Vec fineSol)
9146 {
9147   PetscObject    shifto;
9148   Vec            shift;
9149 
9150   PetscErrorCode ierr;
9151 
9152   PetscFunctionBegin;
9153   if (!interp) {
9154     Vec rscale;
9155 
9156     ierr = DMCreateInterpolation(coarse, fine, &interp, &rscale);CHKERRQ(ierr);
9157     ierr = VecDestroy(&rscale);CHKERRQ(ierr);
9158   } else {
9159     ierr = PetscObjectReference((PetscObject)interp);CHKERRQ(ierr);
9160   }
9161   ierr = PetscObjectQuery((PetscObject)interp, "_DMInterpolateSolution_Plex_Vec", &shifto);CHKERRQ(ierr);
9162   if (!shifto) {
9163     ierr = DMCreateAffineInterpolationCorrection_Plex(coarse, fine, &shift);CHKERRQ(ierr);
9164     ierr = PetscObjectCompose((PetscObject)interp, "_DMInterpolateSolution_Plex_Vec", (PetscObject) shift);CHKERRQ(ierr);
9165     shifto = (PetscObject) shift;
9166     ierr = VecDestroy(&shift);CHKERRQ(ierr);
9167   }
9168   shift = (Vec) shifto;
9169   ierr = MatInterpolate(interp, coarseSol, fineSol);CHKERRQ(ierr);
9170   ierr = VecAXPY(fineSol, 1.0, shift);CHKERRQ(ierr);
9171   ierr = MatDestroy(&interp);CHKERRQ(ierr);
9172   PetscFunctionReturn(0);
9173 }
9174 
9175 /* Pointwise interpolation
9176      Just code FEM for now
9177      u^f = I u^c
9178      sum_k u^f_k phi^f_k = I sum_j u^c_j phi^c_j
9179      u^f_i = sum_j psi^f_i I phi^c_j u^c_j
9180      I_{ij} = psi^f_i phi^c_j
9181 */
9182 PetscErrorCode DMCreateInterpolation_Plex(DM dmCoarse, DM dmFine, Mat *interpolation, Vec *scaling)
9183 {
9184   PetscSection   gsc, gsf;
9185   PetscInt       m, n;
9186   void          *ctx;
9187   DM             cdm;
9188   PetscBool      regular, ismatis, isRefined = dmCoarse->data == dmFine->data ? PETSC_FALSE : PETSC_TRUE;
9189   PetscErrorCode ierr;
9190 
9191   PetscFunctionBegin;
9192   ierr = DMGetGlobalSection(dmFine, &gsf);CHKERRQ(ierr);
9193   ierr = PetscSectionGetConstrainedStorageSize(gsf, &m);CHKERRQ(ierr);
9194   ierr = DMGetGlobalSection(dmCoarse, &gsc);CHKERRQ(ierr);
9195   ierr = PetscSectionGetConstrainedStorageSize(gsc, &n);CHKERRQ(ierr);
9196 
9197   ierr = PetscStrcmp(dmCoarse->mattype, MATIS, &ismatis);CHKERRQ(ierr);
9198   ierr = MatCreate(PetscObjectComm((PetscObject) dmCoarse), interpolation);CHKERRQ(ierr);
9199   ierr = MatSetSizes(*interpolation, m, n, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
9200   ierr = MatSetType(*interpolation, ismatis ? MATAIJ : dmCoarse->mattype);CHKERRQ(ierr);
9201   ierr = DMGetApplicationContext(dmFine, &ctx);CHKERRQ(ierr);
9202 
9203   ierr = DMGetCoarseDM(dmFine, &cdm);CHKERRQ(ierr);
9204   ierr = DMPlexGetRegularRefinement(dmFine, &regular);CHKERRQ(ierr);
9205   if (!isRefined || (regular && cdm == dmCoarse)) {ierr = DMPlexComputeInterpolatorNested(dmCoarse, dmFine, isRefined, *interpolation, ctx);CHKERRQ(ierr);}
9206   else                                            {ierr = DMPlexComputeInterpolatorGeneral(dmCoarse, dmFine, *interpolation, ctx);CHKERRQ(ierr);}
9207   ierr = MatViewFromOptions(*interpolation, NULL, "-interp_mat_view");CHKERRQ(ierr);
9208   if (scaling) {
9209     /* Use naive scaling */
9210     ierr = DMCreateInterpolationScale(dmCoarse, dmFine, *interpolation, scaling);CHKERRQ(ierr);
9211   }
9212   PetscFunctionReturn(0);
9213 }
9214 
9215 PetscErrorCode DMCreateInjection_Plex(DM dmCoarse, DM dmFine, Mat *mat)
9216 {
9217   PetscErrorCode ierr;
9218   VecScatter     ctx;
9219 
9220   PetscFunctionBegin;
9221   ierr = DMPlexComputeInjectorFEM(dmCoarse, dmFine, &ctx, NULL);CHKERRQ(ierr);
9222   ierr = MatCreateScatter(PetscObjectComm((PetscObject)ctx), ctx, mat);CHKERRQ(ierr);
9223   ierr = VecScatterDestroy(&ctx);CHKERRQ(ierr);
9224   PetscFunctionReturn(0);
9225 }
9226 
9227 static void g0_identity_private(PetscInt dim, PetscInt Nf, PetscInt NfAux,
9228                                 const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[],
9229                                 const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[],
9230                                 PetscReal t, PetscReal u_tShift, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar g0[])
9231 {
9232   g0[0] = 1.0;
9233 }
9234 
9235 PetscErrorCode DMCreateMassMatrix_Plex(DM dmCoarse, DM dmFine, Mat *mass)
9236 {
9237   PetscSection   gsc, gsf;
9238   PetscInt       m, n;
9239   void          *ctx;
9240   DM             cdm;
9241   PetscBool      regular;
9242   PetscErrorCode ierr;
9243 
9244   PetscFunctionBegin;
9245   if (dmFine == dmCoarse) {
9246     DM       dmc;
9247     PetscDS  ds;
9248     Vec      u;
9249     IS       cellIS;
9250     PetscFormKey key;
9251     PetscInt depth;
9252 
9253     ierr = DMClone(dmFine, &dmc);CHKERRQ(ierr);
9254     ierr = DMCopyDisc(dmFine, dmc);CHKERRQ(ierr);
9255     ierr = DMGetDS(dmc, &ds);CHKERRQ(ierr);
9256     ierr = PetscDSSetJacobian(ds, 0, 0, g0_identity_private, NULL, NULL, NULL);CHKERRQ(ierr);
9257     ierr = DMCreateMatrix(dmc, mass);CHKERRQ(ierr);
9258     ierr = DMGetGlobalVector(dmc, &u);CHKERRQ(ierr);
9259     ierr = DMPlexGetDepth(dmc, &depth);CHKERRQ(ierr);
9260     ierr = DMGetStratumIS(dmc, "depth", depth, &cellIS);CHKERRQ(ierr);
9261     ierr = MatZeroEntries(*mass);CHKERRQ(ierr);
9262     key.label = NULL;
9263     key.value = 0;
9264     key.field = 0;
9265     key.part  = 0;
9266     ierr = DMPlexComputeJacobian_Internal(dmc, key, cellIS, 0.0, 0.0, u, NULL, *mass, *mass, NULL);CHKERRQ(ierr);
9267     ierr = ISDestroy(&cellIS);CHKERRQ(ierr);
9268     ierr = DMRestoreGlobalVector(dmc, &u);CHKERRQ(ierr);
9269     ierr = DMDestroy(&dmc);CHKERRQ(ierr);
9270   } else {
9271     ierr = DMGetGlobalSection(dmFine, &gsf);CHKERRQ(ierr);
9272     ierr = PetscSectionGetConstrainedStorageSize(gsf, &m);CHKERRQ(ierr);
9273     ierr = DMGetGlobalSection(dmCoarse, &gsc);CHKERRQ(ierr);
9274     ierr = PetscSectionGetConstrainedStorageSize(gsc, &n);CHKERRQ(ierr);
9275 
9276     ierr = MatCreate(PetscObjectComm((PetscObject) dmCoarse), mass);CHKERRQ(ierr);
9277     ierr = MatSetSizes(*mass, m, n, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
9278     ierr = MatSetType(*mass, dmCoarse->mattype);CHKERRQ(ierr);
9279     ierr = DMGetApplicationContext(dmFine, &ctx);CHKERRQ(ierr);
9280 
9281     ierr = DMGetCoarseDM(dmFine, &cdm);CHKERRQ(ierr);
9282     ierr = DMPlexGetRegularRefinement(dmFine, &regular);CHKERRQ(ierr);
9283     if (regular && cdm == dmCoarse) {ierr = DMPlexComputeMassMatrixNested(dmCoarse, dmFine, *mass, ctx);CHKERRQ(ierr);}
9284     else                            {ierr = DMPlexComputeMassMatrixGeneral(dmCoarse, dmFine, *mass, ctx);CHKERRQ(ierr);}
9285   }
9286   ierr = MatViewFromOptions(*mass, NULL, "-mass_mat_view");CHKERRQ(ierr);
9287   PetscFunctionReturn(0);
9288 }
9289 
9290 /*@
9291   DMPlexGetRegularRefinement - Get the flag indicating that this mesh was obtained by regular refinement from its coarse mesh
9292 
9293   Input Parameter:
9294 . dm - The DMPlex object
9295 
9296   Output Parameter:
9297 . regular - The flag
9298 
9299   Level: intermediate
9300 
9301 .seealso: DMPlexSetRegularRefinement()
9302 @*/
9303 PetscErrorCode DMPlexGetRegularRefinement(DM dm, PetscBool *regular)
9304 {
9305   PetscFunctionBegin;
9306   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9307   PetscValidPointer(regular, 2);
9308   *regular = ((DM_Plex *) dm->data)->regularRefinement;
9309   PetscFunctionReturn(0);
9310 }
9311 
9312 /*@
9313   DMPlexSetRegularRefinement - Set the flag indicating that this mesh was obtained by regular refinement from its coarse mesh
9314 
9315   Input Parameters:
9316 + dm - The DMPlex object
9317 - regular - The flag
9318 
9319   Level: intermediate
9320 
9321 .seealso: DMPlexGetRegularRefinement()
9322 @*/
9323 PetscErrorCode DMPlexSetRegularRefinement(DM dm, PetscBool regular)
9324 {
9325   PetscFunctionBegin;
9326   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9327   ((DM_Plex *) dm->data)->regularRefinement = regular;
9328   PetscFunctionReturn(0);
9329 }
9330 
9331 /* anchors */
9332 /*@
9333   DMPlexGetAnchors - Get the layout of the anchor (point-to-point) constraints.  Typically, the user will not have to
9334   call DMPlexGetAnchors() directly: if there are anchors, then DMPlexGetAnchors() is called during DMGetConstraints().
9335 
9336   not collective
9337 
9338   Input Parameter:
9339 . dm - The DMPlex object
9340 
9341   Output Parameters:
9342 + anchorSection - If not NULL, set to the section describing which points anchor the constrained points.
9343 - anchorIS - If not NULL, set to the list of anchors indexed by anchorSection
9344 
9345   Level: intermediate
9346 
9347 .seealso: DMPlexSetAnchors(), DMGetConstraints(), DMSetConstraints()
9348 @*/
9349 PetscErrorCode DMPlexGetAnchors(DM dm, PetscSection *anchorSection, IS *anchorIS)
9350 {
9351   DM_Plex *plex = (DM_Plex *)dm->data;
9352   PetscErrorCode ierr;
9353 
9354   PetscFunctionBegin;
9355   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9356   if (!plex->anchorSection && !plex->anchorIS && plex->createanchors) {ierr = (*plex->createanchors)(dm);CHKERRQ(ierr);}
9357   if (anchorSection) *anchorSection = plex->anchorSection;
9358   if (anchorIS) *anchorIS = plex->anchorIS;
9359   PetscFunctionReturn(0);
9360 }
9361 
9362 /*@
9363   DMPlexSetAnchors - Set the layout of the local anchor (point-to-point) constraints.  Unlike boundary conditions,
9364   when a point's degrees of freedom in a section are constrained to an outside value, the anchor constraints set a
9365   point's degrees of freedom to be a linear combination of other points' degrees of freedom.
9366 
9367   After specifying the layout of constraints with DMPlexSetAnchors(), one specifies the constraints by calling
9368   DMGetConstraints() and filling in the entries in the constraint matrix.
9369 
9370   collective on dm
9371 
9372   Input Parameters:
9373 + dm - The DMPlex object
9374 . anchorSection - The section that describes the mapping from constrained points to the anchor points listed in anchorIS.  Must have a local communicator (PETSC_COMM_SELF or derivative).
9375 - anchorIS - The list of all anchor points.  Must have a local communicator (PETSC_COMM_SELF or derivative).
9376 
9377   The reference counts of anchorSection and anchorIS are incremented.
9378 
9379   Level: intermediate
9380 
9381 .seealso: DMPlexGetAnchors(), DMGetConstraints(), DMSetConstraints()
9382 @*/
9383 PetscErrorCode DMPlexSetAnchors(DM dm, PetscSection anchorSection, IS anchorIS)
9384 {
9385   DM_Plex        *plex = (DM_Plex *)dm->data;
9386   PetscMPIInt    result;
9387   PetscErrorCode ierr;
9388 
9389   PetscFunctionBegin;
9390   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9391   if (anchorSection) {
9392     PetscValidHeaderSpecific(anchorSection,PETSC_SECTION_CLASSID,2);
9393     ierr = MPI_Comm_compare(PETSC_COMM_SELF,PetscObjectComm((PetscObject)anchorSection),&result);CHKERRMPI(ierr);
9394     if (result != MPI_CONGRUENT && result != MPI_IDENT) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_NOTSAMECOMM,"anchor section must have local communicator");
9395   }
9396   if (anchorIS) {
9397     PetscValidHeaderSpecific(anchorIS,IS_CLASSID,3);
9398     ierr = MPI_Comm_compare(PETSC_COMM_SELF,PetscObjectComm((PetscObject)anchorIS),&result);CHKERRMPI(ierr);
9399     if (result != MPI_CONGRUENT && result != MPI_IDENT) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_NOTSAMECOMM,"anchor IS must have local communicator");
9400   }
9401 
9402   ierr = PetscObjectReference((PetscObject)anchorSection);CHKERRQ(ierr);
9403   ierr = PetscSectionDestroy(&plex->anchorSection);CHKERRQ(ierr);
9404   plex->anchorSection = anchorSection;
9405 
9406   ierr = PetscObjectReference((PetscObject)anchorIS);CHKERRQ(ierr);
9407   ierr = ISDestroy(&plex->anchorIS);CHKERRQ(ierr);
9408   plex->anchorIS = anchorIS;
9409 
9410   if (PetscUnlikelyDebug(anchorIS && anchorSection)) {
9411     PetscInt size, a, pStart, pEnd;
9412     const PetscInt *anchors;
9413 
9414     ierr = PetscSectionGetChart(anchorSection,&pStart,&pEnd);CHKERRQ(ierr);
9415     ierr = ISGetLocalSize(anchorIS,&size);CHKERRQ(ierr);
9416     ierr = ISGetIndices(anchorIS,&anchors);CHKERRQ(ierr);
9417     for (a = 0; a < size; a++) {
9418       PetscInt p;
9419 
9420       p = anchors[a];
9421       if (p >= pStart && p < pEnd) {
9422         PetscInt dof;
9423 
9424         ierr = PetscSectionGetDof(anchorSection,p,&dof);CHKERRQ(ierr);
9425         if (dof) {
9426           PetscErrorCode ierr2;
9427 
9428           ierr2 = ISRestoreIndices(anchorIS,&anchors);CHKERRQ(ierr2);
9429           SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_INCOMP,"Point %D cannot be constrained and an anchor",p);
9430         }
9431       }
9432     }
9433     ierr = ISRestoreIndices(anchorIS,&anchors);CHKERRQ(ierr);
9434   }
9435   /* reset the generic constraints */
9436   ierr = DMSetDefaultConstraints(dm,NULL,NULL);CHKERRQ(ierr);
9437   PetscFunctionReturn(0);
9438 }
9439 
9440 static PetscErrorCode DMPlexCreateConstraintSection_Anchors(DM dm, PetscSection section, PetscSection *cSec)
9441 {
9442   PetscSection anchorSection;
9443   PetscInt pStart, pEnd, sStart, sEnd, p, dof, numFields, f;
9444   PetscErrorCode ierr;
9445 
9446   PetscFunctionBegin;
9447   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9448   ierr = DMPlexGetAnchors(dm,&anchorSection,NULL);CHKERRQ(ierr);
9449   ierr = PetscSectionCreate(PETSC_COMM_SELF,cSec);CHKERRQ(ierr);
9450   ierr = PetscSectionGetNumFields(section,&numFields);CHKERRQ(ierr);
9451   if (numFields) {
9452     PetscInt f;
9453     ierr = PetscSectionSetNumFields(*cSec,numFields);CHKERRQ(ierr);
9454 
9455     for (f = 0; f < numFields; f++) {
9456       PetscInt numComp;
9457 
9458       ierr = PetscSectionGetFieldComponents(section,f,&numComp);CHKERRQ(ierr);
9459       ierr = PetscSectionSetFieldComponents(*cSec,f,numComp);CHKERRQ(ierr);
9460     }
9461   }
9462   ierr = PetscSectionGetChart(anchorSection,&pStart,&pEnd);CHKERRQ(ierr);
9463   ierr = PetscSectionGetChart(section,&sStart,&sEnd);CHKERRQ(ierr);
9464   pStart = PetscMax(pStart,sStart);
9465   pEnd   = PetscMin(pEnd,sEnd);
9466   pEnd   = PetscMax(pStart,pEnd);
9467   ierr = PetscSectionSetChart(*cSec,pStart,pEnd);CHKERRQ(ierr);
9468   for (p = pStart; p < pEnd; p++) {
9469     ierr = PetscSectionGetDof(anchorSection,p,&dof);CHKERRQ(ierr);
9470     if (dof) {
9471       ierr = PetscSectionGetDof(section,p,&dof);CHKERRQ(ierr);
9472       ierr = PetscSectionSetDof(*cSec,p,dof);CHKERRQ(ierr);
9473       for (f = 0; f < numFields; f++) {
9474         ierr = PetscSectionGetFieldDof(section,p,f,&dof);CHKERRQ(ierr);
9475         ierr = PetscSectionSetFieldDof(*cSec,p,f,dof);CHKERRQ(ierr);
9476       }
9477     }
9478   }
9479   ierr = PetscSectionSetUp(*cSec);CHKERRQ(ierr);
9480   ierr = PetscObjectSetName((PetscObject) *cSec, "Constraint Section");CHKERRQ(ierr);
9481   PetscFunctionReturn(0);
9482 }
9483 
9484 static PetscErrorCode DMPlexCreateConstraintMatrix_Anchors(DM dm, PetscSection section, PetscSection cSec, Mat *cMat)
9485 {
9486   PetscSection   aSec;
9487   PetscInt       pStart, pEnd, p, sStart, sEnd, dof, aDof, aOff, off, nnz, annz, m, n, q, a, offset, *i, *j;
9488   const PetscInt *anchors;
9489   PetscInt       numFields, f;
9490   IS             aIS;
9491   PetscErrorCode ierr;
9492   MatType        mtype;
9493   PetscBool      iscuda,iskokkos;
9494 
9495   PetscFunctionBegin;
9496   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9497   ierr = PetscSectionGetStorageSize(cSec, &m);CHKERRQ(ierr);
9498   ierr = PetscSectionGetStorageSize(section, &n);CHKERRQ(ierr);
9499   ierr = MatCreate(PETSC_COMM_SELF,cMat);CHKERRQ(ierr);
9500   ierr = MatSetSizes(*cMat,m,n,m,n);CHKERRQ(ierr);
9501   ierr = PetscStrcmp(dm->mattype,MATSEQAIJCUSPARSE,&iscuda);CHKERRQ(ierr);
9502   if (!iscuda) { ierr = PetscStrcmp(dm->mattype,MATMPIAIJCUSPARSE,&iscuda);CHKERRQ(ierr); }
9503   ierr = PetscStrcmp(dm->mattype,MATSEQAIJKOKKOS,&iskokkos);CHKERRQ(ierr);
9504   if (!iskokkos) { ierr = PetscStrcmp(dm->mattype,MATMPIAIJKOKKOS,&iskokkos);CHKERRQ(ierr); }
9505   if (iscuda) mtype = MATSEQAIJCUSPARSE;
9506   else if (iskokkos) mtype = MATSEQAIJKOKKOS;
9507   else mtype = MATSEQAIJ;
9508   ierr = MatSetType(*cMat,mtype);CHKERRQ(ierr);
9509   ierr = DMPlexGetAnchors(dm,&aSec,&aIS);CHKERRQ(ierr);
9510   ierr = ISGetIndices(aIS,&anchors);CHKERRQ(ierr);
9511   /* cSec will be a subset of aSec and section */
9512   ierr = PetscSectionGetChart(cSec,&pStart,&pEnd);CHKERRQ(ierr);
9513   ierr = PetscSectionGetChart(section,&sStart,&sEnd);CHKERRQ(ierr);
9514   ierr = PetscMalloc1(m+1,&i);CHKERRQ(ierr);
9515   i[0] = 0;
9516   ierr = PetscSectionGetNumFields(section,&numFields);CHKERRQ(ierr);
9517   for (p = pStart; p < pEnd; p++) {
9518     PetscInt rDof, rOff, r;
9519 
9520     ierr = PetscSectionGetDof(aSec,p,&rDof);CHKERRQ(ierr);
9521     if (!rDof) continue;
9522     ierr = PetscSectionGetOffset(aSec,p,&rOff);CHKERRQ(ierr);
9523     if (numFields) {
9524       for (f = 0; f < numFields; f++) {
9525         annz = 0;
9526         for (r = 0; r < rDof; r++) {
9527           a = anchors[rOff + r];
9528           if (a < sStart || a >= sEnd) continue;
9529           ierr = PetscSectionGetFieldDof(section,a,f,&aDof);CHKERRQ(ierr);
9530           annz += aDof;
9531         }
9532         ierr = PetscSectionGetFieldDof(cSec,p,f,&dof);CHKERRQ(ierr);
9533         ierr = PetscSectionGetFieldOffset(cSec,p,f,&off);CHKERRQ(ierr);
9534         for (q = 0; q < dof; q++) {
9535           i[off + q + 1] = i[off + q] + annz;
9536         }
9537       }
9538     } else {
9539       annz = 0;
9540       ierr = PetscSectionGetDof(cSec,p,&dof);CHKERRQ(ierr);
9541       for (q = 0; q < dof; q++) {
9542         a = anchors[rOff + q];
9543         if (a < sStart || a >= sEnd) continue;
9544         ierr = PetscSectionGetDof(section,a,&aDof);CHKERRQ(ierr);
9545         annz += aDof;
9546       }
9547       ierr = PetscSectionGetDof(cSec,p,&dof);CHKERRQ(ierr);
9548       ierr = PetscSectionGetOffset(cSec,p,&off);CHKERRQ(ierr);
9549       for (q = 0; q < dof; q++) {
9550         i[off + q + 1] = i[off + q] + annz;
9551       }
9552     }
9553   }
9554   nnz = i[m];
9555   ierr = PetscMalloc1(nnz,&j);CHKERRQ(ierr);
9556   offset = 0;
9557   for (p = pStart; p < pEnd; p++) {
9558     if (numFields) {
9559       for (f = 0; f < numFields; f++) {
9560         ierr = PetscSectionGetFieldDof(cSec,p,f,&dof);CHKERRQ(ierr);
9561         for (q = 0; q < dof; q++) {
9562           PetscInt rDof, rOff, r;
9563           ierr = PetscSectionGetDof(aSec,p,&rDof);CHKERRQ(ierr);
9564           ierr = PetscSectionGetOffset(aSec,p,&rOff);CHKERRQ(ierr);
9565           for (r = 0; r < rDof; r++) {
9566             PetscInt s;
9567 
9568             a = anchors[rOff + r];
9569             if (a < sStart || a >= sEnd) continue;
9570             ierr = PetscSectionGetFieldDof(section,a,f,&aDof);CHKERRQ(ierr);
9571             ierr = PetscSectionGetFieldOffset(section,a,f,&aOff);CHKERRQ(ierr);
9572             for (s = 0; s < aDof; s++) {
9573               j[offset++] = aOff + s;
9574             }
9575           }
9576         }
9577       }
9578     } else {
9579       ierr = PetscSectionGetDof(cSec,p,&dof);CHKERRQ(ierr);
9580       for (q = 0; q < dof; q++) {
9581         PetscInt rDof, rOff, r;
9582         ierr = PetscSectionGetDof(aSec,p,&rDof);CHKERRQ(ierr);
9583         ierr = PetscSectionGetOffset(aSec,p,&rOff);CHKERRQ(ierr);
9584         for (r = 0; r < rDof; r++) {
9585           PetscInt s;
9586 
9587           a = anchors[rOff + r];
9588           if (a < sStart || a >= sEnd) continue;
9589           ierr = PetscSectionGetDof(section,a,&aDof);CHKERRQ(ierr);
9590           ierr = PetscSectionGetOffset(section,a,&aOff);CHKERRQ(ierr);
9591           for (s = 0; s < aDof; s++) {
9592             j[offset++] = aOff + s;
9593           }
9594         }
9595       }
9596     }
9597   }
9598   ierr = MatSeqAIJSetPreallocationCSR(*cMat,i,j,NULL);CHKERRQ(ierr);
9599   ierr = PetscFree(i);CHKERRQ(ierr);
9600   ierr = PetscFree(j);CHKERRQ(ierr);
9601   ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);
9602   PetscFunctionReturn(0);
9603 }
9604 
9605 PetscErrorCode DMCreateDefaultConstraints_Plex(DM dm)
9606 {
9607   DM_Plex        *plex = (DM_Plex *)dm->data;
9608   PetscSection   anchorSection, section, cSec;
9609   Mat            cMat;
9610   PetscErrorCode ierr;
9611 
9612   PetscFunctionBegin;
9613   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9614   ierr = DMPlexGetAnchors(dm,&anchorSection,NULL);CHKERRQ(ierr);
9615   if (anchorSection) {
9616     PetscInt Nf;
9617 
9618     ierr = DMGetLocalSection(dm,&section);CHKERRQ(ierr);
9619     ierr = DMPlexCreateConstraintSection_Anchors(dm,section,&cSec);CHKERRQ(ierr);
9620     ierr = DMPlexCreateConstraintMatrix_Anchors(dm,section,cSec,&cMat);CHKERRQ(ierr);
9621     ierr = DMGetNumFields(dm,&Nf);CHKERRQ(ierr);
9622     if (Nf && plex->computeanchormatrix) {ierr = (*plex->computeanchormatrix)(dm,section,cSec,cMat);CHKERRQ(ierr);}
9623     ierr = DMSetDefaultConstraints(dm,cSec,cMat);CHKERRQ(ierr);
9624     ierr = PetscSectionDestroy(&cSec);CHKERRQ(ierr);
9625     ierr = MatDestroy(&cMat);CHKERRQ(ierr);
9626   }
9627   PetscFunctionReturn(0);
9628 }
9629 
9630 PetscErrorCode DMCreateSubDomainDM_Plex(DM dm, DMLabel label, PetscInt value, IS *is, DM *subdm)
9631 {
9632   IS             subis;
9633   PetscSection   section, subsection;
9634   PetscErrorCode ierr;
9635 
9636   PetscFunctionBegin;
9637   ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);
9638   if (!section) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Must set default section for DM before splitting subdomain");
9639   if (!subdm)   SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Must set output subDM for splitting subdomain");
9640   /* Create subdomain */
9641   ierr = DMPlexFilter(dm, label, value, subdm);CHKERRQ(ierr);
9642   /* Create submodel */
9643   ierr = DMPlexGetSubpointIS(*subdm, &subis);CHKERRQ(ierr);
9644   ierr = PetscSectionCreateSubmeshSection(section, subis, &subsection);CHKERRQ(ierr);
9645   ierr = DMSetLocalSection(*subdm, subsection);CHKERRQ(ierr);
9646   ierr = PetscSectionDestroy(&subsection);CHKERRQ(ierr);
9647   ierr = DMCopyDisc(dm, *subdm);CHKERRQ(ierr);
9648   /* Create map from submodel to global model */
9649   if (is) {
9650     PetscSection    sectionGlobal, subsectionGlobal;
9651     IS              spIS;
9652     const PetscInt *spmap;
9653     PetscInt       *subIndices;
9654     PetscInt        subSize = 0, subOff = 0, pStart, pEnd, p;
9655     PetscInt        Nf, f, bs = -1, bsLocal[2], bsMinMax[2];
9656 
9657     ierr = DMPlexGetSubpointIS(*subdm, &spIS);CHKERRQ(ierr);
9658     ierr = ISGetIndices(spIS, &spmap);CHKERRQ(ierr);
9659     ierr = PetscSectionGetNumFields(section, &Nf);CHKERRQ(ierr);
9660     ierr = DMGetGlobalSection(dm, &sectionGlobal);CHKERRQ(ierr);
9661     ierr = DMGetGlobalSection(*subdm, &subsectionGlobal);CHKERRQ(ierr);
9662     ierr = PetscSectionGetChart(subsection, &pStart, &pEnd);CHKERRQ(ierr);
9663     for (p = pStart; p < pEnd; ++p) {
9664       PetscInt gdof, pSubSize  = 0;
9665 
9666       ierr = PetscSectionGetDof(sectionGlobal, p, &gdof);CHKERRQ(ierr);
9667       if (gdof > 0) {
9668         for (f = 0; f < Nf; ++f) {
9669           PetscInt fdof, fcdof;
9670 
9671           ierr     = PetscSectionGetFieldDof(subsection, p, f, &fdof);CHKERRQ(ierr);
9672           ierr     = PetscSectionGetFieldConstraintDof(subsection, p, f, &fcdof);CHKERRQ(ierr);
9673           pSubSize += fdof-fcdof;
9674         }
9675         subSize += pSubSize;
9676         if (pSubSize) {
9677           if (bs < 0) {
9678             bs = pSubSize;
9679           } else if (bs != pSubSize) {
9680             /* Layout does not admit a pointwise block size */
9681             bs = 1;
9682           }
9683         }
9684       }
9685     }
9686     /* Must have same blocksize on all procs (some might have no points) */
9687     bsLocal[0] = bs < 0 ? PETSC_MAX_INT : bs; bsLocal[1] = bs;
9688     ierr = PetscGlobalMinMaxInt(PetscObjectComm((PetscObject) dm), bsLocal, bsMinMax);CHKERRQ(ierr);
9689     if (bsMinMax[0] != bsMinMax[1]) {bs = 1;}
9690     else                            {bs = bsMinMax[0];}
9691     ierr = PetscMalloc1(subSize, &subIndices);CHKERRQ(ierr);
9692     for (p = pStart; p < pEnd; ++p) {
9693       PetscInt gdof, goff;
9694 
9695       ierr = PetscSectionGetDof(subsectionGlobal, p, &gdof);CHKERRQ(ierr);
9696       if (gdof > 0) {
9697         const PetscInt point = spmap[p];
9698 
9699         ierr = PetscSectionGetOffset(sectionGlobal, point, &goff);CHKERRQ(ierr);
9700         for (f = 0; f < Nf; ++f) {
9701           PetscInt fdof, fcdof, fc, f2, poff = 0;
9702 
9703           /* Can get rid of this loop by storing field information in the global section */
9704           for (f2 = 0; f2 < f; ++f2) {
9705             ierr  = PetscSectionGetFieldDof(section, p, f2, &fdof);CHKERRQ(ierr);
9706             ierr  = PetscSectionGetFieldConstraintDof(section, p, f2, &fcdof);CHKERRQ(ierr);
9707             poff += fdof-fcdof;
9708           }
9709           ierr = PetscSectionGetFieldDof(section, p, f, &fdof);CHKERRQ(ierr);
9710           ierr = PetscSectionGetFieldConstraintDof(section, p, f, &fcdof);CHKERRQ(ierr);
9711           for (fc = 0; fc < fdof-fcdof; ++fc, ++subOff) {
9712             subIndices[subOff] = goff+poff+fc;
9713           }
9714         }
9715       }
9716     }
9717     ierr = ISRestoreIndices(spIS, &spmap);CHKERRQ(ierr);
9718     ierr = ISCreateGeneral(PetscObjectComm((PetscObject)dm), subSize, subIndices, PETSC_OWN_POINTER, is);CHKERRQ(ierr);
9719     if (bs > 1) {
9720       /* We need to check that the block size does not come from non-contiguous fields */
9721       PetscInt i, j, set = 1;
9722       for (i = 0; i < subSize; i += bs) {
9723         for (j = 0; j < bs; ++j) {
9724           if (subIndices[i+j] != subIndices[i]+j) {set = 0; break;}
9725         }
9726       }
9727       if (set) {ierr = ISSetBlockSize(*is, bs);CHKERRQ(ierr);}
9728     }
9729     /* Attach nullspace */
9730     for (f = 0; f < Nf; ++f) {
9731       (*subdm)->nullspaceConstructors[f] = dm->nullspaceConstructors[f];
9732       if ((*subdm)->nullspaceConstructors[f]) break;
9733     }
9734     if (f < Nf) {
9735       MatNullSpace nullSpace;
9736       ierr = (*(*subdm)->nullspaceConstructors[f])(*subdm, f, f, &nullSpace);CHKERRQ(ierr);
9737 
9738       ierr = PetscObjectCompose((PetscObject) *is, "nullspace", (PetscObject) nullSpace);CHKERRQ(ierr);
9739       ierr = MatNullSpaceDestroy(&nullSpace);CHKERRQ(ierr);
9740     }
9741   }
9742   PetscFunctionReturn(0);
9743 }
9744 
9745 /*@
9746   DMPlexMonitorThroughput - Report the cell throughput of FE integration
9747 
9748   Input Parameter:
9749 - dm - The DM
9750 
9751   Level: developer
9752 
9753   Options Database Keys:
9754 . -dm_plex_monitor_throughput - Activate the monitor
9755 
9756 .seealso: DMSetFromOptions(), DMPlexCreate()
9757 @*/
9758 PetscErrorCode DMPlexMonitorThroughput(DM dm, void *dummy)
9759 {
9760 #if defined(PETSC_USE_LOG)
9761   PetscStageLog      stageLog;
9762   PetscLogEvent      event;
9763   PetscLogStage      stage;
9764   PetscEventPerfInfo eventInfo;
9765   PetscReal          cellRate, flopRate;
9766   PetscInt           cStart, cEnd, Nf, N;
9767   const char        *name;
9768   PetscErrorCode     ierr;
9769 #endif
9770 
9771   PetscFunctionBegin;
9772   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9773 #if defined(PETSC_USE_LOG)
9774   ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
9775   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
9776   ierr = DMGetNumFields(dm, &Nf);CHKERRQ(ierr);
9777   ierr = PetscLogGetStageLog(&stageLog);CHKERRQ(ierr);
9778   ierr = PetscStageLogGetCurrent(stageLog, &stage);CHKERRQ(ierr);
9779   ierr = PetscLogEventGetId("DMPlexResidualFE", &event);CHKERRQ(ierr);
9780   ierr = PetscLogEventGetPerfInfo(stage, event, &eventInfo);CHKERRQ(ierr);
9781   N        = (cEnd - cStart)*Nf*eventInfo.count;
9782   flopRate = eventInfo.flops/eventInfo.time;
9783   cellRate = N/eventInfo.time;
9784   ierr = PetscPrintf(PetscObjectComm((PetscObject) dm), "DM (%s) FE Residual Integration: %D integrals %D reps\n  Cell rate: %.2g/s flop rate: %.2g MF/s\n", name ? name : "unknown", N, eventInfo.count, (double) cellRate, (double) (flopRate/1.e6));CHKERRQ(ierr);
9785 #else
9786   SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Plex Throughput Monitor is not supported if logging is turned off. Reconfigure using --with-log.");
9787 #endif
9788   PetscFunctionReturn(0);
9789 }
9790