xref: /petsc/src/dm/impls/plex/plex.c (revision 6913077d92d45dc85e2cd558d8cf01c8c45125fc) ! 
1 #include <petsc/private/dmpleximpl.h>   /*I      "petscdmplex.h"   I*/
2 #include <petsc/private/isimpl.h>
3 #include <petsc/private/vecimpl.h>
4 #include <petsc/private/glvisvecimpl.h>
5 #include <petscsf.h>
6 #include <petscds.h>
7 #include <petscdraw.h>
8 #include <petscdmfield.h>
9 #include <petscdmplextransform.h>
10 
11 /* Logging support */
12 PetscLogEvent DMPLEX_Interpolate, DMPLEX_Partition, DMPLEX_Distribute, DMPLEX_DistributeCones, DMPLEX_DistributeLabels, DMPLEX_DistributeSF, DMPLEX_DistributeOverlap, DMPLEX_DistributeField, DMPLEX_DistributeData, DMPLEX_Migrate, DMPLEX_InterpolateSF, DMPLEX_GlobalToNaturalBegin, DMPLEX_GlobalToNaturalEnd, DMPLEX_NaturalToGlobalBegin, DMPLEX_NaturalToGlobalEnd, DMPLEX_Stratify, DMPLEX_Symmetrize, DMPLEX_Preallocate, DMPLEX_ResidualFEM, DMPLEX_JacobianFEM, DMPLEX_InterpolatorFEM, DMPLEX_InjectorFEM, DMPLEX_IntegralFEM, DMPLEX_CreateGmsh, DMPLEX_RebalanceSharedPoints, DMPLEX_PartSelf, DMPLEX_PartLabelInvert, DMPLEX_PartLabelCreateSF, DMPLEX_PartStratSF, DMPLEX_CreatePointSF,DMPLEX_LocatePoints,DMPLEX_TopologyView,DMPLEX_LabelsView,DMPLEX_CoordinatesView,DMPLEX_SectionView,DMPLEX_GlobalVectorView,DMPLEX_LocalVectorView,DMPLEX_TopologyLoad,DMPLEX_LabelsLoad,DMPLEX_CoordinatesLoad,DMPLEX_SectionLoad,DMPLEX_GlobalVectorLoad,DMPLEX_LocalVectorLoad;
13 
14 PETSC_EXTERN PetscErrorCode VecView_MPI(Vec, PetscViewer);
15 
16 /*@
17   DMPlexIsSimplex - Is the first cell in this mesh a simplex?
18 
19   Input Parameter:
20 . dm      - The DMPlex object
21 
22   Output Parameter:
23 . simplex - Flag checking for a simplex
24 
25   Note: This just gives the first range of cells found. If the mesh has several cell types, it will only give the first.
26   If the mesh has no cells, this returns PETSC_FALSE.
27 
28   Level: intermediate
29 
30 .seealso DMPlexGetSimplexOrBoxCells(), DMPlexGetCellType(), DMPlexGetHeightStratum(), DMPolytopeTypeGetNumVertices()
31 @*/
32 PetscErrorCode DMPlexIsSimplex(DM dm, PetscBool *simplex)
33 {
34   DMPolytopeType ct;
35   PetscInt       cStart, cEnd;
36   PetscErrorCode ierr;
37 
38   PetscFunctionBegin;
39   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
40   if (cEnd <= cStart) {*simplex = PETSC_FALSE; PetscFunctionReturn(0);}
41   ierr = DMPlexGetCellType(dm, cStart, &ct);CHKERRQ(ierr);
42   *simplex = DMPolytopeTypeGetNumVertices(ct) == DMPolytopeTypeGetDim(ct)+1 ? PETSC_TRUE : PETSC_FALSE;
43   PetscFunctionReturn(0);
44 }
45 
46 /*@
47   DMPlexGetSimplexOrBoxCells - Get the range of cells which are neither prisms nor ghost FV cells
48 
49   Input Parameters:
50 + dm     - The DMPlex object
51 - height - The cell height in the Plex, 0 is the default
52 
53   Output Parameters:
54 + cStart - The first "normal" cell
55 - cEnd   - The upper bound on "normal"" cells
56 
57   Note: This just gives the first range of cells found. If the mesh has several cell types, it will only give the first.
58 
59   Level: developer
60 
61 .seealso DMPlexConstructGhostCells(), DMPlexGetGhostCellStratum()
62 @*/
63 PetscErrorCode DMPlexGetSimplexOrBoxCells(DM dm, PetscInt height, PetscInt *cStart, PetscInt *cEnd)
64 {
65   DMPolytopeType ct = DM_POLYTOPE_UNKNOWN;
66   PetscInt       cS, cE, c;
67   PetscErrorCode ierr;
68 
69   PetscFunctionBegin;
70   ierr = DMPlexGetHeightStratum(dm, PetscMax(height, 0), &cS, &cE);CHKERRQ(ierr);
71   for (c = cS; c < cE; ++c) {
72     DMPolytopeType cct;
73 
74     ierr = DMPlexGetCellType(dm, c, &cct);CHKERRQ(ierr);
75     if ((PetscInt) cct < 0) break;
76     switch (cct) {
77       case DM_POLYTOPE_POINT:
78       case DM_POLYTOPE_SEGMENT:
79       case DM_POLYTOPE_TRIANGLE:
80       case DM_POLYTOPE_QUADRILATERAL:
81       case DM_POLYTOPE_TETRAHEDRON:
82       case DM_POLYTOPE_HEXAHEDRON:
83         ct = cct;
84         break;
85       default: break;
86     }
87     if (ct != DM_POLYTOPE_UNKNOWN) break;
88   }
89   if (ct != DM_POLYTOPE_UNKNOWN) {
90     DMLabel ctLabel;
91 
92     ierr = DMPlexGetCellTypeLabel(dm, &ctLabel);CHKERRQ(ierr);
93     ierr = DMLabelGetStratumBounds(ctLabel, ct, &cS, &cE);CHKERRQ(ierr);
94   }
95   if (cStart) *cStart = cS;
96   if (cEnd)   *cEnd   = cE;
97   PetscFunctionReturn(0);
98 }
99 
100 PetscErrorCode DMPlexGetFieldType_Internal(DM dm, PetscSection section, PetscInt field, PetscInt *sStart, PetscInt *sEnd, PetscViewerVTKFieldType *ft)
101 {
102   PetscInt       cdim, pStart, pEnd, vStart, vEnd, cStart, cEnd;
103   PetscInt       vcdof[2] = {0,0}, globalvcdof[2];
104   PetscErrorCode ierr;
105 
106   PetscFunctionBegin;
107   *ft  = PETSC_VTK_INVALID;
108   ierr = DMGetCoordinateDim(dm, &cdim);CHKERRQ(ierr);
109   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
110   ierr = DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
111   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
112   if (field >= 0) {
113     if ((vStart >= pStart) && (vStart < pEnd)) {ierr = PetscSectionGetFieldDof(section, vStart, field, &vcdof[0]);CHKERRQ(ierr);}
114     if ((cStart >= pStart) && (cStart < pEnd)) {ierr = PetscSectionGetFieldDof(section, cStart, field, &vcdof[1]);CHKERRQ(ierr);}
115   } else {
116     if ((vStart >= pStart) && (vStart < pEnd)) {ierr = PetscSectionGetDof(section, vStart, &vcdof[0]);CHKERRQ(ierr);}
117     if ((cStart >= pStart) && (cStart < pEnd)) {ierr = PetscSectionGetDof(section, cStart, &vcdof[1]);CHKERRQ(ierr);}
118   }
119   ierr = MPI_Allreduce(vcdof, globalvcdof, 2, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject)dm));CHKERRMPI(ierr);
120   if (globalvcdof[0]) {
121     *sStart = vStart;
122     *sEnd   = vEnd;
123     if (globalvcdof[0] == cdim) *ft = PETSC_VTK_POINT_VECTOR_FIELD;
124     else                        *ft = PETSC_VTK_POINT_FIELD;
125   } else if (globalvcdof[1]) {
126     *sStart = cStart;
127     *sEnd   = cEnd;
128     if (globalvcdof[1] == cdim) *ft = PETSC_VTK_CELL_VECTOR_FIELD;
129     else                        *ft = PETSC_VTK_CELL_FIELD;
130   } else {
131     if (field >= 0) {
132       const char *fieldname;
133 
134       ierr = PetscSectionGetFieldName(section, field, &fieldname);CHKERRQ(ierr);
135       ierr = PetscInfo((PetscObject) dm, "Could not classify VTK output type of section field %D \"%s\"\n", field, fieldname);CHKERRQ(ierr);
136     } else {
137       ierr = PetscInfo((PetscObject) dm, "Could not classify VTK output type of section\"%s\"\n");CHKERRQ(ierr);
138     }
139   }
140   PetscFunctionReturn(0);
141 }
142 
143 /*@
144   DMPlexVecView1D - Plot many 1D solutions on the same line graph
145 
146   Collective on dm
147 
148   Input Parameters:
149 + dm - The DMPlex
150 . n  - The number of vectors
151 . u  - The array of local vectors
152 - viewer - The Draw viewer
153 
154   Level: advanced
155 
156 .seealso: VecViewFromOptions(), VecView()
157 @*/
158 PetscErrorCode DMPlexVecView1D(DM dm, PetscInt n, Vec u[], PetscViewer viewer)
159 {
160   PetscDS            ds;
161   PetscDraw          draw = NULL;
162   PetscDrawLG        lg;
163   Vec                coordinates;
164   const PetscScalar *coords, **sol;
165   PetscReal         *vals;
166   PetscInt          *Nc;
167   PetscInt           Nf, f, c, Nl, l, i, vStart, vEnd, v;
168   char             **names;
169   PetscErrorCode     ierr;
170 
171   PetscFunctionBegin;
172   ierr = DMGetDS(dm, &ds);CHKERRQ(ierr);
173   ierr = PetscDSGetNumFields(ds, &Nf);CHKERRQ(ierr);
174   ierr = PetscDSGetTotalComponents(ds, &Nl);CHKERRQ(ierr);
175   ierr = PetscDSGetComponents(ds, &Nc);CHKERRQ(ierr);
176 
177   ierr = PetscViewerDrawGetDraw(viewer, 0, &draw);CHKERRQ(ierr);
178   if (!draw) PetscFunctionReturn(0);
179   ierr = PetscDrawLGCreate(draw, n*Nl, &lg);CHKERRQ(ierr);
180 
181   ierr = PetscMalloc3(n, &sol, n*Nl, &names, n*Nl, &vals);CHKERRQ(ierr);
182   for (i = 0, l = 0; i < n; ++i) {
183     const char *vname;
184 
185     ierr = PetscObjectGetName((PetscObject) u[i], &vname);CHKERRQ(ierr);
186     for (f = 0; f < Nf; ++f) {
187       PetscObject disc;
188       const char *fname;
189       char        tmpname[PETSC_MAX_PATH_LEN];
190 
191       ierr = PetscDSGetDiscretization(ds, f, &disc);CHKERRQ(ierr);
192       /* TODO Create names for components */
193       for (c = 0; c < Nc[f]; ++c, ++l) {
194         ierr = PetscObjectGetName(disc, &fname);CHKERRQ(ierr);
195         ierr = PetscStrcpy(tmpname, vname);CHKERRQ(ierr);
196         ierr = PetscStrlcat(tmpname, ":", PETSC_MAX_PATH_LEN);CHKERRQ(ierr);
197         ierr = PetscStrlcat(tmpname, fname, PETSC_MAX_PATH_LEN);CHKERRQ(ierr);
198         ierr = PetscStrallocpy(tmpname, &names[l]);CHKERRQ(ierr);
199       }
200     }
201   }
202   ierr = PetscDrawLGSetLegend(lg, (const char *const *) names);CHKERRQ(ierr);
203   /* Just add P_1 support for now */
204   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
205   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
206   ierr = VecGetArrayRead(coordinates, &coords);CHKERRQ(ierr);
207   for (i = 0; i < n; ++i) {ierr = VecGetArrayRead(u[i], &sol[i]);CHKERRQ(ierr);}
208   for (v = vStart; v < vEnd; ++v) {
209     PetscScalar *x, *svals;
210 
211     ierr = DMPlexPointLocalRead(dm, v, coords, &x);CHKERRQ(ierr);
212     for (i = 0; i < n; ++i) {
213       ierr = DMPlexPointLocalRead(dm, v, sol[i], &svals);CHKERRQ(ierr);
214       for (l = 0; l < Nl; ++l) vals[i*Nl + l] = PetscRealPart(svals[l]);
215     }
216     ierr = PetscDrawLGAddCommonPoint(lg, PetscRealPart(x[0]), vals);CHKERRQ(ierr);
217   }
218   ierr = VecRestoreArrayRead(coordinates, &coords);CHKERRQ(ierr);
219   for (i = 0; i < n; ++i) {ierr = VecRestoreArrayRead(u[i], &sol[i]);CHKERRQ(ierr);}
220   for (l = 0; l < n*Nl; ++l) {ierr = PetscFree(names[l]);CHKERRQ(ierr);}
221   ierr = PetscFree3(sol, names, vals);CHKERRQ(ierr);
222 
223   ierr = PetscDrawLGDraw(lg);CHKERRQ(ierr);
224   ierr = PetscDrawLGDestroy(&lg);CHKERRQ(ierr);
225   PetscFunctionReturn(0);
226 }
227 
228 static PetscErrorCode VecView_Plex_Local_Draw_1D(Vec u, PetscViewer viewer)
229 {
230   DM             dm;
231   PetscErrorCode ierr;
232 
233   PetscFunctionBegin;
234   ierr = VecGetDM(u, &dm);CHKERRQ(ierr);
235   ierr = DMPlexVecView1D(dm, 1, &u, viewer);CHKERRQ(ierr);
236   PetscFunctionReturn(0);
237 }
238 
239 static PetscErrorCode VecView_Plex_Local_Draw_2D(Vec v, PetscViewer viewer)
240 {
241   DM                 dm;
242   PetscSection       s;
243   PetscDraw          draw, popup;
244   DM                 cdm;
245   PetscSection       coordSection;
246   Vec                coordinates;
247   const PetscScalar *coords, *array;
248   PetscReal          bound[4] = {PETSC_MAX_REAL, PETSC_MAX_REAL, PETSC_MIN_REAL, PETSC_MIN_REAL};
249   PetscReal          vbound[2], time;
250   PetscBool          flg;
251   PetscInt           dim, Nf, f, Nc, comp, vStart, vEnd, cStart, cEnd, c, N, level, step, w = 0;
252   const char        *name;
253   char               title[PETSC_MAX_PATH_LEN];
254   PetscErrorCode     ierr;
255 
256   PetscFunctionBegin;
257   ierr = PetscViewerDrawGetDraw(viewer, 0, &draw);CHKERRQ(ierr);
258   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
259   ierr = DMGetCoordinateDim(dm, &dim);CHKERRQ(ierr);
260   ierr = DMGetLocalSection(dm, &s);CHKERRQ(ierr);
261   ierr = PetscSectionGetNumFields(s, &Nf);CHKERRQ(ierr);
262   ierr = DMGetCoarsenLevel(dm, &level);CHKERRQ(ierr);
263   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
264   ierr = DMGetLocalSection(cdm, &coordSection);CHKERRQ(ierr);
265   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
266   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
267   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
268 
269   ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
270   ierr = DMGetOutputSequenceNumber(dm, &step, &time);CHKERRQ(ierr);
271 
272   ierr = VecGetLocalSize(coordinates, &N);CHKERRQ(ierr);
273   ierr = VecGetArrayRead(coordinates, &coords);CHKERRQ(ierr);
274   for (c = 0; c < N; c += dim) {
275     bound[0] = PetscMin(bound[0], PetscRealPart(coords[c]));   bound[2] = PetscMax(bound[2], PetscRealPart(coords[c]));
276     bound[1] = PetscMin(bound[1], PetscRealPart(coords[c+1])); bound[3] = PetscMax(bound[3], PetscRealPart(coords[c+1]));
277   }
278   ierr = VecRestoreArrayRead(coordinates, &coords);CHKERRQ(ierr);
279   ierr = PetscDrawClear(draw);CHKERRQ(ierr);
280 
281   /* Could implement something like DMDASelectFields() */
282   for (f = 0; f < Nf; ++f) {
283     DM   fdm = dm;
284     Vec  fv  = v;
285     IS   fis;
286     char prefix[PETSC_MAX_PATH_LEN];
287     const char *fname;
288 
289     ierr = PetscSectionGetFieldComponents(s, f, &Nc);CHKERRQ(ierr);
290     ierr = PetscSectionGetFieldName(s, f, &fname);CHKERRQ(ierr);
291 
292     if (v->hdr.prefix) {ierr = PetscStrncpy(prefix, v->hdr.prefix,sizeof(prefix));CHKERRQ(ierr);}
293     else               {prefix[0] = '\0';}
294     if (Nf > 1) {
295       ierr = DMCreateSubDM(dm, 1, &f, &fis, &fdm);CHKERRQ(ierr);
296       ierr = VecGetSubVector(v, fis, &fv);CHKERRQ(ierr);
297       ierr = PetscStrlcat(prefix, fname,sizeof(prefix));CHKERRQ(ierr);
298       ierr = PetscStrlcat(prefix, "_",sizeof(prefix));CHKERRQ(ierr);
299     }
300     for (comp = 0; comp < Nc; ++comp, ++w) {
301       PetscInt nmax = 2;
302 
303       ierr = PetscViewerDrawGetDraw(viewer, w, &draw);CHKERRQ(ierr);
304       if (Nc > 1) {ierr = PetscSNPrintf(title, sizeof(title), "%s:%s_%D Step: %D Time: %.4g", name, fname, comp, step, time);CHKERRQ(ierr);}
305       else        {ierr = PetscSNPrintf(title, sizeof(title), "%s:%s Step: %D Time: %.4g", name, fname, step, time);CHKERRQ(ierr);}
306       ierr = PetscDrawSetTitle(draw, title);CHKERRQ(ierr);
307 
308       /* TODO Get max and min only for this component */
309       ierr = PetscOptionsGetRealArray(NULL, prefix, "-vec_view_bounds", vbound, &nmax, &flg);CHKERRQ(ierr);
310       if (!flg) {
311         ierr = VecMin(fv, NULL, &vbound[0]);CHKERRQ(ierr);
312         ierr = VecMax(fv, NULL, &vbound[1]);CHKERRQ(ierr);
313         if (vbound[1] <= vbound[0]) vbound[1] = vbound[0] + 1.0;
314       }
315       ierr = PetscDrawGetPopup(draw, &popup);CHKERRQ(ierr);
316       ierr = PetscDrawScalePopup(popup, vbound[0], vbound[1]);CHKERRQ(ierr);
317       ierr = PetscDrawSetCoordinates(draw, bound[0], bound[1], bound[2], bound[3]);CHKERRQ(ierr);
318 
319       ierr = VecGetArrayRead(fv, &array);CHKERRQ(ierr);
320       for (c = cStart; c < cEnd; ++c) {
321         PetscScalar *coords = NULL, *a = NULL;
322         PetscInt     numCoords, color[4] = {-1,-1,-1,-1};
323 
324         ierr = DMPlexPointLocalRead(fdm, c, array, &a);CHKERRQ(ierr);
325         if (a) {
326           color[0] = PetscDrawRealToColor(PetscRealPart(a[comp]), vbound[0], vbound[1]);
327           color[1] = color[2] = color[3] = color[0];
328         } else {
329           PetscScalar *vals = NULL;
330           PetscInt     numVals, va;
331 
332           ierr = DMPlexVecGetClosure(fdm, NULL, fv, c, &numVals, &vals);CHKERRQ(ierr);
333           PetscCheckFalse(numVals % Nc,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "The number of components %D does not divide the number of values in the closure %D", Nc, numVals);
334           switch (numVals/Nc) {
335           case 3: /* P1 Triangle */
336           case 4: /* P1 Quadrangle */
337             for (va = 0; va < numVals/Nc; ++va) color[va] = PetscDrawRealToColor(PetscRealPart(vals[va*Nc+comp]), vbound[0], vbound[1]);
338             break;
339           case 6: /* P2 Triangle */
340           case 8: /* P2 Quadrangle */
341             for (va = 0; va < numVals/(Nc*2); ++va) color[va] = PetscDrawRealToColor(PetscRealPart(vals[va*Nc+comp + numVals/(Nc*2)]), vbound[0], vbound[1]);
342             break;
343           default: SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of values for cell closure %D cannot be handled", numVals/Nc);
344           }
345           ierr = DMPlexVecRestoreClosure(fdm, NULL, fv, c, &numVals, &vals);CHKERRQ(ierr);
346         }
347         ierr = DMPlexVecGetClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
348         switch (numCoords) {
349         case 6:
350         case 12: /* Localized triangle */
351           ierr = PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), color[0], color[1], color[2]);CHKERRQ(ierr);
352           break;
353         case 8:
354         case 16: /* Localized quadrilateral */
355           ierr = PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), color[0], color[1], color[2]);CHKERRQ(ierr);
356           ierr = PetscDrawTriangle(draw, PetscRealPart(coords[4]), PetscRealPart(coords[5]), PetscRealPart(coords[6]), PetscRealPart(coords[7]), PetscRealPart(coords[0]), PetscRealPart(coords[1]), color[2], color[3], color[0]);CHKERRQ(ierr);
357           break;
358         default: SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Cannot draw cells with %D coordinates", numCoords);
359         }
360         ierr = DMPlexVecRestoreClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
361       }
362       ierr = VecRestoreArrayRead(fv, &array);CHKERRQ(ierr);
363       ierr = PetscDrawFlush(draw);CHKERRQ(ierr);
364       ierr = PetscDrawPause(draw);CHKERRQ(ierr);
365       ierr = PetscDrawSave(draw);CHKERRQ(ierr);
366     }
367     if (Nf > 1) {
368       ierr = VecRestoreSubVector(v, fis, &fv);CHKERRQ(ierr);
369       ierr = ISDestroy(&fis);CHKERRQ(ierr);
370       ierr = DMDestroy(&fdm);CHKERRQ(ierr);
371     }
372   }
373   PetscFunctionReturn(0);
374 }
375 
376 static PetscErrorCode VecView_Plex_Local_Draw(Vec v, PetscViewer viewer)
377 {
378   DM             dm;
379   PetscDraw      draw;
380   PetscInt       dim;
381   PetscBool      isnull;
382   PetscErrorCode ierr;
383 
384   PetscFunctionBegin;
385   ierr = PetscViewerDrawGetDraw(viewer, 0, &draw);CHKERRQ(ierr);
386   ierr = PetscDrawIsNull(draw, &isnull);CHKERRQ(ierr);
387   if (isnull) PetscFunctionReturn(0);
388 
389   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
390   ierr = DMGetCoordinateDim(dm, &dim);CHKERRQ(ierr);
391   switch (dim) {
392     case 1: VecView_Plex_Local_Draw_1D(v, viewer);CHKERRQ(ierr);break;
393     case 2: VecView_Plex_Local_Draw_2D(v, viewer);CHKERRQ(ierr);break;
394     default: SETERRQ(PetscObjectComm((PetscObject) v), PETSC_ERR_SUP, "Cannot draw meshes of dimension %D. Try PETSCVIEWERGLVIS", dim);
395   }
396   PetscFunctionReturn(0);
397 }
398 
399 static PetscErrorCode VecView_Plex_Local_VTK(Vec v, PetscViewer viewer)
400 {
401   DM                      dm;
402   Vec                     locv;
403   const char              *name;
404   PetscSection            section;
405   PetscInt                pStart, pEnd;
406   PetscInt                numFields;
407   PetscViewerVTKFieldType ft;
408   PetscErrorCode          ierr;
409 
410   PetscFunctionBegin;
411   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
412   ierr = DMCreateLocalVector(dm, &locv);CHKERRQ(ierr); /* VTK viewer requires exclusive ownership of the vector */
413   ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
414   ierr = PetscObjectSetName((PetscObject) locv, name);CHKERRQ(ierr);
415   ierr = VecCopy(v, locv);CHKERRQ(ierr);
416   ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);
417   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
418   if (!numFields) {
419     ierr = DMPlexGetFieldType_Internal(dm, section, PETSC_DETERMINE, &pStart, &pEnd, &ft);CHKERRQ(ierr);
420     ierr = PetscViewerVTKAddField(viewer, (PetscObject) dm, DMPlexVTKWriteAll, PETSC_DEFAULT, ft, PETSC_TRUE,(PetscObject) locv);CHKERRQ(ierr);
421   } else {
422     PetscInt f;
423 
424     for (f = 0; f < numFields; f++) {
425       ierr = DMPlexGetFieldType_Internal(dm, section, f, &pStart, &pEnd, &ft);CHKERRQ(ierr);
426       if (ft == PETSC_VTK_INVALID) continue;
427       ierr = PetscObjectReference((PetscObject)locv);CHKERRQ(ierr);
428       ierr = PetscViewerVTKAddField(viewer, (PetscObject) dm, DMPlexVTKWriteAll, f, ft, PETSC_TRUE,(PetscObject) locv);CHKERRQ(ierr);
429     }
430     ierr = VecDestroy(&locv);CHKERRQ(ierr);
431   }
432   PetscFunctionReturn(0);
433 }
434 
435 PetscErrorCode VecView_Plex_Local(Vec v, PetscViewer viewer)
436 {
437   DM             dm;
438   PetscBool      isvtk, ishdf5, isdraw, isglvis;
439   PetscErrorCode ierr;
440 
441   PetscFunctionBegin;
442   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
443   PetscCheckFalse(!dm,PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
444   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,   &isvtk);CHKERRQ(ierr);
445   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,  &ishdf5);CHKERRQ(ierr);
446   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERDRAW,  &isdraw);CHKERRQ(ierr);
447   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERGLVIS, &isglvis);CHKERRQ(ierr);
448   if (isvtk || ishdf5 || isdraw || isglvis) {
449     PetscInt    i,numFields;
450     PetscObject fe;
451     PetscBool   fem = PETSC_FALSE;
452     Vec         locv = v;
453     const char  *name;
454     PetscInt    step;
455     PetscReal   time;
456 
457     ierr = DMGetNumFields(dm, &numFields);CHKERRQ(ierr);
458     for (i=0; i<numFields; i++) {
459       ierr = DMGetField(dm, i, NULL, &fe);CHKERRQ(ierr);
460       if (fe->classid == PETSCFE_CLASSID) { fem = PETSC_TRUE; break; }
461     }
462     if (fem) {
463       PetscObject isZero;
464 
465       ierr = DMGetLocalVector(dm, &locv);CHKERRQ(ierr);
466       ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
467       ierr = PetscObjectSetName((PetscObject) locv, name);CHKERRQ(ierr);
468       ierr = PetscObjectQuery((PetscObject) v, "__Vec_bc_zero__", &isZero);CHKERRQ(ierr);
469       ierr = PetscObjectCompose((PetscObject) locv, "__Vec_bc_zero__", isZero);CHKERRQ(ierr);
470       ierr = VecCopy(v, locv);CHKERRQ(ierr);
471       ierr = DMGetOutputSequenceNumber(dm, NULL, &time);CHKERRQ(ierr);
472       ierr = DMPlexInsertBoundaryValues(dm, PETSC_TRUE, locv, time, NULL, NULL, NULL);CHKERRQ(ierr);
473     }
474     if (isvtk) {
475       ierr = VecView_Plex_Local_VTK(locv, viewer);CHKERRQ(ierr);
476     } else if (ishdf5) {
477 #if defined(PETSC_HAVE_HDF5)
478       ierr = VecView_Plex_Local_HDF5_Internal(locv, viewer);CHKERRQ(ierr);
479 #else
480       SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
481 #endif
482     } else if (isdraw) {
483       ierr = VecView_Plex_Local_Draw(locv, viewer);CHKERRQ(ierr);
484     } else if (isglvis) {
485       ierr = DMGetOutputSequenceNumber(dm, &step, NULL);CHKERRQ(ierr);
486       ierr = PetscViewerGLVisSetSnapId(viewer, step);CHKERRQ(ierr);
487       ierr = VecView_GLVis(locv, viewer);CHKERRQ(ierr);
488     }
489     if (fem) {
490       ierr = PetscObjectCompose((PetscObject) locv, "__Vec_bc_zero__", NULL);CHKERRQ(ierr);
491       ierr = DMRestoreLocalVector(dm, &locv);CHKERRQ(ierr);
492     }
493   } else {
494     PetscBool isseq;
495 
496     ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
497     if (isseq) {ierr = VecView_Seq(v, viewer);CHKERRQ(ierr);}
498     else       {ierr = VecView_MPI(v, viewer);CHKERRQ(ierr);}
499   }
500   PetscFunctionReturn(0);
501 }
502 
503 PetscErrorCode VecView_Plex(Vec v, PetscViewer viewer)
504 {
505   DM             dm;
506   PetscBool      isvtk, ishdf5, isdraw, isglvis, isexodusii;
507   PetscErrorCode ierr;
508 
509   PetscFunctionBegin;
510   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
511   PetscCheckFalse(!dm,PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
512   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,      &isvtk);CHKERRQ(ierr);
513   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,     &ishdf5);CHKERRQ(ierr);
514   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERDRAW,     &isdraw);CHKERRQ(ierr);
515   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERGLVIS,    &isglvis);CHKERRQ(ierr);
516   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWEREXODUSII, &isexodusii);CHKERRQ(ierr);
517   if (isvtk || isdraw || isglvis) {
518     Vec         locv;
519     PetscObject isZero;
520     const char *name;
521 
522     ierr = DMGetLocalVector(dm, &locv);CHKERRQ(ierr);
523     ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
524     ierr = PetscObjectSetName((PetscObject) locv, name);CHKERRQ(ierr);
525     ierr = DMGlobalToLocalBegin(dm, v, INSERT_VALUES, locv);CHKERRQ(ierr);
526     ierr = DMGlobalToLocalEnd(dm, v, INSERT_VALUES, locv);CHKERRQ(ierr);
527     ierr = PetscObjectQuery((PetscObject) v, "__Vec_bc_zero__", &isZero);CHKERRQ(ierr);
528     ierr = PetscObjectCompose((PetscObject) locv, "__Vec_bc_zero__", isZero);CHKERRQ(ierr);
529     ierr = VecView_Plex_Local(locv, viewer);CHKERRQ(ierr);
530     ierr = PetscObjectCompose((PetscObject) locv, "__Vec_bc_zero__", NULL);CHKERRQ(ierr);
531     ierr = DMRestoreLocalVector(dm, &locv);CHKERRQ(ierr);
532   } else if (ishdf5) {
533 #if defined(PETSC_HAVE_HDF5)
534     ierr = VecView_Plex_HDF5_Internal(v, viewer);CHKERRQ(ierr);
535 #else
536     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
537 #endif
538   } else if (isexodusii) {
539 #if defined(PETSC_HAVE_EXODUSII)
540     ierr = VecView_PlexExodusII_Internal(v, viewer);CHKERRQ(ierr);
541 #else
542     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "ExodusII not supported in this build.\nPlease reconfigure using --download-exodusii");
543 #endif
544   } else {
545     PetscBool isseq;
546 
547     ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
548     if (isseq) {ierr = VecView_Seq(v, viewer);CHKERRQ(ierr);}
549     else       {ierr = VecView_MPI(v, viewer);CHKERRQ(ierr);}
550   }
551   PetscFunctionReturn(0);
552 }
553 
554 PetscErrorCode VecView_Plex_Native(Vec originalv, PetscViewer viewer)
555 {
556   DM                dm;
557   MPI_Comm          comm;
558   PetscViewerFormat format;
559   Vec               v;
560   PetscBool         isvtk, ishdf5;
561   PetscErrorCode    ierr;
562 
563   PetscFunctionBegin;
564   ierr = VecGetDM(originalv, &dm);CHKERRQ(ierr);
565   ierr = PetscObjectGetComm((PetscObject) originalv, &comm);CHKERRQ(ierr);
566   PetscCheckFalse(!dm,comm, PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
567   ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
568   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
569   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,  &isvtk);CHKERRQ(ierr);
570   if (format == PETSC_VIEWER_NATIVE) {
571     /* Natural ordering is the common case for DMDA, NATIVE means plain vector, for PLEX is the opposite */
572     /* this need a better fix */
573     if (dm->useNatural) {
574       if (dm->sfNatural) {
575         const char *vecname;
576         PetscInt    n, nroots;
577 
578         ierr = VecGetLocalSize(originalv, &n);CHKERRQ(ierr);
579         ierr = PetscSFGetGraph(dm->sfNatural, &nroots, NULL, NULL, NULL);CHKERRQ(ierr);
580         if (n == nroots) {
581           ierr = DMGetGlobalVector(dm, &v);CHKERRQ(ierr);
582           ierr = DMPlexGlobalToNaturalBegin(dm, originalv, v);CHKERRQ(ierr);
583           ierr = DMPlexGlobalToNaturalEnd(dm, originalv, v);CHKERRQ(ierr);
584           ierr = PetscObjectGetName((PetscObject) originalv, &vecname);CHKERRQ(ierr);
585           ierr = PetscObjectSetName((PetscObject) v, vecname);CHKERRQ(ierr);
586         } else SETERRQ(comm, PETSC_ERR_ARG_WRONG, "DM global to natural SF only handles global vectors");
587       } else SETERRQ(comm, PETSC_ERR_ARG_WRONGSTATE, "DM global to natural SF was not created");
588     } else v = originalv;
589   } else v = originalv;
590 
591   if (ishdf5) {
592 #if defined(PETSC_HAVE_HDF5)
593     ierr = VecView_Plex_HDF5_Native_Internal(v, viewer);CHKERRQ(ierr);
594 #else
595     SETERRQ(comm, PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
596 #endif
597   } else if (isvtk) {
598     SETERRQ(comm, PETSC_ERR_SUP, "VTK format does not support viewing in natural order. Please switch to HDF5.");
599   } else {
600     PetscBool isseq;
601 
602     ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
603     if (isseq) {ierr = VecView_Seq(v, viewer);CHKERRQ(ierr);}
604     else       {ierr = VecView_MPI(v, viewer);CHKERRQ(ierr);}
605   }
606   if (v != originalv) {ierr = DMRestoreGlobalVector(dm, &v);CHKERRQ(ierr);}
607   PetscFunctionReturn(0);
608 }
609 
610 PetscErrorCode VecLoad_Plex_Local(Vec v, PetscViewer viewer)
611 {
612   DM             dm;
613   PetscBool      ishdf5;
614   PetscErrorCode ierr;
615 
616   PetscFunctionBegin;
617   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
618   PetscCheckFalse(!dm,PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
619   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
620   if (ishdf5) {
621     DM          dmBC;
622     Vec         gv;
623     const char *name;
624 
625     ierr = DMGetOutputDM(dm, &dmBC);CHKERRQ(ierr);
626     ierr = DMGetGlobalVector(dmBC, &gv);CHKERRQ(ierr);
627     ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
628     ierr = PetscObjectSetName((PetscObject) gv, name);CHKERRQ(ierr);
629     ierr = VecLoad_Default(gv, viewer);CHKERRQ(ierr);
630     ierr = DMGlobalToLocalBegin(dmBC, gv, INSERT_VALUES, v);CHKERRQ(ierr);
631     ierr = DMGlobalToLocalEnd(dmBC, gv, INSERT_VALUES, v);CHKERRQ(ierr);
632     ierr = DMRestoreGlobalVector(dmBC, &gv);CHKERRQ(ierr);
633   } else {
634     ierr = VecLoad_Default(v, viewer);CHKERRQ(ierr);
635   }
636   PetscFunctionReturn(0);
637 }
638 
639 PetscErrorCode VecLoad_Plex(Vec v, PetscViewer viewer)
640 {
641   DM             dm;
642   PetscBool      ishdf5,isexodusii;
643   PetscErrorCode ierr;
644 
645   PetscFunctionBegin;
646   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
647   PetscCheckFalse(!dm,PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
648   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,     &ishdf5);CHKERRQ(ierr);
649   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWEREXODUSII, &isexodusii);CHKERRQ(ierr);
650   if (ishdf5) {
651 #if defined(PETSC_HAVE_HDF5)
652     ierr = VecLoad_Plex_HDF5_Internal(v, viewer);CHKERRQ(ierr);
653 #else
654     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
655 #endif
656   } else if (isexodusii) {
657 #if defined(PETSC_HAVE_EXODUSII)
658     ierr = VecLoad_PlexExodusII_Internal(v, viewer);CHKERRQ(ierr);
659 #else
660     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "ExodusII not supported in this build.\nPlease reconfigure using --download-exodusii");
661 #endif
662   } else {
663     ierr = VecLoad_Default(v, viewer);CHKERRQ(ierr);
664   }
665   PetscFunctionReturn(0);
666 }
667 
668 PetscErrorCode VecLoad_Plex_Native(Vec originalv, PetscViewer viewer)
669 {
670   DM                dm;
671   PetscViewerFormat format;
672   PetscBool         ishdf5;
673   PetscErrorCode    ierr;
674 
675   PetscFunctionBegin;
676   ierr = VecGetDM(originalv, &dm);CHKERRQ(ierr);
677   PetscCheckFalse(!dm,PetscObjectComm((PetscObject) originalv), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
678   ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
679   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
680   if (format == PETSC_VIEWER_NATIVE) {
681     if (dm->useNatural) {
682       if (dm->sfNatural) {
683         if (ishdf5) {
684 #if defined(PETSC_HAVE_HDF5)
685           Vec         v;
686           const char *vecname;
687 
688           ierr = DMGetGlobalVector(dm, &v);CHKERRQ(ierr);
689           ierr = PetscObjectGetName((PetscObject) originalv, &vecname);CHKERRQ(ierr);
690           ierr = PetscObjectSetName((PetscObject) v, vecname);CHKERRQ(ierr);
691           ierr = VecLoad_Plex_HDF5_Native_Internal(v, viewer);CHKERRQ(ierr);
692           ierr = DMPlexNaturalToGlobalBegin(dm, v, originalv);CHKERRQ(ierr);
693           ierr = DMPlexNaturalToGlobalEnd(dm, v, originalv);CHKERRQ(ierr);
694           ierr = DMRestoreGlobalVector(dm, &v);CHKERRQ(ierr);
695 #else
696           SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
697 #endif
698         } else SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Reading in natural order is not supported for anything but HDF5.");
699       }
700     } else {
701       ierr = VecLoad_Default(originalv, viewer);CHKERRQ(ierr);
702     }
703   }
704   PetscFunctionReturn(0);
705 }
706 
707 PETSC_UNUSED static PetscErrorCode DMPlexView_Ascii_Geometry(DM dm, PetscViewer viewer)
708 {
709   PetscSection       coordSection;
710   Vec                coordinates;
711   DMLabel            depthLabel, celltypeLabel;
712   const char        *name[4];
713   const PetscScalar *a;
714   PetscInt           dim, pStart, pEnd, cStart, cEnd, c;
715   PetscErrorCode     ierr;
716 
717   PetscFunctionBegin;
718   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
719   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
720   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
721   ierr = DMPlexGetDepthLabel(dm, &depthLabel);CHKERRQ(ierr);
722   ierr = DMPlexGetCellTypeLabel(dm, &celltypeLabel);CHKERRQ(ierr);
723   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
724   ierr = PetscSectionGetChart(coordSection, &pStart, &pEnd);CHKERRQ(ierr);
725   ierr = VecGetArrayRead(coordinates, &a);CHKERRQ(ierr);
726   name[0]     = "vertex";
727   name[1]     = "edge";
728   name[dim-1] = "face";
729   name[dim]   = "cell";
730   for (c = cStart; c < cEnd; ++c) {
731     PetscInt *closure = NULL;
732     PetscInt  closureSize, cl, ct;
733 
734     ierr = DMLabelGetValue(celltypeLabel, c, &ct);CHKERRQ(ierr);
735     ierr = PetscViewerASCIIPrintf(viewer, "Geometry for cell %D polytope type %s:\n", c, DMPolytopeTypes[ct]);CHKERRQ(ierr);
736     ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
737     ierr = PetscViewerASCIIPushTab(viewer);CHKERRQ(ierr);
738     for (cl = 0; cl < closureSize*2; cl += 2) {
739       PetscInt point = closure[cl], depth, dof, off, d, p;
740 
741       if ((point < pStart) || (point >= pEnd)) continue;
742       ierr = PetscSectionGetDof(coordSection, point, &dof);CHKERRQ(ierr);
743       if (!dof) continue;
744       ierr = DMLabelGetValue(depthLabel, point, &depth);CHKERRQ(ierr);
745       ierr = PetscSectionGetOffset(coordSection, point, &off);CHKERRQ(ierr);
746       ierr = PetscViewerASCIIPrintf(viewer, "%s %D coords:", name[depth], point);CHKERRQ(ierr);
747       for (p = 0; p < dof/dim; ++p) {
748         ierr = PetscViewerASCIIPrintf(viewer, " (");CHKERRQ(ierr);
749         for (d = 0; d < dim; ++d) {
750           if (d > 0) {ierr = PetscViewerASCIIPrintf(viewer, ", ");CHKERRQ(ierr);}
751           ierr = PetscViewerASCIIPrintf(viewer, "%g", (double) PetscRealPart(a[off+p*dim+d]));CHKERRQ(ierr);
752         }
753         ierr = PetscViewerASCIIPrintf(viewer, ")");CHKERRQ(ierr);
754       }
755       ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
756     }
757     ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
758     ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr);
759   }
760   ierr = VecRestoreArrayRead(coordinates, &a);CHKERRQ(ierr);
761   PetscFunctionReturn(0);
762 }
763 
764 typedef enum {CS_CARTESIAN, CS_POLAR, CS_CYLINDRICAL, CS_SPHERICAL} CoordSystem;
765 const char *CoordSystems[] = {"cartesian", "polar", "cylindrical", "spherical", "CoordSystem", "CS_", NULL};
766 
767 static PetscErrorCode DMPlexView_Ascii_Coordinates(PetscViewer viewer, CoordSystem cs, PetscInt dim, const PetscScalar x[])
768 {
769   PetscInt       i;
770   PetscErrorCode ierr;
771 
772   PetscFunctionBegin;
773   if (dim > 3) {
774     for (i = 0; i < dim; ++i) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, " %g", (double) PetscRealPart(x[i]));CHKERRQ(ierr);}
775   } else {
776     PetscReal coords[3], trcoords[3];
777 
778     for (i = 0; i < dim; ++i) coords[i] = PetscRealPart(x[i]);
779     switch (cs) {
780       case CS_CARTESIAN: for (i = 0; i < dim; ++i) trcoords[i] = coords[i];break;
781       case CS_POLAR:
782         PetscCheckFalse(dim != 2,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Polar coordinates are for 2 dimension, not %D", dim);
783         trcoords[0] = PetscSqrtReal(PetscSqr(coords[0]) + PetscSqr(coords[1]));
784         trcoords[1] = PetscAtan2Real(coords[1], coords[0]);
785         break;
786       case CS_CYLINDRICAL:
787         PetscCheckFalse(dim != 3,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cylindrical coordinates are for 3 dimension, not %D", dim);
788         trcoords[0] = PetscSqrtReal(PetscSqr(coords[0]) + PetscSqr(coords[1]));
789         trcoords[1] = PetscAtan2Real(coords[1], coords[0]);
790         trcoords[2] = coords[2];
791         break;
792       case CS_SPHERICAL:
793         PetscCheckFalse(dim != 3,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Spherical coordinates are for 3 dimension, not %D", dim);
794         trcoords[0] = PetscSqrtReal(PetscSqr(coords[0]) + PetscSqr(coords[1]) + PetscSqr(coords[2]));
795         trcoords[1] = PetscAtan2Real(PetscSqrtReal(PetscSqr(coords[0]) + PetscSqr(coords[1])), coords[2]);
796         trcoords[2] = PetscAtan2Real(coords[1], coords[0]);
797         break;
798     }
799     for (i = 0; i < dim; ++i) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, " %g", (double) trcoords[i]);CHKERRQ(ierr);}
800   }
801   PetscFunctionReturn(0);
802 }
803 
804 static PetscErrorCode DMPlexView_Ascii(DM dm, PetscViewer viewer)
805 {
806   DM_Plex          *mesh = (DM_Plex*) dm->data;
807   DM                cdm;
808   PetscSection      coordSection;
809   Vec               coordinates;
810   PetscViewerFormat format;
811   PetscErrorCode    ierr;
812 
813   PetscFunctionBegin;
814   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
815   ierr = DMGetLocalSection(cdm, &coordSection);CHKERRQ(ierr);
816   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
817   ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
818   if (format == PETSC_VIEWER_ASCII_INFO_DETAIL) {
819     const char *name;
820     PetscInt    dim, cellHeight, maxConeSize, maxSupportSize;
821     PetscInt    pStart, pEnd, p, numLabels, l;
822     PetscMPIInt rank, size;
823 
824     ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRMPI(ierr);
825     ierr = MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size);CHKERRMPI(ierr);
826     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
827     ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
828     ierr = DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize);CHKERRQ(ierr);
829     ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
830     ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
831     if (name) {ierr = PetscViewerASCIIPrintf(viewer, "%s in %D dimension%s:\n", name, dim, dim == 1 ? "" : "s");CHKERRQ(ierr);}
832     else      {ierr = PetscViewerASCIIPrintf(viewer, "Mesh in %D dimension%s:\n", dim, dim == 1 ? "" : "s");CHKERRQ(ierr);}
833     if (cellHeight) {ierr = PetscViewerASCIIPrintf(viewer, "  Cells are at height %D\n", cellHeight);CHKERRQ(ierr);}
834     ierr = PetscViewerASCIIPrintf(viewer, "Supports:\n", name);CHKERRQ(ierr);
835     ierr = PetscViewerASCIIPushSynchronized(viewer);CHKERRQ(ierr);
836     ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%d] Max support size: %D\n", rank, maxSupportSize);CHKERRQ(ierr);
837     for (p = pStart; p < pEnd; ++p) {
838       PetscInt dof, off, s;
839 
840       ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
841       ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
842       for (s = off; s < off+dof; ++s) {
843         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%d]: %D ----> %D\n", rank, p, mesh->supports[s]);CHKERRQ(ierr);
844       }
845     }
846     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
847     ierr = PetscViewerASCIIPrintf(viewer, "Cones:\n", name);CHKERRQ(ierr);
848     ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%d] Max cone size: %D\n", rank, maxConeSize);CHKERRQ(ierr);
849     for (p = pStart; p < pEnd; ++p) {
850       PetscInt dof, off, c;
851 
852       ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
853       ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
854       for (c = off; c < off+dof; ++c) {
855         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%d]: %D <---- %D (%D)\n", rank, p, mesh->cones[c], mesh->coneOrientations[c]);CHKERRQ(ierr);
856       }
857     }
858     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
859     ierr = PetscViewerASCIIPopSynchronized(viewer);CHKERRQ(ierr);
860     if (coordSection && coordinates) {
861       CoordSystem        cs = CS_CARTESIAN;
862       const PetscScalar *array;
863       PetscInt           Nf, Nc, pStart, pEnd, p;
864       PetscMPIInt        rank;
865       const char        *name;
866 
867       ierr = PetscOptionsGetEnum(((PetscObject) viewer)->options, ((PetscObject) viewer)->prefix, "-dm_plex_view_coord_system", CoordSystems, (PetscEnum *) &cs, NULL);CHKERRQ(ierr);
868       ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)viewer), &rank);CHKERRMPI(ierr);
869       ierr = PetscSectionGetNumFields(coordSection, &Nf);CHKERRQ(ierr);
870       PetscCheckFalse(Nf != 1,PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Coordinate section should have 1 field, not %D", Nf);
871       ierr = PetscSectionGetFieldComponents(coordSection, 0, &Nc);CHKERRQ(ierr);
872       ierr = PetscSectionGetChart(coordSection, &pStart, &pEnd);CHKERRQ(ierr);
873       ierr = PetscObjectGetName((PetscObject) coordinates, &name);CHKERRQ(ierr);
874       ierr = PetscViewerASCIIPrintf(viewer, "%s with %D fields\n", name, Nf);CHKERRQ(ierr);
875       ierr = PetscViewerASCIIPrintf(viewer, "  field 0 with %D components\n", Nc);CHKERRQ(ierr);
876       if (cs != CS_CARTESIAN) {ierr = PetscViewerASCIIPrintf(viewer, "  output coordinate system: %s\n", CoordSystems[cs]);CHKERRQ(ierr);}
877 
878       ierr = VecGetArrayRead(coordinates, &array);CHKERRQ(ierr);
879       ierr = PetscViewerASCIIPushSynchronized(viewer);CHKERRQ(ierr);
880       ierr = PetscViewerASCIISynchronizedPrintf(viewer, "Process %d:\n", rank);CHKERRQ(ierr);
881       for (p = pStart; p < pEnd; ++p) {
882         PetscInt dof, off;
883 
884         ierr = PetscSectionGetDof(coordSection, p, &dof);CHKERRQ(ierr);
885         ierr = PetscSectionGetOffset(coordSection, p, &off);CHKERRQ(ierr);
886         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "  (%4D) dim %2D offset %3D", p, dof, off);CHKERRQ(ierr);
887         ierr = DMPlexView_Ascii_Coordinates(viewer, cs, dof, &array[off]);CHKERRQ(ierr);
888         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\n");CHKERRQ(ierr);
889       }
890       ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
891       ierr = PetscViewerASCIIPopSynchronized(viewer);CHKERRQ(ierr);
892       ierr = VecRestoreArrayRead(coordinates, &array);CHKERRQ(ierr);
893     }
894     ierr = DMGetNumLabels(dm, &numLabels);CHKERRQ(ierr);
895     if (numLabels) {ierr = PetscViewerASCIIPrintf(viewer, "Labels:\n");CHKERRQ(ierr);}
896     for (l = 0; l < numLabels; ++l) {
897       DMLabel     label;
898       PetscBool   isdepth;
899       const char *name;
900 
901       ierr = DMGetLabelName(dm, l, &name);CHKERRQ(ierr);
902       ierr = PetscStrcmp(name, "depth", &isdepth);CHKERRQ(ierr);
903       if (isdepth) continue;
904       ierr = DMGetLabel(dm, name, &label);CHKERRQ(ierr);
905       ierr = DMLabelView(label, viewer);CHKERRQ(ierr);
906     }
907     if (size > 1) {
908       PetscSF sf;
909 
910       ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr);
911       ierr = PetscSFView(sf, viewer);CHKERRQ(ierr);
912     }
913     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
914   } else if (format == PETSC_VIEWER_ASCII_LATEX) {
915     const char  *name, *color;
916     const char  *defcolors[3]  = {"gray", "orange", "green"};
917     const char  *deflcolors[4] = {"blue", "cyan", "red", "magenta"};
918     char         lname[PETSC_MAX_PATH_LEN];
919     PetscReal    scale         = 2.0;
920     PetscReal    tikzscale     = 1.0;
921     PetscBool    useNumbers    = PETSC_TRUE, drawNumbers[4], drawColors[4], useLabels, useColors, plotEdges, drawHasse = PETSC_FALSE;
922     double       tcoords[3];
923     PetscScalar *coords;
924     PetscInt     numLabels, l, numColors, numLColors, dim, d, depth, cStart, cEnd, c, vStart, vEnd, v, eStart = 0, eEnd = 0, e, p, n;
925     PetscMPIInt  rank, size;
926     char         **names, **colors, **lcolors;
927     PetscBool    flg, lflg;
928     PetscBT      wp = NULL;
929     PetscInt     pEnd, pStart;
930 
931     ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
932     ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
933     ierr = DMGetNumLabels(dm, &numLabels);CHKERRQ(ierr);
934     numLabels  = PetscMax(numLabels, 10);
935     numColors  = 10;
936     numLColors = 10;
937     ierr = PetscCalloc3(numLabels, &names, numColors, &colors, numLColors, &lcolors);CHKERRQ(ierr);
938     ierr = PetscOptionsGetReal(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_scale", &scale, NULL);CHKERRQ(ierr);
939     ierr = PetscOptionsGetReal(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_tikzscale", &tikzscale, NULL);CHKERRQ(ierr);
940     ierr = PetscOptionsGetBool(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_numbers", &useNumbers, NULL);CHKERRQ(ierr);
941     for (d = 0; d < 4; ++d) drawNumbers[d] = useNumbers;
942     for (d = 0; d < 4; ++d) drawColors[d]  = PETSC_TRUE;
943     n = 4;
944     ierr = PetscOptionsGetBoolArray(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_numbers_depth", drawNumbers, &n, &flg);CHKERRQ(ierr);
945     PetscCheckFalse(flg && n != dim+1,PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_SIZ, "Number of flags %D != %D dim+1", n, dim+1);
946     ierr = PetscOptionsGetBoolArray(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_colors_depth", drawColors, &n, &flg);CHKERRQ(ierr);
947     PetscCheckFalse(flg && n != dim+1,PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_SIZ, "Number of flags %D != %D dim+1", n, dim+1);
948     ierr = PetscOptionsGetStringArray(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_labels", names, &numLabels, &useLabels);CHKERRQ(ierr);
949     if (!useLabels) numLabels = 0;
950     ierr = PetscOptionsGetStringArray(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_colors", colors, &numColors, &useColors);CHKERRQ(ierr);
951     if (!useColors) {
952       numColors = 3;
953       for (c = 0; c < numColors; ++c) {ierr = PetscStrallocpy(defcolors[c], &colors[c]);CHKERRQ(ierr);}
954     }
955     ierr = PetscOptionsGetStringArray(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_lcolors", lcolors, &numLColors, &useColors);CHKERRQ(ierr);
956     if (!useColors) {
957       numLColors = 4;
958       for (c = 0; c < numLColors; ++c) {ierr = PetscStrallocpy(deflcolors[c], &lcolors[c]);CHKERRQ(ierr);}
959     }
960     ierr = PetscOptionsGetString(((PetscObject) viewer)->options, ((PetscObject) viewer)->prefix, "-dm_plex_view_label_filter", lname, sizeof(lname), &lflg);CHKERRQ(ierr);
961     plotEdges = (PetscBool)(depth > 1 && drawNumbers[1] && dim < 3);
962     ierr = PetscOptionsGetBool(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_edges", &plotEdges, &flg);CHKERRQ(ierr);
963     PetscCheckFalse(flg && plotEdges && depth < dim,PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Mesh must be interpolated");
964     if (depth < dim) plotEdges = PETSC_FALSE;
965     ierr = PetscOptionsGetBool(((PetscObject) viewer)->options, ((PetscObject) viewer)->prefix, "-dm_plex_view_hasse", &drawHasse, NULL);CHKERRQ(ierr);
966 
967     /* filter points with labelvalue != labeldefaultvalue */
968     ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
969     ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
970     ierr = DMPlexGetDepthStratum(dm, 1, &eStart, &eEnd);CHKERRQ(ierr);
971     ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
972     if (lflg) {
973       DMLabel lbl;
974 
975       ierr = DMGetLabel(dm, lname, &lbl);CHKERRQ(ierr);
976       if (lbl) {
977         PetscInt val, defval;
978 
979         ierr = DMLabelGetDefaultValue(lbl, &defval);CHKERRQ(ierr);
980         ierr = PetscBTCreate(pEnd-pStart, &wp);CHKERRQ(ierr);
981         for (c = pStart;  c < pEnd; c++) {
982           PetscInt *closure = NULL;
983           PetscInt  closureSize;
984 
985           ierr = DMLabelGetValue(lbl, c, &val);CHKERRQ(ierr);
986           if (val == defval) continue;
987 
988           ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
989           for (p = 0; p < closureSize*2; p += 2) {
990             ierr = PetscBTSet(wp, closure[p] - pStart);CHKERRQ(ierr);
991           }
992           ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
993         }
994       }
995     }
996 
997     ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRMPI(ierr);
998     ierr = MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size);CHKERRMPI(ierr);
999     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
1000     ierr = PetscViewerASCIIPrintf(viewer, "\
1001 \\documentclass[tikz]{standalone}\n\n\
1002 \\usepackage{pgflibraryshapes}\n\
1003 \\usetikzlibrary{backgrounds}\n\
1004 \\usetikzlibrary{arrows}\n\
1005 \\begin{document}\n");CHKERRQ(ierr);
1006     if (size > 1) {
1007       ierr = PetscViewerASCIIPrintf(viewer, "%s for process ", name);CHKERRQ(ierr);
1008       for (p = 0; p < size; ++p) {
1009         if (p > 0 && p == size-1) {
1010           ierr = PetscViewerASCIIPrintf(viewer, ", and ", colors[p%numColors], p);CHKERRQ(ierr);
1011         } else if (p > 0) {
1012           ierr = PetscViewerASCIIPrintf(viewer, ", ", colors[p%numColors], p);CHKERRQ(ierr);
1013         }
1014         ierr = PetscViewerASCIIPrintf(viewer, "{\\textcolor{%s}%D}", colors[p%numColors], p);CHKERRQ(ierr);
1015       }
1016       ierr = PetscViewerASCIIPrintf(viewer, ".\n\n\n");CHKERRQ(ierr);
1017     }
1018     if (drawHasse) {
1019       PetscInt maxStratum = PetscMax(vEnd-vStart, PetscMax(eEnd-eStart, cEnd-cStart));
1020 
1021       ierr = PetscViewerASCIIPrintf(viewer, "\\newcommand{\\vStart}{%D}\n", vStart);CHKERRQ(ierr);
1022       ierr = PetscViewerASCIIPrintf(viewer, "\\newcommand{\\vEnd}{%D}\n", vEnd-1);CHKERRQ(ierr);
1023       ierr = PetscViewerASCIIPrintf(viewer, "\\newcommand{\\numVertices}{%D}\n", vEnd-vStart);CHKERRQ(ierr);
1024       ierr = PetscViewerASCIIPrintf(viewer, "\\newcommand{\\vShift}{%.2f}\n", 3 + (maxStratum-(vEnd-vStart))/2.);CHKERRQ(ierr);
1025       ierr = PetscViewerASCIIPrintf(viewer, "\\newcommand{\\eStart}{%D}\n", eStart);CHKERRQ(ierr);
1026       ierr = PetscViewerASCIIPrintf(viewer, "\\newcommand{\\eEnd}{%D}\n", eEnd-1);CHKERRQ(ierr);
1027       ierr = PetscViewerASCIIPrintf(viewer, "\\newcommand{\\eShift}{%.2f}\n", 3 + (maxStratum-(eEnd-eStart))/2.);CHKERRQ(ierr);
1028       ierr = PetscViewerASCIIPrintf(viewer, "\\newcommand{\\numEdges}{%D}\n", eEnd-eStart);CHKERRQ(ierr);
1029       ierr = PetscViewerASCIIPrintf(viewer, "\\newcommand{\\cStart}{%D}\n", cStart);CHKERRQ(ierr);
1030       ierr = PetscViewerASCIIPrintf(viewer, "\\newcommand{\\cEnd}{%D}\n", cEnd-1);CHKERRQ(ierr);
1031       ierr = PetscViewerASCIIPrintf(viewer, "\\newcommand{\\numCells}{%D}\n", cEnd-cStart);CHKERRQ(ierr);
1032       ierr = PetscViewerASCIIPrintf(viewer, "\\newcommand{\\cShift}{%.2f}\n", 3 + (maxStratum-(cEnd-cStart))/2.);CHKERRQ(ierr);
1033     }
1034     ierr = PetscViewerASCIIPrintf(viewer, "\\begin{tikzpicture}[scale = %g,font=\\fontsize{8}{8}\\selectfont]\n", (double) tikzscale);CHKERRQ(ierr);
1035 
1036     /* Plot vertices */
1037     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
1038     ierr = PetscViewerASCIIPushSynchronized(viewer);CHKERRQ(ierr);
1039     for (v = vStart; v < vEnd; ++v) {
1040       PetscInt  off, dof, d;
1041       PetscBool isLabeled = PETSC_FALSE;
1042 
1043       if (wp && !PetscBTLookup(wp,v - pStart)) continue;
1044       ierr = PetscSectionGetDof(coordSection, v, &dof);CHKERRQ(ierr);
1045       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
1046       ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\path (");CHKERRQ(ierr);
1047       PetscCheckFalse(dof > 3,PETSC_COMM_SELF,PETSC_ERR_PLIB,"coordSection vertex %D has dof %D > 3",v,dof);
1048       for (d = 0; d < dof; ++d) {
1049         tcoords[d] = (double) (scale*PetscRealPart(coords[off+d]));
1050         tcoords[d] = PetscAbs(tcoords[d]) < 1e-10 ? 0.0 : tcoords[d];
1051       }
1052       /* Rotate coordinates since PGF makes z point out of the page instead of up */
1053       if (dim == 3) {PetscReal tmp = tcoords[1]; tcoords[1] = tcoords[2]; tcoords[2] = -tmp;}
1054       for (d = 0; d < dof; ++d) {
1055         if (d > 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, ",");CHKERRQ(ierr);}
1056         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "%g", (double) tcoords[d]);CHKERRQ(ierr);
1057       }
1058       if (drawHasse) color = colors[0%numColors];
1059       else           color = colors[rank%numColors];
1060       for (l = 0; l < numLabels; ++l) {
1061         PetscInt val;
1062         ierr = DMGetLabelValue(dm, names[l], v, &val);CHKERRQ(ierr);
1063         if (val >= 0) {color = lcolors[l%numLColors]; isLabeled = PETSC_TRUE; break;}
1064       }
1065       if (drawNumbers[0]) {
1066         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [draw,shape=circle,color=%s] {%D};\n", v, rank, color, v);CHKERRQ(ierr);
1067       } else if (drawColors[0]) {
1068         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [fill,inner sep=%dpt,shape=circle,color=%s] {};\n", v, rank, !isLabeled ? 1 : 2, color);CHKERRQ(ierr);
1069       } else {
1070         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [] {};\n", v, rank);CHKERRQ(ierr);
1071       }
1072     }
1073     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
1074     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
1075     /* Plot edges */
1076     if (plotEdges) {
1077       ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
1078       ierr = PetscViewerASCIIPrintf(viewer, "\\path\n");CHKERRQ(ierr);
1079       for (e = eStart; e < eEnd; ++e) {
1080         const PetscInt *cone;
1081         PetscInt        coneSize, offA, offB, dof, d;
1082 
1083         if (wp && !PetscBTLookup(wp,e - pStart)) continue;
1084         ierr = DMPlexGetConeSize(dm, e, &coneSize);CHKERRQ(ierr);
1085         PetscCheckFalse(coneSize != 2,PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Edge %D cone should have two vertices, not %D", e, coneSize);
1086         ierr = DMPlexGetCone(dm, e, &cone);CHKERRQ(ierr);
1087         ierr = PetscSectionGetDof(coordSection, cone[0], &dof);CHKERRQ(ierr);
1088         ierr = PetscSectionGetOffset(coordSection, cone[0], &offA);CHKERRQ(ierr);
1089         ierr = PetscSectionGetOffset(coordSection, cone[1], &offB);CHKERRQ(ierr);
1090         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "(");CHKERRQ(ierr);
1091         for (d = 0; d < dof; ++d) {
1092           tcoords[d] = (double) (0.5*scale*PetscRealPart(coords[offA+d]+coords[offB+d]));
1093           tcoords[d] = PetscAbs(tcoords[d]) < 1e-10 ? 0.0 : tcoords[d];
1094         }
1095         /* Rotate coordinates since PGF makes z point out of the page instead of up */
1096         if (dim == 3) {PetscReal tmp = tcoords[1]; tcoords[1] = tcoords[2]; tcoords[2] = -tmp;}
1097         for (d = 0; d < dof; ++d) {
1098           if (d > 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, ",");CHKERRQ(ierr);}
1099           ierr = PetscViewerASCIISynchronizedPrintf(viewer, "%g", (double)tcoords[d]);CHKERRQ(ierr);
1100         }
1101         if (drawHasse) color = colors[1%numColors];
1102         else           color = colors[rank%numColors];
1103         for (l = 0; l < numLabels; ++l) {
1104           PetscInt val;
1105           ierr = DMGetLabelValue(dm, names[l], v, &val);CHKERRQ(ierr);
1106           if (val >= 0) {color = lcolors[l%numLColors]; break;}
1107         }
1108         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [draw,shape=circle,color=%s] {%D} --\n", e, rank, color, e);CHKERRQ(ierr);
1109       }
1110       ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
1111       ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
1112       ierr = PetscViewerASCIIPrintf(viewer, "(0,0);\n");CHKERRQ(ierr);
1113     }
1114     /* Plot cells */
1115     if (dim == 3 || !drawNumbers[1]) {
1116       for (e = eStart; e < eEnd; ++e) {
1117         const PetscInt *cone;
1118 
1119         if (wp && !PetscBTLookup(wp,e - pStart)) continue;
1120         color = colors[rank%numColors];
1121         for (l = 0; l < numLabels; ++l) {
1122           PetscInt val;
1123           ierr = DMGetLabelValue(dm, names[l], e, &val);CHKERRQ(ierr);
1124           if (val >= 0) {color = lcolors[l%numLColors]; break;}
1125         }
1126         ierr = DMPlexGetCone(dm, e, &cone);CHKERRQ(ierr);
1127         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\draw[color=%s] (%D_%d) -- (%D_%d);\n", color, cone[0], rank, cone[1], rank);CHKERRQ(ierr);
1128       }
1129     } else {
1130        DMPolytopeType ct;
1131 
1132       /* Drawing a 2D polygon */
1133       for (c = cStart; c < cEnd; ++c) {
1134         if (wp && !PetscBTLookup(wp, c - pStart)) continue;
1135         ierr = DMPlexGetCellType(dm, c, &ct);CHKERRQ(ierr);
1136         if (ct == DM_POLYTOPE_SEG_PRISM_TENSOR ||
1137             ct == DM_POLYTOPE_TRI_PRISM_TENSOR ||
1138             ct == DM_POLYTOPE_QUAD_PRISM_TENSOR) {
1139           const PetscInt *cone;
1140           PetscInt        coneSize, e;
1141 
1142           ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
1143           ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
1144           for (e = 0; e < coneSize; ++e) {
1145             const PetscInt *econe;
1146 
1147             ierr = DMPlexGetCone(dm, cone[e], &econe);CHKERRQ(ierr);
1148             ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\draw[color=%s] (%D_%d) -- (%D_%d) -- (%D_%d);\n", colors[rank%numColors], econe[0], rank, cone[e], rank, econe[1], rank);CHKERRQ(ierr);
1149           }
1150         } else {
1151           PetscInt *closure = NULL;
1152           PetscInt  closureSize, Nv = 0, v;
1153 
1154           ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1155           for (p = 0; p < closureSize*2; p += 2) {
1156             const PetscInt point = closure[p];
1157 
1158             if ((point >= vStart) && (point < vEnd)) closure[Nv++] = point;
1159           }
1160           ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\draw[color=%s] ", colors[rank%numColors]);CHKERRQ(ierr);
1161           for (v = 0; v <= Nv; ++v) {
1162             const PetscInt vertex = closure[v%Nv];
1163 
1164             if (v > 0) {
1165               if (plotEdges) {
1166                 const PetscInt *edge;
1167                 PetscInt        endpoints[2], ne;
1168 
1169                 endpoints[0] = closure[v-1]; endpoints[1] = vertex;
1170                 ierr = DMPlexGetJoin(dm, 2, endpoints, &ne, &edge);CHKERRQ(ierr);
1171                 PetscCheckFalse(ne != 1,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Could not find edge for vertices %D, %D", endpoints[0], endpoints[1]);
1172                 ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- (%D_%d) -- ", edge[0], rank);CHKERRQ(ierr);
1173                 ierr = DMPlexRestoreJoin(dm, 2, endpoints, &ne, &edge);CHKERRQ(ierr);
1174               } else {
1175                 ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- ");CHKERRQ(ierr);
1176               }
1177             }
1178             ierr = PetscViewerASCIISynchronizedPrintf(viewer, "(%D_%d)", vertex, rank);CHKERRQ(ierr);
1179           }
1180           ierr = PetscViewerASCIISynchronizedPrintf(viewer, ";\n");CHKERRQ(ierr);
1181           ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1182         }
1183       }
1184     }
1185     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
1186     for (c = cStart; c < cEnd; ++c) {
1187       double    ccoords[3] = {0.0, 0.0, 0.0};
1188       PetscBool isLabeled  = PETSC_FALSE;
1189       PetscInt *closure    = NULL;
1190       PetscInt  closureSize, dof, d, n = 0;
1191 
1192       if (wp && !PetscBTLookup(wp,c - pStart)) continue;
1193       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1194       ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\path (");CHKERRQ(ierr);
1195       for (p = 0; p < closureSize*2; p += 2) {
1196         const PetscInt point = closure[p];
1197         PetscInt       off;
1198 
1199         if ((point < vStart) || (point >= vEnd)) continue;
1200         ierr = PetscSectionGetDof(coordSection, point, &dof);CHKERRQ(ierr);
1201         ierr = PetscSectionGetOffset(coordSection, point, &off);CHKERRQ(ierr);
1202         for (d = 0; d < dof; ++d) {
1203           tcoords[d] = (double) (scale*PetscRealPart(coords[off+d]));
1204           tcoords[d] = PetscAbs(tcoords[d]) < 1e-10 ? 0.0 : tcoords[d];
1205         }
1206         /* Rotate coordinates since PGF makes z point out of the page instead of up */
1207         if (dof == 3) {PetscReal tmp = tcoords[1]; tcoords[1] = tcoords[2]; tcoords[2] = -tmp;}
1208         for (d = 0; d < dof; ++d) {ccoords[d] += tcoords[d];}
1209         ++n;
1210       }
1211       for (d = 0; d < dof; ++d) {ccoords[d] /= n;}
1212       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1213       for (d = 0; d < dof; ++d) {
1214         if (d > 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, ",");CHKERRQ(ierr);}
1215         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "%g", (double) ccoords[d]);CHKERRQ(ierr);
1216       }
1217       if (drawHasse) color = colors[depth%numColors];
1218       else           color = colors[rank%numColors];
1219       for (l = 0; l < numLabels; ++l) {
1220         PetscInt val;
1221         ierr = DMGetLabelValue(dm, names[l], c, &val);CHKERRQ(ierr);
1222         if (val >= 0) {color = lcolors[l%numLColors]; isLabeled = PETSC_TRUE; break;}
1223       }
1224       if (drawNumbers[dim]) {
1225         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [draw,shape=circle,color=%s] {%D};\n", c, rank, color, c);CHKERRQ(ierr);
1226       } else if (drawColors[dim]) {
1227         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [fill,inner sep=%dpt,shape=circle,color=%s] {};\n", c, rank, !isLabeled ? 1 : 2, color);CHKERRQ(ierr);
1228       } else {
1229         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [] {};\n", c, rank);CHKERRQ(ierr);
1230       }
1231     }
1232     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
1233     if (drawHasse) {
1234       color = colors[depth%numColors];
1235       ierr = PetscViewerASCIIPrintf(viewer, "%% Cells\n");CHKERRQ(ierr);
1236       ierr = PetscViewerASCIIPrintf(viewer, "\\foreach \\c in {\\cStart,...,\\cEnd}\n");CHKERRQ(ierr);
1237       ierr = PetscViewerASCIIPrintf(viewer, "{\n");CHKERRQ(ierr);
1238       ierr = PetscViewerASCIIPrintf(viewer, "  \\node(\\c_%d) [draw,shape=circle,color=%s,minimum size = 6mm] at (\\cShift+\\c-\\cStart,0) {\\c};\n", rank, color);CHKERRQ(ierr);
1239       ierr = PetscViewerASCIIPrintf(viewer, "}\n");CHKERRQ(ierr);
1240 
1241       color = colors[1%numColors];
1242       ierr = PetscViewerASCIIPrintf(viewer, "%% Edges\n");CHKERRQ(ierr);
1243       ierr = PetscViewerASCIIPrintf(viewer, "\\foreach \\e in {\\eStart,...,\\eEnd}\n");CHKERRQ(ierr);
1244       ierr = PetscViewerASCIIPrintf(viewer, "{\n");CHKERRQ(ierr);
1245       ierr = PetscViewerASCIIPrintf(viewer, "  \\node(\\e_%d) [draw,shape=circle,color=%s,minimum size = 6mm] at (\\eShift+\\e-\\eStart,1) {\\e};\n", rank, color);CHKERRQ(ierr);
1246       ierr = PetscViewerASCIIPrintf(viewer, "}\n");CHKERRQ(ierr);
1247 
1248       color = colors[0%numColors];
1249       ierr = PetscViewerASCIIPrintf(viewer, "%% Vertices\n");CHKERRQ(ierr);
1250       ierr = PetscViewerASCIIPrintf(viewer, "\\foreach \\v in {\\vStart,...,\\vEnd}\n");CHKERRQ(ierr);
1251       ierr = PetscViewerASCIIPrintf(viewer, "{\n");CHKERRQ(ierr);
1252       ierr = PetscViewerASCIIPrintf(viewer, "  \\node(\\v_%d) [draw,shape=circle,color=%s,minimum size = 6mm] at (\\vShift+\\v-\\vStart,2) {\\v};\n", rank, color);CHKERRQ(ierr);
1253       ierr = PetscViewerASCIIPrintf(viewer, "}\n");CHKERRQ(ierr);
1254 
1255       for (p = pStart; p < pEnd; ++p) {
1256         const PetscInt *cone;
1257         PetscInt        coneSize, cp;
1258 
1259         ierr = DMPlexGetCone(dm, p, &cone);CHKERRQ(ierr);
1260         ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
1261         for (cp = 0; cp < coneSize; ++cp) {
1262           ierr = PetscViewerASCIIPrintf(viewer, "\\draw[->, shorten >=1pt] (%D_%d) -- (%D_%d);\n", cone[cp], rank, p, rank);CHKERRQ(ierr);
1263         }
1264       }
1265     }
1266     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
1267     ierr = PetscViewerASCIIPopSynchronized(viewer);CHKERRQ(ierr);
1268     ierr = PetscViewerASCIIPrintf(viewer, "\\end{tikzpicture}\n");CHKERRQ(ierr);
1269     ierr = PetscViewerASCIIPrintf(viewer, "\\end{document}\n", name);CHKERRQ(ierr);
1270     for (l = 0; l < numLabels;  ++l) {ierr = PetscFree(names[l]);CHKERRQ(ierr);}
1271     for (c = 0; c < numColors;  ++c) {ierr = PetscFree(colors[c]);CHKERRQ(ierr);}
1272     for (c = 0; c < numLColors; ++c) {ierr = PetscFree(lcolors[c]);CHKERRQ(ierr);}
1273     ierr = PetscFree3(names, colors, lcolors);CHKERRQ(ierr);
1274     ierr = PetscBTDestroy(&wp);CHKERRQ(ierr);
1275   } else if (format == PETSC_VIEWER_LOAD_BALANCE) {
1276     Vec                    cown,acown;
1277     VecScatter             sct;
1278     ISLocalToGlobalMapping g2l;
1279     IS                     gid,acis;
1280     MPI_Comm               comm,ncomm = MPI_COMM_NULL;
1281     MPI_Group              ggroup,ngroup;
1282     PetscScalar            *array,nid;
1283     const PetscInt         *idxs;
1284     PetscInt               *idxs2,*start,*adjacency,*work;
1285     PetscInt64             lm[3],gm[3];
1286     PetscInt               i,c,cStart,cEnd,cum,numVertices,ect,ectn,cellHeight;
1287     PetscMPIInt            d1,d2,rank;
1288 
1289     ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
1290     ierr = MPI_Comm_rank(comm,&rank);CHKERRMPI(ierr);
1291 #if defined(PETSC_HAVE_MPI_PROCESS_SHARED_MEMORY)
1292     ierr = MPI_Comm_split_type(comm,MPI_COMM_TYPE_SHARED,rank,MPI_INFO_NULL,&ncomm);CHKERRMPI(ierr);
1293 #endif
1294     if (ncomm != MPI_COMM_NULL) {
1295       ierr = MPI_Comm_group(comm,&ggroup);CHKERRMPI(ierr);
1296       ierr = MPI_Comm_group(ncomm,&ngroup);CHKERRMPI(ierr);
1297       d1   = 0;
1298       ierr = MPI_Group_translate_ranks(ngroup,1,&d1,ggroup,&d2);CHKERRMPI(ierr);
1299       nid  = d2;
1300       ierr = MPI_Group_free(&ggroup);CHKERRMPI(ierr);
1301       ierr = MPI_Group_free(&ngroup);CHKERRMPI(ierr);
1302       ierr = MPI_Comm_free(&ncomm);CHKERRMPI(ierr);
1303     } else nid = 0.0;
1304 
1305     /* Get connectivity */
1306     ierr = DMPlexGetVTKCellHeight(dm,&cellHeight);CHKERRQ(ierr);
1307     ierr = DMPlexCreatePartitionerGraph(dm,cellHeight,&numVertices,&start,&adjacency,&gid);CHKERRQ(ierr);
1308 
1309     /* filter overlapped local cells */
1310     ierr = DMPlexGetHeightStratum(dm,cellHeight,&cStart,&cEnd);CHKERRQ(ierr);
1311     ierr = ISGetIndices(gid,&idxs);CHKERRQ(ierr);
1312     ierr = ISGetLocalSize(gid,&cum);CHKERRQ(ierr);
1313     ierr = PetscMalloc1(cum,&idxs2);CHKERRQ(ierr);
1314     for (c = cStart, cum = 0; c < cEnd; c++) {
1315       if (idxs[c-cStart] < 0) continue;
1316       idxs2[cum++] = idxs[c-cStart];
1317     }
1318     ierr = ISRestoreIndices(gid,&idxs);CHKERRQ(ierr);
1319     PetscCheckFalse(numVertices != cum,PETSC_COMM_SELF,PETSC_ERR_PLIB,"Unexpected %D != %D",numVertices,cum);
1320     ierr = ISDestroy(&gid);CHKERRQ(ierr);
1321     ierr = ISCreateGeneral(comm,numVertices,idxs2,PETSC_OWN_POINTER,&gid);CHKERRQ(ierr);
1322 
1323     /* support for node-aware cell locality */
1324     ierr = ISCreateGeneral(comm,start[numVertices],adjacency,PETSC_USE_POINTER,&acis);CHKERRQ(ierr);
1325     ierr = VecCreateSeq(PETSC_COMM_SELF,start[numVertices],&acown);CHKERRQ(ierr);
1326     ierr = VecCreateMPI(comm,numVertices,PETSC_DECIDE,&cown);CHKERRQ(ierr);
1327     ierr = VecGetArray(cown,&array);CHKERRQ(ierr);
1328     for (c = 0; c < numVertices; c++) array[c] = nid;
1329     ierr = VecRestoreArray(cown,&array);CHKERRQ(ierr);
1330     ierr = VecScatterCreate(cown,acis,acown,NULL,&sct);CHKERRQ(ierr);
1331     ierr = VecScatterBegin(sct,cown,acown,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
1332     ierr = VecScatterEnd(sct,cown,acown,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
1333     ierr = ISDestroy(&acis);CHKERRQ(ierr);
1334     ierr = VecScatterDestroy(&sct);CHKERRQ(ierr);
1335     ierr = VecDestroy(&cown);CHKERRQ(ierr);
1336 
1337     /* compute edgeCut */
1338     for (c = 0, cum = 0; c < numVertices; c++) cum = PetscMax(cum,start[c+1]-start[c]);
1339     ierr = PetscMalloc1(cum,&work);CHKERRQ(ierr);
1340     ierr = ISLocalToGlobalMappingCreateIS(gid,&g2l);CHKERRQ(ierr);
1341     ierr = ISLocalToGlobalMappingSetType(g2l,ISLOCALTOGLOBALMAPPINGHASH);CHKERRQ(ierr);
1342     ierr = ISDestroy(&gid);CHKERRQ(ierr);
1343     ierr = VecGetArray(acown,&array);CHKERRQ(ierr);
1344     for (c = 0, ect = 0, ectn = 0; c < numVertices; c++) {
1345       PetscInt totl;
1346 
1347       totl = start[c+1]-start[c];
1348       ierr = ISGlobalToLocalMappingApply(g2l,IS_GTOLM_MASK,totl,adjacency+start[c],NULL,work);CHKERRQ(ierr);
1349       for (i = 0; i < totl; i++) {
1350         if (work[i] < 0) {
1351           ect  += 1;
1352           ectn += (array[i + start[c]] != nid) ? 0 : 1;
1353         }
1354       }
1355     }
1356     ierr  = PetscFree(work);CHKERRQ(ierr);
1357     ierr  = VecRestoreArray(acown,&array);CHKERRQ(ierr);
1358     lm[0] = numVertices > 0 ?  numVertices : PETSC_MAX_INT;
1359     lm[1] = -numVertices;
1360     ierr  = MPIU_Allreduce(lm,gm,2,MPIU_INT64,MPI_MIN,comm);CHKERRMPI(ierr);
1361     ierr  = PetscViewerASCIIPrintf(viewer,"  Cell balance: %.2f (max %D, min %D",-((double)gm[1])/((double)gm[0]),-(PetscInt)gm[1],(PetscInt)gm[0]);CHKERRQ(ierr);
1362     lm[0] = ect; /* edgeCut */
1363     lm[1] = ectn; /* node-aware edgeCut */
1364     lm[2] = numVertices > 0 ? 0 : 1; /* empty processes */
1365     ierr  = MPIU_Allreduce(lm,gm,3,MPIU_INT64,MPI_SUM,comm);CHKERRMPI(ierr);
1366     ierr  = PetscViewerASCIIPrintf(viewer,", empty %D)\n",(PetscInt)gm[2]);CHKERRQ(ierr);
1367 #if defined(PETSC_HAVE_MPI_PROCESS_SHARED_MEMORY)
1368     ierr  = PetscViewerASCIIPrintf(viewer,"  Edge Cut: %D (on node %.3f)\n",(PetscInt)(gm[0]/2),gm[0] ? ((double)(gm[1]))/((double)gm[0]) : 1.);CHKERRQ(ierr);
1369 #else
1370     ierr  = PetscViewerASCIIPrintf(viewer,"  Edge Cut: %D (on node %.3f)\n",(PetscInt)(gm[0]/2),0.0);CHKERRQ(ierr);
1371 #endif
1372     ierr  = ISLocalToGlobalMappingDestroy(&g2l);CHKERRQ(ierr);
1373     ierr  = PetscFree(start);CHKERRQ(ierr);
1374     ierr  = PetscFree(adjacency);CHKERRQ(ierr);
1375     ierr  = VecDestroy(&acown);CHKERRQ(ierr);
1376   } else {
1377     const char    *name;
1378     PetscInt      *sizes, *hybsizes, *ghostsizes;
1379     PetscInt       locDepth, depth, cellHeight, dim, d;
1380     PetscInt       pStart, pEnd, p, gcStart, gcEnd, gcNum;
1381     PetscInt       numLabels, l, maxSize = 17;
1382     DMPolytopeType ct0 = DM_POLYTOPE_UNKNOWN;
1383     MPI_Comm       comm;
1384     PetscMPIInt    size, rank;
1385 
1386     ierr = PetscObjectGetComm((PetscObject) dm, &comm);CHKERRQ(ierr);
1387     ierr = MPI_Comm_size(comm, &size);CHKERRMPI(ierr);
1388     ierr = MPI_Comm_rank(comm, &rank);CHKERRMPI(ierr);
1389     ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
1390     ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
1391     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
1392     if (name) {ierr = PetscViewerASCIIPrintf(viewer, "%s in %D dimension%s:\n", name, dim, dim == 1 ? "" : "s");CHKERRQ(ierr);}
1393     else      {ierr = PetscViewerASCIIPrintf(viewer, "Mesh in %D dimension%s:\n", dim, dim == 1 ? "" : "s");CHKERRQ(ierr);}
1394     if (cellHeight) {ierr = PetscViewerASCIIPrintf(viewer, "  Cells are at height %D\n", cellHeight);CHKERRQ(ierr);}
1395     ierr = DMPlexGetDepth(dm, &locDepth);CHKERRQ(ierr);
1396     ierr = MPIU_Allreduce(&locDepth, &depth, 1, MPIU_INT, MPI_MAX, comm);CHKERRMPI(ierr);
1397     ierr = DMPlexGetGhostCellStratum(dm, &gcStart, &gcEnd);CHKERRQ(ierr);
1398     gcNum = gcEnd - gcStart;
1399     if (size < maxSize) {ierr = PetscCalloc3(size, &sizes, size, &hybsizes, size, &ghostsizes);CHKERRQ(ierr);}
1400     else                {ierr = PetscCalloc3(3,    &sizes, 3,    &hybsizes, 3,    &ghostsizes);CHKERRQ(ierr);}
1401     for (d = 0; d <= depth; d++) {
1402       PetscInt Nc[2] = {0, 0}, ict;
1403 
1404       ierr = DMPlexGetDepthStratum(dm, d, &pStart, &pEnd);CHKERRQ(ierr);
1405       if (pStart < pEnd) {ierr = DMPlexGetCellType(dm, pStart, &ct0);CHKERRQ(ierr);}
1406       ict  = ct0;
1407       ierr = MPI_Bcast(&ict, 1, MPIU_INT, 0, comm);CHKERRMPI(ierr);
1408       ct0  = (DMPolytopeType) ict;
1409       for (p = pStart; p < pEnd; ++p) {
1410         DMPolytopeType ct;
1411 
1412         ierr = DMPlexGetCellType(dm, p, &ct);CHKERRQ(ierr);
1413         if (ct == ct0) ++Nc[0];
1414         else           ++Nc[1];
1415       }
1416       if (size < maxSize) {
1417         ierr = MPI_Gather(&Nc[0], 1, MPIU_INT, sizes,    1, MPIU_INT, 0, comm);CHKERRMPI(ierr);
1418         ierr = MPI_Gather(&Nc[1], 1, MPIU_INT, hybsizes, 1, MPIU_INT, 0, comm);CHKERRMPI(ierr);
1419         if (d == depth) {ierr = MPI_Gather(&gcNum, 1, MPIU_INT, ghostsizes, 1, MPIU_INT, 0, comm);CHKERRMPI(ierr);}
1420         ierr = PetscViewerASCIIPrintf(viewer, "  Number of %D-cells per rank:", (depth == 1) && d ? dim : d);CHKERRQ(ierr);
1421         for (p = 0; p < size; ++p) {
1422           if (rank == 0) {
1423             ierr = PetscViewerASCIIPrintf(viewer, " %D", sizes[p]+hybsizes[p]);CHKERRQ(ierr);
1424             if (hybsizes[p]   > 0) {ierr = PetscViewerASCIIPrintf(viewer, " (%D)", hybsizes[p]);CHKERRQ(ierr);}
1425             if (ghostsizes[p] > 0) {ierr = PetscViewerASCIIPrintf(viewer, " [%D]", ghostsizes[p]);CHKERRQ(ierr);}
1426           }
1427         }
1428       } else {
1429         PetscInt locMinMax[2];
1430 
1431         locMinMax[0] = Nc[0]+Nc[1]; locMinMax[1] = Nc[0]+Nc[1];
1432         ierr = PetscGlobalMinMaxInt(comm, locMinMax, sizes);CHKERRQ(ierr);
1433         locMinMax[0] = Nc[1]; locMinMax[1] = Nc[1];
1434         ierr = PetscGlobalMinMaxInt(comm, locMinMax, hybsizes);CHKERRQ(ierr);
1435         if (d == depth) {
1436           locMinMax[0] = gcNum; locMinMax[1] = gcNum;
1437           ierr = PetscGlobalMinMaxInt(comm, locMinMax, ghostsizes);CHKERRQ(ierr);
1438         }
1439         ierr = PetscViewerASCIIPrintf(viewer, "  Min/Max of %D-cells per rank:", (depth == 1) && d ? dim : d);CHKERRQ(ierr);
1440         ierr = PetscViewerASCIIPrintf(viewer, " %" PetscInt_FMT "/%" PetscInt_FMT, sizes[0], sizes[1]);CHKERRQ(ierr);
1441         if (hybsizes[0]   > 0) {ierr = PetscViewerASCIIPrintf(viewer, " (%" PetscInt_FMT "/%" PetscInt_FMT ")", hybsizes[0], hybsizes[1]);CHKERRQ(ierr);}
1442         if (ghostsizes[0] > 0) {ierr = PetscViewerASCIIPrintf(viewer, " [%" PetscInt_FMT "/%" PetscInt_FMT "]", ghostsizes[0], ghostsizes[1]);CHKERRQ(ierr);}
1443       }
1444       ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
1445     }
1446     ierr = PetscFree3(sizes, hybsizes, ghostsizes);CHKERRQ(ierr);
1447     {
1448       const PetscReal      *maxCell;
1449       const PetscReal      *L;
1450       const DMBoundaryType *bd;
1451       PetscBool             per, localized;
1452 
1453       ierr = DMGetPeriodicity(dm, &per, &maxCell, &L, &bd);CHKERRQ(ierr);
1454       ierr = DMGetCoordinatesLocalized(dm, &localized);CHKERRQ(ierr);
1455       if (per) {
1456         ierr = PetscViewerASCIIPrintf(viewer, "Periodic mesh (");CHKERRQ(ierr);
1457         ierr = PetscViewerASCIIUseTabs(viewer, PETSC_FALSE);CHKERRQ(ierr);
1458         for (d = 0; d < dim; ++d) {
1459           if (bd && d > 0) {ierr = PetscViewerASCIIPrintf(viewer, ", ");CHKERRQ(ierr);}
1460           if (bd)    {ierr = PetscViewerASCIIPrintf(viewer, "%s", DMBoundaryTypes[bd[d]]);CHKERRQ(ierr);}
1461         }
1462         ierr = PetscViewerASCIIPrintf(viewer, ") coordinates %s\n", localized ? "localized" : "not localized");CHKERRQ(ierr);
1463         ierr = PetscViewerASCIIUseTabs(viewer, PETSC_TRUE);CHKERRQ(ierr);
1464       }
1465     }
1466     ierr = DMGetNumLabels(dm, &numLabels);CHKERRQ(ierr);
1467     if (numLabels) {ierr = PetscViewerASCIIPrintf(viewer, "Labels:\n");CHKERRQ(ierr);}
1468     for (l = 0; l < numLabels; ++l) {
1469       DMLabel         label;
1470       const char     *name;
1471       IS              valueIS;
1472       const PetscInt *values;
1473       PetscInt        numValues, v;
1474 
1475       ierr = DMGetLabelName(dm, l, &name);CHKERRQ(ierr);
1476       ierr = DMGetLabel(dm, name, &label);CHKERRQ(ierr);
1477       ierr = DMLabelGetNumValues(label, &numValues);CHKERRQ(ierr);
1478       ierr = PetscViewerASCIIPrintf(viewer, "  %s: %D strata with value/size (", name, numValues);CHKERRQ(ierr);
1479       ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
1480       ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
1481       ierr = PetscViewerASCIIUseTabs(viewer, PETSC_FALSE);CHKERRQ(ierr);
1482       for (v = 0; v < numValues; ++v) {
1483         PetscInt size;
1484 
1485         ierr = DMLabelGetStratumSize(label, values[v], &size);CHKERRQ(ierr);
1486         if (v > 0) {ierr = PetscViewerASCIIPrintf(viewer, ", ");CHKERRQ(ierr);}
1487         ierr = PetscViewerASCIIPrintf(viewer, "%D (%D)", values[v], size);CHKERRQ(ierr);
1488       }
1489       ierr = PetscViewerASCIIPrintf(viewer, ")\n");CHKERRQ(ierr);
1490       ierr = PetscViewerASCIIUseTabs(viewer, PETSC_TRUE);CHKERRQ(ierr);
1491       ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
1492       ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
1493     }
1494     {
1495       char    **labelNames;
1496       PetscInt  Nl = numLabels;
1497       PetscBool flg;
1498 
1499       ierr = PetscMalloc1(Nl, &labelNames);CHKERRQ(ierr);
1500       ierr = PetscOptionsGetStringArray(((PetscObject) dm)->options, ((PetscObject) dm)->prefix, "-dm_plex_view_labels", labelNames, &Nl, &flg);CHKERRQ(ierr);
1501       for (l = 0; l < Nl; ++l) {
1502         DMLabel label;
1503 
1504         ierr = DMHasLabel(dm, labelNames[l], &flg);CHKERRQ(ierr);
1505         if (flg) {
1506           ierr = DMGetLabel(dm, labelNames[l], &label);CHKERRQ(ierr);
1507           ierr = DMLabelView(label, viewer);CHKERRQ(ierr);
1508         }
1509         ierr = PetscFree(labelNames[l]);CHKERRQ(ierr);
1510       }
1511       ierr = PetscFree(labelNames);CHKERRQ(ierr);
1512     }
1513     /* If no fields are specified, people do not want to see adjacency */
1514     if (dm->Nf) {
1515       PetscInt f;
1516 
1517       for (f = 0; f < dm->Nf; ++f) {
1518         const char *name;
1519 
1520         ierr = PetscObjectGetName(dm->fields[f].disc, &name);CHKERRQ(ierr);
1521         if (numLabels) {ierr = PetscViewerASCIIPrintf(viewer, "Field %s:\n", name);CHKERRQ(ierr);}
1522         ierr = PetscViewerASCIIPushTab(viewer);CHKERRQ(ierr);
1523         if (dm->fields[f].label) {ierr = DMLabelView(dm->fields[f].label, viewer);CHKERRQ(ierr);}
1524         if (dm->fields[f].adjacency[0]) {
1525           if (dm->fields[f].adjacency[1]) {ierr = PetscViewerASCIIPrintf(viewer, "adjacency FVM++\n");CHKERRQ(ierr);}
1526           else                            {ierr = PetscViewerASCIIPrintf(viewer, "adjacency FVM\n");CHKERRQ(ierr);}
1527         } else {
1528           if (dm->fields[f].adjacency[1]) {ierr = PetscViewerASCIIPrintf(viewer, "adjacency FEM\n");CHKERRQ(ierr);}
1529           else                            {ierr = PetscViewerASCIIPrintf(viewer, "adjacency FUNKY\n");CHKERRQ(ierr);}
1530         }
1531         ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr);
1532       }
1533     }
1534     ierr = DMGetCoarseDM(dm, &cdm);CHKERRQ(ierr);
1535     if (cdm) {
1536       ierr = PetscViewerASCIIPushTab(viewer);CHKERRQ(ierr);
1537       ierr = DMPlexView_Ascii(cdm, viewer);CHKERRQ(ierr);
1538       ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr);
1539     }
1540   }
1541   PetscFunctionReturn(0);
1542 }
1543 
1544 static PetscErrorCode DMPlexDrawCell(DM dm, PetscDraw draw, PetscInt cell, const PetscScalar coords[])
1545 {
1546   DMPolytopeType ct;
1547   PetscMPIInt    rank;
1548   PetscInt       cdim;
1549   PetscErrorCode ierr;
1550 
1551   PetscFunctionBegin;
1552   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRMPI(ierr);
1553   ierr = DMPlexGetCellType(dm, cell, &ct);CHKERRQ(ierr);
1554   ierr = DMGetCoordinateDim(dm, &cdim);CHKERRQ(ierr);
1555   switch (ct) {
1556   case DM_POLYTOPE_SEGMENT:
1557   case DM_POLYTOPE_POINT_PRISM_TENSOR:
1558     switch (cdim) {
1559     case 1:
1560     {
1561       const PetscReal y  = 0.5;  /* TODO Put it in the middle of the viewport */
1562       const PetscReal dy = 0.05; /* TODO Make it a fraction of the total length */
1563 
1564       ierr = PetscDrawLine(draw, PetscRealPart(coords[0]), y,    PetscRealPart(coords[1]), y,    PETSC_DRAW_BLACK);CHKERRQ(ierr);
1565       ierr = PetscDrawLine(draw, PetscRealPart(coords[0]), y+dy, PetscRealPart(coords[0]), y-dy, PETSC_DRAW_BLACK);CHKERRQ(ierr);
1566       ierr = PetscDrawLine(draw, PetscRealPart(coords[1]), y+dy, PetscRealPart(coords[1]), y-dy, PETSC_DRAW_BLACK);CHKERRQ(ierr);
1567     }
1568     break;
1569     case 2:
1570     {
1571       const PetscReal dx = (PetscRealPart(coords[3]) - PetscRealPart(coords[1]));
1572       const PetscReal dy = (PetscRealPart(coords[2]) - PetscRealPart(coords[0]));
1573       const PetscReal l  = 0.1/PetscSqrtReal(dx*dx + dy*dy);
1574 
1575       ierr = PetscDrawLine(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1576       ierr = PetscDrawLine(draw, PetscRealPart(coords[0])+l*dx, PetscRealPart(coords[1])+l*dy, PetscRealPart(coords[0])-l*dx, PetscRealPart(coords[1])-l*dy, PETSC_DRAW_BLACK);CHKERRQ(ierr);
1577       ierr = PetscDrawLine(draw, PetscRealPart(coords[2])+l*dx, PetscRealPart(coords[3])+l*dy, PetscRealPart(coords[2])-l*dx, PetscRealPart(coords[3])-l*dy, PETSC_DRAW_BLACK);CHKERRQ(ierr);
1578     }
1579     break;
1580     default: SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Cannot draw cells of dimension %D", cdim);
1581     }
1582     break;
1583   case DM_POLYTOPE_TRIANGLE:
1584     ierr = PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]),
1585                              PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1586                              PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1587                              PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2);CHKERRQ(ierr);
1588     ierr = PetscDrawLine(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1589     ierr = PetscDrawLine(draw, PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1590     ierr = PetscDrawLine(draw, PetscRealPart(coords[4]), PetscRealPart(coords[5]), PetscRealPart(coords[0]), PetscRealPart(coords[1]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1591     break;
1592   case DM_POLYTOPE_QUADRILATERAL:
1593     ierr = PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]),
1594                               PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1595                               PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1596                               PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2);CHKERRQ(ierr);
1597     ierr = PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), PetscRealPart(coords[6]), PetscRealPart(coords[7]),
1598                               PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1599                               PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1600                               PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2);CHKERRQ(ierr);
1601     ierr = PetscDrawLine(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1602     ierr = PetscDrawLine(draw, PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1603     ierr = PetscDrawLine(draw, PetscRealPart(coords[4]), PetscRealPart(coords[5]), PetscRealPart(coords[6]), PetscRealPart(coords[7]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1604     ierr = PetscDrawLine(draw, PetscRealPart(coords[6]), PetscRealPart(coords[7]), PetscRealPart(coords[0]), PetscRealPart(coords[1]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1605     break;
1606   default: SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Cannot draw cells of type %s", DMPolytopeTypes[ct]);
1607   }
1608   PetscFunctionReturn(0);
1609 }
1610 
1611 static PetscErrorCode DMPlexDrawCellHighOrder(DM dm, PetscDraw draw, PetscInt cell, const PetscScalar coords[], PetscInt edgeDiv, PetscReal refCoords[], PetscReal edgeCoords[])
1612 {
1613   DMPolytopeType ct;
1614   PetscReal      centroid[2] = {0., 0.};
1615   PetscMPIInt    rank;
1616   PetscInt       fillColor, v, e, d;
1617   PetscErrorCode ierr;
1618 
1619   PetscFunctionBegin;
1620   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRMPI(ierr);
1621   ierr = DMPlexGetCellType(dm, cell, &ct);CHKERRQ(ierr);
1622   fillColor = PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2;
1623   switch (ct) {
1624   case DM_POLYTOPE_TRIANGLE:
1625     {
1626       PetscReal refVertices[6] = {-1., -1., 1., -1., -1., 1.};
1627 
1628       for (v = 0; v < 3; ++v) {centroid[0] += PetscRealPart(coords[v*2+0])/3.;centroid[1] += PetscRealPart(coords[v*2+1])/3.;}
1629       for (e = 0; e < 3; ++e) {
1630         refCoords[0] = refVertices[e*2+0];
1631         refCoords[1] = refVertices[e*2+1];
1632         for (d = 1; d <= edgeDiv; ++d) {
1633           refCoords[d*2+0] = refCoords[0] + (refVertices[(e+1)%3 * 2 + 0] - refCoords[0])*d/edgeDiv;
1634           refCoords[d*2+1] = refCoords[1] + (refVertices[(e+1)%3 * 2 + 1] - refCoords[1])*d/edgeDiv;
1635         }
1636         ierr = DMPlexReferenceToCoordinates(dm, cell, edgeDiv+1, refCoords, edgeCoords);CHKERRQ(ierr);
1637         for (d = 0; d < edgeDiv; ++d) {
1638           ierr = PetscDrawTriangle(draw, centroid[0], centroid[1], edgeCoords[d*2+0], edgeCoords[d*2+1], edgeCoords[(d+1)*2+0], edgeCoords[(d+1)*2+1], fillColor, fillColor, fillColor);CHKERRQ(ierr);
1639           ierr = PetscDrawLine(draw, edgeCoords[d*2+0], edgeCoords[d*2+1], edgeCoords[(d+1)*2+0], edgeCoords[(d+1)*2+1], PETSC_DRAW_BLACK);CHKERRQ(ierr);
1640         }
1641       }
1642     }
1643     break;
1644   default: SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Cannot draw cells of type %s", DMPolytopeTypes[ct]);
1645   }
1646   PetscFunctionReturn(0);
1647 }
1648 
1649 static PetscErrorCode DMPlexView_Draw(DM dm, PetscViewer viewer)
1650 {
1651   PetscDraw          draw;
1652   DM                 cdm;
1653   PetscSection       coordSection;
1654   Vec                coordinates;
1655   const PetscScalar *coords;
1656   PetscReal          xyl[2],xyr[2],bound[4] = {PETSC_MAX_REAL, PETSC_MAX_REAL, PETSC_MIN_REAL, PETSC_MIN_REAL};
1657   PetscReal         *refCoords, *edgeCoords;
1658   PetscBool          isnull, drawAffine = PETSC_TRUE;
1659   PetscInt           dim, vStart, vEnd, cStart, cEnd, c, N, edgeDiv = 4;
1660   PetscErrorCode     ierr;
1661 
1662   PetscFunctionBegin;
1663   ierr = DMGetCoordinateDim(dm, &dim);CHKERRQ(ierr);
1664   PetscCheckFalse(dim > 2,PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Cannot draw meshes of dimension %D", dim);
1665   ierr = PetscOptionsGetBool(((PetscObject) dm)->options, ((PetscObject) dm)->prefix, "-dm_view_draw_affine", &drawAffine, NULL);CHKERRQ(ierr);
1666   if (!drawAffine) {ierr = PetscMalloc2((edgeDiv+1)*dim, &refCoords, (edgeDiv+1)*dim, &edgeCoords);CHKERRQ(ierr);}
1667   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
1668   ierr = DMGetLocalSection(cdm, &coordSection);CHKERRQ(ierr);
1669   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
1670   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
1671   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
1672 
1673   ierr = PetscViewerDrawGetDraw(viewer, 0, &draw);CHKERRQ(ierr);
1674   ierr = PetscDrawIsNull(draw, &isnull);CHKERRQ(ierr);
1675   if (isnull) PetscFunctionReturn(0);
1676   ierr = PetscDrawSetTitle(draw, "Mesh");CHKERRQ(ierr);
1677 
1678   ierr = VecGetLocalSize(coordinates, &N);CHKERRQ(ierr);
1679   ierr = VecGetArrayRead(coordinates, &coords);CHKERRQ(ierr);
1680   for (c = 0; c < N; c += dim) {
1681     bound[0] = PetscMin(bound[0], PetscRealPart(coords[c]));   bound[2] = PetscMax(bound[2], PetscRealPart(coords[c]));
1682     bound[1] = PetscMin(bound[1], PetscRealPart(coords[c+1])); bound[3] = PetscMax(bound[3], PetscRealPart(coords[c+1]));
1683   }
1684   ierr = VecRestoreArrayRead(coordinates, &coords);CHKERRQ(ierr);
1685   ierr = MPIU_Allreduce(&bound[0],xyl,2,MPIU_REAL,MPIU_MIN,PetscObjectComm((PetscObject)dm));CHKERRMPI(ierr);
1686   ierr = MPIU_Allreduce(&bound[2],xyr,2,MPIU_REAL,MPIU_MAX,PetscObjectComm((PetscObject)dm));CHKERRMPI(ierr);
1687   ierr = PetscDrawSetCoordinates(draw, xyl[0], xyl[1], xyr[0], xyr[1]);CHKERRQ(ierr);
1688   ierr = PetscDrawClear(draw);CHKERRQ(ierr);
1689 
1690   for (c = cStart; c < cEnd; ++c) {
1691     PetscScalar *coords = NULL;
1692     PetscInt     numCoords;
1693 
1694     ierr = DMPlexVecGetClosureAtDepth_Internal(dm, coordSection, coordinates, c, 0, &numCoords, &coords);CHKERRQ(ierr);
1695     if (drawAffine) {
1696       ierr = DMPlexDrawCell(dm, draw, c, coords);CHKERRQ(ierr);
1697     } else {
1698       ierr = DMPlexDrawCellHighOrder(dm, draw, c, coords, edgeDiv, refCoords, edgeCoords);CHKERRQ(ierr);
1699     }
1700     ierr = DMPlexVecRestoreClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
1701   }
1702   if (!drawAffine) {ierr = PetscFree2(refCoords, edgeCoords);CHKERRQ(ierr);}
1703   ierr = PetscDrawFlush(draw);CHKERRQ(ierr);
1704   ierr = PetscDrawPause(draw);CHKERRQ(ierr);
1705   ierr = PetscDrawSave(draw);CHKERRQ(ierr);
1706   PetscFunctionReturn(0);
1707 }
1708 
1709 #if defined(PETSC_HAVE_EXODUSII)
1710 #include <exodusII.h>
1711 #include <petscviewerexodusii.h>
1712 #endif
1713 
1714 PetscErrorCode DMView_Plex(DM dm, PetscViewer viewer)
1715 {
1716   PetscBool      iascii, ishdf5, isvtk, isdraw, flg, isglvis, isexodus;
1717   char           name[PETSC_MAX_PATH_LEN];
1718   PetscErrorCode ierr;
1719 
1720   PetscFunctionBegin;
1721   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1722   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1723   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERASCII,    &iascii);CHKERRQ(ierr);
1724   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,      &isvtk);CHKERRQ(ierr);
1725   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,     &ishdf5);CHKERRQ(ierr);
1726   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERDRAW,     &isdraw);CHKERRQ(ierr);
1727   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERGLVIS,    &isglvis);CHKERRQ(ierr);
1728   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWEREXODUSII, &isexodus);CHKERRQ(ierr);
1729   if (iascii) {
1730     PetscViewerFormat format;
1731     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
1732     if (format == PETSC_VIEWER_ASCII_GLVIS) {
1733       ierr = DMPlexView_GLVis(dm, viewer);CHKERRQ(ierr);
1734     } else {
1735       ierr = DMPlexView_Ascii(dm, viewer);CHKERRQ(ierr);
1736     }
1737   } else if (ishdf5) {
1738 #if defined(PETSC_HAVE_HDF5)
1739     ierr = DMPlexView_HDF5_Internal(dm, viewer);CHKERRQ(ierr);
1740 #else
1741     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1742 #endif
1743   } else if (isvtk) {
1744     ierr = DMPlexVTKWriteAll((PetscObject) dm,viewer);CHKERRQ(ierr);
1745   } else if (isdraw) {
1746     ierr = DMPlexView_Draw(dm, viewer);CHKERRQ(ierr);
1747   } else if (isglvis) {
1748     ierr = DMPlexView_GLVis(dm, viewer);CHKERRQ(ierr);
1749 #if defined(PETSC_HAVE_EXODUSII)
1750   } else if (isexodus) {
1751 /*
1752       exodusII requires that all sets be part of exactly one cell set.
1753       If the dm does not have a "Cell Sets" label defined, we create one
1754       with ID 1, containig all cells.
1755       Note that if the Cell Sets label is defined but does not cover all cells,
1756       we may still have a problem. This should probably be checked here or in the viewer;
1757     */
1758     PetscInt numCS;
1759     ierr = DMGetLabelSize(dm,"Cell Sets",&numCS);CHKERRQ(ierr);
1760     if (!numCS) {
1761       PetscInt cStart, cEnd, c;
1762       ierr = DMCreateLabel(dm, "Cell Sets");CHKERRQ(ierr);
1763       ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
1764       for (c = cStart; c < cEnd; ++c) {ierr = DMSetLabelValue(dm, "Cell Sets", c, 1);CHKERRQ(ierr);}
1765     }
1766     ierr = DMView_PlexExodusII(dm, viewer);CHKERRQ(ierr);
1767 #endif
1768   } else {
1769     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Viewer type %s not yet supported for DMPlex writing", ((PetscObject)viewer)->type_name);
1770   }
1771   /* Optionally view the partition */
1772   ierr = PetscOptionsHasName(((PetscObject) dm)->options, ((PetscObject) dm)->prefix, "-dm_partition_view", &flg);CHKERRQ(ierr);
1773   if (flg) {
1774     Vec ranks;
1775     ierr = DMPlexCreateRankField(dm, &ranks);CHKERRQ(ierr);
1776     ierr = VecView(ranks, viewer);CHKERRQ(ierr);
1777     ierr = VecDestroy(&ranks);CHKERRQ(ierr);
1778   }
1779   /* Optionally view a label */
1780   ierr = PetscOptionsGetString(((PetscObject) dm)->options, ((PetscObject) dm)->prefix, "-dm_label_view", name, sizeof(name), &flg);CHKERRQ(ierr);
1781   if (flg) {
1782     DMLabel label;
1783     Vec     val;
1784 
1785     ierr = DMGetLabel(dm, name, &label);CHKERRQ(ierr);
1786     PetscCheckFalse(!label,PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Label %s provided to -dm_label_view does not exist in this DM", name);
1787     ierr = DMPlexCreateLabelField(dm, label, &val);CHKERRQ(ierr);
1788     ierr = VecView(val, viewer);CHKERRQ(ierr);
1789     ierr = VecDestroy(&val);CHKERRQ(ierr);
1790   }
1791   PetscFunctionReturn(0);
1792 }
1793 
1794 /*@
1795   DMPlexTopologyView - Saves a DMPlex topology into a file
1796 
1797   Collective on DM
1798 
1799   Input Parameters:
1800 + dm     - The DM whose topology is to be saved
1801 - viewer - The PetscViewer for saving
1802 
1803   Level: advanced
1804 
1805 .seealso: DMView(), DMPlexCoordinatesView(), DMPlexLabelsView(), DMPlexTopologyLoad()
1806 @*/
1807 PetscErrorCode DMPlexTopologyView(DM dm, PetscViewer viewer)
1808 {
1809   PetscBool      ishdf5;
1810   PetscErrorCode ierr;
1811 
1812   PetscFunctionBegin;
1813   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1814   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1815   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
1816   ierr = PetscLogEventBegin(DMPLEX_TopologyView,viewer,0,0,0);CHKERRQ(ierr);
1817   if (ishdf5) {
1818 #if defined(PETSC_HAVE_HDF5)
1819     PetscViewerFormat format;
1820     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
1821     if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
1822       IS globalPointNumbering;
1823 
1824       ierr = DMPlexCreatePointNumbering(dm, &globalPointNumbering);CHKERRQ(ierr);
1825       ierr = DMPlexTopologyView_HDF5_Internal(dm, globalPointNumbering, viewer);CHKERRQ(ierr);
1826       ierr = ISDestroy(&globalPointNumbering);CHKERRQ(ierr);
1827     } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 output.", PetscViewerFormats[format]);
1828 #else
1829     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1830 #endif
1831   }
1832   ierr = PetscLogEventEnd(DMPLEX_TopologyView,viewer,0,0,0);CHKERRQ(ierr);
1833   PetscFunctionReturn(0);
1834 }
1835 
1836 /*@
1837   DMPlexCoordinatesView - Saves DMPlex coordinates into a file
1838 
1839   Collective on DM
1840 
1841   Input Parameters:
1842 + dm     - The DM whose coordinates are to be saved
1843 - viewer - The PetscViewer for saving
1844 
1845   Level: advanced
1846 
1847 .seealso: DMView(), DMPlexTopologyView(), DMPlexLabelsView(), DMPlexCoordinatesLoad()
1848 @*/
1849 PetscErrorCode DMPlexCoordinatesView(DM dm, PetscViewer viewer)
1850 {
1851   PetscBool      ishdf5;
1852   PetscErrorCode ierr;
1853 
1854   PetscFunctionBegin;
1855   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1856   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1857   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
1858   ierr = PetscLogEventBegin(DMPLEX_CoordinatesView,viewer,0,0,0);CHKERRQ(ierr);
1859   if (ishdf5) {
1860 #if defined(PETSC_HAVE_HDF5)
1861     PetscViewerFormat format;
1862     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
1863     if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
1864       ierr = DMPlexCoordinatesView_HDF5_Internal(dm, viewer);CHKERRQ(ierr);
1865     } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 input.", PetscViewerFormats[format]);
1866 #else
1867     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1868 #endif
1869   }
1870   ierr = PetscLogEventEnd(DMPLEX_CoordinatesView,viewer,0,0,0);CHKERRQ(ierr);
1871   PetscFunctionReturn(0);
1872 }
1873 
1874 /*@
1875   DMPlexLabelsView - Saves DMPlex labels into a file
1876 
1877   Collective on DM
1878 
1879   Input Parameters:
1880 + dm     - The DM whose labels are to be saved
1881 - viewer - The PetscViewer for saving
1882 
1883   Level: advanced
1884 
1885 .seealso: DMView(), DMPlexTopologyView(), DMPlexCoordinatesView(), DMPlexLabelsLoad()
1886 @*/
1887 PetscErrorCode DMPlexLabelsView(DM dm, PetscViewer viewer)
1888 {
1889   PetscBool      ishdf5;
1890   PetscErrorCode ierr;
1891 
1892   PetscFunctionBegin;
1893   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1894   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1895   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
1896   ierr = PetscLogEventBegin(DMPLEX_LabelsView,viewer,0,0,0);CHKERRQ(ierr);
1897   if (ishdf5) {
1898 #if defined(PETSC_HAVE_HDF5)
1899     IS                globalPointNumbering;
1900     PetscViewerFormat format;
1901 
1902     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
1903     if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
1904       ierr = DMPlexCreatePointNumbering(dm, &globalPointNumbering);CHKERRQ(ierr);
1905       ierr = DMPlexLabelsView_HDF5_Internal(dm, globalPointNumbering, viewer);CHKERRQ(ierr);
1906       ierr = ISDestroy(&globalPointNumbering);CHKERRQ(ierr);
1907     } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 input.", PetscViewerFormats[format]);
1908 #else
1909     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1910 #endif
1911   }
1912   ierr = PetscLogEventEnd(DMPLEX_LabelsView,viewer,0,0,0);CHKERRQ(ierr);
1913   PetscFunctionReturn(0);
1914 }
1915 
1916 /*@
1917   DMPlexSectionView - Saves a section associated with a DMPlex
1918 
1919   Collective on DM
1920 
1921   Input Parameters:
1922 + dm         - The DM that contains the topology on which the section to be saved is defined
1923 . viewer     - The PetscViewer for saving
1924 - sectiondm  - The DM that contains the section to be saved
1925 
1926   Level: advanced
1927 
1928   Notes:
1929   This function is a wrapper around PetscSectionView(); in addition to the raw section, it saves information that associates the section points to the topology (dm) points. When the topology (dm) and the section are later loaded with DMPlexTopologyLoad() and DMPlexSectionLoad(), respectively, this information is used to match section points with topology points.
1930 
1931   In general dm and sectiondm are two different objects, the former carrying the topology and the latter carrying the section, and have been given a topology name and a section name, respectively, with PetscObjectSetName(). In practice, however, they can be the same object if it carries both topology and section; in that case the name of the object is used as both the topology name and the section name.
1932 
1933 .seealso: DMView(), DMPlexTopologyView(), DMPlexCoordinatesView(), DMPlexLabelsView(), DMPlexGlobalVectorView(), DMPlexLocalVectorView(), PetscSectionView(), DMPlexSectionLoad()
1934 @*/
1935 PetscErrorCode DMPlexSectionView(DM dm, PetscViewer viewer, DM sectiondm)
1936 {
1937   PetscBool      ishdf5;
1938   PetscErrorCode ierr;
1939 
1940   PetscFunctionBegin;
1941   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1942   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1943   PetscValidHeaderSpecific(sectiondm, DM_CLASSID, 3);
1944   ierr = PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERHDF5,&ishdf5);CHKERRQ(ierr);
1945   ierr = PetscLogEventBegin(DMPLEX_SectionView,viewer,0,0,0);CHKERRQ(ierr);
1946   if (ishdf5) {
1947 #if defined(PETSC_HAVE_HDF5)
1948     ierr = DMPlexSectionView_HDF5_Internal(dm, viewer, sectiondm);CHKERRQ(ierr);
1949 #else
1950     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1951 #endif
1952   }
1953   ierr = PetscLogEventEnd(DMPLEX_SectionView,viewer,0,0,0);CHKERRQ(ierr);
1954   PetscFunctionReturn(0);
1955 }
1956 
1957 /*@
1958   DMPlexGlobalVectorView - Saves a global vector
1959 
1960   Collective on DM
1961 
1962   Input Parameters:
1963 + dm        - The DM that represents the topology
1964 . viewer    - The PetscViewer to save data with
1965 . sectiondm - The DM that contains the global section on which vec is defined
1966 - vec       - The global vector to be saved
1967 
1968   Level: advanced
1969 
1970   Notes:
1971   In general dm and sectiondm are two different objects, the former carrying the topology and the latter carrying the section, and have been given a topology name and a section name, respectively, with PetscObjectSetName(). In practice, however, they can be the same object if it carries both topology and section; in that case the name of the object is used as both the topology name and the section name.
1972 
1973   Typical calling sequence
1974 $       DMCreate(PETSC_COMM_WORLD, &dm);
1975 $       DMSetType(dm, DMPLEX);
1976 $       PetscObjectSetName((PetscObject)dm, "topologydm_name");
1977 $       DMClone(dm, &sectiondm);
1978 $       PetscObjectSetName((PetscObject)sectiondm, "sectiondm_name");
1979 $       PetscSectionCreate(PETSC_COMM_WORLD, &section);
1980 $       DMPlexGetChart(sectiondm, &pStart, &pEnd);
1981 $       PetscSectionSetChart(section, pStart, pEnd);
1982 $       PetscSectionSetUp(section);
1983 $       DMSetLocalSection(sectiondm, section);
1984 $       PetscSectionDestroy(&section);
1985 $       DMGetGlobalVector(sectiondm, &vec);
1986 $       PetscObjectSetName((PetscObject)vec, "vec_name");
1987 $       DMPlexTopologyView(dm, viewer);
1988 $       DMPlexSectionView(dm, viewer, sectiondm);
1989 $       DMPlexGlobalVectorView(dm, viewer, sectiondm, vec);
1990 $       DMRestoreGlobalVector(sectiondm, &vec);
1991 $       DMDestroy(&sectiondm);
1992 $       DMDestroy(&dm);
1993 
1994 .seealso: DMPlexTopologyView(), DMPlexSectionView(), DMPlexLocalVectorView(), DMPlexGlobalVectorLoad(), DMPlexLocalVectorLoad()
1995 @*/
1996 PetscErrorCode DMPlexGlobalVectorView(DM dm, PetscViewer viewer, DM sectiondm, Vec vec)
1997 {
1998   PetscBool       ishdf5;
1999   PetscErrorCode  ierr;
2000 
2001   PetscFunctionBegin;
2002   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2003   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
2004   PetscValidHeaderSpecific(sectiondm, DM_CLASSID, 3);
2005   PetscValidHeaderSpecific(vec, VEC_CLASSID, 4);
2006   /* Check consistency */
2007   {
2008     PetscSection  section;
2009     PetscBool     includesConstraints;
2010     PetscInt      m, m1;
2011 
2012     ierr = VecGetLocalSize(vec, &m1);CHKERRQ(ierr);
2013     ierr = DMGetGlobalSection(sectiondm, &section);CHKERRQ(ierr);
2014     ierr = PetscSectionGetIncludesConstraints(section, &includesConstraints);CHKERRQ(ierr);
2015     if (includesConstraints) {ierr = PetscSectionGetStorageSize(section, &m);CHKERRQ(ierr);}
2016     else {ierr = PetscSectionGetConstrainedStorageSize(section, &m);CHKERRQ(ierr);}
2017     PetscCheckFalse(m1 != m,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Global vector size (%D) != global section storage size (%D)", m1, m);
2018   }
2019   ierr = PetscObjectTypeCompare((PetscObject)viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
2020   ierr = PetscLogEventBegin(DMPLEX_GlobalVectorView,viewer,0,0,0);CHKERRQ(ierr);
2021   if (ishdf5) {
2022 #if defined(PETSC_HAVE_HDF5)
2023     ierr = DMPlexGlobalVectorView_HDF5_Internal(dm, viewer, sectiondm, vec);CHKERRQ(ierr);
2024 #else
2025     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
2026 #endif
2027   }
2028   ierr = PetscLogEventEnd(DMPLEX_GlobalVectorView,viewer,0,0,0);CHKERRQ(ierr);
2029   PetscFunctionReturn(0);
2030 }
2031 
2032 /*@
2033   DMPlexLocalVectorView - Saves a local vector
2034 
2035   Collective on DM
2036 
2037   Input Parameters:
2038 + dm        - The DM that represents the topology
2039 . viewer    - The PetscViewer to save data with
2040 . sectiondm - The DM that contains the local section on which vec is defined; may be the same as dm
2041 - vec       - The local vector to be saved
2042 
2043   Level: advanced
2044 
2045   Notes:
2046   In general dm and sectiondm are two different objects, the former carrying the topology and the latter carrying the section, and have been given a topology name and a section name, respectively, with PetscObjectSetName(). In practice, however, they can be the same object if it carries both topology and section; in that case the name of the object is used as both the topology name and the section name.
2047 
2048   Typical calling sequence
2049 $       DMCreate(PETSC_COMM_WORLD, &dm);
2050 $       DMSetType(dm, DMPLEX);
2051 $       PetscObjectSetName((PetscObject)dm, "topologydm_name");
2052 $       DMClone(dm, &sectiondm);
2053 $       PetscObjectSetName((PetscObject)sectiondm, "sectiondm_name");
2054 $       PetscSectionCreate(PETSC_COMM_WORLD, &section);
2055 $       DMPlexGetChart(sectiondm, &pStart, &pEnd);
2056 $       PetscSectionSetChart(section, pStart, pEnd);
2057 $       PetscSectionSetUp(section);
2058 $       DMSetLocalSection(sectiondm, section);
2059 $       DMGetLocalVector(sectiondm, &vec);
2060 $       PetscObjectSetName((PetscObject)vec, "vec_name");
2061 $       DMPlexTopologyView(dm, viewer);
2062 $       DMPlexSectionView(dm, viewer, sectiondm);
2063 $       DMPlexLocalVectorView(dm, viewer, sectiondm, vec);
2064 $       DMRestoreLocalVector(sectiondm, &vec);
2065 $       DMDestroy(&sectiondm);
2066 $       DMDestroy(&dm);
2067 
2068 .seealso: DMPlexTopologyView(), DMPlexSectionView(), DMPlexGlobalVectorView(), DMPlexGlobalVectorLoad(), DMPlexLocalVectorLoad()
2069 @*/
2070 PetscErrorCode DMPlexLocalVectorView(DM dm, PetscViewer viewer, DM sectiondm, Vec vec)
2071 {
2072   PetscBool       ishdf5;
2073   PetscErrorCode  ierr;
2074 
2075   PetscFunctionBegin;
2076   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2077   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
2078   PetscValidHeaderSpecific(sectiondm, DM_CLASSID, 3);
2079   PetscValidHeaderSpecific(vec, VEC_CLASSID, 4);
2080   /* Check consistency */
2081   {
2082     PetscSection  section;
2083     PetscBool     includesConstraints;
2084     PetscInt      m, m1;
2085 
2086     ierr = VecGetLocalSize(vec, &m1);CHKERRQ(ierr);
2087     ierr = DMGetLocalSection(sectiondm, &section);CHKERRQ(ierr);
2088     ierr = PetscSectionGetIncludesConstraints(section, &includesConstraints);CHKERRQ(ierr);
2089     if (includesConstraints) {ierr = PetscSectionGetStorageSize(section, &m);CHKERRQ(ierr);}
2090     else {ierr = PetscSectionGetConstrainedStorageSize(section, &m);CHKERRQ(ierr);}
2091     PetscCheckFalse(m1 != m,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Local vector size (%D) != local section storage size (%D)", m1, m);
2092   }
2093   ierr = PetscObjectTypeCompare((PetscObject)viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
2094   ierr = PetscLogEventBegin(DMPLEX_LocalVectorView,viewer,0,0,0);CHKERRQ(ierr);
2095   if (ishdf5) {
2096 #if defined(PETSC_HAVE_HDF5)
2097     ierr = DMPlexLocalVectorView_HDF5_Internal(dm, viewer, sectiondm, vec);CHKERRQ(ierr);
2098 #else
2099     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
2100 #endif
2101   }
2102   ierr = PetscLogEventEnd(DMPLEX_LocalVectorView,viewer,0,0,0);CHKERRQ(ierr);
2103   PetscFunctionReturn(0);
2104 }
2105 
2106 PetscErrorCode DMLoad_Plex(DM dm, PetscViewer viewer)
2107 {
2108   PetscBool      ishdf5;
2109   PetscErrorCode ierr;
2110 
2111   PetscFunctionBegin;
2112   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2113   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
2114   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,   &ishdf5);CHKERRQ(ierr);
2115   if (ishdf5) {
2116 #if defined(PETSC_HAVE_HDF5)
2117     PetscViewerFormat format;
2118     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
2119     if (format == PETSC_VIEWER_HDF5_XDMF || format == PETSC_VIEWER_HDF5_VIZ) {
2120       ierr = DMPlexLoad_HDF5_Xdmf_Internal(dm, viewer);CHKERRQ(ierr);
2121     } else if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
2122       ierr = DMPlexLoad_HDF5_Internal(dm, viewer);CHKERRQ(ierr);
2123     } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 input.", PetscViewerFormats[format]);
2124     PetscFunctionReturn(0);
2125 #else
2126     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
2127 #endif
2128   } else SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Viewer type %s not yet supported for DMPlex loading", ((PetscObject)viewer)->type_name);
2129 }
2130 
2131 /*@
2132   DMPlexTopologyLoad - Loads a topology into a DMPlex
2133 
2134   Collective on DM
2135 
2136   Input Parameters:
2137 + dm     - The DM into which the topology is loaded
2138 - viewer - The PetscViewer for the saved topology
2139 
2140   Output Parameters:
2141 . globalToLocalPointSF - The PetscSF that pushes points in [0, N) to the associated points in the loaded plex, where N is the global number of points; NULL if unneeded
2142 
2143   Level: advanced
2144 
2145 .seealso: DMLoad(), DMPlexCoordinatesLoad(), DMPlexLabelsLoad(), DMView(), PetscViewerHDF5Open(), PetscViewerPushFormat()
2146 @*/
2147 PetscErrorCode DMPlexTopologyLoad(DM dm, PetscViewer viewer, PetscSF *globalToLocalPointSF)
2148 {
2149   PetscBool      ishdf5;
2150   PetscErrorCode ierr;
2151 
2152   PetscFunctionBegin;
2153   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2154   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
2155   if (globalToLocalPointSF) PetscValidPointer(globalToLocalPointSF, 3);
2156   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
2157   ierr = PetscLogEventBegin(DMPLEX_TopologyLoad,viewer,0,0,0);CHKERRQ(ierr);
2158   if (ishdf5) {
2159 #if defined(PETSC_HAVE_HDF5)
2160     PetscViewerFormat format;
2161     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
2162     if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
2163       ierr = DMPlexTopologyLoad_HDF5_Internal(dm, viewer, globalToLocalPointSF);CHKERRQ(ierr);
2164     } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 input.", PetscViewerFormats[format]);
2165 #else
2166     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
2167 #endif
2168   }
2169   ierr = PetscLogEventEnd(DMPLEX_TopologyLoad,viewer,0,0,0);CHKERRQ(ierr);
2170   PetscFunctionReturn(0);
2171 }
2172 
2173 /*@
2174   DMPlexCoordinatesLoad - Loads coordinates into a DMPlex
2175 
2176   Collective on DM
2177 
2178   Input Parameters:
2179 + dm     - The DM into which the coordinates are loaded
2180 . viewer - The PetscViewer for the saved coordinates
2181 - globalToLocalPointSF - The SF returned by DMPlexTopologyLoad() when loading dm from viewer
2182 
2183   Level: advanced
2184 
2185 .seealso: DMLoad(), DMPlexTopologyLoad(), DMPlexLabelsLoad(), DMView(), PetscViewerHDF5Open(), PetscViewerPushFormat()
2186 @*/
2187 PetscErrorCode DMPlexCoordinatesLoad(DM dm, PetscViewer viewer, PetscSF globalToLocalPointSF)
2188 {
2189   PetscBool      ishdf5;
2190   PetscErrorCode ierr;
2191 
2192   PetscFunctionBegin;
2193   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2194   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
2195   PetscValidHeaderSpecific(globalToLocalPointSF, PETSCSF_CLASSID, 3);
2196   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
2197   ierr = PetscLogEventBegin(DMPLEX_CoordinatesLoad,viewer,0,0,0);CHKERRQ(ierr);
2198   if (ishdf5) {
2199 #if defined(PETSC_HAVE_HDF5)
2200     PetscViewerFormat format;
2201     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
2202     if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
2203       ierr = DMPlexCoordinatesLoad_HDF5_Internal(dm, viewer, globalToLocalPointSF);CHKERRQ(ierr);
2204     } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 input.", PetscViewerFormats[format]);
2205 #else
2206     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
2207 #endif
2208   }
2209   ierr = PetscLogEventEnd(DMPLEX_CoordinatesLoad,viewer,0,0,0);CHKERRQ(ierr);
2210   PetscFunctionReturn(0);
2211 }
2212 
2213 /*@
2214   DMPlexLabelsLoad - Loads labels into a DMPlex
2215 
2216   Collective on DM
2217 
2218   Input Parameters:
2219 + dm     - The DM into which the labels are loaded
2220 . viewer - The PetscViewer for the saved labels
2221 - globalToLocalPointSF - The SF returned by DMPlexTopologyLoad() when loading dm from viewer
2222 
2223   Level: advanced
2224 
2225   Notes:
2226   The PetscSF argument must not be NULL if the DM is distributed, otherwise an error occurs.
2227 
2228 .seealso: DMLoad(), DMPlexTopologyLoad(), DMPlexCoordinatesLoad(), DMView(), PetscViewerHDF5Open(), PetscViewerPushFormat()
2229 @*/
2230 PetscErrorCode DMPlexLabelsLoad(DM dm, PetscViewer viewer, PetscSF globalToLocalPointSF)
2231 {
2232   PetscBool      ishdf5;
2233   PetscErrorCode ierr;
2234 
2235   PetscFunctionBegin;
2236   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2237   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
2238   if (globalToLocalPointSF) PetscValidHeaderSpecific(globalToLocalPointSF, PETSCSF_CLASSID, 3);
2239   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
2240   ierr = PetscLogEventBegin(DMPLEX_LabelsLoad,viewer,0,0,0);CHKERRQ(ierr);
2241   if (ishdf5) {
2242 #if defined(PETSC_HAVE_HDF5)
2243     PetscViewerFormat format;
2244 
2245     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
2246     if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
2247       ierr = DMPlexLabelsLoad_HDF5_Internal(dm, viewer, globalToLocalPointSF);CHKERRQ(ierr);
2248     } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 input.", PetscViewerFormats[format]);
2249 #else
2250     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
2251 #endif
2252   }
2253   ierr = PetscLogEventEnd(DMPLEX_LabelsLoad,viewer,0,0,0);CHKERRQ(ierr);
2254   PetscFunctionReturn(0);
2255 }
2256 
2257 /*@
2258   DMPlexSectionLoad - Loads section into a DMPlex
2259 
2260   Collective on DM
2261 
2262   Input Parameters:
2263 + dm          - The DM that represents the topology
2264 . viewer      - The PetscViewer that represents the on-disk section (sectionA)
2265 . sectiondm   - The DM into which the on-disk section (sectionA) is migrated
2266 - globalToLocalPointSF - The SF returned by DMPlexTopologyLoad() when loading dm from viewer
2267 
2268   Output Parameters
2269 + globalDofSF - The SF that migrates any on-disk Vec data associated with sectionA into a global Vec associated with the sectiondm's global section (NULL if not needed)
2270 - localDofSF  - The SF that migrates any on-disk Vec data associated with sectionA into a local Vec associated with the sectiondm's local section (NULL if not needed)
2271 
2272   Level: advanced
2273 
2274   Notes:
2275   This function is a wrapper around PetscSectionLoad(); it loads, in addition to the raw section, a list of global point numbers that associates each on-disk section point with a global point number in [0, NX), where NX is the number of topology points in dm. Noting that globalToLocalPointSF associates each topology point in dm with a global number in [0, NX), one can readily establish an association of the on-disk section points with the topology points.
2276 
2277   In general dm and sectiondm are two different objects, the former carrying the topology and the latter carrying the section, and have been given a topology name and a section name, respectively, with PetscObjectSetName(). In practice, however, they can be the same object if it carries both topology and section; in that case the name of the object is used as both the topology name and the section name.
2278 
2279   The output parameter, globalDofSF (localDofSF), can later be used with DMPlexGlobalVectorLoad() (DMPlexLocalVectorLoad()) to load on-disk vectors into global (local) vectors associated with sectiondm's global (local) section.
2280 
2281   Example using 2 processes:
2282 $  NX (number of points on dm): 4
2283 $  sectionA                   : the on-disk section
2284 $  vecA                       : a vector associated with sectionA
2285 $  sectionB                   : sectiondm's local section constructed in this function
2286 $  vecB (local)               : a vector associated with sectiondm's local section
2287 $  vecB (global)              : a vector associated with sectiondm's global section
2288 $
2289 $                                     rank 0    rank 1
2290 $  vecA (global)                  : [.0 .4 .1 | .2 .3]        <- to be loaded in DMPlexGlobalVectorLoad() or DMPlexLocalVectorLoad()
2291 $  sectionA->atlasOff             :       0 2 | 1             <- loaded in PetscSectionLoad()
2292 $  sectionA->atlasDof             :       1 3 | 1             <- loaded in PetscSectionLoad()
2293 $  sectionA's global point numbers:       0 2 | 3             <- loaded in DMPlexSectionLoad()
2294 $  [0, NX)                        :       0 1 | 2 3           <- conceptual partition used in globalToLocalPointSF
2295 $  sectionB's global point numbers:     0 1 3 | 3 2           <- associated with [0, NX) by globalToLocalPointSF
2296 $  sectionB->atlasDof             :     1 0 1 | 1 3
2297 $  sectionB->atlasOff (no perm)   :     0 1 1 | 0 1
2298 $  vecB (local)                   :   [.0 .4] | [.4 .1 .2 .3] <- to be constructed by calling DMPlexLocalVectorLoad() with localDofSF
2299 $  vecB (global)                  :    [.0 .4 | .1 .2 .3]     <- to be constructed by calling DMPlexGlobalVectorLoad() with globalDofSF
2300 $
2301 $  where "|" represents a partition of loaded data, and global point 3 is assumed to be owned by rank 0.
2302 
2303 .seealso: DMLoad(), DMPlexTopologyLoad(), DMPlexCoordinatesLoad(), DMPlexLabelsLoad(), DMPlexGlobalVectorLoad(), DMPlexLocalVectorLoad(), PetscSectionLoad(), DMPlexSectionView()
2304 @*/
2305 PetscErrorCode DMPlexSectionLoad(DM dm, PetscViewer viewer, DM sectiondm, PetscSF globalToLocalPointSF, PetscSF *globalDofSF, PetscSF *localDofSF)
2306 {
2307   PetscBool      ishdf5;
2308   PetscErrorCode ierr;
2309 
2310   PetscFunctionBegin;
2311   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2312   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
2313   PetscValidHeaderSpecific(sectiondm, DM_CLASSID, 3);
2314   PetscValidHeaderSpecific(globalToLocalPointSF, PETSCSF_CLASSID, 4);
2315   if (globalDofSF) PetscValidPointer(globalDofSF, 5);
2316   if (localDofSF) PetscValidPointer(localDofSF, 6);
2317   ierr = PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERHDF5,&ishdf5);CHKERRQ(ierr);
2318   ierr = PetscLogEventBegin(DMPLEX_SectionLoad,viewer,0,0,0);CHKERRQ(ierr);
2319   if (ishdf5) {
2320 #if defined(PETSC_HAVE_HDF5)
2321     ierr = DMPlexSectionLoad_HDF5_Internal(dm, viewer, sectiondm, globalToLocalPointSF, globalDofSF, localDofSF);CHKERRQ(ierr);
2322 #else
2323     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
2324 #endif
2325   }
2326   ierr = PetscLogEventEnd(DMPLEX_SectionLoad,viewer,0,0,0);CHKERRQ(ierr);
2327   PetscFunctionReturn(0);
2328 }
2329 
2330 /*@
2331   DMPlexGlobalVectorLoad - Loads on-disk vector data into a global vector
2332 
2333   Collective on DM
2334 
2335   Input Parameters:
2336 + dm        - The DM that represents the topology
2337 . viewer    - The PetscViewer that represents the on-disk vector data
2338 . sectiondm - The DM that contains the global section on which vec is defined
2339 . sf        - The SF that migrates the on-disk vector data into vec
2340 - vec       - The global vector to set values of
2341 
2342   Level: advanced
2343 
2344   Notes:
2345   In general dm and sectiondm are two different objects, the former carrying the topology and the latter carrying the section, and have been given a topology name and a section name, respectively, with PetscObjectSetName(). In practice, however, they can be the same object if it carries both topology and section; in that case the name of the object is used as both the topology name and the section name.
2346 
2347   Typical calling sequence
2348 $       DMCreate(PETSC_COMM_WORLD, &dm);
2349 $       DMSetType(dm, DMPLEX);
2350 $       PetscObjectSetName((PetscObject)dm, "topologydm_name");
2351 $       DMPlexTopologyLoad(dm, viewer, &sfX);
2352 $       DMClone(dm, &sectiondm);
2353 $       PetscObjectSetName((PetscObject)sectiondm, "sectiondm_name");
2354 $       DMPlexSectionLoad(dm, viewer, sectiondm, sfX, &gsf, NULL);
2355 $       DMGetGlobalVector(sectiondm, &vec);
2356 $       PetscObjectSetName((PetscObject)vec, "vec_name");
2357 $       DMPlexGlobalVectorLoad(dm, viewer, sectiondm, gsf, vec);
2358 $       DMRestoreGlobalVector(sectiondm, &vec);
2359 $       PetscSFDestroy(&gsf);
2360 $       PetscSFDestroy(&sfX);
2361 $       DMDestroy(&sectiondm);
2362 $       DMDestroy(&dm);
2363 
2364 .seealso: DMPlexTopologyLoad(), DMPlexSectionLoad(), DMPlexLocalVectorLoad(), DMPlexGlobalVectorView(), DMPlexLocalVectorView()
2365 @*/
2366 PetscErrorCode DMPlexGlobalVectorLoad(DM dm, PetscViewer viewer, DM sectiondm, PetscSF sf, Vec vec)
2367 {
2368   PetscBool       ishdf5;
2369   PetscErrorCode  ierr;
2370 
2371   PetscFunctionBegin;
2372   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2373   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
2374   PetscValidHeaderSpecific(sectiondm, DM_CLASSID, 3);
2375   PetscValidHeaderSpecific(sf, PETSCSF_CLASSID, 4);
2376   PetscValidHeaderSpecific(vec, VEC_CLASSID, 5);
2377   /* Check consistency */
2378   {
2379     PetscSection  section;
2380     PetscBool     includesConstraints;
2381     PetscInt      m, m1;
2382 
2383     ierr = VecGetLocalSize(vec, &m1);CHKERRQ(ierr);
2384     ierr = DMGetGlobalSection(sectiondm, &section);CHKERRQ(ierr);
2385     ierr = PetscSectionGetIncludesConstraints(section, &includesConstraints);CHKERRQ(ierr);
2386     if (includesConstraints) {ierr = PetscSectionGetStorageSize(section, &m);CHKERRQ(ierr);}
2387     else {ierr = PetscSectionGetConstrainedStorageSize(section, &m);CHKERRQ(ierr);}
2388     PetscCheckFalse(m1 != m,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Global vector size (%D) != global section storage size (%D)", m1, m);
2389   }
2390   ierr = PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERHDF5,&ishdf5);CHKERRQ(ierr);
2391   ierr = PetscLogEventBegin(DMPLEX_GlobalVectorLoad,viewer,0,0,0);CHKERRQ(ierr);
2392   if (ishdf5) {
2393 #if defined(PETSC_HAVE_HDF5)
2394     ierr = DMPlexVecLoad_HDF5_Internal(dm, viewer, sectiondm, sf, vec);CHKERRQ(ierr);
2395 #else
2396     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
2397 #endif
2398   }
2399   ierr = PetscLogEventEnd(DMPLEX_GlobalVectorLoad,viewer,0,0,0);CHKERRQ(ierr);
2400   PetscFunctionReturn(0);
2401 }
2402 
2403 /*@
2404   DMPlexLocalVectorLoad - Loads on-disk vector data into a local vector
2405 
2406   Collective on DM
2407 
2408   Input Parameters:
2409 + dm        - The DM that represents the topology
2410 . viewer    - The PetscViewer that represents the on-disk vector data
2411 . sectiondm - The DM that contains the local section on which vec is defined
2412 . sf        - The SF that migrates the on-disk vector data into vec
2413 - vec       - The local vector to set values of
2414 
2415   Level: advanced
2416 
2417   Notes:
2418   In general dm and sectiondm are two different objects, the former carrying the topology and the latter carrying the section, and have been given a topology name and a section name, respectively, with PetscObjectSetName(). In practice, however, they can be the same object if it carries both topology and section; in that case the name of the object is used as both the topology name and the section name.
2419 
2420   Typical calling sequence
2421 $       DMCreate(PETSC_COMM_WORLD, &dm);
2422 $       DMSetType(dm, DMPLEX);
2423 $       PetscObjectSetName((PetscObject)dm, "topologydm_name");
2424 $       DMPlexTopologyLoad(dm, viewer, &sfX);
2425 $       DMClone(dm, &sectiondm);
2426 $       PetscObjectSetName((PetscObject)sectiondm, "sectiondm_name");
2427 $       DMPlexSectionLoad(dm, viewer, sectiondm, sfX, NULL, &lsf);
2428 $       DMGetLocalVector(sectiondm, &vec);
2429 $       PetscObjectSetName((PetscObject)vec, "vec_name");
2430 $       DMPlexLocalVectorLoad(dm, viewer, sectiondm, lsf, vec);
2431 $       DMRestoreLocalVector(sectiondm, &vec);
2432 $       PetscSFDestroy(&lsf);
2433 $       PetscSFDestroy(&sfX);
2434 $       DMDestroy(&sectiondm);
2435 $       DMDestroy(&dm);
2436 
2437 .seealso: DMPlexTopologyLoad(), DMPlexSectionLoad(), DMPlexGlobalVectorLoad(), DMPlexGlobalVectorView(), DMPlexLocalVectorView()
2438 @*/
2439 PetscErrorCode DMPlexLocalVectorLoad(DM dm, PetscViewer viewer, DM sectiondm, PetscSF sf, Vec vec)
2440 {
2441   PetscBool       ishdf5;
2442   PetscErrorCode  ierr;
2443 
2444   PetscFunctionBegin;
2445   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2446   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
2447   PetscValidHeaderSpecific(sectiondm, DM_CLASSID, 3);
2448   PetscValidHeaderSpecific(sf, PETSCSF_CLASSID, 4);
2449   PetscValidHeaderSpecific(vec, VEC_CLASSID, 5);
2450   /* Check consistency */
2451   {
2452     PetscSection  section;
2453     PetscBool     includesConstraints;
2454     PetscInt      m, m1;
2455 
2456     ierr = VecGetLocalSize(vec, &m1);CHKERRQ(ierr);
2457     ierr = DMGetLocalSection(sectiondm, &section);CHKERRQ(ierr);
2458     ierr = PetscSectionGetIncludesConstraints(section, &includesConstraints);CHKERRQ(ierr);
2459     if (includesConstraints) {ierr = PetscSectionGetStorageSize(section, &m);CHKERRQ(ierr);}
2460     else {ierr = PetscSectionGetConstrainedStorageSize(section, &m);CHKERRQ(ierr);}
2461     PetscCheckFalse(m1 != m,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Local vector size (%D) != local section storage size (%D)", m1, m);
2462   }
2463   ierr = PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERHDF5,&ishdf5);CHKERRQ(ierr);
2464   ierr = PetscLogEventBegin(DMPLEX_LocalVectorLoad,viewer,0,0,0);CHKERRQ(ierr);
2465   if (ishdf5) {
2466 #if defined(PETSC_HAVE_HDF5)
2467     ierr = DMPlexVecLoad_HDF5_Internal(dm, viewer, sectiondm, sf, vec);CHKERRQ(ierr);
2468 #else
2469     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
2470 #endif
2471   }
2472   ierr = PetscLogEventEnd(DMPLEX_LocalVectorLoad,viewer,0,0,0);CHKERRQ(ierr);
2473   PetscFunctionReturn(0);
2474 }
2475 
2476 PetscErrorCode DMDestroy_Plex(DM dm)
2477 {
2478   DM_Plex       *mesh = (DM_Plex*) dm->data;
2479   PetscErrorCode ierr;
2480 
2481   PetscFunctionBegin;
2482   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMSetUpGLVisViewer_C",NULL);CHKERRQ(ierr);
2483   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMPlexInsertBoundaryValues_C", NULL);CHKERRQ(ierr);
2484   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMCreateNeumannOverlap_C", NULL);CHKERRQ(ierr);
2485   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMInterpolateSolution_C", NULL);CHKERRQ(ierr);
2486   if (--mesh->refct > 0) PetscFunctionReturn(0);
2487   ierr = PetscSectionDestroy(&mesh->coneSection);CHKERRQ(ierr);
2488   ierr = PetscFree(mesh->cones);CHKERRQ(ierr);
2489   ierr = PetscFree(mesh->coneOrientations);CHKERRQ(ierr);
2490   ierr = PetscSectionDestroy(&mesh->supportSection);CHKERRQ(ierr);
2491   ierr = PetscSectionDestroy(&mesh->subdomainSection);CHKERRQ(ierr);
2492   ierr = PetscFree(mesh->supports);CHKERRQ(ierr);
2493   ierr = PetscFree(mesh->facesTmp);CHKERRQ(ierr);
2494   ierr = PetscFree(mesh->tetgenOpts);CHKERRQ(ierr);
2495   ierr = PetscFree(mesh->triangleOpts);CHKERRQ(ierr);
2496   ierr = PetscFree(mesh->transformType);CHKERRQ(ierr);
2497   ierr = PetscPartitionerDestroy(&mesh->partitioner);CHKERRQ(ierr);
2498   ierr = DMLabelDestroy(&mesh->subpointMap);CHKERRQ(ierr);
2499   ierr = ISDestroy(&mesh->subpointIS);CHKERRQ(ierr);
2500   ierr = ISDestroy(&mesh->globalVertexNumbers);CHKERRQ(ierr);
2501   ierr = ISDestroy(&mesh->globalCellNumbers);CHKERRQ(ierr);
2502   ierr = PetscSectionDestroy(&mesh->anchorSection);CHKERRQ(ierr);
2503   ierr = ISDestroy(&mesh->anchorIS);CHKERRQ(ierr);
2504   ierr = PetscSectionDestroy(&mesh->parentSection);CHKERRQ(ierr);
2505   ierr = PetscFree(mesh->parents);CHKERRQ(ierr);
2506   ierr = PetscFree(mesh->childIDs);CHKERRQ(ierr);
2507   ierr = PetscSectionDestroy(&mesh->childSection);CHKERRQ(ierr);
2508   ierr = PetscFree(mesh->children);CHKERRQ(ierr);
2509   ierr = DMDestroy(&mesh->referenceTree);CHKERRQ(ierr);
2510   ierr = PetscGridHashDestroy(&mesh->lbox);CHKERRQ(ierr);
2511   ierr = PetscFree(mesh->neighbors);CHKERRQ(ierr);
2512   if (mesh->metricCtx) { ierr = PetscFree(mesh->metricCtx);CHKERRQ(ierr); }
2513   /* This was originally freed in DMDestroy(), but that prevents reference counting of backend objects */
2514   ierr = PetscFree(mesh);CHKERRQ(ierr);
2515   PetscFunctionReturn(0);
2516 }
2517 
2518 PetscErrorCode DMCreateMatrix_Plex(DM dm, Mat *J)
2519 {
2520   PetscSection           sectionGlobal;
2521   PetscInt               bs = -1, mbs;
2522   PetscInt               localSize;
2523   PetscBool              isShell, isBlock, isSeqBlock, isMPIBlock, isSymBlock, isSymSeqBlock, isSymMPIBlock, isMatIS;
2524   PetscErrorCode         ierr;
2525   MatType                mtype;
2526   ISLocalToGlobalMapping ltog;
2527 
2528   PetscFunctionBegin;
2529   ierr = MatInitializePackage();CHKERRQ(ierr);
2530   mtype = dm->mattype;
2531   ierr = DMGetGlobalSection(dm, &sectionGlobal);CHKERRQ(ierr);
2532   /* ierr = PetscSectionGetStorageSize(sectionGlobal, &localSize);CHKERRQ(ierr); */
2533   ierr = PetscSectionGetConstrainedStorageSize(sectionGlobal, &localSize);CHKERRQ(ierr);
2534   ierr = MatCreate(PetscObjectComm((PetscObject)dm), J);CHKERRQ(ierr);
2535   ierr = MatSetSizes(*J, localSize, localSize, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
2536   ierr = MatSetType(*J, mtype);CHKERRQ(ierr);
2537   ierr = MatSetFromOptions(*J);CHKERRQ(ierr);
2538   ierr = MatGetBlockSize(*J, &mbs);CHKERRQ(ierr);
2539   if (mbs > 1) bs = mbs;
2540   ierr = PetscStrcmp(mtype, MATSHELL, &isShell);CHKERRQ(ierr);
2541   ierr = PetscStrcmp(mtype, MATBAIJ, &isBlock);CHKERRQ(ierr);
2542   ierr = PetscStrcmp(mtype, MATSEQBAIJ, &isSeqBlock);CHKERRQ(ierr);
2543   ierr = PetscStrcmp(mtype, MATMPIBAIJ, &isMPIBlock);CHKERRQ(ierr);
2544   ierr = PetscStrcmp(mtype, MATSBAIJ, &isSymBlock);CHKERRQ(ierr);
2545   ierr = PetscStrcmp(mtype, MATSEQSBAIJ, &isSymSeqBlock);CHKERRQ(ierr);
2546   ierr = PetscStrcmp(mtype, MATMPISBAIJ, &isSymMPIBlock);CHKERRQ(ierr);
2547   ierr = PetscStrcmp(mtype, MATIS, &isMatIS);CHKERRQ(ierr);
2548   if (!isShell) {
2549     PetscBool fillMatrix = (PetscBool)(!dm->prealloc_only && !isMatIS);
2550     PetscInt  *dnz, *onz, *dnzu, *onzu, bsLocal[2], bsMinMax[2];
2551     PetscInt  pStart, pEnd, p, dof, cdof;
2552 
2553     ierr = DMGetLocalToGlobalMapping(dm,&ltog);CHKERRQ(ierr);
2554     ierr = PetscSectionGetChart(sectionGlobal, &pStart, &pEnd);CHKERRQ(ierr);
2555     for (p = pStart; p < pEnd; ++p) {
2556       PetscInt bdof;
2557 
2558       ierr = PetscSectionGetDof(sectionGlobal, p, &dof);CHKERRQ(ierr);
2559       ierr = PetscSectionGetConstraintDof(sectionGlobal, p, &cdof);CHKERRQ(ierr);
2560       dof  = dof < 0 ? -(dof+1) : dof;
2561       bdof = cdof && (dof-cdof) ? 1 : dof;
2562       if (dof) {
2563         if (bs < 0)          {bs = bdof;}
2564         else if (bs != bdof) {bs = 1; break;}
2565       }
2566     }
2567     /* Must have same blocksize on all procs (some might have no points) */
2568     bsLocal[0] = bs < 0 ? PETSC_MAX_INT : bs;
2569     bsLocal[1] = bs;
2570     ierr = PetscGlobalMinMaxInt(PetscObjectComm((PetscObject) dm), bsLocal, bsMinMax);CHKERRQ(ierr);
2571     if (bsMinMax[0] != bsMinMax[1]) bs = 1;
2572     else bs = bsMinMax[0];
2573     bs = PetscMax(1,bs);
2574     ierr = MatSetLocalToGlobalMapping(*J,ltog,ltog);CHKERRQ(ierr);
2575     if (dm->prealloc_skip) { // User will likely use MatSetPreallocationCOO(), but still set structural parameters
2576       ierr = MatSetBlockSize(*J, bs);CHKERRQ(ierr);
2577       ierr = MatSetUp(*J);CHKERRQ(ierr);
2578     } else {
2579       ierr = PetscCalloc4(localSize/bs, &dnz, localSize/bs, &onz, localSize/bs, &dnzu, localSize/bs, &onzu);CHKERRQ(ierr);
2580       ierr = DMPlexPreallocateOperator(dm, bs, dnz, onz, dnzu, onzu, *J, fillMatrix);CHKERRQ(ierr);
2581       ierr = PetscFree4(dnz, onz, dnzu, onzu);CHKERRQ(ierr);
2582     }
2583   }
2584   ierr = MatSetDM(*J, dm);CHKERRQ(ierr);
2585   PetscFunctionReturn(0);
2586 }
2587 
2588 /*@
2589   DMPlexGetSubdomainSection - Returns the section associated with the subdomain
2590 
2591   Not collective
2592 
2593   Input Parameter:
2594 . mesh - The DMPlex
2595 
2596   Output Parameters:
2597 . subsection - The subdomain section
2598 
2599   Level: developer
2600 
2601 .seealso:
2602 @*/
2603 PetscErrorCode DMPlexGetSubdomainSection(DM dm, PetscSection *subsection)
2604 {
2605   DM_Plex       *mesh = (DM_Plex*) dm->data;
2606   PetscErrorCode ierr;
2607 
2608   PetscFunctionBegin;
2609   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2610   if (!mesh->subdomainSection) {
2611     PetscSection section;
2612     PetscSF      sf;
2613 
2614     ierr = PetscSFCreate(PETSC_COMM_SELF,&sf);CHKERRQ(ierr);
2615     ierr = DMGetLocalSection(dm,&section);CHKERRQ(ierr);
2616     ierr = PetscSectionCreateGlobalSection(section,sf,PETSC_FALSE,PETSC_TRUE,&mesh->subdomainSection);CHKERRQ(ierr);
2617     ierr = PetscSFDestroy(&sf);CHKERRQ(ierr);
2618   }
2619   *subsection = mesh->subdomainSection;
2620   PetscFunctionReturn(0);
2621 }
2622 
2623 /*@
2624   DMPlexGetChart - Return the interval for all mesh points [pStart, pEnd)
2625 
2626   Not collective
2627 
2628   Input Parameter:
2629 . mesh - The DMPlex
2630 
2631   Output Parameters:
2632 + pStart - The first mesh point
2633 - pEnd   - The upper bound for mesh points
2634 
2635   Level: beginner
2636 
2637 .seealso: DMPlexCreate(), DMPlexSetChart()
2638 @*/
2639 PetscErrorCode DMPlexGetChart(DM dm, PetscInt *pStart, PetscInt *pEnd)
2640 {
2641   DM_Plex       *mesh = (DM_Plex*) dm->data;
2642   PetscErrorCode ierr;
2643 
2644   PetscFunctionBegin;
2645   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2646   ierr = PetscSectionGetChart(mesh->coneSection, pStart, pEnd);CHKERRQ(ierr);
2647   PetscFunctionReturn(0);
2648 }
2649 
2650 /*@
2651   DMPlexSetChart - Set the interval for all mesh points [pStart, pEnd)
2652 
2653   Not collective
2654 
2655   Input Parameters:
2656 + mesh - The DMPlex
2657 . pStart - The first mesh point
2658 - pEnd   - The upper bound for mesh points
2659 
2660   Output Parameters:
2661 
2662   Level: beginner
2663 
2664 .seealso: DMPlexCreate(), DMPlexGetChart()
2665 @*/
2666 PetscErrorCode DMPlexSetChart(DM dm, PetscInt pStart, PetscInt pEnd)
2667 {
2668   DM_Plex       *mesh = (DM_Plex*) dm->data;
2669   PetscErrorCode ierr;
2670 
2671   PetscFunctionBegin;
2672   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2673   ierr = PetscSectionSetChart(mesh->coneSection, pStart, pEnd);CHKERRQ(ierr);
2674   ierr = PetscSectionSetChart(mesh->supportSection, pStart, pEnd);CHKERRQ(ierr);
2675   PetscFunctionReturn(0);
2676 }
2677 
2678 /*@
2679   DMPlexGetConeSize - Return the number of in-edges for this point in the DAG
2680 
2681   Not collective
2682 
2683   Input Parameters:
2684 + mesh - The DMPlex
2685 - p - The point, which must lie in the chart set with DMPlexSetChart()
2686 
2687   Output Parameter:
2688 . size - The cone size for point p
2689 
2690   Level: beginner
2691 
2692 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart()
2693 @*/
2694 PetscErrorCode DMPlexGetConeSize(DM dm, PetscInt p, PetscInt *size)
2695 {
2696   DM_Plex       *mesh = (DM_Plex*) dm->data;
2697   PetscErrorCode ierr;
2698 
2699   PetscFunctionBegin;
2700   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2701   PetscValidPointer(size, 3);
2702   ierr = PetscSectionGetDof(mesh->coneSection, p, size);CHKERRQ(ierr);
2703   PetscFunctionReturn(0);
2704 }
2705 
2706 /*@
2707   DMPlexSetConeSize - Set the number of in-edges for this point in the DAG
2708 
2709   Not collective
2710 
2711   Input Parameters:
2712 + mesh - The DMPlex
2713 . p - The point, which must lie in the chart set with DMPlexSetChart()
2714 - size - The cone size for point p
2715 
2716   Output Parameter:
2717 
2718   Note:
2719   This should be called after DMPlexSetChart().
2720 
2721   Level: beginner
2722 
2723 .seealso: DMPlexCreate(), DMPlexGetConeSize(), DMPlexSetChart()
2724 @*/
2725 PetscErrorCode DMPlexSetConeSize(DM dm, PetscInt p, PetscInt size)
2726 {
2727   DM_Plex       *mesh = (DM_Plex*) dm->data;
2728   PetscErrorCode ierr;
2729 
2730   PetscFunctionBegin;
2731   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2732   ierr = PetscSectionSetDof(mesh->coneSection, p, size);CHKERRQ(ierr);
2733 
2734   mesh->maxConeSize = PetscMax(mesh->maxConeSize, size);
2735   PetscFunctionReturn(0);
2736 }
2737 
2738 /*@
2739   DMPlexAddConeSize - Add the given number of in-edges to this point in the DAG
2740 
2741   Not collective
2742 
2743   Input Parameters:
2744 + mesh - The DMPlex
2745 . p - The point, which must lie in the chart set with DMPlexSetChart()
2746 - size - The additional cone size for point p
2747 
2748   Output Parameter:
2749 
2750   Note:
2751   This should be called after DMPlexSetChart().
2752 
2753   Level: beginner
2754 
2755 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexGetConeSize(), DMPlexSetChart()
2756 @*/
2757 PetscErrorCode DMPlexAddConeSize(DM dm, PetscInt p, PetscInt size)
2758 {
2759   DM_Plex       *mesh = (DM_Plex*) dm->data;
2760   PetscInt       csize;
2761   PetscErrorCode ierr;
2762 
2763   PetscFunctionBegin;
2764   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2765   ierr = PetscSectionAddDof(mesh->coneSection, p, size);CHKERRQ(ierr);
2766   ierr = PetscSectionGetDof(mesh->coneSection, p, &csize);CHKERRQ(ierr);
2767 
2768   mesh->maxConeSize = PetscMax(mesh->maxConeSize, csize);
2769   PetscFunctionReturn(0);
2770 }
2771 
2772 /*@C
2773   DMPlexGetCone - Return the points on the in-edges for this point in the DAG
2774 
2775   Not collective
2776 
2777   Input Parameters:
2778 + dm - The DMPlex
2779 - p - The point, which must lie in the chart set with DMPlexSetChart()
2780 
2781   Output Parameter:
2782 . cone - An array of points which are on the in-edges for point p
2783 
2784   Level: beginner
2785 
2786   Fortran Notes:
2787   Since it returns an array, this routine is only available in Fortran 90, and you must
2788   include petsc.h90 in your code.
2789   You must also call DMPlexRestoreCone() after you finish using the returned array.
2790   DMPlexRestoreCone() is not needed/available in C.
2791 
2792 .seealso: DMPlexGetConeSize(), DMPlexSetCone(), DMPlexGetConeTuple(), DMPlexSetChart()
2793 @*/
2794 PetscErrorCode DMPlexGetCone(DM dm, PetscInt p, const PetscInt *cone[])
2795 {
2796   DM_Plex       *mesh = (DM_Plex*) dm->data;
2797   PetscInt       off;
2798   PetscErrorCode ierr;
2799 
2800   PetscFunctionBegin;
2801   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2802   PetscValidPointer(cone, 3);
2803   ierr  = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2804   *cone = &mesh->cones[off];
2805   PetscFunctionReturn(0);
2806 }
2807 
2808 /*@C
2809   DMPlexGetConeTuple - Return the points on the in-edges of several points in the DAG
2810 
2811   Not collective
2812 
2813   Input Parameters:
2814 + dm - The DMPlex
2815 - p - The IS of points, which must lie in the chart set with DMPlexSetChart()
2816 
2817   Output Parameters:
2818 + pConesSection - PetscSection describing the layout of pCones
2819 - pCones - An array of points which are on the in-edges for the point set p
2820 
2821   Level: intermediate
2822 
2823 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexGetConeRecursive(), DMPlexSetChart()
2824 @*/
2825 PetscErrorCode DMPlexGetConeTuple(DM dm, IS p, PetscSection *pConesSection, IS *pCones)
2826 {
2827   PetscSection        cs, newcs;
2828   PetscInt            *cones;
2829   PetscInt            *newarr=NULL;
2830   PetscInt            n;
2831   PetscErrorCode      ierr;
2832 
2833   PetscFunctionBegin;
2834   ierr = DMPlexGetCones(dm, &cones);CHKERRQ(ierr);
2835   ierr = DMPlexGetConeSection(dm, &cs);CHKERRQ(ierr);
2836   ierr = PetscSectionExtractDofsFromArray(cs, MPIU_INT, cones, p, &newcs, pCones ? ((void**)&newarr) : NULL);CHKERRQ(ierr);
2837   if (pConesSection) *pConesSection = newcs;
2838   if (pCones) {
2839     ierr = PetscSectionGetStorageSize(newcs, &n);CHKERRQ(ierr);
2840     ierr = ISCreateGeneral(PetscObjectComm((PetscObject)p), n, newarr, PETSC_OWN_POINTER, pCones);CHKERRQ(ierr);
2841   }
2842   PetscFunctionReturn(0);
2843 }
2844 
2845 /*@
2846   DMPlexGetConeRecursiveVertices - Expand each given point into its cone points and do that recursively until we end up just with vertices.
2847 
2848   Not collective
2849 
2850   Input Parameters:
2851 + dm - The DMPlex
2852 - points - The IS of points, which must lie in the chart set with DMPlexSetChart()
2853 
2854   Output Parameter:
2855 . expandedPoints - An array of vertices recursively expanded from input points
2856 
2857   Level: advanced
2858 
2859   Notes:
2860   Like DMPlexGetConeRecursive but returns only the 0-depth IS (i.e. vertices only) and no sections.
2861   There is no corresponding Restore function, just call ISDestroy() on the returned IS to deallocate.
2862 
2863 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexGetConeTuple(), DMPlexGetConeRecursive(), DMPlexRestoreConeRecursive(), DMPlexGetDepth()
2864 @*/
2865 PetscErrorCode DMPlexGetConeRecursiveVertices(DM dm, IS points, IS *expandedPoints)
2866 {
2867   IS                  *expandedPointsAll;
2868   PetscInt            depth;
2869   PetscErrorCode      ierr;
2870 
2871   PetscFunctionBegin;
2872   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2873   PetscValidHeaderSpecific(points, IS_CLASSID, 2);
2874   PetscValidPointer(expandedPoints, 3);
2875   ierr = DMPlexGetConeRecursive(dm, points, &depth, &expandedPointsAll, NULL);CHKERRQ(ierr);
2876   *expandedPoints = expandedPointsAll[0];
2877   ierr = PetscObjectReference((PetscObject)expandedPointsAll[0]);CHKERRQ(ierr);
2878   ierr = DMPlexRestoreConeRecursive(dm, points, &depth, &expandedPointsAll, NULL);CHKERRQ(ierr);
2879   PetscFunctionReturn(0);
2880 }
2881 
2882 /*@
2883   DMPlexGetConeRecursive - Expand each given point into its cone points and do that recursively until we end up just with vertices (DAG points of depth 0, i.e. without cones).
2884 
2885   Not collective
2886 
2887   Input Parameters:
2888 + dm - The DMPlex
2889 - points - The IS of points, which must lie in the chart set with DMPlexSetChart()
2890 
2891   Output Parameters:
2892 + depth - (optional) Size of the output arrays, equal to DMPlex depth, returned by DMPlexGetDepth()
2893 . expandedPoints - (optional) An array of index sets with recursively expanded cones
2894 - sections - (optional) An array of sections which describe mappings from points to their cone points
2895 
2896   Level: advanced
2897 
2898   Notes:
2899   Like DMPlexGetConeTuple() but recursive.
2900 
2901   Array expandedPoints has size equal to depth. Each expandedPoints[d] contains DAG points with maximum depth d, recursively cone-wise expanded from the input points.
2902   For example, for d=0 it contains only vertices, for d=1 it can contain vertices and edges, etc.
2903 
2904   Array section has size equal to depth.  Each PetscSection sections[d] realizes mapping from expandedPoints[d+1] (section points) to expandedPoints[d] (section dofs) as follows:
2905   (1) DAG points in expandedPoints[d+1] with depth d+1 to their cone points in expandedPoints[d];
2906   (2) DAG points in expandedPoints[d+1] with depth in [0,d] to the same points in expandedPoints[d].
2907 
2908 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexGetConeTuple(), DMPlexRestoreConeRecursive(), DMPlexGetConeRecursiveVertices(), DMPlexGetDepth()
2909 @*/
2910 PetscErrorCode DMPlexGetConeRecursive(DM dm, IS points, PetscInt *depth, IS *expandedPoints[], PetscSection *sections[])
2911 {
2912   const PetscInt      *arr0=NULL, *cone=NULL;
2913   PetscInt            *arr=NULL, *newarr=NULL;
2914   PetscInt            d, depth_, i, n, newn, cn, co, start, end;
2915   IS                  *expandedPoints_;
2916   PetscSection        *sections_;
2917   PetscErrorCode      ierr;
2918 
2919   PetscFunctionBegin;
2920   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2921   PetscValidHeaderSpecific(points, IS_CLASSID, 2);
2922   if (depth) PetscValidIntPointer(depth, 3);
2923   if (expandedPoints) PetscValidPointer(expandedPoints, 4);
2924   if (sections) PetscValidPointer(sections, 5);
2925   ierr = ISGetLocalSize(points, &n);CHKERRQ(ierr);
2926   ierr = ISGetIndices(points, &arr0);CHKERRQ(ierr);
2927   ierr = DMPlexGetDepth(dm, &depth_);CHKERRQ(ierr);
2928   ierr = PetscCalloc1(depth_, &expandedPoints_);CHKERRQ(ierr);
2929   ierr = PetscCalloc1(depth_, &sections_);CHKERRQ(ierr);
2930   arr = (PetscInt*) arr0; /* this is ok because first generation of arr is not modified */
2931   for (d=depth_-1; d>=0; d--) {
2932     ierr = PetscSectionCreate(PETSC_COMM_SELF, &sections_[d]);CHKERRQ(ierr);
2933     ierr = PetscSectionSetChart(sections_[d], 0, n);CHKERRQ(ierr);
2934     for (i=0; i<n; i++) {
2935       ierr = DMPlexGetDepthStratum(dm, d+1, &start, &end);CHKERRQ(ierr);
2936       if (arr[i] >= start && arr[i] < end) {
2937         ierr = DMPlexGetConeSize(dm, arr[i], &cn);CHKERRQ(ierr);
2938         ierr = PetscSectionSetDof(sections_[d], i, cn);CHKERRQ(ierr);
2939       } else {
2940         ierr = PetscSectionSetDof(sections_[d], i, 1);CHKERRQ(ierr);
2941       }
2942     }
2943     ierr = PetscSectionSetUp(sections_[d]);CHKERRQ(ierr);
2944     ierr = PetscSectionGetStorageSize(sections_[d], &newn);CHKERRQ(ierr);
2945     ierr = PetscMalloc1(newn, &newarr);CHKERRQ(ierr);
2946     for (i=0; i<n; i++) {
2947       ierr = PetscSectionGetDof(sections_[d], i, &cn);CHKERRQ(ierr);
2948       ierr = PetscSectionGetOffset(sections_[d], i, &co);CHKERRQ(ierr);
2949       if (cn > 1) {
2950         ierr = DMPlexGetCone(dm, arr[i], &cone);CHKERRQ(ierr);
2951         ierr = PetscMemcpy(&newarr[co], cone, cn*sizeof(PetscInt));CHKERRQ(ierr);
2952       } else {
2953         newarr[co] = arr[i];
2954       }
2955     }
2956     ierr = ISCreateGeneral(PETSC_COMM_SELF, newn, newarr, PETSC_OWN_POINTER, &expandedPoints_[d]);CHKERRQ(ierr);
2957     arr = newarr;
2958     n = newn;
2959   }
2960   ierr = ISRestoreIndices(points, &arr0);CHKERRQ(ierr);
2961   *depth = depth_;
2962   if (expandedPoints) *expandedPoints = expandedPoints_;
2963   else {
2964     for (d=0; d<depth_; d++) {ierr = ISDestroy(&expandedPoints_[d]);CHKERRQ(ierr);}
2965     ierr = PetscFree(expandedPoints_);CHKERRQ(ierr);
2966   }
2967   if (sections) *sections = sections_;
2968   else {
2969     for (d=0; d<depth_; d++) {ierr = PetscSectionDestroy(&sections_[d]);CHKERRQ(ierr);}
2970     ierr = PetscFree(sections_);CHKERRQ(ierr);
2971   }
2972   PetscFunctionReturn(0);
2973 }
2974 
2975 /*@
2976   DMPlexRestoreConeRecursive - Deallocates arrays created by DMPlexGetConeRecursive
2977 
2978   Not collective
2979 
2980   Input Parameters:
2981 + dm - The DMPlex
2982 - points - The IS of points, which must lie in the chart set with DMPlexSetChart()
2983 
2984   Output Parameters:
2985 + depth - (optional) Size of the output arrays, equal to DMPlex depth, returned by DMPlexGetDepth()
2986 . expandedPoints - (optional) An array of recursively expanded cones
2987 - sections - (optional) An array of sections which describe mappings from points to their cone points
2988 
2989   Level: advanced
2990 
2991   Notes:
2992   See DMPlexGetConeRecursive() for details.
2993 
2994 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexGetConeTuple(), DMPlexGetConeRecursive(), DMPlexGetConeRecursiveVertices(), DMPlexGetDepth()
2995 @*/
2996 PetscErrorCode DMPlexRestoreConeRecursive(DM dm, IS points, PetscInt *depth, IS *expandedPoints[], PetscSection *sections[])
2997 {
2998   PetscInt            d, depth_;
2999   PetscErrorCode      ierr;
3000 
3001   PetscFunctionBegin;
3002   ierr = DMPlexGetDepth(dm, &depth_);CHKERRQ(ierr);
3003   PetscCheckFalse(depth && *depth != depth_,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "depth changed since last call to DMPlexGetConeRecursive");
3004   if (depth) *depth = 0;
3005   if (expandedPoints) {
3006     for (d=0; d<depth_; d++) {ierr = ISDestroy(&((*expandedPoints)[d]));CHKERRQ(ierr);}
3007     ierr = PetscFree(*expandedPoints);CHKERRQ(ierr);
3008   }
3009   if (sections)  {
3010     for (d=0; d<depth_; d++) {ierr = PetscSectionDestroy(&((*sections)[d]));CHKERRQ(ierr);}
3011     ierr = PetscFree(*sections);CHKERRQ(ierr);
3012   }
3013   PetscFunctionReturn(0);
3014 }
3015 
3016 /*@
3017   DMPlexSetCone - Set the points on the in-edges for this point in the DAG; that is these are the points that cover the specific point
3018 
3019   Not collective
3020 
3021   Input Parameters:
3022 + mesh - The DMPlex
3023 . p - The point, which must lie in the chart set with DMPlexSetChart()
3024 - cone - An array of points which are on the in-edges for point p
3025 
3026   Output Parameter:
3027 
3028   Note:
3029   This should be called after all calls to DMPlexSetConeSize() and DMSetUp().
3030 
3031   Level: beginner
3032 
3033 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp(), DMPlexSetSupport(), DMPlexSetSupportSize()
3034 @*/
3035 PetscErrorCode DMPlexSetCone(DM dm, PetscInt p, const PetscInt cone[])
3036 {
3037   DM_Plex       *mesh = (DM_Plex*) dm->data;
3038   PetscInt       pStart, pEnd;
3039   PetscInt       dof, off, c;
3040   PetscErrorCode ierr;
3041 
3042   PetscFunctionBegin;
3043   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3044   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
3045   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
3046   if (dof) PetscValidPointer(cone, 3);
3047   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
3048   PetscCheckFalse((p < pStart) || (p >= pEnd),PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
3049   for (c = 0; c < dof; ++c) {
3050     PetscCheckFalse((cone[c] < pStart) || (cone[c] >= pEnd),PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone point %D is not in the valid range [%D, %D)", cone[c], pStart, pEnd);
3051     mesh->cones[off+c] = cone[c];
3052   }
3053   PetscFunctionReturn(0);
3054 }
3055 
3056 /*@C
3057   DMPlexGetConeOrientation - Return the orientations on the in-edges for this point in the DAG
3058 
3059   Not collective
3060 
3061   Input Parameters:
3062 + mesh - The DMPlex
3063 - p - The point, which must lie in the chart set with DMPlexSetChart()
3064 
3065   Output Parameter:
3066 . coneOrientation - An array of orientations which are on the in-edges for point p. An orientation is an
3067                     integer giving the prescription for cone traversal.
3068 
3069   Level: beginner
3070 
3071   Notes:
3072   The number indexes the symmetry transformations for the cell type (see manual). Orientation 0 is always
3073   the identity transformation. Negative orientation indicates reflection so that -(o+1) is the reflection
3074   of o, however it is not necessarily the inverse. To get the inverse, use DMPolytopeTypeComposeOrientationInv()
3075   with the identity.
3076 
3077   Fortran Notes:
3078   Since it returns an array, this routine is only available in Fortran 90, and you must
3079   include petsc.h90 in your code.
3080   You must also call DMPlexRestoreConeOrientation() after you finish using the returned array.
3081   DMPlexRestoreConeOrientation() is not needed/available in C.
3082 
3083 .seealso: DMPolytopeTypeComposeOrientation(), DMPolytopeTypeComposeOrientationInv(), DMPlexCreate(), DMPlexGetCone(), DMPlexSetCone(), DMPlexSetChart()
3084 @*/
3085 PetscErrorCode DMPlexGetConeOrientation(DM dm, PetscInt p, const PetscInt *coneOrientation[])
3086 {
3087   DM_Plex       *mesh = (DM_Plex*) dm->data;
3088   PetscInt       off;
3089   PetscErrorCode ierr;
3090 
3091   PetscFunctionBegin;
3092   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3093   if (PetscDefined(USE_DEBUG)) {
3094     PetscInt dof;
3095     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
3096     if (dof) PetscValidPointer(coneOrientation, 3);
3097   }
3098   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
3099 
3100   *coneOrientation = &mesh->coneOrientations[off];
3101   PetscFunctionReturn(0);
3102 }
3103 
3104 /*@
3105   DMPlexSetConeOrientation - Set the orientations on the in-edges for this point in the DAG
3106 
3107   Not collective
3108 
3109   Input Parameters:
3110 + mesh - The DMPlex
3111 . p - The point, which must lie in the chart set with DMPlexSetChart()
3112 - coneOrientation - An array of orientations
3113   Output Parameter:
3114 
3115   Notes:
3116   This should be called after all calls to DMPlexSetConeSize() and DMSetUp().
3117 
3118   The meaning of coneOrientation is detailed in DMPlexGetConeOrientation().
3119 
3120   Level: beginner
3121 
3122 .seealso: DMPlexCreate(), DMPlexGetConeOrientation(), DMPlexSetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
3123 @*/
3124 PetscErrorCode DMPlexSetConeOrientation(DM dm, PetscInt p, const PetscInt coneOrientation[])
3125 {
3126   DM_Plex       *mesh = (DM_Plex*) dm->data;
3127   PetscInt       pStart, pEnd;
3128   PetscInt       dof, off, c;
3129   PetscErrorCode ierr;
3130 
3131   PetscFunctionBegin;
3132   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3133   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
3134   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
3135   if (dof) PetscValidPointer(coneOrientation, 3);
3136   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
3137   PetscCheckFalse((p < pStart) || (p >= pEnd),PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
3138   for (c = 0; c < dof; ++c) {
3139     PetscInt cdof, o = coneOrientation[c];
3140 
3141     ierr = PetscSectionGetDof(mesh->coneSection, mesh->cones[off+c], &cdof);CHKERRQ(ierr);
3142     PetscCheckFalse(o && ((o < -(cdof+1)) || (o >= cdof)),PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone orientation %D is not in the valid range [%D. %D)", o, -(cdof+1), cdof);
3143     mesh->coneOrientations[off+c] = o;
3144   }
3145   PetscFunctionReturn(0);
3146 }
3147 
3148 /*@
3149   DMPlexInsertCone - Insert a point into the in-edges for the point p in the DAG
3150 
3151   Not collective
3152 
3153   Input Parameters:
3154 + mesh - The DMPlex
3155 . p - The point, which must lie in the chart set with DMPlexSetChart()
3156 . conePos - The local index in the cone where the point should be put
3157 - conePoint - The mesh point to insert
3158 
3159   Level: beginner
3160 
3161 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
3162 @*/
3163 PetscErrorCode DMPlexInsertCone(DM dm, PetscInt p, PetscInt conePos, PetscInt conePoint)
3164 {
3165   DM_Plex       *mesh = (DM_Plex*) dm->data;
3166   PetscInt       pStart, pEnd;
3167   PetscInt       dof, off;
3168   PetscErrorCode ierr;
3169 
3170   PetscFunctionBegin;
3171   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3172   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
3173   PetscCheckFalse((p < pStart) || (p >= pEnd),PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
3174   PetscCheckFalse((conePoint < pStart) || (conePoint >= pEnd),PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone point %D is not in the valid range [%D, %D)", conePoint, pStart, pEnd);
3175   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
3176   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
3177   PetscCheckFalse((conePos < 0) || (conePos >= dof),PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone position %D of point %D is not in the valid range [0, %D)", conePos, p, dof);
3178   mesh->cones[off+conePos] = conePoint;
3179   PetscFunctionReturn(0);
3180 }
3181 
3182 /*@
3183   DMPlexInsertConeOrientation - Insert a point orientation for the in-edge for the point p in the DAG
3184 
3185   Not collective
3186 
3187   Input Parameters:
3188 + mesh - The DMPlex
3189 . p - The point, which must lie in the chart set with DMPlexSetChart()
3190 . conePos - The local index in the cone where the point should be put
3191 - coneOrientation - The point orientation to insert
3192 
3193   Level: beginner
3194 
3195   Notes:
3196   The meaning of coneOrientation values is detailed in DMPlexGetConeOrientation().
3197 
3198 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
3199 @*/
3200 PetscErrorCode DMPlexInsertConeOrientation(DM dm, PetscInt p, PetscInt conePos, PetscInt coneOrientation)
3201 {
3202   DM_Plex       *mesh = (DM_Plex*) dm->data;
3203   PetscInt       pStart, pEnd;
3204   PetscInt       dof, off;
3205   PetscErrorCode ierr;
3206 
3207   PetscFunctionBegin;
3208   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3209   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
3210   PetscCheckFalse((p < pStart) || (p >= pEnd),PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
3211   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
3212   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
3213   PetscCheckFalse((conePos < 0) || (conePos >= dof),PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone position %D of point %D is not in the valid range [0, %D)", conePos, p, dof);
3214   mesh->coneOrientations[off+conePos] = coneOrientation;
3215   PetscFunctionReturn(0);
3216 }
3217 
3218 /*@
3219   DMPlexGetSupportSize - Return the number of out-edges for this point in the DAG
3220 
3221   Not collective
3222 
3223   Input Parameters:
3224 + mesh - The DMPlex
3225 - p - The point, which must lie in the chart set with DMPlexSetChart()
3226 
3227   Output Parameter:
3228 . size - The support size for point p
3229 
3230   Level: beginner
3231 
3232 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart(), DMPlexGetConeSize()
3233 @*/
3234 PetscErrorCode DMPlexGetSupportSize(DM dm, PetscInt p, PetscInt *size)
3235 {
3236   DM_Plex       *mesh = (DM_Plex*) dm->data;
3237   PetscErrorCode ierr;
3238 
3239   PetscFunctionBegin;
3240   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3241   PetscValidPointer(size, 3);
3242   ierr = PetscSectionGetDof(mesh->supportSection, p, size);CHKERRQ(ierr);
3243   PetscFunctionReturn(0);
3244 }
3245 
3246 /*@
3247   DMPlexSetSupportSize - Set the number of out-edges for this point in the DAG
3248 
3249   Not collective
3250 
3251   Input Parameters:
3252 + mesh - The DMPlex
3253 . p - The point, which must lie in the chart set with DMPlexSetChart()
3254 - size - The support size for point p
3255 
3256   Output Parameter:
3257 
3258   Note:
3259   This should be called after DMPlexSetChart().
3260 
3261   Level: beginner
3262 
3263 .seealso: DMPlexCreate(), DMPlexGetSupportSize(), DMPlexSetChart()
3264 @*/
3265 PetscErrorCode DMPlexSetSupportSize(DM dm, PetscInt p, PetscInt size)
3266 {
3267   DM_Plex       *mesh = (DM_Plex*) dm->data;
3268   PetscErrorCode ierr;
3269 
3270   PetscFunctionBegin;
3271   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3272   ierr = PetscSectionSetDof(mesh->supportSection, p, size);CHKERRQ(ierr);
3273 
3274   mesh->maxSupportSize = PetscMax(mesh->maxSupportSize, size);
3275   PetscFunctionReturn(0);
3276 }
3277 
3278 /*@C
3279   DMPlexGetSupport - Return the points on the out-edges for this point in the DAG
3280 
3281   Not collective
3282 
3283   Input Parameters:
3284 + mesh - The DMPlex
3285 - p - The point, which must lie in the chart set with DMPlexSetChart()
3286 
3287   Output Parameter:
3288 . support - An array of points which are on the out-edges for point p
3289 
3290   Level: beginner
3291 
3292   Fortran Notes:
3293   Since it returns an array, this routine is only available in Fortran 90, and you must
3294   include petsc.h90 in your code.
3295   You must also call DMPlexRestoreSupport() after you finish using the returned array.
3296   DMPlexRestoreSupport() is not needed/available in C.
3297 
3298 .seealso: DMPlexGetSupportSize(), DMPlexSetSupport(), DMPlexGetCone(), DMPlexSetChart()
3299 @*/
3300 PetscErrorCode DMPlexGetSupport(DM dm, PetscInt p, const PetscInt *support[])
3301 {
3302   DM_Plex       *mesh = (DM_Plex*) dm->data;
3303   PetscInt       off;
3304   PetscErrorCode ierr;
3305 
3306   PetscFunctionBegin;
3307   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3308   PetscValidPointer(support, 3);
3309   ierr     = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
3310   *support = &mesh->supports[off];
3311   PetscFunctionReturn(0);
3312 }
3313 
3314 /*@
3315   DMPlexSetSupport - Set the points on the out-edges for this point in the DAG, that is the list of points that this point covers
3316 
3317   Not collective
3318 
3319   Input Parameters:
3320 + mesh - The DMPlex
3321 . p - The point, which must lie in the chart set with DMPlexSetChart()
3322 - support - An array of points which are on the out-edges for point p
3323 
3324   Output Parameter:
3325 
3326   Note:
3327   This should be called after all calls to DMPlexSetSupportSize() and DMSetUp().
3328 
3329   Level: beginner
3330 
3331 .seealso: DMPlexSetCone(), DMPlexSetConeSize(), DMPlexCreate(), DMPlexGetSupport(), DMPlexSetChart(), DMPlexSetSupportSize(), DMSetUp()
3332 @*/
3333 PetscErrorCode DMPlexSetSupport(DM dm, PetscInt p, const PetscInt support[])
3334 {
3335   DM_Plex       *mesh = (DM_Plex*) dm->data;
3336   PetscInt       pStart, pEnd;
3337   PetscInt       dof, off, c;
3338   PetscErrorCode ierr;
3339 
3340   PetscFunctionBegin;
3341   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3342   ierr = PetscSectionGetChart(mesh->supportSection, &pStart, &pEnd);CHKERRQ(ierr);
3343   ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
3344   if (dof) PetscValidPointer(support, 3);
3345   ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
3346   PetscCheckFalse((p < pStart) || (p >= pEnd),PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
3347   for (c = 0; c < dof; ++c) {
3348     PetscCheckFalse((support[c] < pStart) || (support[c] >= pEnd),PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support point %D is not in the valid range [%D, %D)", support[c], pStart, pEnd);
3349     mesh->supports[off+c] = support[c];
3350   }
3351   PetscFunctionReturn(0);
3352 }
3353 
3354 /*@
3355   DMPlexInsertSupport - Insert a point into the out-edges for the point p in the DAG
3356 
3357   Not collective
3358 
3359   Input Parameters:
3360 + mesh - The DMPlex
3361 . p - The point, which must lie in the chart set with DMPlexSetChart()
3362 . supportPos - The local index in the cone where the point should be put
3363 - supportPoint - The mesh point to insert
3364 
3365   Level: beginner
3366 
3367 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
3368 @*/
3369 PetscErrorCode DMPlexInsertSupport(DM dm, PetscInt p, PetscInt supportPos, PetscInt supportPoint)
3370 {
3371   DM_Plex       *mesh = (DM_Plex*) dm->data;
3372   PetscInt       pStart, pEnd;
3373   PetscInt       dof, off;
3374   PetscErrorCode ierr;
3375 
3376   PetscFunctionBegin;
3377   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3378   ierr = PetscSectionGetChart(mesh->supportSection, &pStart, &pEnd);CHKERRQ(ierr);
3379   ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
3380   ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
3381   PetscCheckFalse((p < pStart) || (p >= pEnd),PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
3382   PetscCheckFalse((supportPoint < pStart) || (supportPoint >= pEnd),PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support point %D is not in the valid range [%D, %D)", supportPoint, pStart, pEnd);
3383   PetscCheckFalse(supportPos >= dof,PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support position %D of point %D is not in the valid range [0, %D)", supportPos, p, dof);
3384   mesh->supports[off+supportPos] = supportPoint;
3385   PetscFunctionReturn(0);
3386 }
3387 
3388 /* Converts an orientation o in the current numbering to the previous scheme used in Plex */
3389 PetscInt DMPolytopeConvertNewOrientation_Internal(DMPolytopeType ct, PetscInt o)
3390 {
3391   switch (ct) {
3392     case DM_POLYTOPE_SEGMENT:
3393       if (o == -1) return -2;
3394       break;
3395     case DM_POLYTOPE_TRIANGLE:
3396       if (o == -3) return -1;
3397       if (o == -2) return -3;
3398       if (o == -1) return -2;
3399       break;
3400     case DM_POLYTOPE_QUADRILATERAL:
3401       if (o == -4) return -2;
3402       if (o == -3) return -1;
3403       if (o == -2) return -4;
3404       if (o == -1) return -3;
3405       break;
3406     default: return o;
3407   }
3408   return o;
3409 }
3410 
3411 /* Converts an orientation o in the previous scheme used in Plex to the current numbering */
3412 PetscInt DMPolytopeConvertOldOrientation_Internal(DMPolytopeType ct, PetscInt o)
3413 {
3414   switch (ct) {
3415     case DM_POLYTOPE_SEGMENT:
3416       if ((o == -2) || (o == 1)) return -1;
3417       if (o == -1) return 0;
3418       break;
3419     case DM_POLYTOPE_TRIANGLE:
3420       if (o == -3) return -2;
3421       if (o == -2) return -1;
3422       if (o == -1) return -3;
3423       break;
3424     case DM_POLYTOPE_QUADRILATERAL:
3425       if (o == -4) return -2;
3426       if (o == -3) return -1;
3427       if (o == -2) return -4;
3428       if (o == -1) return -3;
3429       break;
3430     default: return o;
3431   }
3432   return o;
3433 }
3434 
3435 /* Takes in a mesh whose orientations are in the previous scheme and converts them all to the current numbering */
3436 PetscErrorCode DMPlexConvertOldOrientations_Internal(DM dm)
3437 {
3438   PetscInt       pStart, pEnd, p;
3439   PetscErrorCode ierr;
3440 
3441   PetscFunctionBegin;
3442   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
3443   for (p = pStart; p < pEnd; ++p) {
3444     const PetscInt *cone, *ornt;
3445     PetscInt        coneSize, c;
3446 
3447     ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
3448     ierr = DMPlexGetCone(dm, p, &cone);CHKERRQ(ierr);
3449     ierr = DMPlexGetConeOrientation(dm, p, &ornt);CHKERRQ(ierr);
3450     for (c = 0; c < coneSize; ++c) {
3451       DMPolytopeType ct;
3452       const PetscInt o = ornt[c];
3453 
3454       ierr = DMPlexGetCellType(dm, cone[c], &ct);CHKERRQ(ierr);
3455       switch (ct) {
3456         case DM_POLYTOPE_SEGMENT:
3457           if ((o == -2) || (o == 1)) {ierr = DMPlexInsertConeOrientation(dm, p, c, -1);CHKERRQ(ierr);}
3458           if (o == -1) {ierr = DMPlexInsertConeOrientation(dm, p, c, 0);CHKERRQ(ierr);}
3459           break;
3460         case DM_POLYTOPE_TRIANGLE:
3461           if (o == -3) {ierr = DMPlexInsertConeOrientation(dm, p, c, -2);CHKERRQ(ierr);}
3462           if (o == -2) {ierr = DMPlexInsertConeOrientation(dm, p, c, -1);CHKERRQ(ierr);}
3463           if (o == -1) {ierr = DMPlexInsertConeOrientation(dm, p, c, -3);CHKERRQ(ierr);}
3464           break;
3465         case DM_POLYTOPE_QUADRILATERAL:
3466           if (o == -4) {ierr = DMPlexInsertConeOrientation(dm, p, c, -2);CHKERRQ(ierr);}
3467           if (o == -3) {ierr = DMPlexInsertConeOrientation(dm, p, c, -1);CHKERRQ(ierr);}
3468           if (o == -2) {ierr = DMPlexInsertConeOrientation(dm, p, c, -4);CHKERRQ(ierr);}
3469           if (o == -1) {ierr = DMPlexInsertConeOrientation(dm, p, c, -3);CHKERRQ(ierr);}
3470           break;
3471         default: break;
3472       }
3473     }
3474   }
3475   PetscFunctionReturn(0);
3476 }
3477 
3478 static PetscErrorCode DMPlexGetTransitiveClosure_Depth1_Private(DM dm, PetscInt p, PetscInt ornt, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
3479 {
3480   DMPolytopeType  ct = DM_POLYTOPE_UNKNOWN;
3481   PetscInt       *closure;
3482   const PetscInt *tmp = NULL, *tmpO = NULL;
3483   PetscInt        off = 0, tmpSize, t;
3484   PetscErrorCode  ierr;
3485 
3486   PetscFunctionBeginHot;
3487   if (ornt) {
3488     ierr = DMPlexGetCellType(dm, p, &ct);CHKERRQ(ierr);
3489     if (ct == DM_POLYTOPE_FV_GHOST || ct == DM_POLYTOPE_INTERIOR_GHOST || ct == DM_POLYTOPE_UNKNOWN) ct = DM_POLYTOPE_UNKNOWN;
3490   }
3491   if (*points) {
3492     closure = *points;
3493   } else {
3494     PetscInt maxConeSize, maxSupportSize;
3495     ierr = DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize);CHKERRQ(ierr);
3496     ierr = DMGetWorkArray(dm, 2*(PetscMax(maxConeSize, maxSupportSize)+1), MPIU_INT, &closure);CHKERRQ(ierr);
3497   }
3498   if (useCone) {
3499     ierr = DMPlexGetConeSize(dm, p, &tmpSize);CHKERRQ(ierr);
3500     ierr = DMPlexGetCone(dm, p, &tmp);CHKERRQ(ierr);
3501     ierr = DMPlexGetConeOrientation(dm, p, &tmpO);CHKERRQ(ierr);
3502   } else {
3503     ierr = DMPlexGetSupportSize(dm, p, &tmpSize);CHKERRQ(ierr);
3504     ierr = DMPlexGetSupport(dm, p, &tmp);CHKERRQ(ierr);
3505   }
3506   if (ct == DM_POLYTOPE_UNKNOWN) {
3507     closure[off++] = p;
3508     closure[off++] = 0;
3509     for (t = 0; t < tmpSize; ++t) {
3510       closure[off++] = tmp[t];
3511       closure[off++] = tmpO ? tmpO[t] : 0;
3512     }
3513   } else {
3514     const PetscInt *arr = DMPolytopeTypeGetArrangment(ct, ornt);CHKERRQ(ierr);
3515 
3516     /* We assume that cells with a valid type have faces with a valid type */
3517     closure[off++] = p;
3518     closure[off++] = ornt;
3519     for (t = 0; t < tmpSize; ++t) {
3520       DMPolytopeType ft;
3521 
3522       ierr = DMPlexGetCellType(dm, tmp[t], &ft);CHKERRQ(ierr);
3523       closure[off++] = tmp[arr[t]];
3524       closure[off++] = tmpO ? DMPolytopeTypeComposeOrientation(ft, ornt, tmpO[t]) : 0;
3525     }
3526   }
3527   if (numPoints) *numPoints = tmpSize+1;
3528   if (points)    *points    = closure;
3529   PetscFunctionReturn(0);
3530 }
3531 
3532 /* We need a special tensor verison becasue we want to allow duplicate points in the endcaps for hybrid cells */
3533 static PetscErrorCode DMPlexTransitiveClosure_Tensor_Internal(DM dm, PetscInt point, DMPolytopeType ct, PetscInt o, PetscBool useCone, PetscInt *numPoints, PetscInt **points)
3534 {
3535   const PetscInt *arr = DMPolytopeTypeGetArrangment(ct, o);
3536   const PetscInt *cone, *ornt;
3537   PetscInt       *pts,  *closure = NULL;
3538   DMPolytopeType  ft;
3539   PetscInt        maxConeSize, maxSupportSize, coneSeries, supportSeries, maxSize;
3540   PetscInt        dim, coneSize, c, d, clSize, cl;
3541   PetscErrorCode  ierr;
3542 
3543   PetscFunctionBeginHot;
3544   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
3545   ierr = DMPlexGetConeSize(dm, point, &coneSize);CHKERRQ(ierr);
3546   ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
3547   ierr = DMPlexGetConeOrientation(dm, point, &ornt);CHKERRQ(ierr);
3548   ierr = DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize);CHKERRQ(ierr);
3549   coneSeries    = (maxConeSize    > 1) ? ((PetscPowInt(maxConeSize,    dim+1)-1)/(maxConeSize-1))    : dim+1;
3550   supportSeries = (maxSupportSize > 1) ? ((PetscPowInt(maxSupportSize, dim+1)-1)/(maxSupportSize-1)) : dim+1;
3551   maxSize       = PetscMax(coneSeries, supportSeries);
3552   if (*points) {pts  = *points;}
3553   else         {ierr = DMGetWorkArray(dm, 2*maxSize, MPIU_INT, &pts);CHKERRQ(ierr);}
3554   c    = 0;
3555   pts[c++] = point;
3556   pts[c++] = o;
3557   ierr = DMPlexGetCellType(dm, cone[arr[0*2+0]], &ft);CHKERRQ(ierr);
3558   ierr = DMPlexGetTransitiveClosure_Internal(dm, cone[arr[0*2+0]], DMPolytopeTypeComposeOrientation(ft, arr[0*2+1], ornt[0]), useCone, &clSize, &closure);CHKERRQ(ierr);
3559   for (cl = 0; cl < clSize*2; cl += 2) {pts[c++] = closure[cl]; pts[c++] = closure[cl+1];}
3560   ierr = DMPlexGetTransitiveClosure_Internal(dm, cone[arr[1*2+0]], DMPolytopeTypeComposeOrientation(ft, arr[1*2+1], ornt[1]), useCone, &clSize, &closure);CHKERRQ(ierr);
3561   for (cl = 0; cl < clSize*2; cl += 2) {pts[c++] = closure[cl]; pts[c++] = closure[cl+1];}
3562   ierr = DMPlexRestoreTransitiveClosure(dm, cone[0], useCone, &clSize, &closure);CHKERRQ(ierr);
3563   for (d = 2; d < coneSize; ++d) {
3564     ierr = DMPlexGetCellType(dm, cone[arr[d*2+0]], &ft);CHKERRQ(ierr);
3565     pts[c++] = cone[arr[d*2+0]];
3566     pts[c++] = DMPolytopeTypeComposeOrientation(ft, arr[d*2+1], ornt[d]);
3567   }
3568   if (dim >= 3) {
3569     for (d = 2; d < coneSize; ++d) {
3570       const PetscInt  fpoint = cone[arr[d*2+0]];
3571       const PetscInt *fcone, *fornt;
3572       PetscInt        fconeSize, fc, i;
3573 
3574       ierr = DMPlexGetCellType(dm, fpoint, &ft);CHKERRQ(ierr);
3575       const PetscInt *farr = DMPolytopeTypeGetArrangment(ft, DMPolytopeTypeComposeOrientation(ft, arr[d*2+1], ornt[d]));
3576       ierr = DMPlexGetConeSize(dm, fpoint, &fconeSize);CHKERRQ(ierr);
3577       ierr = DMPlexGetCone(dm, fpoint, &fcone);CHKERRQ(ierr);
3578       ierr = DMPlexGetConeOrientation(dm, fpoint, &fornt);CHKERRQ(ierr);
3579       for (fc = 0; fc < fconeSize; ++fc) {
3580         const PetscInt cp = fcone[farr[fc*2+0]];
3581         const PetscInt co = farr[fc*2+1];
3582 
3583         for (i = 0; i < c; i += 2) if (pts[i] == cp) break;
3584         if (i == c) {
3585           ierr = DMPlexGetCellType(dm, cp, &ft);CHKERRQ(ierr);
3586           pts[c++] = cp;
3587           pts[c++] = DMPolytopeTypeComposeOrientation(ft, co, fornt[farr[fc*2+0]]);
3588         }
3589       }
3590     }
3591   }
3592   *numPoints = c/2;
3593   *points    = pts;
3594   PetscFunctionReturn(0);
3595 }
3596 
3597 PetscErrorCode DMPlexGetTransitiveClosure_Internal(DM dm, PetscInt p, PetscInt ornt, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
3598 {
3599   DMPolytopeType ct;
3600   PetscInt      *closure, *fifo;
3601   PetscInt       closureSize = 0, fifoStart = 0, fifoSize = 0;
3602   PetscInt       maxConeSize, maxSupportSize, coneSeries, supportSeries;
3603   PetscInt       depth, maxSize;
3604   PetscErrorCode ierr;
3605 
3606   PetscFunctionBeginHot;
3607   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3608   if (depth == 1) {
3609     ierr = DMPlexGetTransitiveClosure_Depth1_Private(dm, p, ornt, useCone, numPoints, points);CHKERRQ(ierr);
3610     PetscFunctionReturn(0);
3611   }
3612   ierr = DMPlexGetCellType(dm, p, &ct);CHKERRQ(ierr);
3613   if (ct == DM_POLYTOPE_FV_GHOST || ct == DM_POLYTOPE_INTERIOR_GHOST || ct == DM_POLYTOPE_UNKNOWN) ct = DM_POLYTOPE_UNKNOWN;
3614   if (ct == DM_POLYTOPE_SEG_PRISM_TENSOR || ct == DM_POLYTOPE_TRI_PRISM_TENSOR || ct == DM_POLYTOPE_QUAD_PRISM_TENSOR) {
3615     ierr = DMPlexTransitiveClosure_Tensor_Internal(dm, p, ct, ornt, useCone, numPoints, points);CHKERRQ(ierr);
3616     PetscFunctionReturn(0);
3617   }
3618   ierr = DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize);CHKERRQ(ierr);
3619   coneSeries    = (maxConeSize    > 1) ? ((PetscPowInt(maxConeSize,    depth+1)-1)/(maxConeSize-1))    : depth+1;
3620   supportSeries = (maxSupportSize > 1) ? ((PetscPowInt(maxSupportSize, depth+1)-1)/(maxSupportSize-1)) : depth+1;
3621   maxSize       = PetscMax(coneSeries, supportSeries);
3622   ierr = DMGetWorkArray(dm, 3*maxSize, MPIU_INT, &fifo);CHKERRQ(ierr);
3623   if (*points) {closure = *points;}
3624   else         {ierr = DMGetWorkArray(dm, 2*maxSize, MPIU_INT, &closure);CHKERRQ(ierr);}
3625   closure[closureSize++] = p;
3626   closure[closureSize++] = ornt;
3627   fifo[fifoSize++]       = p;
3628   fifo[fifoSize++]       = ornt;
3629   fifo[fifoSize++]       = ct;
3630   /* Should kick out early when depth is reached, rather than checking all vertices for empty cones */
3631   while (fifoSize - fifoStart) {
3632     const PetscInt       q    = fifo[fifoStart++];
3633     const PetscInt       o    = fifo[fifoStart++];
3634     const DMPolytopeType qt   = (DMPolytopeType) fifo[fifoStart++];
3635     const PetscInt      *qarr = DMPolytopeTypeGetArrangment(qt, o);
3636     const PetscInt      *tmp, *tmpO;
3637     PetscInt             tmpSize, t;
3638 
3639     if (PetscDefined(USE_DEBUG)) {
3640       PetscInt nO = DMPolytopeTypeGetNumArrangments(qt)/2;
3641       PetscCheckFalse(o && (o >= nO || o < -nO),PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid orientation %D not in [%D,%D) for %s %D", o, -nO, nO, DMPolytopeTypes[qt], q);
3642     }
3643     if (useCone) {
3644       ierr = DMPlexGetConeSize(dm, q, &tmpSize);CHKERRQ(ierr);
3645       ierr = DMPlexGetCone(dm, q, &tmp);CHKERRQ(ierr);
3646       ierr = DMPlexGetConeOrientation(dm, q, &tmpO);CHKERRQ(ierr);
3647     } else {
3648       ierr = DMPlexGetSupportSize(dm, q, &tmpSize);CHKERRQ(ierr);
3649       ierr = DMPlexGetSupport(dm, q, &tmp);CHKERRQ(ierr);
3650       tmpO = NULL;
3651     }
3652     for (t = 0; t < tmpSize; ++t) {
3653       const PetscInt ip = useCone && qarr ? qarr[t*2]   : t;
3654       const PetscInt io = useCone && qarr ? qarr[t*2+1] : 0;
3655       const PetscInt cp = tmp[ip];
3656       ierr = DMPlexGetCellType(dm, cp, &ct);CHKERRQ(ierr);
3657       const PetscInt co = tmpO ? DMPolytopeTypeComposeOrientation(ct, io, tmpO[ip]) : 0;
3658       PetscInt       c;
3659 
3660       /* Check for duplicate */
3661       for (c = 0; c < closureSize; c += 2) {
3662         if (closure[c] == cp) break;
3663       }
3664       if (c == closureSize) {
3665         closure[closureSize++] = cp;
3666         closure[closureSize++] = co;
3667         fifo[fifoSize++]       = cp;
3668         fifo[fifoSize++]       = co;
3669         fifo[fifoSize++]       = ct;
3670       }
3671     }
3672   }
3673   ierr = DMRestoreWorkArray(dm, 3*maxSize, MPIU_INT, &fifo);CHKERRQ(ierr);
3674   if (numPoints) *numPoints = closureSize/2;
3675   if (points)    *points    = closure;
3676   PetscFunctionReturn(0);
3677 }
3678 
3679 /*@C
3680   DMPlexGetTransitiveClosure - Return the points on the transitive closure of the in-edges or out-edges for this point in the DAG
3681 
3682   Not collective
3683 
3684   Input Parameters:
3685 + dm      - The DMPlex
3686 . p       - The mesh point
3687 - useCone - PETSC_TRUE for the closure, otherwise return the star
3688 
3689   Input/Output Parameter:
3690 . points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...];
3691            if NULL on input, internal storage will be returned, otherwise the provided array is used
3692 
3693   Output Parameter:
3694 . numPoints - The number of points in the closure, so points[] is of size 2*numPoints
3695 
3696   Note:
3697   If using internal storage (points is NULL on input), each call overwrites the last output.
3698 
3699   Fortran Notes:
3700   Since it returns an array, this routine is only available in Fortran 90, and you must include petsc.h90 in your code.
3701 
3702   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3703 
3704   Level: beginner
3705 
3706 .seealso: DMPlexRestoreTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
3707 @*/
3708 PetscErrorCode DMPlexGetTransitiveClosure(DM dm, PetscInt p, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
3709 {
3710   PetscErrorCode ierr;
3711 
3712   PetscFunctionBeginHot;
3713   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3714   if (numPoints) PetscValidIntPointer(numPoints, 4);
3715   if (points)    PetscValidPointer(points, 5);
3716   ierr = DMPlexGetTransitiveClosure_Internal(dm, p, 0, useCone, numPoints, points);CHKERRQ(ierr);
3717   PetscFunctionReturn(0);
3718 }
3719 
3720 /*@C
3721   DMPlexRestoreTransitiveClosure - Restore the array of points on the transitive closure of the in-edges or out-edges for this point in the DAG
3722 
3723   Not collective
3724 
3725   Input Parameters:
3726 + dm        - The DMPlex
3727 . p         - The mesh point
3728 . useCone   - PETSC_TRUE for the closure, otherwise return the star
3729 . numPoints - The number of points in the closure, so points[] is of size 2*numPoints
3730 - points    - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...]
3731 
3732   Note:
3733   If not using internal storage (points is not NULL on input), this call is unnecessary
3734 
3735   Fortran Notes:
3736   Since it returns an array, this routine is only available in Fortran 90, and you must include petsc.h90 in your code.
3737 
3738   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3739 
3740   Level: beginner
3741 
3742 .seealso: DMPlexGetTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
3743 @*/
3744 PetscErrorCode DMPlexRestoreTransitiveClosure(DM dm, PetscInt p, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
3745 {
3746   PetscErrorCode ierr;
3747 
3748   PetscFunctionBeginHot;
3749   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3750   if (numPoints) *numPoints = 0;
3751   ierr = DMRestoreWorkArray(dm, 0, MPIU_INT, points);CHKERRQ(ierr);
3752   PetscFunctionReturn(0);
3753 }
3754 
3755 /*@
3756   DMPlexGetMaxSizes - Return the maximum number of in-edges (cone) and out-edges (support) for any point in the DAG
3757 
3758   Not collective
3759 
3760   Input Parameter:
3761 . mesh - The DMPlex
3762 
3763   Output Parameters:
3764 + maxConeSize - The maximum number of in-edges
3765 - maxSupportSize - The maximum number of out-edges
3766 
3767   Level: beginner
3768 
3769 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart()
3770 @*/
3771 PetscErrorCode DMPlexGetMaxSizes(DM dm, PetscInt *maxConeSize, PetscInt *maxSupportSize)
3772 {
3773   DM_Plex *mesh = (DM_Plex*) dm->data;
3774 
3775   PetscFunctionBegin;
3776   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3777   if (maxConeSize)    *maxConeSize    = mesh->maxConeSize;
3778   if (maxSupportSize) *maxSupportSize = mesh->maxSupportSize;
3779   PetscFunctionReturn(0);
3780 }
3781 
3782 PetscErrorCode DMSetUp_Plex(DM dm)
3783 {
3784   DM_Plex       *mesh = (DM_Plex*) dm->data;
3785   PetscInt       size;
3786   PetscErrorCode ierr;
3787 
3788   PetscFunctionBegin;
3789   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3790   ierr = PetscSectionSetUp(mesh->coneSection);CHKERRQ(ierr);
3791   ierr = PetscSectionGetStorageSize(mesh->coneSection, &size);CHKERRQ(ierr);
3792   ierr = PetscMalloc1(size, &mesh->cones);CHKERRQ(ierr);
3793   ierr = PetscCalloc1(size, &mesh->coneOrientations);CHKERRQ(ierr);
3794   ierr = PetscLogObjectMemory((PetscObject) dm, size*2*sizeof(PetscInt));CHKERRQ(ierr);
3795   if (mesh->maxSupportSize) {
3796     ierr = PetscSectionSetUp(mesh->supportSection);CHKERRQ(ierr);
3797     ierr = PetscSectionGetStorageSize(mesh->supportSection, &size);CHKERRQ(ierr);
3798     ierr = PetscMalloc1(size, &mesh->supports);CHKERRQ(ierr);
3799     ierr = PetscLogObjectMemory((PetscObject) dm, size*sizeof(PetscInt));CHKERRQ(ierr);
3800   }
3801   PetscFunctionReturn(0);
3802 }
3803 
3804 PetscErrorCode DMCreateSubDM_Plex(DM dm, PetscInt numFields, const PetscInt fields[], IS *is, DM *subdm)
3805 {
3806   PetscErrorCode ierr;
3807 
3808   PetscFunctionBegin;
3809   if (subdm) {ierr = DMClone(dm, subdm);CHKERRQ(ierr);}
3810   ierr = DMCreateSectionSubDM(dm, numFields, fields, is, subdm);CHKERRQ(ierr);
3811   if (subdm) {(*subdm)->useNatural = dm->useNatural;}
3812   if (dm->useNatural && dm->sfMigration) {
3813     PetscSF        sfMigrationInv,sfNatural;
3814     PetscSection   section, sectionSeq;
3815 
3816     (*subdm)->sfMigration = dm->sfMigration;
3817     ierr = PetscObjectReference((PetscObject) dm->sfMigration);CHKERRQ(ierr);
3818     ierr = DMGetLocalSection((*subdm), &section);CHKERRQ(ierr);
3819     ierr = PetscSFCreateInverseSF((*subdm)->sfMigration, &sfMigrationInv);CHKERRQ(ierr);
3820     ierr = PetscSectionCreate(PetscObjectComm((PetscObject) (*subdm)), &sectionSeq);CHKERRQ(ierr);
3821     ierr = PetscSFDistributeSection(sfMigrationInv, section, NULL, sectionSeq);CHKERRQ(ierr);
3822 
3823     ierr = DMPlexCreateGlobalToNaturalSF(*subdm, sectionSeq, (*subdm)->sfMigration, &sfNatural);CHKERRQ(ierr);
3824     (*subdm)->sfNatural = sfNatural;
3825     ierr = PetscSectionDestroy(&sectionSeq);CHKERRQ(ierr);
3826     ierr = PetscSFDestroy(&sfMigrationInv);CHKERRQ(ierr);
3827   }
3828   PetscFunctionReturn(0);
3829 }
3830 
3831 PetscErrorCode DMCreateSuperDM_Plex(DM dms[], PetscInt len, IS **is, DM *superdm)
3832 {
3833   PetscErrorCode ierr;
3834   PetscInt       i = 0;
3835 
3836   PetscFunctionBegin;
3837   ierr = DMClone(dms[0], superdm);CHKERRQ(ierr);
3838   ierr = DMCreateSectionSuperDM(dms, len, is, superdm);CHKERRQ(ierr);
3839   (*superdm)->useNatural = PETSC_FALSE;
3840   for (i = 0; i < len; i++) {
3841     if (dms[i]->useNatural && dms[i]->sfMigration) {
3842       PetscSF        sfMigrationInv,sfNatural;
3843       PetscSection   section, sectionSeq;
3844 
3845       (*superdm)->sfMigration = dms[i]->sfMigration;
3846       ierr = PetscObjectReference((PetscObject) dms[i]->sfMigration);CHKERRQ(ierr);
3847       (*superdm)->useNatural = PETSC_TRUE;
3848       ierr = DMGetLocalSection((*superdm), &section);CHKERRQ(ierr);
3849       ierr = PetscSFCreateInverseSF((*superdm)->sfMigration, &sfMigrationInv);CHKERRQ(ierr);
3850       ierr = PetscSectionCreate(PetscObjectComm((PetscObject) (*superdm)), &sectionSeq);CHKERRQ(ierr);
3851       ierr = PetscSFDistributeSection(sfMigrationInv, section, NULL, sectionSeq);CHKERRQ(ierr);
3852 
3853       ierr = DMPlexCreateGlobalToNaturalSF(*superdm, sectionSeq, (*superdm)->sfMigration, &sfNatural);CHKERRQ(ierr);
3854       (*superdm)->sfNatural = sfNatural;
3855       ierr = PetscSectionDestroy(&sectionSeq);CHKERRQ(ierr);
3856       ierr = PetscSFDestroy(&sfMigrationInv);CHKERRQ(ierr);
3857       break;
3858     }
3859   }
3860   PetscFunctionReturn(0);
3861 }
3862 
3863 /*@
3864   DMPlexSymmetrize - Create support (out-edge) information from cone (in-edge) information
3865 
3866   Not collective
3867 
3868   Input Parameter:
3869 . mesh - The DMPlex
3870 
3871   Output Parameter:
3872 
3873   Note:
3874   This should be called after all calls to DMPlexSetCone()
3875 
3876   Level: beginner
3877 
3878 .seealso: DMPlexCreate(), DMPlexSetChart(), DMPlexSetConeSize(), DMPlexSetCone()
3879 @*/
3880 PetscErrorCode DMPlexSymmetrize(DM dm)
3881 {
3882   DM_Plex       *mesh = (DM_Plex*) dm->data;
3883   PetscInt      *offsets;
3884   PetscInt       supportSize;
3885   PetscInt       pStart, pEnd, p;
3886   PetscErrorCode ierr;
3887 
3888   PetscFunctionBegin;
3889   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3890   PetscCheckFalse(mesh->supports,PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONGSTATE, "Supports were already setup in this DMPlex");
3891   ierr = PetscLogEventBegin(DMPLEX_Symmetrize,dm,0,0,0);CHKERRQ(ierr);
3892   /* Calculate support sizes */
3893   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
3894   for (p = pStart; p < pEnd; ++p) {
3895     PetscInt dof, off, c;
3896 
3897     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
3898     ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
3899     for (c = off; c < off+dof; ++c) {
3900       ierr = PetscSectionAddDof(mesh->supportSection, mesh->cones[c], 1);CHKERRQ(ierr);
3901     }
3902   }
3903   for (p = pStart; p < pEnd; ++p) {
3904     PetscInt dof;
3905 
3906     ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
3907 
3908     mesh->maxSupportSize = PetscMax(mesh->maxSupportSize, dof);
3909   }
3910   ierr = PetscSectionSetUp(mesh->supportSection);CHKERRQ(ierr);
3911   /* Calculate supports */
3912   ierr = PetscSectionGetStorageSize(mesh->supportSection, &supportSize);CHKERRQ(ierr);
3913   ierr = PetscMalloc1(supportSize, &mesh->supports);CHKERRQ(ierr);
3914   ierr = PetscCalloc1(pEnd - pStart, &offsets);CHKERRQ(ierr);
3915   for (p = pStart; p < pEnd; ++p) {
3916     PetscInt dof, off, c;
3917 
3918     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
3919     ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
3920     for (c = off; c < off+dof; ++c) {
3921       const PetscInt q = mesh->cones[c];
3922       PetscInt       offS;
3923 
3924       ierr = PetscSectionGetOffset(mesh->supportSection, q, &offS);CHKERRQ(ierr);
3925 
3926       mesh->supports[offS+offsets[q]] = p;
3927       ++offsets[q];
3928     }
3929   }
3930   ierr = PetscFree(offsets);CHKERRQ(ierr);
3931   ierr = PetscLogEventEnd(DMPLEX_Symmetrize,dm,0,0,0);CHKERRQ(ierr);
3932   PetscFunctionReturn(0);
3933 }
3934 
3935 static PetscErrorCode DMPlexCreateDepthStratum(DM dm, DMLabel label, PetscInt depth, PetscInt pStart, PetscInt pEnd)
3936 {
3937   IS             stratumIS;
3938   PetscErrorCode ierr;
3939 
3940   PetscFunctionBegin;
3941   if (pStart >= pEnd) PetscFunctionReturn(0);
3942   if (PetscDefined(USE_DEBUG)) {
3943     PetscInt  qStart, qEnd, numLevels, level;
3944     PetscBool overlap = PETSC_FALSE;
3945     ierr = DMLabelGetNumValues(label, &numLevels);CHKERRQ(ierr);
3946     for (level = 0; level < numLevels; level++) {
3947       ierr = DMLabelGetStratumBounds(label, level, &qStart, &qEnd);CHKERRQ(ierr);
3948       if ((pStart >= qStart && pStart < qEnd) || (pEnd > qStart && pEnd <= qEnd)) {overlap = PETSC_TRUE; break;}
3949     }
3950     PetscCheckFalse(overlap,PETSC_COMM_SELF, PETSC_ERR_PLIB, "New depth %D range [%D,%D) overlaps with depth %D range [%D,%D)", depth, pStart, pEnd, level, qStart, qEnd);
3951   }
3952   ierr = ISCreateStride(PETSC_COMM_SELF, pEnd-pStart, pStart, 1, &stratumIS);CHKERRQ(ierr);
3953   ierr = DMLabelSetStratumIS(label, depth, stratumIS);CHKERRQ(ierr);
3954   ierr = ISDestroy(&stratumIS);CHKERRQ(ierr);
3955   PetscFunctionReturn(0);
3956 }
3957 
3958 /*@
3959   DMPlexStratify - The DAG for most topologies is a graded poset (https://en.wikipedia.org/wiki/Graded_poset), and
3960   can be illustrated by a Hasse Diagram (https://en.wikipedia.org/wiki/Hasse_diagram). The strata group all points of the
3961   same grade, and this function calculates the strata. This grade can be seen as the height (or depth) of the point in
3962   the DAG.
3963 
3964   Collective on dm
3965 
3966   Input Parameter:
3967 . mesh - The DMPlex
3968 
3969   Output Parameter:
3970 
3971   Notes:
3972   Concretely, DMPlexStratify() creates a new label named "depth" containing the depth in the DAG of each point. For cell-vertex
3973   meshes, vertices are depth 0 and cells are depth 1. For fully interpolated meshes, depth 0 for vertices, 1 for edges, and so on
3974   until cells have depth equal to the dimension of the mesh. The depth label can be accessed through DMPlexGetDepthLabel() or DMPlexGetDepthStratum(), or
3975   manually via DMGetLabel().  The height is defined implicitly by height = maxDimension - depth, and can be accessed
3976   via DMPlexGetHeightStratum().  For example, cells have height 0 and faces have height 1.
3977 
3978   The depth of a point is calculated by executing a breadth-first search (BFS) on the DAG. This could produce surprising results
3979   if run on a partially interpolated mesh, meaning one that had some edges and faces, but not others. For example, suppose that
3980   we had a mesh consisting of one triangle (c0) and three vertices (v0, v1, v2), and only one edge is on the boundary so we choose
3981   to interpolate only that one (e0), so that
3982 $  cone(c0) = {e0, v2}
3983 $  cone(e0) = {v0, v1}
3984   If DMPlexStratify() is run on this mesh, it will give depths
3985 $  depth 0 = {v0, v1, v2}
3986 $  depth 1 = {e0, c0}
3987   where the triangle has been given depth 1, instead of 2, because it is reachable from vertex v2.
3988 
3989   DMPlexStratify() should be called after all calls to DMPlexSymmetrize()
3990 
3991   Level: beginner
3992 
3993 .seealso: DMPlexCreate(), DMPlexSymmetrize(), DMPlexComputeCellTypes()
3994 @*/
3995 PetscErrorCode DMPlexStratify(DM dm)
3996 {
3997   DM_Plex       *mesh = (DM_Plex*) dm->data;
3998   DMLabel        label;
3999   PetscInt       pStart, pEnd, p;
4000   PetscInt       numRoots = 0, numLeaves = 0;
4001   PetscErrorCode ierr;
4002 
4003   PetscFunctionBegin;
4004   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4005   ierr = PetscLogEventBegin(DMPLEX_Stratify,dm,0,0,0);CHKERRQ(ierr);
4006 
4007   /* Create depth label */
4008   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
4009   ierr = DMCreateLabel(dm, "depth");CHKERRQ(ierr);
4010   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
4011 
4012   {
4013     /* Initialize roots and count leaves */
4014     PetscInt sMin = PETSC_MAX_INT;
4015     PetscInt sMax = PETSC_MIN_INT;
4016     PetscInt coneSize, supportSize;
4017 
4018     for (p = pStart; p < pEnd; ++p) {
4019       ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
4020       ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
4021       if (!coneSize && supportSize) {
4022         sMin = PetscMin(p, sMin);
4023         sMax = PetscMax(p, sMax);
4024         ++numRoots;
4025       } else if (!supportSize && coneSize) {
4026         ++numLeaves;
4027       } else if (!supportSize && !coneSize) {
4028         /* Isolated points */
4029         sMin = PetscMin(p, sMin);
4030         sMax = PetscMax(p, sMax);
4031       }
4032     }
4033     ierr = DMPlexCreateDepthStratum(dm, label, 0, sMin, sMax+1);CHKERRQ(ierr);
4034   }
4035 
4036   if (numRoots + numLeaves == (pEnd - pStart)) {
4037     PetscInt sMin = PETSC_MAX_INT;
4038     PetscInt sMax = PETSC_MIN_INT;
4039     PetscInt coneSize, supportSize;
4040 
4041     for (p = pStart; p < pEnd; ++p) {
4042       ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
4043       ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
4044       if (!supportSize && coneSize) {
4045         sMin = PetscMin(p, sMin);
4046         sMax = PetscMax(p, sMax);
4047       }
4048     }
4049     ierr = DMPlexCreateDepthStratum(dm, label, 1, sMin, sMax+1);CHKERRQ(ierr);
4050   } else {
4051     PetscInt level = 0;
4052     PetscInt qStart, qEnd, q;
4053 
4054     ierr = DMLabelGetStratumBounds(label, level, &qStart, &qEnd);CHKERRQ(ierr);
4055     while (qEnd > qStart) {
4056       PetscInt sMin = PETSC_MAX_INT;
4057       PetscInt sMax = PETSC_MIN_INT;
4058 
4059       for (q = qStart; q < qEnd; ++q) {
4060         const PetscInt *support;
4061         PetscInt        supportSize, s;
4062 
4063         ierr = DMPlexGetSupportSize(dm, q, &supportSize);CHKERRQ(ierr);
4064         ierr = DMPlexGetSupport(dm, q, &support);CHKERRQ(ierr);
4065         for (s = 0; s < supportSize; ++s) {
4066           sMin = PetscMin(support[s], sMin);
4067           sMax = PetscMax(support[s], sMax);
4068         }
4069       }
4070       ierr = DMLabelGetNumValues(label, &level);CHKERRQ(ierr);
4071       ierr = DMPlexCreateDepthStratum(dm, label, level, sMin, sMax+1);CHKERRQ(ierr);
4072       ierr = DMLabelGetStratumBounds(label, level, &qStart, &qEnd);CHKERRQ(ierr);
4073     }
4074   }
4075   { /* just in case there is an empty process */
4076     PetscInt numValues, maxValues = 0, v;
4077 
4078     ierr = DMLabelGetNumValues(label, &numValues);CHKERRQ(ierr);
4079     ierr = MPI_Allreduce(&numValues,&maxValues,1,MPIU_INT,MPI_MAX,PetscObjectComm((PetscObject)dm));CHKERRMPI(ierr);
4080     for (v = numValues; v < maxValues; v++) {
4081       ierr = DMLabelAddStratum(label, v);CHKERRQ(ierr);
4082     }
4083   }
4084   ierr = PetscObjectStateGet((PetscObject) label, &mesh->depthState);CHKERRQ(ierr);
4085   ierr = PetscLogEventEnd(DMPLEX_Stratify,dm,0,0,0);CHKERRQ(ierr);
4086   PetscFunctionReturn(0);
4087 }
4088 
4089 PetscErrorCode DMPlexComputeCellType_Internal(DM dm, PetscInt p, PetscInt pdepth, DMPolytopeType *pt)
4090 {
4091   DMPolytopeType ct = DM_POLYTOPE_UNKNOWN;
4092   PetscInt       dim, depth, pheight, coneSize;
4093   PetscErrorCode ierr;
4094 
4095   PetscFunctionBeginHot;
4096   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
4097   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
4098   ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
4099   pheight = depth - pdepth;
4100   if (depth <= 1) {
4101     switch (pdepth) {
4102       case 0: ct = DM_POLYTOPE_POINT;break;
4103       case 1:
4104         switch (coneSize) {
4105           case 2: ct = DM_POLYTOPE_SEGMENT;break;
4106           case 3: ct = DM_POLYTOPE_TRIANGLE;break;
4107           case 4:
4108           switch (dim) {
4109             case 2: ct = DM_POLYTOPE_QUADRILATERAL;break;
4110             case 3: ct = DM_POLYTOPE_TETRAHEDRON;break;
4111             default: break;
4112           }
4113           break;
4114         case 5: ct = DM_POLYTOPE_PYRAMID;break;
4115         case 6: ct = DM_POLYTOPE_TRI_PRISM_TENSOR;break;
4116         case 8: ct = DM_POLYTOPE_HEXAHEDRON;break;
4117         default: break;
4118       }
4119     }
4120   } else {
4121     if (pdepth == 0) {
4122       ct = DM_POLYTOPE_POINT;
4123     } else if (pheight == 0) {
4124       switch (dim) {
4125         case 1:
4126           switch (coneSize) {
4127             case 2: ct = DM_POLYTOPE_SEGMENT;break;
4128             default: break;
4129           }
4130           break;
4131         case 2:
4132           switch (coneSize) {
4133             case 3: ct = DM_POLYTOPE_TRIANGLE;break;
4134             case 4: ct = DM_POLYTOPE_QUADRILATERAL;break;
4135             default: break;
4136           }
4137           break;
4138         case 3:
4139           switch (coneSize) {
4140             case 4: ct = DM_POLYTOPE_TETRAHEDRON;break;
4141             case 5:
4142             {
4143               const PetscInt *cone;
4144               PetscInt        faceConeSize;
4145 
4146               ierr = DMPlexGetCone(dm, p, &cone);CHKERRQ(ierr);
4147               ierr = DMPlexGetConeSize(dm, cone[0], &faceConeSize);CHKERRQ(ierr);
4148               switch (faceConeSize) {
4149                 case 3: ct = DM_POLYTOPE_TRI_PRISM_TENSOR;break;
4150                 case 4: ct = DM_POLYTOPE_PYRAMID;break;
4151               }
4152             }
4153             break;
4154             case 6: ct = DM_POLYTOPE_HEXAHEDRON;break;
4155             default: break;
4156           }
4157           break;
4158         default: break;
4159       }
4160     } else if (pheight > 0) {
4161       switch (coneSize) {
4162         case 2: ct = DM_POLYTOPE_SEGMENT;break;
4163         case 3: ct = DM_POLYTOPE_TRIANGLE;break;
4164         case 4: ct = DM_POLYTOPE_QUADRILATERAL;break;
4165         default: break;
4166       }
4167     }
4168   }
4169   *pt = ct;
4170   PetscFunctionReturn(0);
4171 }
4172 
4173 /*@
4174   DMPlexComputeCellTypes - Infer the polytope type of every cell using its dimension and cone size.
4175 
4176   Collective on dm
4177 
4178   Input Parameter:
4179 . mesh - The DMPlex
4180 
4181   DMPlexComputeCellTypes() should be called after all calls to DMPlexSymmetrize() and DMPlexStratify()
4182 
4183   Level: developer
4184 
4185   Note: This function is normally called automatically by Plex when a cell type is requested. It creates an
4186   internal DMLabel named "celltype" which can be directly accessed using DMGetLabel(). A user may disable
4187   automatic creation by creating the label manually, using DMCreateLabel(dm, "celltype").
4188 
4189 .seealso: DMPlexCreate(), DMPlexSymmetrize(), DMPlexStratify(), DMGetLabel(), DMCreateLabel()
4190 @*/
4191 PetscErrorCode DMPlexComputeCellTypes(DM dm)
4192 {
4193   DM_Plex       *mesh;
4194   DMLabel        ctLabel;
4195   PetscInt       pStart, pEnd, p;
4196   PetscErrorCode ierr;
4197 
4198   PetscFunctionBegin;
4199   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4200   mesh = (DM_Plex *) dm->data;
4201   ierr = DMCreateLabel(dm, "celltype");CHKERRQ(ierr);
4202   ierr = DMPlexGetCellTypeLabel(dm, &ctLabel);CHKERRQ(ierr);
4203   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
4204   for (p = pStart; p < pEnd; ++p) {
4205     DMPolytopeType ct = DM_POLYTOPE_UNKNOWN;
4206     PetscInt       pdepth;
4207 
4208     ierr = DMPlexGetPointDepth(dm, p, &pdepth);CHKERRQ(ierr);
4209     ierr = DMPlexComputeCellType_Internal(dm, p, pdepth, &ct);CHKERRQ(ierr);
4210     PetscCheckFalse(ct == DM_POLYTOPE_UNKNOWN,PETSC_COMM_SELF, PETSC_ERR_SUP, "Point %D is screwed up", p);
4211     ierr = DMLabelSetValue(ctLabel, p, ct);CHKERRQ(ierr);
4212   }
4213   ierr = PetscObjectStateGet((PetscObject) ctLabel, &mesh->celltypeState);CHKERRQ(ierr);
4214   ierr = PetscObjectViewFromOptions((PetscObject) ctLabel, NULL, "-dm_plex_celltypes_view");CHKERRQ(ierr);
4215   PetscFunctionReturn(0);
4216 }
4217 
4218 /*@C
4219   DMPlexGetJoin - Get an array for the join of the set of points
4220 
4221   Not Collective
4222 
4223   Input Parameters:
4224 + dm - The DMPlex object
4225 . numPoints - The number of input points for the join
4226 - points - The input points
4227 
4228   Output Parameters:
4229 + numCoveredPoints - The number of points in the join
4230 - coveredPoints - The points in the join
4231 
4232   Level: intermediate
4233 
4234   Note: Currently, this is restricted to a single level join
4235 
4236   Fortran Notes:
4237   Since it returns an array, this routine is only available in Fortran 90, and you must
4238   include petsc.h90 in your code.
4239 
4240   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
4241 
4242 .seealso: DMPlexRestoreJoin(), DMPlexGetMeet()
4243 @*/
4244 PetscErrorCode DMPlexGetJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
4245 {
4246   DM_Plex       *mesh = (DM_Plex*) dm->data;
4247   PetscInt      *join[2];
4248   PetscInt       joinSize, i = 0;
4249   PetscInt       dof, off, p, c, m;
4250   PetscErrorCode ierr;
4251 
4252   PetscFunctionBegin;
4253   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4254   PetscValidIntPointer(points, 3);
4255   PetscValidIntPointer(numCoveredPoints, 4);
4256   PetscValidPointer(coveredPoints, 5);
4257   ierr = DMGetWorkArray(dm, mesh->maxSupportSize, MPIU_INT, &join[0]);CHKERRQ(ierr);
4258   ierr = DMGetWorkArray(dm, mesh->maxSupportSize, MPIU_INT, &join[1]);CHKERRQ(ierr);
4259   /* Copy in support of first point */
4260   ierr = PetscSectionGetDof(mesh->supportSection, points[0], &dof);CHKERRQ(ierr);
4261   ierr = PetscSectionGetOffset(mesh->supportSection, points[0], &off);CHKERRQ(ierr);
4262   for (joinSize = 0; joinSize < dof; ++joinSize) {
4263     join[i][joinSize] = mesh->supports[off+joinSize];
4264   }
4265   /* Check each successive support */
4266   for (p = 1; p < numPoints; ++p) {
4267     PetscInt newJoinSize = 0;
4268 
4269     ierr = PetscSectionGetDof(mesh->supportSection, points[p], &dof);CHKERRQ(ierr);
4270     ierr = PetscSectionGetOffset(mesh->supportSection, points[p], &off);CHKERRQ(ierr);
4271     for (c = 0; c < dof; ++c) {
4272       const PetscInt point = mesh->supports[off+c];
4273 
4274       for (m = 0; m < joinSize; ++m) {
4275         if (point == join[i][m]) {
4276           join[1-i][newJoinSize++] = point;
4277           break;
4278         }
4279       }
4280     }
4281     joinSize = newJoinSize;
4282     i        = 1-i;
4283   }
4284   *numCoveredPoints = joinSize;
4285   *coveredPoints    = join[i];
4286   ierr              = DMRestoreWorkArray(dm, mesh->maxSupportSize, MPIU_INT, &join[1-i]);CHKERRQ(ierr);
4287   PetscFunctionReturn(0);
4288 }
4289 
4290 /*@C
4291   DMPlexRestoreJoin - Restore an array for the join of the set of points
4292 
4293   Not Collective
4294 
4295   Input Parameters:
4296 + dm - The DMPlex object
4297 . numPoints - The number of input points for the join
4298 - points - The input points
4299 
4300   Output Parameters:
4301 + numCoveredPoints - The number of points in the join
4302 - coveredPoints - The points in the join
4303 
4304   Fortran Notes:
4305   Since it returns an array, this routine is only available in Fortran 90, and you must
4306   include petsc.h90 in your code.
4307 
4308   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
4309 
4310   Level: intermediate
4311 
4312 .seealso: DMPlexGetJoin(), DMPlexGetFullJoin(), DMPlexGetMeet()
4313 @*/
4314 PetscErrorCode DMPlexRestoreJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
4315 {
4316   PetscErrorCode ierr;
4317 
4318   PetscFunctionBegin;
4319   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4320   if (points) PetscValidIntPointer(points,3);
4321   if (numCoveredPoints) PetscValidIntPointer(numCoveredPoints,4);
4322   PetscValidPointer(coveredPoints, 5);
4323   ierr = DMRestoreWorkArray(dm, 0, MPIU_INT, (void*) coveredPoints);CHKERRQ(ierr);
4324   if (numCoveredPoints) *numCoveredPoints = 0;
4325   PetscFunctionReturn(0);
4326 }
4327 
4328 /*@C
4329   DMPlexGetFullJoin - Get an array for the join of the set of points
4330 
4331   Not Collective
4332 
4333   Input Parameters:
4334 + dm - The DMPlex object
4335 . numPoints - The number of input points for the join
4336 - points - The input points
4337 
4338   Output Parameters:
4339 + numCoveredPoints - The number of points in the join
4340 - coveredPoints - The points in the join
4341 
4342   Fortran Notes:
4343   Since it returns an array, this routine is only available in Fortran 90, and you must
4344   include petsc.h90 in your code.
4345 
4346   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
4347 
4348   Level: intermediate
4349 
4350 .seealso: DMPlexGetJoin(), DMPlexRestoreJoin(), DMPlexGetMeet()
4351 @*/
4352 PetscErrorCode DMPlexGetFullJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
4353 {
4354   DM_Plex       *mesh = (DM_Plex*) dm->data;
4355   PetscInt      *offsets, **closures;
4356   PetscInt      *join[2];
4357   PetscInt       depth = 0, maxSize, joinSize = 0, i = 0;
4358   PetscInt       p, d, c, m, ms;
4359   PetscErrorCode ierr;
4360 
4361   PetscFunctionBegin;
4362   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4363   PetscValidIntPointer(points, 3);
4364   PetscValidIntPointer(numCoveredPoints, 4);
4365   PetscValidPointer(coveredPoints, 5);
4366 
4367   ierr    = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
4368   ierr    = PetscCalloc1(numPoints, &closures);CHKERRQ(ierr);
4369   ierr    = DMGetWorkArray(dm, numPoints*(depth+2), MPIU_INT, &offsets);CHKERRQ(ierr);
4370   ms      = mesh->maxSupportSize;
4371   maxSize = (ms > 1) ? ((PetscPowInt(ms,depth+1)-1)/(ms-1)) : depth + 1;
4372   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &join[0]);CHKERRQ(ierr);
4373   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &join[1]);CHKERRQ(ierr);
4374 
4375   for (p = 0; p < numPoints; ++p) {
4376     PetscInt closureSize;
4377 
4378     ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_FALSE, &closureSize, &closures[p]);CHKERRQ(ierr);
4379 
4380     offsets[p*(depth+2)+0] = 0;
4381     for (d = 0; d < depth+1; ++d) {
4382       PetscInt pStart, pEnd, i;
4383 
4384       ierr = DMPlexGetDepthStratum(dm, d, &pStart, &pEnd);CHKERRQ(ierr);
4385       for (i = offsets[p*(depth+2)+d]; i < closureSize; ++i) {
4386         if ((pStart > closures[p][i*2]) || (pEnd <= closures[p][i*2])) {
4387           offsets[p*(depth+2)+d+1] = i;
4388           break;
4389         }
4390       }
4391       if (i == closureSize) offsets[p*(depth+2)+d+1] = i;
4392     }
4393     PetscCheckFalse(offsets[p*(depth+2)+depth+1] != closureSize,PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Total size of closure %D should be %D", offsets[p*(depth+2)+depth+1], closureSize);
4394   }
4395   for (d = 0; d < depth+1; ++d) {
4396     PetscInt dof;
4397 
4398     /* Copy in support of first point */
4399     dof = offsets[d+1] - offsets[d];
4400     for (joinSize = 0; joinSize < dof; ++joinSize) {
4401       join[i][joinSize] = closures[0][(offsets[d]+joinSize)*2];
4402     }
4403     /* Check each successive cone */
4404     for (p = 1; p < numPoints && joinSize; ++p) {
4405       PetscInt newJoinSize = 0;
4406 
4407       dof = offsets[p*(depth+2)+d+1] - offsets[p*(depth+2)+d];
4408       for (c = 0; c < dof; ++c) {
4409         const PetscInt point = closures[p][(offsets[p*(depth+2)+d]+c)*2];
4410 
4411         for (m = 0; m < joinSize; ++m) {
4412           if (point == join[i][m]) {
4413             join[1-i][newJoinSize++] = point;
4414             break;
4415           }
4416         }
4417       }
4418       joinSize = newJoinSize;
4419       i        = 1-i;
4420     }
4421     if (joinSize) break;
4422   }
4423   *numCoveredPoints = joinSize;
4424   *coveredPoints    = join[i];
4425   for (p = 0; p < numPoints; ++p) {
4426     ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_FALSE, NULL, &closures[p]);CHKERRQ(ierr);
4427   }
4428   ierr = PetscFree(closures);CHKERRQ(ierr);
4429   ierr = DMRestoreWorkArray(dm, numPoints*(depth+2), MPIU_INT, &offsets);CHKERRQ(ierr);
4430   ierr = DMRestoreWorkArray(dm, mesh->maxSupportSize, MPIU_INT, &join[1-i]);CHKERRQ(ierr);
4431   PetscFunctionReturn(0);
4432 }
4433 
4434 /*@C
4435   DMPlexGetMeet - Get an array for the meet of the set of points
4436 
4437   Not Collective
4438 
4439   Input Parameters:
4440 + dm - The DMPlex object
4441 . numPoints - The number of input points for the meet
4442 - points - The input points
4443 
4444   Output Parameters:
4445 + numCoveredPoints - The number of points in the meet
4446 - coveredPoints - The points in the meet
4447 
4448   Level: intermediate
4449 
4450   Note: Currently, this is restricted to a single level meet
4451 
4452   Fortran Notes:
4453   Since it returns an array, this routine is only available in Fortran 90, and you must
4454   include petsc.h90 in your code.
4455 
4456   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
4457 
4458 .seealso: DMPlexRestoreMeet(), DMPlexGetJoin()
4459 @*/
4460 PetscErrorCode DMPlexGetMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveringPoints, const PetscInt **coveringPoints)
4461 {
4462   DM_Plex       *mesh = (DM_Plex*) dm->data;
4463   PetscInt      *meet[2];
4464   PetscInt       meetSize, i = 0;
4465   PetscInt       dof, off, p, c, m;
4466   PetscErrorCode ierr;
4467 
4468   PetscFunctionBegin;
4469   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4470   PetscValidPointer(points, 3);
4471   PetscValidPointer(numCoveringPoints, 4);
4472   PetscValidPointer(coveringPoints, 5);
4473   ierr = DMGetWorkArray(dm, mesh->maxConeSize, MPIU_INT, &meet[0]);CHKERRQ(ierr);
4474   ierr = DMGetWorkArray(dm, mesh->maxConeSize, MPIU_INT, &meet[1]);CHKERRQ(ierr);
4475   /* Copy in cone of first point */
4476   ierr = PetscSectionGetDof(mesh->coneSection, points[0], &dof);CHKERRQ(ierr);
4477   ierr = PetscSectionGetOffset(mesh->coneSection, points[0], &off);CHKERRQ(ierr);
4478   for (meetSize = 0; meetSize < dof; ++meetSize) {
4479     meet[i][meetSize] = mesh->cones[off+meetSize];
4480   }
4481   /* Check each successive cone */
4482   for (p = 1; p < numPoints; ++p) {
4483     PetscInt newMeetSize = 0;
4484 
4485     ierr = PetscSectionGetDof(mesh->coneSection, points[p], &dof);CHKERRQ(ierr);
4486     ierr = PetscSectionGetOffset(mesh->coneSection, points[p], &off);CHKERRQ(ierr);
4487     for (c = 0; c < dof; ++c) {
4488       const PetscInt point = mesh->cones[off+c];
4489 
4490       for (m = 0; m < meetSize; ++m) {
4491         if (point == meet[i][m]) {
4492           meet[1-i][newMeetSize++] = point;
4493           break;
4494         }
4495       }
4496     }
4497     meetSize = newMeetSize;
4498     i        = 1-i;
4499   }
4500   *numCoveringPoints = meetSize;
4501   *coveringPoints    = meet[i];
4502   ierr               = DMRestoreWorkArray(dm, mesh->maxConeSize, MPIU_INT, &meet[1-i]);CHKERRQ(ierr);
4503   PetscFunctionReturn(0);
4504 }
4505 
4506 /*@C
4507   DMPlexRestoreMeet - Restore an array for the meet of the set of points
4508 
4509   Not Collective
4510 
4511   Input Parameters:
4512 + dm - The DMPlex object
4513 . numPoints - The number of input points for the meet
4514 - points - The input points
4515 
4516   Output Parameters:
4517 + numCoveredPoints - The number of points in the meet
4518 - coveredPoints - The points in the meet
4519 
4520   Level: intermediate
4521 
4522   Fortran Notes:
4523   Since it returns an array, this routine is only available in Fortran 90, and you must
4524   include petsc.h90 in your code.
4525 
4526   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
4527 
4528 .seealso: DMPlexGetMeet(), DMPlexGetFullMeet(), DMPlexGetJoin()
4529 @*/
4530 PetscErrorCode DMPlexRestoreMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
4531 {
4532   PetscErrorCode ierr;
4533 
4534   PetscFunctionBegin;
4535   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4536   if (points) PetscValidIntPointer(points,3);
4537   if (numCoveredPoints) PetscValidIntPointer(numCoveredPoints,4);
4538   PetscValidPointer(coveredPoints,5);
4539   ierr = DMRestoreWorkArray(dm, 0, MPIU_INT, (void*) coveredPoints);CHKERRQ(ierr);
4540   if (numCoveredPoints) *numCoveredPoints = 0;
4541   PetscFunctionReturn(0);
4542 }
4543 
4544 /*@C
4545   DMPlexGetFullMeet - Get an array for the meet of the set of points
4546 
4547   Not Collective
4548 
4549   Input Parameters:
4550 + dm - The DMPlex object
4551 . numPoints - The number of input points for the meet
4552 - points - The input points
4553 
4554   Output Parameters:
4555 + numCoveredPoints - The number of points in the meet
4556 - coveredPoints - The points in the meet
4557 
4558   Level: intermediate
4559 
4560   Fortran Notes:
4561   Since it returns an array, this routine is only available in Fortran 90, and you must
4562   include petsc.h90 in your code.
4563 
4564   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
4565 
4566 .seealso: DMPlexGetMeet(), DMPlexRestoreMeet(), DMPlexGetJoin()
4567 @*/
4568 PetscErrorCode DMPlexGetFullMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
4569 {
4570   DM_Plex       *mesh = (DM_Plex*) dm->data;
4571   PetscInt      *offsets, **closures;
4572   PetscInt      *meet[2];
4573   PetscInt       height = 0, maxSize, meetSize = 0, i = 0;
4574   PetscInt       p, h, c, m, mc;
4575   PetscErrorCode ierr;
4576 
4577   PetscFunctionBegin;
4578   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4579   PetscValidPointer(points, 3);
4580   PetscValidPointer(numCoveredPoints, 4);
4581   PetscValidPointer(coveredPoints, 5);
4582 
4583   ierr    = DMPlexGetDepth(dm, &height);CHKERRQ(ierr);
4584   ierr    = PetscMalloc1(numPoints, &closures);CHKERRQ(ierr);
4585   ierr    = DMGetWorkArray(dm, numPoints*(height+2), MPIU_INT, &offsets);CHKERRQ(ierr);
4586   mc      = mesh->maxConeSize;
4587   maxSize = (mc > 1) ? ((PetscPowInt(mc,height+1)-1)/(mc-1)) : height + 1;
4588   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &meet[0]);CHKERRQ(ierr);
4589   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &meet[1]);CHKERRQ(ierr);
4590 
4591   for (p = 0; p < numPoints; ++p) {
4592     PetscInt closureSize;
4593 
4594     ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_TRUE, &closureSize, &closures[p]);CHKERRQ(ierr);
4595 
4596     offsets[p*(height+2)+0] = 0;
4597     for (h = 0; h < height+1; ++h) {
4598       PetscInt pStart, pEnd, i;
4599 
4600       ierr = DMPlexGetHeightStratum(dm, h, &pStart, &pEnd);CHKERRQ(ierr);
4601       for (i = offsets[p*(height+2)+h]; i < closureSize; ++i) {
4602         if ((pStart > closures[p][i*2]) || (pEnd <= closures[p][i*2])) {
4603           offsets[p*(height+2)+h+1] = i;
4604           break;
4605         }
4606       }
4607       if (i == closureSize) offsets[p*(height+2)+h+1] = i;
4608     }
4609     PetscCheckFalse(offsets[p*(height+2)+height+1] != closureSize,PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Total size of closure %D should be %D", offsets[p*(height+2)+height+1], closureSize);
4610   }
4611   for (h = 0; h < height+1; ++h) {
4612     PetscInt dof;
4613 
4614     /* Copy in cone of first point */
4615     dof = offsets[h+1] - offsets[h];
4616     for (meetSize = 0; meetSize < dof; ++meetSize) {
4617       meet[i][meetSize] = closures[0][(offsets[h]+meetSize)*2];
4618     }
4619     /* Check each successive cone */
4620     for (p = 1; p < numPoints && meetSize; ++p) {
4621       PetscInt newMeetSize = 0;
4622 
4623       dof = offsets[p*(height+2)+h+1] - offsets[p*(height+2)+h];
4624       for (c = 0; c < dof; ++c) {
4625         const PetscInt point = closures[p][(offsets[p*(height+2)+h]+c)*2];
4626 
4627         for (m = 0; m < meetSize; ++m) {
4628           if (point == meet[i][m]) {
4629             meet[1-i][newMeetSize++] = point;
4630             break;
4631           }
4632         }
4633       }
4634       meetSize = newMeetSize;
4635       i        = 1-i;
4636     }
4637     if (meetSize) break;
4638   }
4639   *numCoveredPoints = meetSize;
4640   *coveredPoints    = meet[i];
4641   for (p = 0; p < numPoints; ++p) {
4642     ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_TRUE, NULL, &closures[p]);CHKERRQ(ierr);
4643   }
4644   ierr = PetscFree(closures);CHKERRQ(ierr);
4645   ierr = DMRestoreWorkArray(dm, numPoints*(height+2), MPIU_INT, &offsets);CHKERRQ(ierr);
4646   ierr = DMRestoreWorkArray(dm, mesh->maxConeSize, MPIU_INT, &meet[1-i]);CHKERRQ(ierr);
4647   PetscFunctionReturn(0);
4648 }
4649 
4650 /*@C
4651   DMPlexEqual - Determine if two DMs have the same topology
4652 
4653   Not Collective
4654 
4655   Input Parameters:
4656 + dmA - A DMPlex object
4657 - dmB - A DMPlex object
4658 
4659   Output Parameters:
4660 . equal - PETSC_TRUE if the topologies are identical
4661 
4662   Level: intermediate
4663 
4664   Notes:
4665   We are not solving graph isomorphism, so we do not permutation.
4666 
4667 .seealso: DMPlexGetCone()
4668 @*/
4669 PetscErrorCode DMPlexEqual(DM dmA, DM dmB, PetscBool *equal)
4670 {
4671   PetscInt       depth, depthB, pStart, pEnd, pStartB, pEndB, p;
4672   PetscErrorCode ierr;
4673 
4674   PetscFunctionBegin;
4675   PetscValidHeaderSpecific(dmA, DM_CLASSID, 1);
4676   PetscValidHeaderSpecific(dmB, DM_CLASSID, 2);
4677   PetscValidPointer(equal, 3);
4678 
4679   *equal = PETSC_FALSE;
4680   ierr = DMPlexGetDepth(dmA, &depth);CHKERRQ(ierr);
4681   ierr = DMPlexGetDepth(dmB, &depthB);CHKERRQ(ierr);
4682   if (depth != depthB) PetscFunctionReturn(0);
4683   ierr = DMPlexGetChart(dmA, &pStart,  &pEnd);CHKERRQ(ierr);
4684   ierr = DMPlexGetChart(dmB, &pStartB, &pEndB);CHKERRQ(ierr);
4685   if ((pStart != pStartB) || (pEnd != pEndB)) PetscFunctionReturn(0);
4686   for (p = pStart; p < pEnd; ++p) {
4687     const PetscInt *cone, *coneB, *ornt, *orntB, *support, *supportB;
4688     PetscInt        coneSize, coneSizeB, c, supportSize, supportSizeB, s;
4689 
4690     ierr = DMPlexGetConeSize(dmA, p, &coneSize);CHKERRQ(ierr);
4691     ierr = DMPlexGetCone(dmA, p, &cone);CHKERRQ(ierr);
4692     ierr = DMPlexGetConeOrientation(dmA, p, &ornt);CHKERRQ(ierr);
4693     ierr = DMPlexGetConeSize(dmB, p, &coneSizeB);CHKERRQ(ierr);
4694     ierr = DMPlexGetCone(dmB, p, &coneB);CHKERRQ(ierr);
4695     ierr = DMPlexGetConeOrientation(dmB, p, &orntB);CHKERRQ(ierr);
4696     if (coneSize != coneSizeB) PetscFunctionReturn(0);
4697     for (c = 0; c < coneSize; ++c) {
4698       if (cone[c] != coneB[c]) PetscFunctionReturn(0);
4699       if (ornt[c] != orntB[c]) PetscFunctionReturn(0);
4700     }
4701     ierr = DMPlexGetSupportSize(dmA, p, &supportSize);CHKERRQ(ierr);
4702     ierr = DMPlexGetSupport(dmA, p, &support);CHKERRQ(ierr);
4703     ierr = DMPlexGetSupportSize(dmB, p, &supportSizeB);CHKERRQ(ierr);
4704     ierr = DMPlexGetSupport(dmB, p, &supportB);CHKERRQ(ierr);
4705     if (supportSize != supportSizeB) PetscFunctionReturn(0);
4706     for (s = 0; s < supportSize; ++s) {
4707       if (support[s] != supportB[s]) PetscFunctionReturn(0);
4708     }
4709   }
4710   *equal = PETSC_TRUE;
4711   PetscFunctionReturn(0);
4712 }
4713 
4714 /*@C
4715   DMPlexGetNumFaceVertices - Returns the number of vertices on a face
4716 
4717   Not Collective
4718 
4719   Input Parameters:
4720 + dm         - The DMPlex
4721 . cellDim    - The cell dimension
4722 - numCorners - The number of vertices on a cell
4723 
4724   Output Parameters:
4725 . numFaceVertices - The number of vertices on a face
4726 
4727   Level: developer
4728 
4729   Notes:
4730   Of course this can only work for a restricted set of symmetric shapes
4731 
4732 .seealso: DMPlexGetCone()
4733 @*/
4734 PetscErrorCode DMPlexGetNumFaceVertices(DM dm, PetscInt cellDim, PetscInt numCorners, PetscInt *numFaceVertices)
4735 {
4736   MPI_Comm       comm;
4737   PetscErrorCode ierr;
4738 
4739   PetscFunctionBegin;
4740   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
4741   PetscValidPointer(numFaceVertices,4);
4742   switch (cellDim) {
4743   case 0:
4744     *numFaceVertices = 0;
4745     break;
4746   case 1:
4747     *numFaceVertices = 1;
4748     break;
4749   case 2:
4750     switch (numCorners) {
4751     case 3: /* triangle */
4752       *numFaceVertices = 2; /* Edge has 2 vertices */
4753       break;
4754     case 4: /* quadrilateral */
4755       *numFaceVertices = 2; /* Edge has 2 vertices */
4756       break;
4757     case 6: /* quadratic triangle, tri and quad cohesive Lagrange cells */
4758       *numFaceVertices = 3; /* Edge has 3 vertices */
4759       break;
4760     case 9: /* quadratic quadrilateral, quadratic quad cohesive Lagrange cells */
4761       *numFaceVertices = 3; /* Edge has 3 vertices */
4762       break;
4763     default:
4764       SETERRQ(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid number of face corners %D for dimension %D", numCorners, cellDim);
4765     }
4766     break;
4767   case 3:
4768     switch (numCorners) {
4769     case 4: /* tetradehdron */
4770       *numFaceVertices = 3; /* Face has 3 vertices */
4771       break;
4772     case 6: /* tet cohesive cells */
4773       *numFaceVertices = 4; /* Face has 4 vertices */
4774       break;
4775     case 8: /* hexahedron */
4776       *numFaceVertices = 4; /* Face has 4 vertices */
4777       break;
4778     case 9: /* tet cohesive Lagrange cells */
4779       *numFaceVertices = 6; /* Face has 6 vertices */
4780       break;
4781     case 10: /* quadratic tetrahedron */
4782       *numFaceVertices = 6; /* Face has 6 vertices */
4783       break;
4784     case 12: /* hex cohesive Lagrange cells */
4785       *numFaceVertices = 6; /* Face has 6 vertices */
4786       break;
4787     case 18: /* quadratic tet cohesive Lagrange cells */
4788       *numFaceVertices = 6; /* Face has 6 vertices */
4789       break;
4790     case 27: /* quadratic hexahedron, quadratic hex cohesive Lagrange cells */
4791       *numFaceVertices = 9; /* Face has 9 vertices */
4792       break;
4793     default:
4794       SETERRQ(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid number of face corners %D for dimension %D", numCorners, cellDim);
4795     }
4796     break;
4797   default:
4798     SETERRQ(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid cell dimension %D", cellDim);
4799   }
4800   PetscFunctionReturn(0);
4801 }
4802 
4803 /*@
4804   DMPlexGetDepthLabel - Get the DMLabel recording the depth of each point
4805 
4806   Not Collective
4807 
4808   Input Parameter:
4809 . dm    - The DMPlex object
4810 
4811   Output Parameter:
4812 . depthLabel - The DMLabel recording point depth
4813 
4814   Level: developer
4815 
4816 .seealso: DMPlexGetDepth(), DMPlexGetHeightStratum(), DMPlexGetDepthStratum(), DMPlexGetPointDepth(),
4817 @*/
4818 PetscErrorCode DMPlexGetDepthLabel(DM dm, DMLabel *depthLabel)
4819 {
4820   PetscFunctionBegin;
4821   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4822   PetscValidPointer(depthLabel, 2);
4823   *depthLabel = dm->depthLabel;
4824   PetscFunctionReturn(0);
4825 }
4826 
4827 /*@
4828   DMPlexGetDepth - Get the depth of the DAG representing this mesh
4829 
4830   Not Collective
4831 
4832   Input Parameter:
4833 . dm    - The DMPlex object
4834 
4835   Output Parameter:
4836 . depth - The number of strata (breadth first levels) in the DAG
4837 
4838   Level: developer
4839 
4840   Notes:
4841   This returns maximum of point depths over all points, i.e. maximum value of the label returned by DMPlexGetDepthLabel().
4842   The point depth is described more in detail in DMPlexGetDepthStratum().
4843   An empty mesh gives -1.
4844 
4845 .seealso: DMPlexGetDepthLabel(), DMPlexGetDepthStratum(), DMPlexGetPointDepth(), DMPlexSymmetrize()
4846 @*/
4847 PetscErrorCode DMPlexGetDepth(DM dm, PetscInt *depth)
4848 {
4849   DMLabel        label;
4850   PetscInt       d = 0;
4851   PetscErrorCode ierr;
4852 
4853   PetscFunctionBegin;
4854   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4855   PetscValidPointer(depth, 2);
4856   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
4857   if (label) {ierr = DMLabelGetNumValues(label, &d);CHKERRQ(ierr);}
4858   *depth = d-1;
4859   PetscFunctionReturn(0);
4860 }
4861 
4862 /*@
4863   DMPlexGetDepthStratum - Get the bounds [start, end) for all points at a certain depth.
4864 
4865   Not Collective
4866 
4867   Input Parameters:
4868 + dm           - The DMPlex object
4869 - stratumValue - The requested depth
4870 
4871   Output Parameters:
4872 + start - The first point at this depth
4873 - end   - One beyond the last point at this depth
4874 
4875   Notes:
4876   Depth indexing is related to topological dimension.  Depth stratum 0 contains the lowest topological dimension points,
4877   often "vertices".  If the mesh is "interpolated" (see DMPlexInterpolate()), then depth stratum 1 contains the next
4878   higher dimension, e.g., "edges".
4879 
4880   Level: developer
4881 
4882 .seealso: DMPlexGetHeightStratum(), DMPlexGetDepth(), DMPlexGetDepthLabel(), DMPlexGetPointDepth(), DMPlexSymmetrize(), DMPlexInterpolate()
4883 @*/
4884 PetscErrorCode DMPlexGetDepthStratum(DM dm, PetscInt stratumValue, PetscInt *start, PetscInt *end)
4885 {
4886   DMLabel        label;
4887   PetscInt       pStart, pEnd;
4888   PetscErrorCode ierr;
4889 
4890   PetscFunctionBegin;
4891   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4892   if (start) {PetscValidPointer(start, 3); *start = 0;}
4893   if (end)   {PetscValidPointer(end,   4); *end   = 0;}
4894   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
4895   if (pStart == pEnd) PetscFunctionReturn(0);
4896   if (stratumValue < 0) {
4897     if (start) *start = pStart;
4898     if (end)   *end   = pEnd;
4899     PetscFunctionReturn(0);
4900   }
4901   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
4902   PetscCheckFalse(!label,PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "No label named depth was found");
4903   ierr = DMLabelGetStratumBounds(label, stratumValue, start, end);CHKERRQ(ierr);
4904   PetscFunctionReturn(0);
4905 }
4906 
4907 /*@
4908   DMPlexGetHeightStratum - Get the bounds [start, end) for all points at a certain height.
4909 
4910   Not Collective
4911 
4912   Input Parameters:
4913 + dm           - The DMPlex object
4914 - stratumValue - The requested height
4915 
4916   Output Parameters:
4917 + start - The first point at this height
4918 - end   - One beyond the last point at this height
4919 
4920   Notes:
4921   Height indexing is related to topological codimension.  Height stratum 0 contains the highest topological dimension
4922   points, often called "cells" or "elements".  If the mesh is "interpolated" (see DMPlexInterpolate()), then height
4923   stratum 1 contains the boundary of these "cells", often called "faces" or "facets".
4924 
4925   Level: developer
4926 
4927 .seealso: DMPlexGetDepthStratum(), DMPlexGetDepth(), DMPlexGetPointHeight()
4928 @*/
4929 PetscErrorCode DMPlexGetHeightStratum(DM dm, PetscInt stratumValue, PetscInt *start, PetscInt *end)
4930 {
4931   DMLabel        label;
4932   PetscInt       depth, pStart, pEnd;
4933   PetscErrorCode ierr;
4934 
4935   PetscFunctionBegin;
4936   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4937   if (start) {PetscValidPointer(start, 3); *start = 0;}
4938   if (end)   {PetscValidPointer(end,   4); *end   = 0;}
4939   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
4940   if (pStart == pEnd) PetscFunctionReturn(0);
4941   if (stratumValue < 0) {
4942     if (start) *start = pStart;
4943     if (end)   *end   = pEnd;
4944     PetscFunctionReturn(0);
4945   }
4946   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
4947   PetscCheckFalse(!label,PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "No label named depth was found");
4948   ierr = DMLabelGetNumValues(label, &depth);CHKERRQ(ierr);
4949   ierr = DMLabelGetStratumBounds(label, depth-1-stratumValue, start, end);CHKERRQ(ierr);
4950   PetscFunctionReturn(0);
4951 }
4952 
4953 /*@
4954   DMPlexGetPointDepth - Get the depth of a given point
4955 
4956   Not Collective
4957 
4958   Input Parameters:
4959 + dm    - The DMPlex object
4960 - point - The point
4961 
4962   Output Parameter:
4963 . depth - The depth of the point
4964 
4965   Level: intermediate
4966 
4967 .seealso: DMPlexGetCellType(), DMPlexGetDepthLabel(), DMPlexGetDepth(), DMPlexGetPointHeight()
4968 @*/
4969 PetscErrorCode DMPlexGetPointDepth(DM dm, PetscInt point, PetscInt *depth)
4970 {
4971   PetscErrorCode ierr;
4972 
4973   PetscFunctionBegin;
4974   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4975   PetscValidIntPointer(depth, 3);
4976   ierr = DMLabelGetValue(dm->depthLabel, point, depth);CHKERRQ(ierr);
4977   PetscFunctionReturn(0);
4978 }
4979 
4980 /*@
4981   DMPlexGetPointHeight - Get the height of a given point
4982 
4983   Not Collective
4984 
4985   Input Parameters:
4986 + dm    - The DMPlex object
4987 - point - The point
4988 
4989   Output Parameter:
4990 . height - The height of the point
4991 
4992   Level: intermediate
4993 
4994 .seealso: DMPlexGetCellType(), DMPlexGetDepthLabel(), DMPlexGetDepth(), DMPlexGetPointDepth()
4995 @*/
4996 PetscErrorCode DMPlexGetPointHeight(DM dm, PetscInt point, PetscInt *height)
4997 {
4998   PetscInt       n, pDepth;
4999   PetscErrorCode ierr;
5000 
5001   PetscFunctionBegin;
5002   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5003   PetscValidIntPointer(height, 3);
5004   ierr = DMLabelGetNumValues(dm->depthLabel, &n);CHKERRQ(ierr);
5005   ierr = DMLabelGetValue(dm->depthLabel, point, &pDepth);CHKERRQ(ierr);
5006   *height = n - 1 - pDepth;  /* DAG depth is n-1 */
5007   PetscFunctionReturn(0);
5008 }
5009 
5010 /*@
5011   DMPlexGetCellTypeLabel - Get the DMLabel recording the polytope type of each cell
5012 
5013   Not Collective
5014 
5015   Input Parameter:
5016 . dm - The DMPlex object
5017 
5018   Output Parameter:
5019 . celltypeLabel - The DMLabel recording cell polytope type
5020 
5021   Note: This function will trigger automatica computation of cell types. This can be disabled by calling
5022   DMCreateLabel(dm, "celltype") beforehand.
5023 
5024   Level: developer
5025 
5026 .seealso: DMPlexGetCellType(), DMPlexGetDepthLabel(), DMCreateLabel()
5027 @*/
5028 PetscErrorCode DMPlexGetCellTypeLabel(DM dm, DMLabel *celltypeLabel)
5029 {
5030   PetscErrorCode ierr;
5031 
5032   PetscFunctionBegin;
5033   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5034   PetscValidPointer(celltypeLabel, 2);
5035   if (!dm->celltypeLabel) {ierr = DMPlexComputeCellTypes(dm);CHKERRQ(ierr);}
5036   *celltypeLabel = dm->celltypeLabel;
5037   PetscFunctionReturn(0);
5038 }
5039 
5040 /*@
5041   DMPlexGetCellType - Get the polytope type of a given cell
5042 
5043   Not Collective
5044 
5045   Input Parameters:
5046 + dm   - The DMPlex object
5047 - cell - The cell
5048 
5049   Output Parameter:
5050 . celltype - The polytope type of the cell
5051 
5052   Level: intermediate
5053 
5054 .seealso: DMPlexGetCellTypeLabel(), DMPlexGetDepthLabel(), DMPlexGetDepth()
5055 @*/
5056 PetscErrorCode DMPlexGetCellType(DM dm, PetscInt cell, DMPolytopeType *celltype)
5057 {
5058   DMLabel        label;
5059   PetscInt       ct;
5060   PetscErrorCode ierr;
5061 
5062   PetscFunctionBegin;
5063   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5064   PetscValidPointer(celltype, 3);
5065   ierr = DMPlexGetCellTypeLabel(dm, &label);CHKERRQ(ierr);
5066   ierr = DMLabelGetValue(label, cell, &ct);CHKERRQ(ierr);
5067   PetscCheckFalse(ct < 0,PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Cell %D has not been assigned a cell type", cell);
5068   *celltype = (DMPolytopeType) ct;
5069   PetscFunctionReturn(0);
5070 }
5071 
5072 /*@
5073   DMPlexSetCellType - Set the polytope type of a given cell
5074 
5075   Not Collective
5076 
5077   Input Parameters:
5078 + dm   - The DMPlex object
5079 . cell - The cell
5080 - celltype - The polytope type of the cell
5081 
5082   Note: By default, cell types will be automatically computed using DMPlexComputeCellTypes() before this function
5083   is executed. This function will override the computed type. However, if automatic classification will not succeed
5084   and a user wants to manually specify all types, the classification must be disabled by calling
5085   DMCreaateLabel(dm, "celltype") before getting or setting any cell types.
5086 
5087   Level: advanced
5088 
5089 .seealso: DMPlexGetCellTypeLabel(), DMPlexGetDepthLabel(), DMPlexGetDepth(), DMPlexComputeCellTypes(), DMCreateLabel()
5090 @*/
5091 PetscErrorCode DMPlexSetCellType(DM dm, PetscInt cell, DMPolytopeType celltype)
5092 {
5093   DMLabel        label;
5094   PetscErrorCode ierr;
5095 
5096   PetscFunctionBegin;
5097   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5098   ierr = DMPlexGetCellTypeLabel(dm, &label);CHKERRQ(ierr);
5099   ierr = DMLabelSetValue(label, cell, celltype);CHKERRQ(ierr);
5100   PetscFunctionReturn(0);
5101 }
5102 
5103 PetscErrorCode DMCreateCoordinateDM_Plex(DM dm, DM *cdm)
5104 {
5105   PetscSection   section, s;
5106   Mat            m;
5107   PetscInt       maxHeight;
5108   PetscErrorCode ierr;
5109 
5110   PetscFunctionBegin;
5111   ierr = DMClone(dm, cdm);CHKERRQ(ierr);
5112   ierr = DMPlexGetMaxProjectionHeight(dm, &maxHeight);CHKERRQ(ierr);
5113   ierr = DMPlexSetMaxProjectionHeight(*cdm, maxHeight);CHKERRQ(ierr);
5114   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &section);CHKERRQ(ierr);
5115   ierr = DMSetLocalSection(*cdm, section);CHKERRQ(ierr);
5116   ierr = PetscSectionDestroy(&section);CHKERRQ(ierr);
5117   ierr = PetscSectionCreate(PETSC_COMM_SELF, &s);CHKERRQ(ierr);
5118   ierr = MatCreate(PETSC_COMM_SELF, &m);CHKERRQ(ierr);
5119   ierr = DMSetDefaultConstraints(*cdm, s, m, NULL);CHKERRQ(ierr);
5120   ierr = PetscSectionDestroy(&s);CHKERRQ(ierr);
5121   ierr = MatDestroy(&m);CHKERRQ(ierr);
5122 
5123   ierr = DMSetNumFields(*cdm, 1);CHKERRQ(ierr);
5124   ierr = DMCreateDS(*cdm);CHKERRQ(ierr);
5125   PetscFunctionReturn(0);
5126 }
5127 
5128 PetscErrorCode DMCreateCoordinateField_Plex(DM dm, DMField *field)
5129 {
5130   Vec            coordsLocal;
5131   DM             coordsDM;
5132   PetscErrorCode ierr;
5133 
5134   PetscFunctionBegin;
5135   *field = NULL;
5136   ierr = DMGetCoordinatesLocal(dm,&coordsLocal);CHKERRQ(ierr);
5137   ierr = DMGetCoordinateDM(dm,&coordsDM);CHKERRQ(ierr);
5138   if (coordsLocal && coordsDM) {
5139     ierr = DMFieldCreateDS(coordsDM, 0, coordsLocal, field);CHKERRQ(ierr);
5140   }
5141   PetscFunctionReturn(0);
5142 }
5143 
5144 /*@C
5145   DMPlexGetConeSection - Return a section which describes the layout of cone data
5146 
5147   Not Collective
5148 
5149   Input Parameters:
5150 . dm        - The DMPlex object
5151 
5152   Output Parameter:
5153 . section - The PetscSection object
5154 
5155   Level: developer
5156 
5157 .seealso: DMPlexGetSupportSection(), DMPlexGetCones(), DMPlexGetConeOrientations()
5158 @*/
5159 PetscErrorCode DMPlexGetConeSection(DM dm, PetscSection *section)
5160 {
5161   DM_Plex *mesh = (DM_Plex*) dm->data;
5162 
5163   PetscFunctionBegin;
5164   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5165   if (section) *section = mesh->coneSection;
5166   PetscFunctionReturn(0);
5167 }
5168 
5169 /*@C
5170   DMPlexGetSupportSection - Return a section which describes the layout of support data
5171 
5172   Not Collective
5173 
5174   Input Parameters:
5175 . dm        - The DMPlex object
5176 
5177   Output Parameter:
5178 . section - The PetscSection object
5179 
5180   Level: developer
5181 
5182 .seealso: DMPlexGetConeSection()
5183 @*/
5184 PetscErrorCode DMPlexGetSupportSection(DM dm, PetscSection *section)
5185 {
5186   DM_Plex *mesh = (DM_Plex*) dm->data;
5187 
5188   PetscFunctionBegin;
5189   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5190   if (section) *section = mesh->supportSection;
5191   PetscFunctionReturn(0);
5192 }
5193 
5194 /*@C
5195   DMPlexGetCones - Return cone data
5196 
5197   Not Collective
5198 
5199   Input Parameters:
5200 . dm        - The DMPlex object
5201 
5202   Output Parameter:
5203 . cones - The cone for each point
5204 
5205   Level: developer
5206 
5207 .seealso: DMPlexGetConeSection()
5208 @*/
5209 PetscErrorCode DMPlexGetCones(DM dm, PetscInt *cones[])
5210 {
5211   DM_Plex *mesh = (DM_Plex*) dm->data;
5212 
5213   PetscFunctionBegin;
5214   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5215   if (cones) *cones = mesh->cones;
5216   PetscFunctionReturn(0);
5217 }
5218 
5219 /*@C
5220   DMPlexGetConeOrientations - Return cone orientation data
5221 
5222   Not Collective
5223 
5224   Input Parameters:
5225 . dm        - The DMPlex object
5226 
5227   Output Parameter:
5228 . coneOrientations - The array of cone orientations for all points
5229 
5230   Level: developer
5231 
5232   Notes:
5233   The PetscSection returned by DMPlexGetConeSection() partitions coneOrientations into cone orientations of particular points as returned by DMPlexGetConeOrientation().
5234 
5235   The meaning of coneOrientations values is detailed in DMPlexGetConeOrientation().
5236 
5237 .seealso: DMPlexGetConeSection(), DMPlexGetConeOrientation()
5238 @*/
5239 PetscErrorCode DMPlexGetConeOrientations(DM dm, PetscInt *coneOrientations[])
5240 {
5241   DM_Plex *mesh = (DM_Plex*) dm->data;
5242 
5243   PetscFunctionBegin;
5244   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5245   if (coneOrientations) *coneOrientations = mesh->coneOrientations;
5246   PetscFunctionReturn(0);
5247 }
5248 
5249 /******************************** FEM Support **********************************/
5250 
5251 /*
5252  Returns number of components and tensor degree for the field.  For interpolated meshes, line should be a point
5253  representing a line in the section.
5254 */
5255 static PetscErrorCode PetscSectionFieldGetTensorDegree_Private(PetscSection section,PetscInt field,PetscInt line,PetscBool vertexchart,PetscInt *Nc,PetscInt *k)
5256 {
5257   PetscErrorCode ierr;
5258 
5259   PetscFunctionBeginHot;
5260   ierr = PetscSectionGetFieldComponents(section, field, Nc);CHKERRQ(ierr);
5261   if (line < 0) {
5262     *k = 0;
5263     *Nc = 0;
5264   } else if (vertexchart) {            /* If we only have a vertex chart, we must have degree k=1 */
5265     *k = 1;
5266   } else {                      /* Assume the full interpolated mesh is in the chart; lines in particular */
5267     /* An order k SEM disc has k-1 dofs on an edge */
5268     ierr = PetscSectionGetFieldDof(section, line, field, k);CHKERRQ(ierr);
5269     *k = *k / *Nc + 1;
5270   }
5271   PetscFunctionReturn(0);
5272 }
5273 
5274 /*@
5275 
5276   DMPlexSetClosurePermutationTensor - Create a permutation from the default (BFS) point ordering in the closure, to a
5277   lexicographic ordering over the tensor product cell (i.e., line, quad, hex, etc.), and set this permutation in the
5278   section provided (or the section of the DM).
5279 
5280   Input Parameters:
5281 + dm      - The DM
5282 . point   - Either a cell (highest dim point) or an edge (dim 1 point), or PETSC_DETERMINE
5283 - section - The PetscSection to reorder, or NULL for the default section
5284 
5285   Note: The point is used to determine the number of dofs/field on an edge. For SEM, this is related to the polynomial
5286   degree of the basis.
5287 
5288   Example:
5289   A typical interpolated single-quad mesh might order points as
5290 .vb
5291   [c0, v1, v2, v3, v4, e5, e6, e7, e8]
5292 
5293   v4 -- e6 -- v3
5294   |           |
5295   e7    c0    e8
5296   |           |
5297   v1 -- e5 -- v2
5298 .ve
5299 
5300   (There is no significance to the ordering described here.)  The default section for a Q3 quad might typically assign
5301   dofs in the order of points, e.g.,
5302 .vb
5303     c0 -> [0,1,2,3]
5304     v1 -> [4]
5305     ...
5306     e5 -> [8, 9]
5307 .ve
5308 
5309   which corresponds to the dofs
5310 .vb
5311     6   10  11  7
5312     13  2   3   15
5313     12  0   1   14
5314     4   8   9   5
5315 .ve
5316 
5317   The closure in BFS ordering works through height strata (cells, edges, vertices) to produce the ordering
5318 .vb
5319   0 1 2 3 8 9 14 15 11 10 13 12 4 5 7 6
5320 .ve
5321 
5322   After calling DMPlexSetClosurePermutationTensor(), the closure will be ordered lexicographically,
5323 .vb
5324    4 8 9 5 12 0 1 14 13 2 3 15 6 10 11 7
5325 .ve
5326 
5327   Level: developer
5328 
5329 .seealso: DMGetLocalSection(), PetscSectionSetClosurePermutation(), DMSetGlobalSection()
5330 @*/
5331 PetscErrorCode DMPlexSetClosurePermutationTensor(DM dm, PetscInt point, PetscSection section)
5332 {
5333   DMLabel        label;
5334   PetscInt       dim, depth = -1, eStart = -1, Nf;
5335   PetscBool      vertexchart;
5336   PetscErrorCode ierr;
5337 
5338   PetscFunctionBegin;
5339   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
5340   if (dim < 1) PetscFunctionReturn(0);
5341   if (point < 0) {
5342     PetscInt sStart,sEnd;
5343 
5344     ierr = DMPlexGetDepthStratum(dm, 1, &sStart, &sEnd);CHKERRQ(ierr);
5345     point = sEnd-sStart ? sStart : point;
5346   }
5347   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
5348   if (point >= 0) { ierr = DMLabelGetValue(label, point, &depth);CHKERRQ(ierr); }
5349   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
5350   if (depth == 1) {eStart = point;}
5351   else if  (depth == dim) {
5352     const PetscInt *cone;
5353 
5354     ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
5355     if (dim == 2) eStart = cone[0];
5356     else if (dim == 3) {
5357       const PetscInt *cone2;
5358       ierr = DMPlexGetCone(dm, cone[0], &cone2);CHKERRQ(ierr);
5359       eStart = cone2[0];
5360     } else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Point %D of depth %D cannot be used to bootstrap spectral ordering for dim %D", point, depth, dim);
5361   } else PetscCheckFalse(depth >= 0,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Point %D of depth %D cannot be used to bootstrap spectral ordering for dim %D", point, depth, dim);
5362   {                             /* Determine whether the chart covers all points or just vertices. */
5363     PetscInt pStart,pEnd,cStart,cEnd;
5364     ierr = DMPlexGetDepthStratum(dm,0,&pStart,&pEnd);CHKERRQ(ierr);
5365     ierr = PetscSectionGetChart(section,&cStart,&cEnd);CHKERRQ(ierr);
5366     if (pStart == cStart && pEnd == cEnd) vertexchart = PETSC_TRUE;      /* Only vertices are in the chart */
5367     else if (cStart <= point && point < cEnd) vertexchart = PETSC_FALSE; /* Some interpolated points exist in the chart */
5368     else vertexchart = PETSC_TRUE;                                       /* Some interpolated points are not in chart; assume dofs only at cells and vertices */
5369   }
5370   ierr = PetscSectionGetNumFields(section, &Nf);CHKERRQ(ierr);
5371   for (PetscInt d=1; d<=dim; d++) {
5372     PetscInt k, f, Nc, c, i, j, size = 0, offset = 0, foffset = 0;
5373     PetscInt *perm;
5374 
5375     for (f = 0; f < Nf; ++f) {
5376       ierr = PetscSectionFieldGetTensorDegree_Private(section,f,eStart,vertexchart,&Nc,&k);CHKERRQ(ierr);
5377       size += PetscPowInt(k+1, d)*Nc;
5378     }
5379     ierr = PetscMalloc1(size, &perm);CHKERRQ(ierr);
5380     for (f = 0; f < Nf; ++f) {
5381       switch (d) {
5382       case 1:
5383         ierr = PetscSectionFieldGetTensorDegree_Private(section,f,eStart,vertexchart,&Nc,&k);CHKERRQ(ierr);
5384         /*
5385          Original ordering is [ edge of length k-1; vtx0; vtx1 ]
5386          We want              [ vtx0; edge of length k-1; vtx1 ]
5387          */
5388         for (c=0; c<Nc; c++,offset++) perm[offset] = (k-1)*Nc + c + foffset;
5389         for (i=0; i<k-1; i++) for (c=0; c<Nc; c++,offset++) perm[offset] = i*Nc + c + foffset;
5390         for (c=0; c<Nc; c++,offset++) perm[offset] = k*Nc + c + foffset;
5391         foffset = offset;
5392         break;
5393       case 2:
5394         /* The original quad closure is oriented clockwise, {f, e_b, e_r, e_t, e_l, v_lb, v_rb, v_tr, v_tl} */
5395         ierr = PetscSectionFieldGetTensorDegree_Private(section,f,eStart,vertexchart,&Nc,&k);CHKERRQ(ierr);
5396         /* The SEM order is
5397 
5398          v_lb, {e_b}, v_rb,
5399          e^{(k-1)-i}_l, {f^{i*(k-1)}}, e^i_r,
5400          v_lt, reverse {e_t}, v_rt
5401          */
5402         {
5403           const PetscInt of   = 0;
5404           const PetscInt oeb  = of   + PetscSqr(k-1);
5405           const PetscInt oer  = oeb  + (k-1);
5406           const PetscInt oet  = oer  + (k-1);
5407           const PetscInt oel  = oet  + (k-1);
5408           const PetscInt ovlb = oel  + (k-1);
5409           const PetscInt ovrb = ovlb + 1;
5410           const PetscInt ovrt = ovrb + 1;
5411           const PetscInt ovlt = ovrt + 1;
5412           PetscInt       o;
5413 
5414           /* bottom */
5415           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovlb*Nc + c + foffset;
5416           for (o = oeb; o < oer; ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5417           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovrb*Nc + c + foffset;
5418           /* middle */
5419           for (i = 0; i < k-1; ++i) {
5420             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oel+(k-2)-i)*Nc + c + foffset;
5421             for (o = of+(k-1)*i; o < of+(k-1)*(i+1); ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5422             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oer+i)*Nc + c + foffset;
5423           }
5424           /* top */
5425           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovlt*Nc + c + foffset;
5426           for (o = oel-1; o >= oet; --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5427           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovrt*Nc + c + foffset;
5428           foffset = offset;
5429         }
5430         break;
5431       case 3:
5432         /* The original hex closure is
5433 
5434          {c,
5435          f_b, f_t, f_f, f_b, f_r, f_l,
5436          e_bl, e_bb, e_br, e_bf,  e_tf, e_tr, e_tb, e_tl,  e_rf, e_lf, e_lb, e_rb,
5437          v_blf, v_blb, v_brb, v_brf, v_tlf, v_trf, v_trb, v_tlb}
5438          */
5439         ierr = PetscSectionFieldGetTensorDegree_Private(section,f,eStart,vertexchart,&Nc,&k);CHKERRQ(ierr);
5440         /* The SEM order is
5441          Bottom Slice
5442          v_blf, {e^{(k-1)-n}_bf}, v_brf,
5443          e^{i}_bl, f^{n*(k-1)+(k-1)-i}_b, e^{(k-1)-i}_br,
5444          v_blb, {e_bb}, v_brb,
5445 
5446          Middle Slice (j)
5447          {e^{(k-1)-j}_lf}, {f^{j*(k-1)+n}_f}, e^j_rf,
5448          f^{i*(k-1)+j}_l, {c^{(j*(k-1) + i)*(k-1)+n}_t}, f^{j*(k-1)+i}_r,
5449          e^j_lb, {f^{j*(k-1)+(k-1)-n}_b}, e^{(k-1)-j}_rb,
5450 
5451          Top Slice
5452          v_tlf, {e_tf}, v_trf,
5453          e^{(k-1)-i}_tl, {f^{i*(k-1)}_t}, e^{i}_tr,
5454          v_tlb, {e^{(k-1)-n}_tb}, v_trb,
5455          */
5456         {
5457           const PetscInt oc    = 0;
5458           const PetscInt ofb   = oc    + PetscSqr(k-1)*(k-1);
5459           const PetscInt oft   = ofb   + PetscSqr(k-1);
5460           const PetscInt off   = oft   + PetscSqr(k-1);
5461           const PetscInt ofk   = off   + PetscSqr(k-1);
5462           const PetscInt ofr   = ofk   + PetscSqr(k-1);
5463           const PetscInt ofl   = ofr   + PetscSqr(k-1);
5464           const PetscInt oebl  = ofl   + PetscSqr(k-1);
5465           const PetscInt oebb  = oebl  + (k-1);
5466           const PetscInt oebr  = oebb  + (k-1);
5467           const PetscInt oebf  = oebr  + (k-1);
5468           const PetscInt oetf  = oebf  + (k-1);
5469           const PetscInt oetr  = oetf  + (k-1);
5470           const PetscInt oetb  = oetr  + (k-1);
5471           const PetscInt oetl  = oetb  + (k-1);
5472           const PetscInt oerf  = oetl  + (k-1);
5473           const PetscInt oelf  = oerf  + (k-1);
5474           const PetscInt oelb  = oelf  + (k-1);
5475           const PetscInt oerb  = oelb  + (k-1);
5476           const PetscInt ovblf = oerb  + (k-1);
5477           const PetscInt ovblb = ovblf + 1;
5478           const PetscInt ovbrb = ovblb + 1;
5479           const PetscInt ovbrf = ovbrb + 1;
5480           const PetscInt ovtlf = ovbrf + 1;
5481           const PetscInt ovtrf = ovtlf + 1;
5482           const PetscInt ovtrb = ovtrf + 1;
5483           const PetscInt ovtlb = ovtrb + 1;
5484           PetscInt       o, n;
5485 
5486           /* Bottom Slice */
5487           /*   bottom */
5488           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovblf*Nc + c + foffset;
5489           for (o = oetf-1; o >= oebf; --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5490           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovbrf*Nc + c + foffset;
5491           /*   middle */
5492           for (i = 0; i < k-1; ++i) {
5493             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oebl+i)*Nc + c + foffset;
5494             for (n = 0; n < k-1; ++n) {o = ofb+n*(k-1)+i; for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;}
5495             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oebr+(k-2)-i)*Nc + c + foffset;
5496           }
5497           /*   top */
5498           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovblb*Nc + c + foffset;
5499           for (o = oebb; o < oebr; ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5500           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovbrb*Nc + c + foffset;
5501 
5502           /* Middle Slice */
5503           for (j = 0; j < k-1; ++j) {
5504             /*   bottom */
5505             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oelf+(k-2)-j)*Nc + c + foffset;
5506             for (o = off+j*(k-1); o < off+(j+1)*(k-1); ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5507             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oerf+j)*Nc + c + foffset;
5508             /*   middle */
5509             for (i = 0; i < k-1; ++i) {
5510               for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (ofl+i*(k-1)+j)*Nc + c + foffset;
5511               for (n = 0; n < k-1; ++n) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oc+(j*(k-1)+i)*(k-1)+n)*Nc + c + foffset;
5512               for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (ofr+j*(k-1)+i)*Nc + c + foffset;
5513             }
5514             /*   top */
5515             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oelb+j)*Nc + c + foffset;
5516             for (o = ofk+j*(k-1)+(k-2); o >= ofk+j*(k-1); --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5517             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oerb+(k-2)-j)*Nc + c + foffset;
5518           }
5519 
5520           /* Top Slice */
5521           /*   bottom */
5522           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtlf*Nc + c + foffset;
5523           for (o = oetf; o < oetr; ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5524           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtrf*Nc + c + foffset;
5525           /*   middle */
5526           for (i = 0; i < k-1; ++i) {
5527             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oetl+(k-2)-i)*Nc + c + foffset;
5528             for (n = 0; n < k-1; ++n) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oft+i*(k-1)+n)*Nc + c + foffset;
5529             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oetr+i)*Nc + c + foffset;
5530           }
5531           /*   top */
5532           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtlb*Nc + c + foffset;
5533           for (o = oetl-1; o >= oetb; --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5534           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtrb*Nc + c + foffset;
5535 
5536           foffset = offset;
5537         }
5538         break;
5539       default: SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_OUTOFRANGE, "No spectral ordering for dimension %D", d);
5540       }
5541     }
5542     PetscCheckFalse(offset != size,PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Number of permutation entries %D != %D", offset, size);
5543     /* Check permutation */
5544     {
5545       PetscInt *check;
5546 
5547       ierr = PetscMalloc1(size, &check);CHKERRQ(ierr);
5548       for (i = 0; i < size; ++i) {check[i] = -1; PetscCheckFalse(perm[i] < 0 || perm[i] >= size,PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Invalid permutation index p[%D] = %D", i, perm[i]);}
5549       for (i = 0; i < size; ++i) check[perm[i]] = i;
5550       for (i = 0; i < size; ++i) {PetscCheckFalse(check[i] < 0,PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Missing permutation index %D", i);}
5551       ierr = PetscFree(check);CHKERRQ(ierr);
5552     }
5553     ierr = PetscSectionSetClosurePermutation_Internal(section, (PetscObject) dm, d, size, PETSC_OWN_POINTER, perm);CHKERRQ(ierr);
5554     if (d == dim) { // Add permutation for localized (in case this is a coordinate DM)
5555       PetscInt *loc_perm;
5556       ierr = PetscMalloc1(size*2, &loc_perm);CHKERRQ(ierr);
5557       for (PetscInt i=0; i<size; i++) {
5558         loc_perm[i] = perm[i];
5559         loc_perm[size+i] = size + perm[i];
5560       }
5561       ierr = PetscSectionSetClosurePermutation_Internal(section, (PetscObject) dm, d, size*2, PETSC_OWN_POINTER, loc_perm);CHKERRQ(ierr);
5562     }
5563   }
5564   PetscFunctionReturn(0);
5565 }
5566 
5567 PetscErrorCode DMPlexGetPointDualSpaceFEM(DM dm, PetscInt point, PetscInt field, PetscDualSpace *dspace)
5568 {
5569   PetscDS        prob;
5570   PetscInt       depth, Nf, h;
5571   DMLabel        label;
5572   PetscErrorCode ierr;
5573 
5574   PetscFunctionBeginHot;
5575   ierr = DMGetDS(dm, &prob);CHKERRQ(ierr);
5576   Nf      = prob->Nf;
5577   label   = dm->depthLabel;
5578   *dspace = NULL;
5579   if (field < Nf) {
5580     PetscObject disc = prob->disc[field];
5581 
5582     if (disc->classid == PETSCFE_CLASSID) {
5583       PetscDualSpace dsp;
5584 
5585       ierr = PetscFEGetDualSpace((PetscFE)disc,&dsp);CHKERRQ(ierr);
5586       ierr = DMLabelGetNumValues(label,&depth);CHKERRQ(ierr);
5587       ierr = DMLabelGetValue(label,point,&h);CHKERRQ(ierr);
5588       h    = depth - 1 - h;
5589       if (h) {
5590         ierr = PetscDualSpaceGetHeightSubspace(dsp,h,dspace);CHKERRQ(ierr);
5591       } else {
5592         *dspace = dsp;
5593       }
5594     }
5595   }
5596   PetscFunctionReturn(0);
5597 }
5598 
5599 static inline PetscErrorCode DMPlexVecGetClosure_Depth1_Static(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
5600 {
5601   PetscScalar    *array, *vArray;
5602   const PetscInt *cone, *coneO;
5603   PetscInt        pStart, pEnd, p, numPoints, size = 0, offset = 0;
5604   PetscErrorCode  ierr;
5605 
5606   PetscFunctionBeginHot;
5607   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
5608   ierr = DMPlexGetConeSize(dm, point, &numPoints);CHKERRQ(ierr);
5609   ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
5610   ierr = DMPlexGetConeOrientation(dm, point, &coneO);CHKERRQ(ierr);
5611   if (!values || !*values) {
5612     if ((point >= pStart) && (point < pEnd)) {
5613       PetscInt dof;
5614 
5615       ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
5616       size += dof;
5617     }
5618     for (p = 0; p < numPoints; ++p) {
5619       const PetscInt cp = cone[p];
5620       PetscInt       dof;
5621 
5622       if ((cp < pStart) || (cp >= pEnd)) continue;
5623       ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
5624       size += dof;
5625     }
5626     if (!values) {
5627       if (csize) *csize = size;
5628       PetscFunctionReturn(0);
5629     }
5630     ierr = DMGetWorkArray(dm, size, MPIU_SCALAR, &array);CHKERRQ(ierr);
5631   } else {
5632     array = *values;
5633   }
5634   size = 0;
5635   ierr = VecGetArray(v, &vArray);CHKERRQ(ierr);
5636   if ((point >= pStart) && (point < pEnd)) {
5637     PetscInt     dof, off, d;
5638     PetscScalar *varr;
5639 
5640     ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
5641     ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
5642     varr = &vArray[off];
5643     for (d = 0; d < dof; ++d, ++offset) {
5644       array[offset] = varr[d];
5645     }
5646     size += dof;
5647   }
5648   for (p = 0; p < numPoints; ++p) {
5649     const PetscInt cp = cone[p];
5650     PetscInt       o  = coneO[p];
5651     PetscInt       dof, off, d;
5652     PetscScalar   *varr;
5653 
5654     if ((cp < pStart) || (cp >= pEnd)) continue;
5655     ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
5656     ierr = PetscSectionGetOffset(section, cp, &off);CHKERRQ(ierr);
5657     varr = &vArray[off];
5658     if (o >= 0) {
5659       for (d = 0; d < dof; ++d, ++offset) {
5660         array[offset] = varr[d];
5661       }
5662     } else {
5663       for (d = dof-1; d >= 0; --d, ++offset) {
5664         array[offset] = varr[d];
5665       }
5666     }
5667     size += dof;
5668   }
5669   ierr = VecRestoreArray(v, &vArray);CHKERRQ(ierr);
5670   if (!*values) {
5671     if (csize) *csize = size;
5672     *values = array;
5673   } else {
5674     PetscCheckFalse(size > *csize,PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Size of input array %D < actual size %D", *csize, size);
5675     *csize = size;
5676   }
5677   PetscFunctionReturn(0);
5678 }
5679 
5680 /* Compress out points not in the section */
5681 static inline PetscErrorCode CompressPoints_Private(PetscSection section, PetscInt *numPoints, PetscInt points[])
5682 {
5683   const PetscInt np = *numPoints;
5684   PetscInt       pStart, pEnd, p, q;
5685   PetscErrorCode ierr;
5686 
5687   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
5688   for (p = 0, q = 0; p < np; ++p) {
5689     const PetscInt r = points[p*2];
5690     if ((r >= pStart) && (r < pEnd)) {
5691       points[q*2]   = r;
5692       points[q*2+1] = points[p*2+1];
5693       ++q;
5694     }
5695   }
5696   *numPoints = q;
5697   return 0;
5698 }
5699 
5700 /* Compressed closure does not apply closure permutation */
5701 PetscErrorCode DMPlexGetCompressedClosure(DM dm, PetscSection section, PetscInt point, PetscInt *numPoints, PetscInt **points, PetscSection *clSec, IS *clPoints, const PetscInt **clp)
5702 {
5703   const PetscInt *cla = NULL;
5704   PetscInt       np, *pts = NULL;
5705   PetscErrorCode ierr;
5706 
5707   PetscFunctionBeginHot;
5708   ierr = PetscSectionGetClosureIndex(section, (PetscObject) dm, clSec, clPoints);CHKERRQ(ierr);
5709   if (*clPoints) {
5710     PetscInt dof, off;
5711 
5712     ierr = PetscSectionGetDof(*clSec, point, &dof);CHKERRQ(ierr);
5713     ierr = PetscSectionGetOffset(*clSec, point, &off);CHKERRQ(ierr);
5714     ierr = ISGetIndices(*clPoints, &cla);CHKERRQ(ierr);
5715     np   = dof/2;
5716     pts  = (PetscInt *) &cla[off];
5717   } else {
5718     ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &np, &pts);CHKERRQ(ierr);
5719     ierr = CompressPoints_Private(section, &np, pts);CHKERRQ(ierr);
5720   }
5721   *numPoints = np;
5722   *points    = pts;
5723   *clp       = cla;
5724   PetscFunctionReturn(0);
5725 }
5726 
5727 PetscErrorCode DMPlexRestoreCompressedClosure(DM dm, PetscSection section, PetscInt point, PetscInt *numPoints, PetscInt **points, PetscSection *clSec, IS *clPoints, const PetscInt **clp)
5728 {
5729   PetscErrorCode ierr;
5730 
5731   PetscFunctionBeginHot;
5732   if (!*clPoints) {
5733     ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, numPoints, points);CHKERRQ(ierr);
5734   } else {
5735     ierr = ISRestoreIndices(*clPoints, clp);CHKERRQ(ierr);
5736   }
5737   *numPoints = 0;
5738   *points    = NULL;
5739   *clSec     = NULL;
5740   *clPoints  = NULL;
5741   *clp       = NULL;
5742   PetscFunctionReturn(0);
5743 }
5744 
5745 static inline PetscErrorCode DMPlexVecGetClosure_Static(DM dm, PetscSection section, PetscInt numPoints, const PetscInt points[], const PetscInt clperm[], const PetscScalar vArray[], PetscInt *size, PetscScalar array[])
5746 {
5747   PetscInt          offset = 0, p;
5748   const PetscInt    **perms = NULL;
5749   const PetscScalar **flips = NULL;
5750   PetscErrorCode    ierr;
5751 
5752   PetscFunctionBeginHot;
5753   *size = 0;
5754   ierr = PetscSectionGetPointSyms(section,numPoints,points,&perms,&flips);CHKERRQ(ierr);
5755   for (p = 0; p < numPoints; p++) {
5756     const PetscInt    point = points[2*p];
5757     const PetscInt    *perm = perms ? perms[p] : NULL;
5758     const PetscScalar *flip = flips ? flips[p] : NULL;
5759     PetscInt          dof, off, d;
5760     const PetscScalar *varr;
5761 
5762     ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
5763     ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
5764     varr = &vArray[off];
5765     if (clperm) {
5766       if (perm) {
5767         for (d = 0; d < dof; d++) array[clperm[offset + perm[d]]]  = varr[d];
5768       } else {
5769         for (d = 0; d < dof; d++) array[clperm[offset +      d ]]  = varr[d];
5770       }
5771       if (flip) {
5772         for (d = 0; d < dof; d++) array[clperm[offset +      d ]] *= flip[d];
5773       }
5774     } else {
5775       if (perm) {
5776         for (d = 0; d < dof; d++) array[offset + perm[d]]  = varr[d];
5777       } else {
5778         for (d = 0; d < dof; d++) array[offset +      d ]  = varr[d];
5779       }
5780       if (flip) {
5781         for (d = 0; d < dof; d++) array[offset +      d ] *= flip[d];
5782       }
5783     }
5784     offset += dof;
5785   }
5786   ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms,&flips);CHKERRQ(ierr);
5787   *size = offset;
5788   PetscFunctionReturn(0);
5789 }
5790 
5791 static inline PetscErrorCode DMPlexVecGetClosure_Fields_Static(DM dm, PetscSection section, PetscInt numPoints, const PetscInt points[], PetscInt numFields, const PetscInt clperm[], const PetscScalar vArray[], PetscInt *size, PetscScalar array[])
5792 {
5793   PetscInt          offset = 0, f;
5794   PetscErrorCode    ierr;
5795 
5796   PetscFunctionBeginHot;
5797   *size = 0;
5798   for (f = 0; f < numFields; ++f) {
5799     PetscInt          p;
5800     const PetscInt    **perms = NULL;
5801     const PetscScalar **flips = NULL;
5802 
5803     ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
5804     for (p = 0; p < numPoints; p++) {
5805       const PetscInt    point = points[2*p];
5806       PetscInt          fdof, foff, b;
5807       const PetscScalar *varr;
5808       const PetscInt    *perm = perms ? perms[p] : NULL;
5809       const PetscScalar *flip = flips ? flips[p] : NULL;
5810 
5811       ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
5812       ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
5813       varr = &vArray[foff];
5814       if (clperm) {
5815         if (perm) {for (b = 0; b < fdof; b++) {array[clperm[offset + perm[b]]]  = varr[b];}}
5816         else      {for (b = 0; b < fdof; b++) {array[clperm[offset +      b ]]  = varr[b];}}
5817         if (flip) {for (b = 0; b < fdof; b++) {array[clperm[offset +      b ]] *= flip[b];}}
5818       } else {
5819         if (perm) {for (b = 0; b < fdof; b++) {array[offset + perm[b]]  = varr[b];}}
5820         else      {for (b = 0; b < fdof; b++) {array[offset +      b ]  = varr[b];}}
5821         if (flip) {for (b = 0; b < fdof; b++) {array[offset +      b ] *= flip[b];}}
5822       }
5823       offset += fdof;
5824     }
5825     ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
5826   }
5827   *size = offset;
5828   PetscFunctionReturn(0);
5829 }
5830 
5831 /*@C
5832   DMPlexVecGetClosure - Get an array of the values on the closure of 'point'
5833 
5834   Not collective
5835 
5836   Input Parameters:
5837 + dm - The DM
5838 . section - The section describing the layout in v, or NULL to use the default section
5839 . v - The local vector
5840 - point - The point in the DM
5841 
5842   Input/Output Parameters:
5843 + csize  - The size of the input values array, or NULL; on output the number of values in the closure
5844 - values - An array to use for the values, or NULL to have it allocated automatically;
5845            if the user provided NULL, it is a borrowed array and should not be freed
5846 
5847 $ Note that DMPlexVecGetClosure/DMPlexVecRestoreClosure only allocates the values array if it set to NULL in the
5848 $ calling function. This is because DMPlexVecGetClosure() is typically called in the inner loop of a Vec or Mat
5849 $ assembly function, and a user may already have allocated storage for this operation.
5850 $
5851 $ A typical use could be
5852 $
5853 $  values = NULL;
5854 $  ierr = DMPlexVecGetClosure(dm, NULL, v, p, &clSize, &values);CHKERRQ(ierr);
5855 $  for (cl = 0; cl < clSize; ++cl) {
5856 $    <Compute on closure>
5857 $  }
5858 $  ierr = DMPlexVecRestoreClosure(dm, NULL, v, p, &clSize, &values);CHKERRQ(ierr);
5859 $
5860 $ or
5861 $
5862 $  PetscMalloc1(clMaxSize, &values);
5863 $  for (p = pStart; p < pEnd; ++p) {
5864 $    clSize = clMaxSize;
5865 $    ierr = DMPlexVecGetClosure(dm, NULL, v, p, &clSize, &values);CHKERRQ(ierr);
5866 $    for (cl = 0; cl < clSize; ++cl) {
5867 $      <Compute on closure>
5868 $    }
5869 $  }
5870 $  PetscFree(values);
5871 
5872   Fortran Notes:
5873   Since it returns an array, this routine is only available in Fortran 90, and you must
5874   include petsc.h90 in your code.
5875 
5876   The csize argument is not present in the Fortran 90 binding since it is internal to the array.
5877 
5878   Level: intermediate
5879 
5880 .seealso DMPlexVecRestoreClosure(), DMPlexVecSetClosure(), DMPlexMatSetClosure()
5881 @*/
5882 PetscErrorCode DMPlexVecGetClosure(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
5883 {
5884   PetscSection       clSection;
5885   IS                 clPoints;
5886   PetscInt          *points = NULL;
5887   const PetscInt    *clp, *perm;
5888   PetscInt           depth, numFields, numPoints, asize;
5889   PetscErrorCode     ierr;
5890 
5891   PetscFunctionBeginHot;
5892   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5893   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
5894   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
5895   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
5896   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
5897   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5898   if (depth == 1 && numFields < 2) {
5899     ierr = DMPlexVecGetClosure_Depth1_Static(dm, section, v, point, csize, values);CHKERRQ(ierr);
5900     PetscFunctionReturn(0);
5901   }
5902   /* Get points */
5903   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5904   /* Get sizes */
5905   asize = 0;
5906   for (PetscInt p = 0; p < numPoints*2; p += 2) {
5907     PetscInt dof;
5908     ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5909     asize += dof;
5910   }
5911   if (values) {
5912     const PetscScalar *vArray;
5913     PetscInt          size;
5914 
5915     if (*values) {
5916       PetscCheckFalse(*csize < asize,PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Provided array size %D not sufficient to hold closure size %D", *csize, asize);
5917     } else {ierr = DMGetWorkArray(dm, asize, MPIU_SCALAR, values);CHKERRQ(ierr);}
5918     ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, depth, asize, &perm);CHKERRQ(ierr);
5919     ierr = VecGetArrayRead(v, &vArray);CHKERRQ(ierr);
5920     /* Get values */
5921     if (numFields > 0) {ierr = DMPlexVecGetClosure_Fields_Static(dm, section, numPoints, points, numFields, perm, vArray, &size, *values);CHKERRQ(ierr);}
5922     else               {ierr = DMPlexVecGetClosure_Static(dm, section, numPoints, points, perm, vArray, &size, *values);CHKERRQ(ierr);}
5923     PetscCheckFalse(asize != size,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Section size %D does not match Vec closure size %D", asize, size);
5924     /* Cleanup array */
5925     ierr = VecRestoreArrayRead(v, &vArray);CHKERRQ(ierr);
5926   }
5927   if (csize) *csize = asize;
5928   /* Cleanup points */
5929   ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5930   PetscFunctionReturn(0);
5931 }
5932 
5933 PetscErrorCode DMPlexVecGetClosureAtDepth_Internal(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt depth, PetscInt *csize, PetscScalar *values[])
5934 {
5935   DMLabel            depthLabel;
5936   PetscSection       clSection;
5937   IS                 clPoints;
5938   PetscScalar       *array;
5939   const PetscScalar *vArray;
5940   PetscInt          *points = NULL;
5941   const PetscInt    *clp, *perm = NULL;
5942   PetscInt           mdepth, numFields, numPoints, Np = 0, p, clsize, size;
5943   PetscErrorCode     ierr;
5944 
5945   PetscFunctionBeginHot;
5946   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5947   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
5948   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
5949   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
5950   ierr = DMPlexGetDepth(dm, &mdepth);CHKERRQ(ierr);
5951   ierr = DMPlexGetDepthLabel(dm, &depthLabel);CHKERRQ(ierr);
5952   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5953   if (mdepth == 1 && numFields < 2) {
5954     ierr = DMPlexVecGetClosure_Depth1_Static(dm, section, v, point, csize, values);CHKERRQ(ierr);
5955     PetscFunctionReturn(0);
5956   }
5957   /* Get points */
5958   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5959   for (clsize=0,p=0; p<Np; p++) {
5960     PetscInt dof;
5961     ierr = PetscSectionGetDof(section, points[2*p], &dof);CHKERRQ(ierr);
5962     clsize += dof;
5963   }
5964   ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, depth, clsize, &perm);CHKERRQ(ierr);
5965   /* Filter points */
5966   for (p = 0; p < numPoints*2; p += 2) {
5967     PetscInt dep;
5968 
5969     ierr = DMLabelGetValue(depthLabel, points[p], &dep);CHKERRQ(ierr);
5970     if (dep != depth) continue;
5971     points[Np*2+0] = points[p];
5972     points[Np*2+1] = points[p+1];
5973     ++Np;
5974   }
5975   /* Get array */
5976   if (!values || !*values) {
5977     PetscInt asize = 0, dof;
5978 
5979     for (p = 0; p < Np*2; p += 2) {
5980       ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5981       asize += dof;
5982     }
5983     if (!values) {
5984       ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5985       if (csize) *csize = asize;
5986       PetscFunctionReturn(0);
5987     }
5988     ierr = DMGetWorkArray(dm, asize, MPIU_SCALAR, &array);CHKERRQ(ierr);
5989   } else {
5990     array = *values;
5991   }
5992   ierr = VecGetArrayRead(v, &vArray);CHKERRQ(ierr);
5993   /* Get values */
5994   if (numFields > 0) {ierr = DMPlexVecGetClosure_Fields_Static(dm, section, Np, points, numFields, perm, vArray, &size, array);CHKERRQ(ierr);}
5995   else               {ierr = DMPlexVecGetClosure_Static(dm, section, Np, points, perm, vArray, &size, array);CHKERRQ(ierr);}
5996   /* Cleanup points */
5997   ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5998   /* Cleanup array */
5999   ierr = VecRestoreArrayRead(v, &vArray);CHKERRQ(ierr);
6000   if (!*values) {
6001     if (csize) *csize = size;
6002     *values = array;
6003   } else {
6004     PetscCheckFalse(size > *csize,PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Size of input array %D < actual size %D", *csize, size);
6005     *csize = size;
6006   }
6007   PetscFunctionReturn(0);
6008 }
6009 
6010 /*@C
6011   DMPlexVecRestoreClosure - Restore the array of the values on the closure of 'point'
6012 
6013   Not collective
6014 
6015   Input Parameters:
6016 + dm - The DM
6017 . section - The section describing the layout in v, or NULL to use the default section
6018 . v - The local vector
6019 . point - The point in the DM
6020 . csize - The number of values in the closure, or NULL
6021 - values - The array of values, which is a borrowed array and should not be freed
6022 
6023   Note that the array values are discarded and not copied back into v. In order to copy values back to v, use DMPlexVecSetClosure()
6024 
6025   Fortran Notes:
6026   Since it returns an array, this routine is only available in Fortran 90, and you must
6027   include petsc.h90 in your code.
6028 
6029   The csize argument is not present in the Fortran 90 binding since it is internal to the array.
6030 
6031   Level: intermediate
6032 
6033 .seealso DMPlexVecGetClosure(), DMPlexVecSetClosure(), DMPlexMatSetClosure()
6034 @*/
6035 PetscErrorCode DMPlexVecRestoreClosure(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
6036 {
6037   PetscInt       size = 0;
6038   PetscErrorCode ierr;
6039 
6040   PetscFunctionBegin;
6041   /* Should work without recalculating size */
6042   ierr = DMRestoreWorkArray(dm, size, MPIU_SCALAR, (void*) values);CHKERRQ(ierr);
6043   *values = NULL;
6044   PetscFunctionReturn(0);
6045 }
6046 
6047 static inline void add   (PetscScalar *x, PetscScalar y) {*x += y;}
6048 static inline void insert(PetscScalar *x, PetscScalar y) {*x  = y;}
6049 
6050 static inline PetscErrorCode updatePoint_private(PetscSection section, PetscInt point, PetscInt dof, void (*fuse)(PetscScalar*, PetscScalar), PetscBool setBC, const PetscInt perm[], const PetscScalar flip[], const PetscInt clperm[], const PetscScalar values[], PetscInt offset, PetscScalar array[])
6051 {
6052   PetscInt        cdof;   /* The number of constraints on this point */
6053   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
6054   PetscScalar    *a;
6055   PetscInt        off, cind = 0, k;
6056   PetscErrorCode  ierr;
6057 
6058   PetscFunctionBegin;
6059   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
6060   ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
6061   a    = &array[off];
6062   if (!cdof || setBC) {
6063     if (clperm) {
6064       if (perm) {for (k = 0; k < dof; ++k) {fuse(&a[k], values[clperm[offset+perm[k]]] * (flip ? flip[perm[k]] : 1.));}}
6065       else      {for (k = 0; k < dof; ++k) {fuse(&a[k], values[clperm[offset+     k ]] * (flip ? flip[     k ] : 1.));}}
6066     } else {
6067       if (perm) {for (k = 0; k < dof; ++k) {fuse(&a[k], values[offset+perm[k]] * (flip ? flip[perm[k]] : 1.));}}
6068       else      {for (k = 0; k < dof; ++k) {fuse(&a[k], values[offset+     k ] * (flip ? flip[     k ] : 1.));}}
6069     }
6070   } else {
6071     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
6072     if (clperm) {
6073       if (perm) {for (k = 0; k < dof; ++k) {
6074           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
6075           fuse(&a[k], values[clperm[offset+perm[k]]] * (flip ? flip[perm[k]] : 1.));
6076         }
6077       } else {
6078         for (k = 0; k < dof; ++k) {
6079           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
6080           fuse(&a[k], values[clperm[offset+     k ]] * (flip ? flip[     k ] : 1.));
6081         }
6082       }
6083     } else {
6084       if (perm) {
6085         for (k = 0; k < dof; ++k) {
6086           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
6087           fuse(&a[k], values[offset+perm[k]] * (flip ? flip[perm[k]] : 1.));
6088         }
6089       } else {
6090         for (k = 0; k < dof; ++k) {
6091           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
6092           fuse(&a[k], values[offset+     k ] * (flip ? flip[     k ] : 1.));
6093         }
6094       }
6095     }
6096   }
6097   PetscFunctionReturn(0);
6098 }
6099 
6100 static inline PetscErrorCode updatePointBC_private(PetscSection section, PetscInt point, PetscInt dof, void (*fuse)(PetscScalar*, PetscScalar), const PetscInt perm[], const PetscScalar flip[], const PetscInt clperm[], const PetscScalar values[], PetscInt offset, PetscScalar array[])
6101 {
6102   PetscInt        cdof;   /* The number of constraints on this point */
6103   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
6104   PetscScalar    *a;
6105   PetscInt        off, cind = 0, k;
6106   PetscErrorCode  ierr;
6107 
6108   PetscFunctionBegin;
6109   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
6110   ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
6111   a    = &array[off];
6112   if (cdof) {
6113     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
6114     if (clperm) {
6115       if (perm) {
6116         for (k = 0; k < dof; ++k) {
6117           if ((cind < cdof) && (k == cdofs[cind])) {
6118             fuse(&a[k], values[clperm[offset+perm[k]]] * (flip ? flip[perm[k]] : 1.));
6119             cind++;
6120           }
6121         }
6122       } else {
6123         for (k = 0; k < dof; ++k) {
6124           if ((cind < cdof) && (k == cdofs[cind])) {
6125             fuse(&a[k], values[clperm[offset+     k ]] * (flip ? flip[     k ] : 1.));
6126             cind++;
6127           }
6128         }
6129       }
6130     } else {
6131       if (perm) {
6132         for (k = 0; k < dof; ++k) {
6133           if ((cind < cdof) && (k == cdofs[cind])) {
6134             fuse(&a[k], values[offset+perm[k]] * (flip ? flip[perm[k]] : 1.));
6135             cind++;
6136           }
6137         }
6138       } else {
6139         for (k = 0; k < dof; ++k) {
6140           if ((cind < cdof) && (k == cdofs[cind])) {
6141             fuse(&a[k], values[offset+     k ] * (flip ? flip[     k ] : 1.));
6142             cind++;
6143           }
6144         }
6145       }
6146     }
6147   }
6148   PetscFunctionReturn(0);
6149 }
6150 
6151 static inline PetscErrorCode updatePointFields_private(PetscSection section, PetscInt point, const PetscInt *perm, const PetscScalar *flip, PetscInt f, void (*fuse)(PetscScalar*, PetscScalar), PetscBool setBC, const PetscInt clperm[], const PetscScalar values[], PetscInt *offset, PetscScalar array[])
6152 {
6153   PetscScalar    *a;
6154   PetscInt        fdof, foff, fcdof, foffset = *offset;
6155   const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
6156   PetscInt        cind = 0, b;
6157   PetscErrorCode  ierr;
6158 
6159   PetscFunctionBegin;
6160   ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
6161   ierr = PetscSectionGetFieldConstraintDof(section, point, f, &fcdof);CHKERRQ(ierr);
6162   ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
6163   a    = &array[foff];
6164   if (!fcdof || setBC) {
6165     if (clperm) {
6166       if (perm) {for (b = 0; b < fdof; b++) {fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));}}
6167       else      {for (b = 0; b < fdof; b++) {fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));}}
6168     } else {
6169       if (perm) {for (b = 0; b < fdof; b++) {fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));}}
6170       else      {for (b = 0; b < fdof; b++) {fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));}}
6171     }
6172   } else {
6173     ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
6174     if (clperm) {
6175       if (perm) {
6176         for (b = 0; b < fdof; b++) {
6177           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
6178           fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));
6179         }
6180       } else {
6181         for (b = 0; b < fdof; b++) {
6182           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
6183           fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));
6184         }
6185       }
6186     } else {
6187       if (perm) {
6188         for (b = 0; b < fdof; b++) {
6189           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
6190           fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));
6191         }
6192       } else {
6193         for (b = 0; b < fdof; b++) {
6194           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
6195           fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));
6196         }
6197       }
6198     }
6199   }
6200   *offset += fdof;
6201   PetscFunctionReturn(0);
6202 }
6203 
6204 static inline PetscErrorCode updatePointFieldsBC_private(PetscSection section, PetscInt point, const PetscInt perm[], const PetscScalar flip[], PetscInt f, PetscInt Ncc, const PetscInt comps[], void (*fuse)(PetscScalar*, PetscScalar), const PetscInt clperm[], const PetscScalar values[], PetscInt *offset, PetscScalar array[])
6205 {
6206   PetscScalar    *a;
6207   PetscInt        fdof, foff, fcdof, foffset = *offset;
6208   const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
6209   PetscInt        Nc, cind = 0, ncind = 0, b;
6210   PetscBool       ncSet, fcSet;
6211   PetscErrorCode  ierr;
6212 
6213   PetscFunctionBegin;
6214   ierr = PetscSectionGetFieldComponents(section, f, &Nc);CHKERRQ(ierr);
6215   ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
6216   ierr = PetscSectionGetFieldConstraintDof(section, point, f, &fcdof);CHKERRQ(ierr);
6217   ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
6218   a    = &array[foff];
6219   if (fcdof) {
6220     /* We just override fcdof and fcdofs with Ncc and comps */
6221     ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
6222     if (clperm) {
6223       if (perm) {
6224         if (comps) {
6225           for (b = 0; b < fdof; b++) {
6226             ncSet = fcSet = PETSC_FALSE;
6227             if (b%Nc == comps[ncind]) {ncind = (ncind+1)%Ncc; ncSet = PETSC_TRUE;}
6228             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
6229             if (ncSet && fcSet) {fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));}
6230           }
6231         } else {
6232           for (b = 0; b < fdof; b++) {
6233             if ((cind < fcdof) && (b == fcdofs[cind])) {
6234               fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));
6235               ++cind;
6236             }
6237           }
6238         }
6239       } else {
6240         if (comps) {
6241           for (b = 0; b < fdof; b++) {
6242             ncSet = fcSet = PETSC_FALSE;
6243             if (b%Nc == comps[ncind]) {ncind = (ncind+1)%Ncc; ncSet = PETSC_TRUE;}
6244             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
6245             if (ncSet && fcSet) {fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));}
6246           }
6247         } else {
6248           for (b = 0; b < fdof; b++) {
6249             if ((cind < fcdof) && (b == fcdofs[cind])) {
6250               fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));
6251               ++cind;
6252             }
6253           }
6254         }
6255       }
6256     } else {
6257       if (perm) {
6258         if (comps) {
6259           for (b = 0; b < fdof; b++) {
6260             ncSet = fcSet = PETSC_FALSE;
6261             if (b%Nc == comps[ncind]) {ncind = (ncind+1)%Ncc; ncSet = PETSC_TRUE;}
6262             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
6263             if (ncSet && fcSet) {fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));}
6264           }
6265         } else {
6266           for (b = 0; b < fdof; b++) {
6267             if ((cind < fcdof) && (b == fcdofs[cind])) {
6268               fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));
6269               ++cind;
6270             }
6271           }
6272         }
6273       } else {
6274         if (comps) {
6275           for (b = 0; b < fdof; b++) {
6276             ncSet = fcSet = PETSC_FALSE;
6277             if (b%Nc == comps[ncind]) {ncind = (ncind+1)%Ncc; ncSet = PETSC_TRUE;}
6278             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
6279             if (ncSet && fcSet) {fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));}
6280           }
6281         } else {
6282           for (b = 0; b < fdof; b++) {
6283             if ((cind < fcdof) && (b == fcdofs[cind])) {
6284               fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));
6285               ++cind;
6286             }
6287           }
6288         }
6289       }
6290     }
6291   }
6292   *offset += fdof;
6293   PetscFunctionReturn(0);
6294 }
6295 
6296 static inline PetscErrorCode DMPlexVecSetClosure_Depth1_Static(DM dm, PetscSection section, Vec v, PetscInt point, const PetscScalar values[], InsertMode mode)
6297 {
6298   PetscScalar    *array;
6299   const PetscInt *cone, *coneO;
6300   PetscInt        pStart, pEnd, p, numPoints, off, dof;
6301   PetscErrorCode  ierr;
6302 
6303   PetscFunctionBeginHot;
6304   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
6305   ierr = DMPlexGetConeSize(dm, point, &numPoints);CHKERRQ(ierr);
6306   ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
6307   ierr = DMPlexGetConeOrientation(dm, point, &coneO);CHKERRQ(ierr);
6308   ierr = VecGetArray(v, &array);CHKERRQ(ierr);
6309   for (p = 0, off = 0; p <= numPoints; ++p, off += dof) {
6310     const PetscInt cp = !p ? point : cone[p-1];
6311     const PetscInt o  = !p ? 0     : coneO[p-1];
6312 
6313     if ((cp < pStart) || (cp >= pEnd)) {dof = 0; continue;}
6314     ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
6315     /* ADD_VALUES */
6316     {
6317       const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
6318       PetscScalar    *a;
6319       PetscInt        cdof, coff, cind = 0, k;
6320 
6321       ierr = PetscSectionGetConstraintDof(section, cp, &cdof);CHKERRQ(ierr);
6322       ierr = PetscSectionGetOffset(section, cp, &coff);CHKERRQ(ierr);
6323       a    = &array[coff];
6324       if (!cdof) {
6325         if (o >= 0) {
6326           for (k = 0; k < dof; ++k) {
6327             a[k] += values[off+k];
6328           }
6329         } else {
6330           for (k = 0; k < dof; ++k) {
6331             a[k] += values[off+dof-k-1];
6332           }
6333         }
6334       } else {
6335         ierr = PetscSectionGetConstraintIndices(section, cp, &cdofs);CHKERRQ(ierr);
6336         if (o >= 0) {
6337           for (k = 0; k < dof; ++k) {
6338             if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
6339             a[k] += values[off+k];
6340           }
6341         } else {
6342           for (k = 0; k < dof; ++k) {
6343             if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
6344             a[k] += values[off+dof-k-1];
6345           }
6346         }
6347       }
6348     }
6349   }
6350   ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
6351   PetscFunctionReturn(0);
6352 }
6353 
6354 /*@C
6355   DMPlexVecSetClosure - Set an array of the values on the closure of 'point'
6356 
6357   Not collective
6358 
6359   Input Parameters:
6360 + dm - The DM
6361 . section - The section describing the layout in v, or NULL to use the default section
6362 . v - The local vector
6363 . point - The point in the DM
6364 . values - The array of values
6365 - mode - The insert mode. One of INSERT_ALL_VALUES, ADD_ALL_VALUES, INSERT_VALUES, ADD_VALUES, INSERT_BC_VALUES, and ADD_BC_VALUES,
6366          where INSERT_ALL_VALUES and ADD_ALL_VALUES also overwrite boundary conditions.
6367 
6368   Fortran Notes:
6369   This routine is only available in Fortran 90, and you must include petsc.h90 in your code.
6370 
6371   Level: intermediate
6372 
6373 .seealso DMPlexVecGetClosure(), DMPlexMatSetClosure()
6374 @*/
6375 PetscErrorCode DMPlexVecSetClosure(DM dm, PetscSection section, Vec v, PetscInt point, const PetscScalar values[], InsertMode mode)
6376 {
6377   PetscSection    clSection;
6378   IS              clPoints;
6379   PetscScalar    *array;
6380   PetscInt       *points = NULL;
6381   const PetscInt *clp, *clperm = NULL;
6382   PetscInt        depth, numFields, numPoints, p, clsize;
6383   PetscErrorCode  ierr;
6384 
6385   PetscFunctionBeginHot;
6386   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6387   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
6388   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
6389   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
6390   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
6391   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
6392   if (depth == 1 && numFields < 2 && mode == ADD_VALUES) {
6393     ierr = DMPlexVecSetClosure_Depth1_Static(dm, section, v, point, values, mode);CHKERRQ(ierr);
6394     PetscFunctionReturn(0);
6395   }
6396   /* Get points */
6397   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
6398   for (clsize=0,p=0; p<numPoints; p++) {
6399     PetscInt dof;
6400     ierr = PetscSectionGetDof(section, points[2*p], &dof);CHKERRQ(ierr);
6401     clsize += dof;
6402   }
6403   ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, depth, clsize, &clperm);CHKERRQ(ierr);
6404   /* Get array */
6405   ierr = VecGetArray(v, &array);CHKERRQ(ierr);
6406   /* Get values */
6407   if (numFields > 0) {
6408     PetscInt offset = 0, f;
6409     for (f = 0; f < numFields; ++f) {
6410       const PetscInt    **perms = NULL;
6411       const PetscScalar **flips = NULL;
6412 
6413       ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
6414       switch (mode) {
6415       case INSERT_VALUES:
6416         for (p = 0; p < numPoints; p++) {
6417           const PetscInt    point = points[2*p];
6418           const PetscInt    *perm = perms ? perms[p] : NULL;
6419           const PetscScalar *flip = flips ? flips[p] : NULL;
6420           updatePointFields_private(section, point, perm, flip, f, insert, PETSC_FALSE, clperm, values, &offset, array);
6421         } break;
6422       case INSERT_ALL_VALUES:
6423         for (p = 0; p < numPoints; p++) {
6424           const PetscInt    point = points[2*p];
6425           const PetscInt    *perm = perms ? perms[p] : NULL;
6426           const PetscScalar *flip = flips ? flips[p] : NULL;
6427           updatePointFields_private(section, point, perm, flip, f, insert, PETSC_TRUE, clperm, values, &offset, array);
6428         } break;
6429       case INSERT_BC_VALUES:
6430         for (p = 0; p < numPoints; p++) {
6431           const PetscInt    point = points[2*p];
6432           const PetscInt    *perm = perms ? perms[p] : NULL;
6433           const PetscScalar *flip = flips ? flips[p] : NULL;
6434           updatePointFieldsBC_private(section, point, perm, flip, f, -1, NULL, insert, clperm, values, &offset, array);
6435         } break;
6436       case ADD_VALUES:
6437         for (p = 0; p < numPoints; p++) {
6438           const PetscInt    point = points[2*p];
6439           const PetscInt    *perm = perms ? perms[p] : NULL;
6440           const PetscScalar *flip = flips ? flips[p] : NULL;
6441           updatePointFields_private(section, point, perm, flip, f, add, PETSC_FALSE, clperm, values, &offset, array);
6442         } break;
6443       case ADD_ALL_VALUES:
6444         for (p = 0; p < numPoints; p++) {
6445           const PetscInt    point = points[2*p];
6446           const PetscInt    *perm = perms ? perms[p] : NULL;
6447           const PetscScalar *flip = flips ? flips[p] : NULL;
6448           updatePointFields_private(section, point, perm, flip, f, add, PETSC_TRUE, clperm, values, &offset, array);
6449         } break;
6450       case ADD_BC_VALUES:
6451         for (p = 0; p < numPoints; p++) {
6452           const PetscInt    point = points[2*p];
6453           const PetscInt    *perm = perms ? perms[p] : NULL;
6454           const PetscScalar *flip = flips ? flips[p] : NULL;
6455           updatePointFieldsBC_private(section, point, perm, flip, f, -1, NULL, add, clperm, values, &offset, array);
6456         } break;
6457       default:
6458         SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %d", mode);
6459       }
6460       ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
6461     }
6462   } else {
6463     PetscInt dof, off;
6464     const PetscInt    **perms = NULL;
6465     const PetscScalar **flips = NULL;
6466 
6467     ierr = PetscSectionGetPointSyms(section,numPoints,points,&perms,&flips);CHKERRQ(ierr);
6468     switch (mode) {
6469     case INSERT_VALUES:
6470       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
6471         const PetscInt    point = points[2*p];
6472         const PetscInt    *perm = perms ? perms[p] : NULL;
6473         const PetscScalar *flip = flips ? flips[p] : NULL;
6474         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
6475         updatePoint_private(section, point, dof, insert, PETSC_FALSE, perm, flip, clperm, values, off, array);
6476       } break;
6477     case INSERT_ALL_VALUES:
6478       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
6479         const PetscInt    point = points[2*p];
6480         const PetscInt    *perm = perms ? perms[p] : NULL;
6481         const PetscScalar *flip = flips ? flips[p] : NULL;
6482         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
6483         updatePoint_private(section, point, dof, insert, PETSC_TRUE,  perm, flip, clperm, values, off, array);
6484       } break;
6485     case INSERT_BC_VALUES:
6486       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
6487         const PetscInt    point = points[2*p];
6488         const PetscInt    *perm = perms ? perms[p] : NULL;
6489         const PetscScalar *flip = flips ? flips[p] : NULL;
6490         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
6491         updatePointBC_private(section, point, dof, insert,  perm, flip, clperm, values, off, array);
6492       } break;
6493     case ADD_VALUES:
6494       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
6495         const PetscInt    point = points[2*p];
6496         const PetscInt    *perm = perms ? perms[p] : NULL;
6497         const PetscScalar *flip = flips ? flips[p] : NULL;
6498         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
6499         updatePoint_private(section, point, dof, add,    PETSC_FALSE, perm, flip, clperm, values, off, array);
6500       } break;
6501     case ADD_ALL_VALUES:
6502       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
6503         const PetscInt    point = points[2*p];
6504         const PetscInt    *perm = perms ? perms[p] : NULL;
6505         const PetscScalar *flip = flips ? flips[p] : NULL;
6506         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
6507         updatePoint_private(section, point, dof, add,    PETSC_TRUE,  perm, flip, clperm, values, off, array);
6508       } break;
6509     case ADD_BC_VALUES:
6510       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
6511         const PetscInt    point = points[2*p];
6512         const PetscInt    *perm = perms ? perms[p] : NULL;
6513         const PetscScalar *flip = flips ? flips[p] : NULL;
6514         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
6515         updatePointBC_private(section, point, dof, add,  perm, flip, clperm, values, off, array);
6516       } break;
6517     default:
6518       SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %d", mode);
6519     }
6520     ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms,&flips);CHKERRQ(ierr);
6521   }
6522   /* Cleanup points */
6523   ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
6524   /* Cleanup array */
6525   ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
6526   PetscFunctionReturn(0);
6527 }
6528 
6529 /* Check whether the given point is in the label. If not, update the offset to skip this point */
6530 static inline PetscErrorCode CheckPoint_Private(DMLabel label, PetscInt labelId, PetscSection section, PetscInt point, PetscInt f, PetscInt *offset)
6531 {
6532   PetscFunctionBegin;
6533   if (label) {
6534     PetscInt       val, fdof;
6535     PetscErrorCode ierr;
6536 
6537     /* There is a problem with this:
6538          Suppose we have two label values, defining surfaces, interecting along a line in 3D. When we add cells to the label, the cells that
6539        touch both surfaces must pick a label value. Thus we miss setting values for the surface with that other value intersecting that cell.
6540        Thus I am only going to check val != -1, not val != labelId
6541     */
6542     ierr = DMLabelGetValue(label, point, &val);CHKERRQ(ierr);
6543     if (val < 0) {
6544       ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
6545       *offset += fdof;
6546       PetscFunctionReturn(1);
6547     }
6548   }
6549   PetscFunctionReturn(0);
6550 }
6551 
6552 /* Unlike DMPlexVecSetClosure(), this uses plex-native closure permutation, not a user-specified permutation such as DMPlexSetClosurePermutationTensor(). */
6553 PetscErrorCode DMPlexVecSetFieldClosure_Internal(DM dm, PetscSection section, Vec v, PetscBool fieldActive[], PetscInt point, PetscInt Ncc, const PetscInt comps[], DMLabel label, PetscInt labelId, const PetscScalar values[], InsertMode mode)
6554 {
6555   PetscSection      clSection;
6556   IS                clPoints;
6557   PetscScalar       *array;
6558   PetscInt          *points = NULL;
6559   const PetscInt    *clp;
6560   PetscInt          numFields, numPoints, p;
6561   PetscInt          offset = 0, f;
6562   PetscErrorCode    ierr;
6563 
6564   PetscFunctionBeginHot;
6565   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6566   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
6567   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
6568   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
6569   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
6570   /* Get points */
6571   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
6572   /* Get array */
6573   ierr = VecGetArray(v, &array);CHKERRQ(ierr);
6574   /* Get values */
6575   for (f = 0; f < numFields; ++f) {
6576     const PetscInt    **perms = NULL;
6577     const PetscScalar **flips = NULL;
6578 
6579     if (!fieldActive[f]) {
6580       for (p = 0; p < numPoints*2; p += 2) {
6581         PetscInt fdof;
6582         ierr = PetscSectionGetFieldDof(section, points[p], f, &fdof);CHKERRQ(ierr);
6583         offset += fdof;
6584       }
6585       continue;
6586     }
6587     ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
6588     switch (mode) {
6589     case INSERT_VALUES:
6590       for (p = 0; p < numPoints; p++) {
6591         const PetscInt    point = points[2*p];
6592         const PetscInt    *perm = perms ? perms[p] : NULL;
6593         const PetscScalar *flip = flips ? flips[p] : NULL;
6594         ierr = CheckPoint_Private(label, labelId, section, point, f, &offset); if (ierr) continue;
6595         updatePointFields_private(section, point, perm, flip, f, insert, PETSC_FALSE, NULL, values, &offset, array);
6596       } break;
6597     case INSERT_ALL_VALUES:
6598       for (p = 0; p < numPoints; p++) {
6599         const PetscInt    point = points[2*p];
6600         const PetscInt    *perm = perms ? perms[p] : NULL;
6601         const PetscScalar *flip = flips ? flips[p] : NULL;
6602         ierr = CheckPoint_Private(label, labelId, section, point, f, &offset); if (ierr) continue;
6603         updatePointFields_private(section, point, perm, flip, f, insert, PETSC_TRUE, NULL, values, &offset, array);
6604       } break;
6605     case INSERT_BC_VALUES:
6606       for (p = 0; p < numPoints; p++) {
6607         const PetscInt    point = points[2*p];
6608         const PetscInt    *perm = perms ? perms[p] : NULL;
6609         const PetscScalar *flip = flips ? flips[p] : NULL;
6610         ierr = CheckPoint_Private(label, labelId, section, point, f, &offset); if (ierr) continue;
6611         updatePointFieldsBC_private(section, point, perm, flip, f, Ncc, comps, insert, NULL, values, &offset, array);
6612       } break;
6613     case ADD_VALUES:
6614       for (p = 0; p < numPoints; p++) {
6615         const PetscInt    point = points[2*p];
6616         const PetscInt    *perm = perms ? perms[p] : NULL;
6617         const PetscScalar *flip = flips ? flips[p] : NULL;
6618         ierr = CheckPoint_Private(label, labelId, section, point, f, &offset); if (ierr) continue;
6619         updatePointFields_private(section, point, perm, flip, f, add, PETSC_FALSE, NULL, values, &offset, array);
6620       } break;
6621     case ADD_ALL_VALUES:
6622       for (p = 0; p < numPoints; p++) {
6623         const PetscInt    point = points[2*p];
6624         const PetscInt    *perm = perms ? perms[p] : NULL;
6625         const PetscScalar *flip = flips ? flips[p] : NULL;
6626         ierr = CheckPoint_Private(label, labelId, section, point, f, &offset); if (ierr) continue;
6627         updatePointFields_private(section, point, perm, flip, f, add, PETSC_TRUE, NULL, values, &offset, array);
6628       } break;
6629     default:
6630       SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %d", mode);
6631     }
6632     ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
6633   }
6634   /* Cleanup points */
6635   ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
6636   /* Cleanup array */
6637   ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
6638   PetscFunctionReturn(0);
6639 }
6640 
6641 static PetscErrorCode DMPlexPrintMatSetValues(PetscViewer viewer, Mat A, PetscInt point, PetscInt numRIndices, const PetscInt rindices[], PetscInt numCIndices, const PetscInt cindices[], const PetscScalar values[])
6642 {
6643   PetscMPIInt    rank;
6644   PetscInt       i, j;
6645   PetscErrorCode ierr;
6646 
6647   PetscFunctionBegin;
6648   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRMPI(ierr);
6649   ierr = PetscViewerASCIIPrintf(viewer, "[%d]mat for point %D\n", rank, point);CHKERRQ(ierr);
6650   for (i = 0; i < numRIndices; i++) {ierr = PetscViewerASCIIPrintf(viewer, "[%d]mat row indices[%D] = %D\n", rank, i, rindices[i]);CHKERRQ(ierr);}
6651   for (i = 0; i < numCIndices; i++) {ierr = PetscViewerASCIIPrintf(viewer, "[%d]mat col indices[%D] = %D\n", rank, i, cindices[i]);CHKERRQ(ierr);}
6652   numCIndices = numCIndices ? numCIndices : numRIndices;
6653   if (!values) PetscFunctionReturn(0);
6654   for (i = 0; i < numRIndices; i++) {
6655     ierr = PetscViewerASCIIPrintf(viewer, "[%d]", rank);CHKERRQ(ierr);
6656     for (j = 0; j < numCIndices; j++) {
6657 #if defined(PETSC_USE_COMPLEX)
6658       ierr = PetscViewerASCIIPrintf(viewer, " (%g,%g)", (double)PetscRealPart(values[i*numCIndices+j]), (double)PetscImaginaryPart(values[i*numCIndices+j]));CHKERRQ(ierr);
6659 #else
6660       ierr = PetscViewerASCIIPrintf(viewer, " %g", (double)values[i*numCIndices+j]);CHKERRQ(ierr);
6661 #endif
6662     }
6663     ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
6664   }
6665   PetscFunctionReturn(0);
6666 }
6667 
6668 /*
6669   DMPlexGetIndicesPoint_Internal - Add the indices for dofs on a point to an index array
6670 
6671   Input Parameters:
6672 + section - The section for this data layout
6673 . islocal - Is the section (and thus indices being requested) local or global?
6674 . point   - The point contributing dofs with these indices
6675 . off     - The global offset of this point
6676 . loff    - The local offset of each field
6677 . setBC   - The flag determining whether to include indices of boundary values
6678 . perm    - A permutation of the dofs on this point, or NULL
6679 - indperm - A permutation of the entire indices array, or NULL
6680 
6681   Output Parameter:
6682 . indices - Indices for dofs on this point
6683 
6684   Level: developer
6685 
6686   Note: The indices could be local or global, depending on the value of 'off'.
6687 */
6688 PetscErrorCode DMPlexGetIndicesPoint_Internal(PetscSection section, PetscBool islocal,PetscInt point, PetscInt off, PetscInt *loff, PetscBool setBC, const PetscInt perm[], const PetscInt indperm[], PetscInt indices[])
6689 {
6690   PetscInt        dof;   /* The number of unknowns on this point */
6691   PetscInt        cdof;  /* The number of constraints on this point */
6692   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
6693   PetscInt        cind = 0, k;
6694   PetscErrorCode  ierr;
6695 
6696   PetscFunctionBegin;
6697   PetscCheckFalse(!islocal && setBC,PetscObjectComm((PetscObject)section),PETSC_ERR_ARG_INCOMP,"setBC incompatible with global indices; use a local section or disable setBC");
6698   ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
6699   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
6700   if (!cdof || setBC) {
6701     for (k = 0; k < dof; ++k) {
6702       const PetscInt preind = perm ? *loff+perm[k] : *loff+k;
6703       const PetscInt ind    = indperm ? indperm[preind] : preind;
6704 
6705       indices[ind] = off + k;
6706     }
6707   } else {
6708     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
6709     for (k = 0; k < dof; ++k) {
6710       const PetscInt preind = perm ? *loff+perm[k] : *loff+k;
6711       const PetscInt ind    = indperm ? indperm[preind] : preind;
6712 
6713       if ((cind < cdof) && (k == cdofs[cind])) {
6714         /* Insert check for returning constrained indices */
6715         indices[ind] = -(off+k+1);
6716         ++cind;
6717       } else {
6718         indices[ind] = off + k - (islocal ? 0 : cind);
6719       }
6720     }
6721   }
6722   *loff += dof;
6723   PetscFunctionReturn(0);
6724 }
6725 
6726 /*
6727  DMPlexGetIndicesPointFields_Internal - gets section indices for a point in its canonical ordering.
6728 
6729  Input Parameters:
6730 + section - a section (global or local)
6731 - islocal - PETSC_TRUE if requesting local indices (i.e., section is local); PETSC_FALSE for global
6732 . point - point within section
6733 . off - The offset of this point in the (local or global) indexed space - should match islocal and (usually) the section
6734 . foffs - array of length numFields containing the offset in canonical point ordering (the location in indices) of each field
6735 . setBC - identify constrained (boundary condition) points via involution.
6736 . perms - perms[f][permsoff][:] is a permutation of dofs within each field
6737 . permsoff - offset
6738 - indperm - index permutation
6739 
6740  Output Parameter:
6741 . foffs - each entry is incremented by the number of (unconstrained if setBC=FALSE) dofs in that field
6742 . indices - array to hold indices (as defined by section) of each dof associated with point
6743 
6744  Notes:
6745  If section is local and setBC=true, there is no distinction between constrained and unconstrained dofs.
6746  If section is local and setBC=false, the indices for constrained points are the involution -(i+1) of their position
6747  in the local vector.
6748 
6749  If section is global and setBC=false, the indices for constrained points are negative (and their value is not
6750  significant).  It is invalid to call with a global section and setBC=true.
6751 
6752  Developer Note:
6753  The section is only used for field layout, so islocal is technically a statement about the offset (off).  At some point
6754  in the future, global sections may have fields set, in which case we could pass the global section and obtain the
6755  offset could be obtained from the section instead of passing it explicitly as we do now.
6756 
6757  Example:
6758  Suppose a point contains one field with three components, and for which the unconstrained indices are {10, 11, 12}.
6759  When the middle component is constrained, we get the array {10, -12, 12} for (islocal=TRUE, setBC=FALSE).
6760  Note that -12 is the involution of 11, so the user can involute negative indices to recover local indices.
6761  The global vector does not store constrained dofs, so when this function returns global indices, say {110, -112, 111}, the value of -112 is an arbitrary flag that should not be interpreted beyond its sign.
6762 
6763  Level: developer
6764 */
6765 PetscErrorCode DMPlexGetIndicesPointFields_Internal(PetscSection section, PetscBool islocal, PetscInt point, PetscInt off, PetscInt foffs[], PetscBool setBC, const PetscInt ***perms, PetscInt permsoff, const PetscInt indperm[], PetscInt indices[])
6766 {
6767   PetscInt       numFields, foff, f;
6768   PetscErrorCode ierr;
6769 
6770   PetscFunctionBegin;
6771   PetscCheckFalse(!islocal && setBC,PetscObjectComm((PetscObject)section),PETSC_ERR_ARG_INCOMP,"setBC incompatible with global indices; use a local section or disable setBC");
6772   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
6773   for (f = 0, foff = 0; f < numFields; ++f) {
6774     PetscInt        fdof, cfdof;
6775     const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
6776     PetscInt        cind = 0, b;
6777     const PetscInt  *perm = (perms && perms[f]) ? perms[f][permsoff] : NULL;
6778 
6779     ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
6780     ierr = PetscSectionGetFieldConstraintDof(section, point, f, &cfdof);CHKERRQ(ierr);
6781     if (!cfdof || setBC) {
6782       for (b = 0; b < fdof; ++b) {
6783         const PetscInt preind = perm ? foffs[f]+perm[b] : foffs[f]+b;
6784         const PetscInt ind    = indperm ? indperm[preind] : preind;
6785 
6786         indices[ind] = off+foff+b;
6787       }
6788     } else {
6789       ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
6790       for (b = 0; b < fdof; ++b) {
6791         const PetscInt preind = perm ? foffs[f]+perm[b] : foffs[f]+b;
6792         const PetscInt ind    = indperm ? indperm[preind] : preind;
6793 
6794         if ((cind < cfdof) && (b == fcdofs[cind])) {
6795           indices[ind] = -(off+foff+b+1);
6796           ++cind;
6797         } else {
6798           indices[ind] = off + foff + b - (islocal ? 0 : cind);
6799         }
6800       }
6801     }
6802     foff     += (setBC || islocal ? fdof : (fdof - cfdof));
6803     foffs[f] += fdof;
6804   }
6805   PetscFunctionReturn(0);
6806 }
6807 
6808 /*
6809   This version believes the globalSection offsets for each field, rather than just the point offset
6810 
6811  . foffs - The offset into 'indices' for each field, since it is segregated by field
6812 
6813  Notes:
6814  The semantics of this function relate to that of setBC=FALSE in DMPlexGetIndicesPointFields_Internal.
6815  Since this function uses global indices, setBC=TRUE would be invalid, so no such argument exists.
6816 */
6817 static PetscErrorCode DMPlexGetIndicesPointFieldsSplit_Internal(PetscSection section, PetscSection globalSection, PetscInt point, PetscInt foffs[], const PetscInt ***perms, PetscInt permsoff, const PetscInt indperm[], PetscInt indices[])
6818 {
6819   PetscInt       numFields, foff, f;
6820   PetscErrorCode ierr;
6821 
6822   PetscFunctionBegin;
6823   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
6824   for (f = 0; f < numFields; ++f) {
6825     PetscInt        fdof, cfdof;
6826     const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
6827     PetscInt        cind = 0, b;
6828     const PetscInt  *perm = (perms && perms[f]) ? perms[f][permsoff] : NULL;
6829 
6830     ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
6831     ierr = PetscSectionGetFieldConstraintDof(section, point, f, &cfdof);CHKERRQ(ierr);
6832     ierr = PetscSectionGetFieldOffset(globalSection, point, f, &foff);CHKERRQ(ierr);
6833     if (!cfdof) {
6834       for (b = 0; b < fdof; ++b) {
6835         const PetscInt preind = perm ? foffs[f]+perm[b] : foffs[f]+b;
6836         const PetscInt ind    = indperm ? indperm[preind] : preind;
6837 
6838         indices[ind] = foff+b;
6839       }
6840     } else {
6841       ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
6842       for (b = 0; b < fdof; ++b) {
6843         const PetscInt preind = perm ? foffs[f]+perm[b] : foffs[f]+b;
6844         const PetscInt ind    = indperm ? indperm[preind] : preind;
6845 
6846         if ((cind < cfdof) && (b == fcdofs[cind])) {
6847           indices[ind] = -(foff+b+1);
6848           ++cind;
6849         } else {
6850           indices[ind] = foff+b-cind;
6851         }
6852       }
6853     }
6854     foffs[f] += fdof;
6855   }
6856   PetscFunctionReturn(0);
6857 }
6858 
6859 PetscErrorCode DMPlexAnchorsModifyMat(DM dm, PetscSection section, PetscInt numPoints, PetscInt numIndices, const PetscInt points[], const PetscInt ***perms, const PetscScalar values[], PetscInt *outNumPoints, PetscInt *outNumIndices, PetscInt *outPoints[], PetscScalar *outValues[], PetscInt offsets[], PetscBool multiplyLeft)
6860 {
6861   Mat             cMat;
6862   PetscSection    aSec, cSec;
6863   IS              aIS;
6864   PetscInt        aStart = -1, aEnd = -1;
6865   const PetscInt  *anchors;
6866   PetscInt        numFields, f, p, q, newP = 0;
6867   PetscInt        newNumPoints = 0, newNumIndices = 0;
6868   PetscInt        *newPoints, *indices, *newIndices;
6869   PetscInt        maxAnchor, maxDof;
6870   PetscInt        newOffsets[32];
6871   PetscInt        *pointMatOffsets[32];
6872   PetscInt        *newPointOffsets[32];
6873   PetscScalar     *pointMat[32];
6874   PetscScalar     *newValues=NULL,*tmpValues;
6875   PetscBool       anyConstrained = PETSC_FALSE;
6876   PetscErrorCode  ierr;
6877 
6878   PetscFunctionBegin;
6879   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6880   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
6881   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
6882 
6883   ierr = DMPlexGetAnchors(dm,&aSec,&aIS);CHKERRQ(ierr);
6884   /* if there are point-to-point constraints */
6885   if (aSec) {
6886     ierr = PetscArrayzero(newOffsets, 32);CHKERRQ(ierr);
6887     ierr = ISGetIndices(aIS,&anchors);CHKERRQ(ierr);
6888     ierr = PetscSectionGetChart(aSec,&aStart,&aEnd);CHKERRQ(ierr);
6889     /* figure out how many points are going to be in the new element matrix
6890      * (we allow double counting, because it's all just going to be summed
6891      * into the global matrix anyway) */
6892     for (p = 0; p < 2*numPoints; p+=2) {
6893       PetscInt b    = points[p];
6894       PetscInt bDof = 0, bSecDof;
6895 
6896       ierr = PetscSectionGetDof(section,b,&bSecDof);CHKERRQ(ierr);
6897       if (!bSecDof) {
6898         continue;
6899       }
6900       if (b >= aStart && b < aEnd) {
6901         ierr = PetscSectionGetDof(aSec,b,&bDof);CHKERRQ(ierr);
6902       }
6903       if (bDof) {
6904         /* this point is constrained */
6905         /* it is going to be replaced by its anchors */
6906         PetscInt bOff, q;
6907 
6908         anyConstrained = PETSC_TRUE;
6909         newNumPoints  += bDof;
6910         ierr = PetscSectionGetOffset(aSec,b,&bOff);CHKERRQ(ierr);
6911         for (q = 0; q < bDof; q++) {
6912           PetscInt a = anchors[bOff + q];
6913           PetscInt aDof;
6914 
6915           ierr           = PetscSectionGetDof(section,a,&aDof);CHKERRQ(ierr);
6916           newNumIndices += aDof;
6917           for (f = 0; f < numFields; ++f) {
6918             PetscInt fDof;
6919 
6920             ierr             = PetscSectionGetFieldDof(section, a, f, &fDof);CHKERRQ(ierr);
6921             newOffsets[f+1] += fDof;
6922           }
6923         }
6924       }
6925       else {
6926         /* this point is not constrained */
6927         newNumPoints++;
6928         newNumIndices += bSecDof;
6929         for (f = 0; f < numFields; ++f) {
6930           PetscInt fDof;
6931 
6932           ierr = PetscSectionGetFieldDof(section, b, f, &fDof);CHKERRQ(ierr);
6933           newOffsets[f+1] += fDof;
6934         }
6935       }
6936     }
6937   }
6938   if (!anyConstrained) {
6939     if (outNumPoints)  *outNumPoints  = 0;
6940     if (outNumIndices) *outNumIndices = 0;
6941     if (outPoints)     *outPoints     = NULL;
6942     if (outValues)     *outValues     = NULL;
6943     if (aSec) {ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);}
6944     PetscFunctionReturn(0);
6945   }
6946 
6947   if (outNumPoints)  *outNumPoints  = newNumPoints;
6948   if (outNumIndices) *outNumIndices = newNumIndices;
6949 
6950   for (f = 0; f < numFields; ++f) newOffsets[f+1] += newOffsets[f];
6951 
6952   if (!outPoints && !outValues) {
6953     if (offsets) {
6954       for (f = 0; f <= numFields; f++) {
6955         offsets[f] = newOffsets[f];
6956       }
6957     }
6958     if (aSec) {ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);}
6959     PetscFunctionReturn(0);
6960   }
6961 
6962   PetscCheckFalse(numFields && newOffsets[numFields] != newNumIndices,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", newOffsets[numFields], newNumIndices);
6963 
6964   ierr = DMGetDefaultConstraints(dm, &cSec, &cMat, NULL);CHKERRQ(ierr);
6965 
6966   /* workspaces */
6967   if (numFields) {
6968     for (f = 0; f < numFields; f++) {
6969       ierr = DMGetWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[f]);CHKERRQ(ierr);
6970       ierr = DMGetWorkArray(dm,numPoints+1,MPIU_INT,&newPointOffsets[f]);CHKERRQ(ierr);
6971     }
6972   }
6973   else {
6974     ierr = DMGetWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[0]);CHKERRQ(ierr);
6975     ierr = DMGetWorkArray(dm,numPoints,MPIU_INT,&newPointOffsets[0]);CHKERRQ(ierr);
6976   }
6977 
6978   /* get workspaces for the point-to-point matrices */
6979   if (numFields) {
6980     PetscInt totalOffset, totalMatOffset;
6981 
6982     for (p = 0; p < numPoints; p++) {
6983       PetscInt b    = points[2*p];
6984       PetscInt bDof = 0, bSecDof;
6985 
6986       ierr = PetscSectionGetDof(section,b,&bSecDof);CHKERRQ(ierr);
6987       if (!bSecDof) {
6988         for (f = 0; f < numFields; f++) {
6989           newPointOffsets[f][p + 1] = 0;
6990           pointMatOffsets[f][p + 1] = 0;
6991         }
6992         continue;
6993       }
6994       if (b >= aStart && b < aEnd) {
6995         ierr = PetscSectionGetDof(aSec, b, &bDof);CHKERRQ(ierr);
6996       }
6997       if (bDof) {
6998         for (f = 0; f < numFields; f++) {
6999           PetscInt fDof, q, bOff, allFDof = 0;
7000 
7001           ierr = PetscSectionGetFieldDof(section, b, f, &fDof);CHKERRQ(ierr);
7002           ierr = PetscSectionGetOffset(aSec, b, &bOff);CHKERRQ(ierr);
7003           for (q = 0; q < bDof; q++) {
7004             PetscInt a = anchors[bOff + q];
7005             PetscInt aFDof;
7006 
7007             ierr     = PetscSectionGetFieldDof(section, a, f, &aFDof);CHKERRQ(ierr);
7008             allFDof += aFDof;
7009           }
7010           newPointOffsets[f][p+1] = allFDof;
7011           pointMatOffsets[f][p+1] = fDof * allFDof;
7012         }
7013       }
7014       else {
7015         for (f = 0; f < numFields; f++) {
7016           PetscInt fDof;
7017 
7018           ierr = PetscSectionGetFieldDof(section, b, f, &fDof);CHKERRQ(ierr);
7019           newPointOffsets[f][p+1] = fDof;
7020           pointMatOffsets[f][p+1] = 0;
7021         }
7022       }
7023     }
7024     for (f = 0, totalOffset = 0, totalMatOffset = 0; f < numFields; f++) {
7025       newPointOffsets[f][0] = totalOffset;
7026       pointMatOffsets[f][0] = totalMatOffset;
7027       for (p = 0; p < numPoints; p++) {
7028         newPointOffsets[f][p+1] += newPointOffsets[f][p];
7029         pointMatOffsets[f][p+1] += pointMatOffsets[f][p];
7030       }
7031       totalOffset    = newPointOffsets[f][numPoints];
7032       totalMatOffset = pointMatOffsets[f][numPoints];
7033       ierr = DMGetWorkArray(dm,pointMatOffsets[f][numPoints],MPIU_SCALAR,&pointMat[f]);CHKERRQ(ierr);
7034     }
7035   }
7036   else {
7037     for (p = 0; p < numPoints; p++) {
7038       PetscInt b    = points[2*p];
7039       PetscInt bDof = 0, bSecDof;
7040 
7041       ierr = PetscSectionGetDof(section,b,&bSecDof);CHKERRQ(ierr);
7042       if (!bSecDof) {
7043         newPointOffsets[0][p + 1] = 0;
7044         pointMatOffsets[0][p + 1] = 0;
7045         continue;
7046       }
7047       if (b >= aStart && b < aEnd) {
7048         ierr = PetscSectionGetDof(aSec, b, &bDof);CHKERRQ(ierr);
7049       }
7050       if (bDof) {
7051         PetscInt bOff, q, allDof = 0;
7052 
7053         ierr = PetscSectionGetOffset(aSec, b, &bOff);CHKERRQ(ierr);
7054         for (q = 0; q < bDof; q++) {
7055           PetscInt a = anchors[bOff + q], aDof;
7056 
7057           ierr    = PetscSectionGetDof(section, a, &aDof);CHKERRQ(ierr);
7058           allDof += aDof;
7059         }
7060         newPointOffsets[0][p+1] = allDof;
7061         pointMatOffsets[0][p+1] = bSecDof * allDof;
7062       }
7063       else {
7064         newPointOffsets[0][p+1] = bSecDof;
7065         pointMatOffsets[0][p+1] = 0;
7066       }
7067     }
7068     newPointOffsets[0][0] = 0;
7069     pointMatOffsets[0][0] = 0;
7070     for (p = 0; p < numPoints; p++) {
7071       newPointOffsets[0][p+1] += newPointOffsets[0][p];
7072       pointMatOffsets[0][p+1] += pointMatOffsets[0][p];
7073     }
7074     ierr = DMGetWorkArray(dm,pointMatOffsets[0][numPoints],MPIU_SCALAR,&pointMat[0]);CHKERRQ(ierr);
7075   }
7076 
7077   /* output arrays */
7078   ierr = DMGetWorkArray(dm,2*newNumPoints,MPIU_INT,&newPoints);CHKERRQ(ierr);
7079 
7080   /* get the point-to-point matrices; construct newPoints */
7081   ierr = PetscSectionGetMaxDof(aSec, &maxAnchor);CHKERRQ(ierr);
7082   ierr = PetscSectionGetMaxDof(section, &maxDof);CHKERRQ(ierr);
7083   ierr = DMGetWorkArray(dm,maxDof,MPIU_INT,&indices);CHKERRQ(ierr);
7084   ierr = DMGetWorkArray(dm,maxAnchor*maxDof,MPIU_INT,&newIndices);CHKERRQ(ierr);
7085   if (numFields) {
7086     for (p = 0, newP = 0; p < numPoints; p++) {
7087       PetscInt b    = points[2*p];
7088       PetscInt o    = points[2*p+1];
7089       PetscInt bDof = 0, bSecDof;
7090 
7091       ierr = PetscSectionGetDof(section, b, &bSecDof);CHKERRQ(ierr);
7092       if (!bSecDof) {
7093         continue;
7094       }
7095       if (b >= aStart && b < aEnd) {
7096         ierr = PetscSectionGetDof(aSec, b, &bDof);CHKERRQ(ierr);
7097       }
7098       if (bDof) {
7099         PetscInt fStart[32], fEnd[32], fAnchorStart[32], fAnchorEnd[32], bOff, q;
7100 
7101         fStart[0] = 0;
7102         fEnd[0]   = 0;
7103         for (f = 0; f < numFields; f++) {
7104           PetscInt fDof;
7105 
7106           ierr        = PetscSectionGetFieldDof(cSec, b, f, &fDof);CHKERRQ(ierr);
7107           fStart[f+1] = fStart[f] + fDof;
7108           fEnd[f+1]   = fStart[f+1];
7109         }
7110         ierr = PetscSectionGetOffset(cSec, b, &bOff);CHKERRQ(ierr);
7111         ierr = DMPlexGetIndicesPointFields_Internal(cSec, PETSC_TRUE, b, bOff, fEnd, PETSC_TRUE, perms, p, NULL, indices);CHKERRQ(ierr);
7112 
7113         fAnchorStart[0] = 0;
7114         fAnchorEnd[0]   = 0;
7115         for (f = 0; f < numFields; f++) {
7116           PetscInt fDof = newPointOffsets[f][p + 1] - newPointOffsets[f][p];
7117 
7118           fAnchorStart[f+1] = fAnchorStart[f] + fDof;
7119           fAnchorEnd[f+1]   = fAnchorStart[f + 1];
7120         }
7121         ierr = PetscSectionGetOffset(aSec, b, &bOff);CHKERRQ(ierr);
7122         for (q = 0; q < bDof; q++) {
7123           PetscInt a = anchors[bOff + q], aOff;
7124 
7125           /* we take the orientation of ap into account in the order that we constructed the indices above: the newly added points have no orientation */
7126           newPoints[2*(newP + q)]     = a;
7127           newPoints[2*(newP + q) + 1] = 0;
7128           ierr = PetscSectionGetOffset(section, a, &aOff);CHKERRQ(ierr);
7129           ierr = DMPlexGetIndicesPointFields_Internal(section, PETSC_TRUE, a, aOff, fAnchorEnd, PETSC_TRUE, NULL, -1, NULL, newIndices);CHKERRQ(ierr);
7130         }
7131         newP += bDof;
7132 
7133         if (outValues) {
7134           /* get the point-to-point submatrix */
7135           for (f = 0; f < numFields; f++) {
7136             ierr = MatGetValues(cMat,fEnd[f]-fStart[f],indices + fStart[f],fAnchorEnd[f] - fAnchorStart[f],newIndices + fAnchorStart[f],pointMat[f] + pointMatOffsets[f][p]);CHKERRQ(ierr);
7137           }
7138         }
7139       }
7140       else {
7141         newPoints[2 * newP]     = b;
7142         newPoints[2 * newP + 1] = o;
7143         newP++;
7144       }
7145     }
7146   } else {
7147     for (p = 0; p < numPoints; p++) {
7148       PetscInt b    = points[2*p];
7149       PetscInt o    = points[2*p+1];
7150       PetscInt bDof = 0, bSecDof;
7151 
7152       ierr = PetscSectionGetDof(section, b, &bSecDof);CHKERRQ(ierr);
7153       if (!bSecDof) {
7154         continue;
7155       }
7156       if (b >= aStart && b < aEnd) {
7157         ierr = PetscSectionGetDof(aSec, b, &bDof);CHKERRQ(ierr);
7158       }
7159       if (bDof) {
7160         PetscInt bEnd = 0, bAnchorEnd = 0, bOff;
7161 
7162         ierr = PetscSectionGetOffset(cSec, b, &bOff);CHKERRQ(ierr);
7163         ierr = DMPlexGetIndicesPoint_Internal(cSec, PETSC_TRUE, b, bOff, &bEnd, PETSC_TRUE, (perms && perms[0]) ? perms[0][p] : NULL, NULL, indices);CHKERRQ(ierr);
7164 
7165         ierr = PetscSectionGetOffset (aSec, b, &bOff);CHKERRQ(ierr);
7166         for (q = 0; q < bDof; q++) {
7167           PetscInt a = anchors[bOff + q], aOff;
7168 
7169           /* we take the orientation of ap into account in the order that we constructed the indices above: the newly added points have no orientation */
7170 
7171           newPoints[2*(newP + q)]     = a;
7172           newPoints[2*(newP + q) + 1] = 0;
7173           ierr = PetscSectionGetOffset(section, a, &aOff);CHKERRQ(ierr);
7174           ierr = DMPlexGetIndicesPoint_Internal(section, PETSC_TRUE, a, aOff, &bAnchorEnd, PETSC_TRUE, NULL, NULL, newIndices);CHKERRQ(ierr);
7175         }
7176         newP += bDof;
7177 
7178         /* get the point-to-point submatrix */
7179         if (outValues) {
7180           ierr = MatGetValues(cMat,bEnd,indices,bAnchorEnd,newIndices,pointMat[0] + pointMatOffsets[0][p]);CHKERRQ(ierr);
7181         }
7182       }
7183       else {
7184         newPoints[2 * newP]     = b;
7185         newPoints[2 * newP + 1] = o;
7186         newP++;
7187       }
7188     }
7189   }
7190 
7191   if (outValues) {
7192     ierr = DMGetWorkArray(dm,newNumIndices*numIndices,MPIU_SCALAR,&tmpValues);CHKERRQ(ierr);
7193     ierr = PetscArrayzero(tmpValues,newNumIndices*numIndices);CHKERRQ(ierr);
7194     /* multiply constraints on the right */
7195     if (numFields) {
7196       for (f = 0; f < numFields; f++) {
7197         PetscInt oldOff = offsets[f];
7198 
7199         for (p = 0; p < numPoints; p++) {
7200           PetscInt cStart = newPointOffsets[f][p];
7201           PetscInt b      = points[2 * p];
7202           PetscInt c, r, k;
7203           PetscInt dof;
7204 
7205           ierr = PetscSectionGetFieldDof(section,b,f,&dof);CHKERRQ(ierr);
7206           if (!dof) {
7207             continue;
7208           }
7209           if (pointMatOffsets[f][p] < pointMatOffsets[f][p + 1]) {
7210             PetscInt nCols         = newPointOffsets[f][p+1]-cStart;
7211             const PetscScalar *mat = pointMat[f] + pointMatOffsets[f][p];
7212 
7213             for (r = 0; r < numIndices; r++) {
7214               for (c = 0; c < nCols; c++) {
7215                 for (k = 0; k < dof; k++) {
7216                   tmpValues[r * newNumIndices + cStart + c] += values[r * numIndices + oldOff + k] * mat[k * nCols + c];
7217                 }
7218               }
7219             }
7220           }
7221           else {
7222             /* copy this column as is */
7223             for (r = 0; r < numIndices; r++) {
7224               for (c = 0; c < dof; c++) {
7225                 tmpValues[r * newNumIndices + cStart + c] = values[r * numIndices + oldOff + c];
7226               }
7227             }
7228           }
7229           oldOff += dof;
7230         }
7231       }
7232     }
7233     else {
7234       PetscInt oldOff = 0;
7235       for (p = 0; p < numPoints; p++) {
7236         PetscInt cStart = newPointOffsets[0][p];
7237         PetscInt b      = points[2 * p];
7238         PetscInt c, r, k;
7239         PetscInt dof;
7240 
7241         ierr = PetscSectionGetDof(section,b,&dof);CHKERRQ(ierr);
7242         if (!dof) {
7243           continue;
7244         }
7245         if (pointMatOffsets[0][p] < pointMatOffsets[0][p + 1]) {
7246           PetscInt nCols         = newPointOffsets[0][p+1]-cStart;
7247           const PetscScalar *mat = pointMat[0] + pointMatOffsets[0][p];
7248 
7249           for (r = 0; r < numIndices; r++) {
7250             for (c = 0; c < nCols; c++) {
7251               for (k = 0; k < dof; k++) {
7252                 tmpValues[r * newNumIndices + cStart + c] += mat[k * nCols + c] * values[r * numIndices + oldOff + k];
7253               }
7254             }
7255           }
7256         }
7257         else {
7258           /* copy this column as is */
7259           for (r = 0; r < numIndices; r++) {
7260             for (c = 0; c < dof; c++) {
7261               tmpValues[r * newNumIndices + cStart + c] = values[r * numIndices + oldOff + c];
7262             }
7263           }
7264         }
7265         oldOff += dof;
7266       }
7267     }
7268 
7269     if (multiplyLeft) {
7270       ierr = DMGetWorkArray(dm,newNumIndices*newNumIndices,MPIU_SCALAR,&newValues);CHKERRQ(ierr);
7271       ierr = PetscArrayzero(newValues,newNumIndices*newNumIndices);CHKERRQ(ierr);
7272       /* multiply constraints transpose on the left */
7273       if (numFields) {
7274         for (f = 0; f < numFields; f++) {
7275           PetscInt oldOff = offsets[f];
7276 
7277           for (p = 0; p < numPoints; p++) {
7278             PetscInt rStart = newPointOffsets[f][p];
7279             PetscInt b      = points[2 * p];
7280             PetscInt c, r, k;
7281             PetscInt dof;
7282 
7283             ierr = PetscSectionGetFieldDof(section,b,f,&dof);CHKERRQ(ierr);
7284             if (pointMatOffsets[f][p] < pointMatOffsets[f][p + 1]) {
7285               PetscInt nRows                        = newPointOffsets[f][p+1]-rStart;
7286               const PetscScalar *PETSC_RESTRICT mat = pointMat[f] + pointMatOffsets[f][p];
7287 
7288               for (r = 0; r < nRows; r++) {
7289                 for (c = 0; c < newNumIndices; c++) {
7290                   for (k = 0; k < dof; k++) {
7291                     newValues[(rStart + r) * newNumIndices + c] += mat[k * nRows + r] * tmpValues[(oldOff + k) * newNumIndices + c];
7292                   }
7293                 }
7294               }
7295             }
7296             else {
7297               /* copy this row as is */
7298               for (r = 0; r < dof; r++) {
7299                 for (c = 0; c < newNumIndices; c++) {
7300                   newValues[(rStart + r) * newNumIndices + c] = tmpValues[(oldOff + r) * newNumIndices + c];
7301                 }
7302               }
7303             }
7304             oldOff += dof;
7305           }
7306         }
7307       }
7308       else {
7309         PetscInt oldOff = 0;
7310 
7311         for (p = 0; p < numPoints; p++) {
7312           PetscInt rStart = newPointOffsets[0][p];
7313           PetscInt b      = points[2 * p];
7314           PetscInt c, r, k;
7315           PetscInt dof;
7316 
7317           ierr = PetscSectionGetDof(section,b,&dof);CHKERRQ(ierr);
7318           if (pointMatOffsets[0][p] < pointMatOffsets[0][p + 1]) {
7319             PetscInt nRows                        = newPointOffsets[0][p+1]-rStart;
7320             const PetscScalar *PETSC_RESTRICT mat = pointMat[0] + pointMatOffsets[0][p];
7321 
7322             for (r = 0; r < nRows; r++) {
7323               for (c = 0; c < newNumIndices; c++) {
7324                 for (k = 0; k < dof; k++) {
7325                   newValues[(rStart + r) * newNumIndices + c] += mat[k * nRows + r] * tmpValues[(oldOff + k) * newNumIndices + c];
7326                 }
7327               }
7328             }
7329           }
7330           else {
7331             /* copy this row as is */
7332             for (r = 0; r < dof; r++) {
7333               for (c = 0; c < newNumIndices; c++) {
7334                 newValues[(rStart + r) * newNumIndices + c] = tmpValues[(oldOff + r) * newNumIndices + c];
7335               }
7336             }
7337           }
7338           oldOff += dof;
7339         }
7340       }
7341 
7342       ierr = DMRestoreWorkArray(dm,newNumIndices*numIndices,MPIU_SCALAR,&tmpValues);CHKERRQ(ierr);
7343     }
7344     else {
7345       newValues = tmpValues;
7346     }
7347   }
7348 
7349   /* clean up */
7350   ierr = DMRestoreWorkArray(dm,maxDof,MPIU_INT,&indices);CHKERRQ(ierr);
7351   ierr = DMRestoreWorkArray(dm,maxAnchor*maxDof,MPIU_INT,&newIndices);CHKERRQ(ierr);
7352 
7353   if (numFields) {
7354     for (f = 0; f < numFields; f++) {
7355       ierr = DMRestoreWorkArray(dm,pointMatOffsets[f][numPoints],MPIU_SCALAR,&pointMat[f]);CHKERRQ(ierr);
7356       ierr = DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[f]);CHKERRQ(ierr);
7357       ierr = DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&newPointOffsets[f]);CHKERRQ(ierr);
7358     }
7359   }
7360   else {
7361     ierr = DMRestoreWorkArray(dm,pointMatOffsets[0][numPoints],MPIU_SCALAR,&pointMat[0]);CHKERRQ(ierr);
7362     ierr = DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[0]);CHKERRQ(ierr);
7363     ierr = DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&newPointOffsets[0]);CHKERRQ(ierr);
7364   }
7365   ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);
7366 
7367   /* output */
7368   if (outPoints) {
7369     *outPoints = newPoints;
7370   }
7371   else {
7372     ierr = DMRestoreWorkArray(dm,2*newNumPoints,MPIU_INT,&newPoints);CHKERRQ(ierr);
7373   }
7374   if (outValues) {
7375     *outValues = newValues;
7376   }
7377   for (f = 0; f <= numFields; f++) {
7378     offsets[f] = newOffsets[f];
7379   }
7380   PetscFunctionReturn(0);
7381 }
7382 
7383 /*@C
7384   DMPlexGetClosureIndices - Gets the global dof indices associated with the closure of the given point within the provided sections.
7385 
7386   Not collective
7387 
7388   Input Parameters:
7389 + dm         - The DM
7390 . section    - The PetscSection describing the points (a local section)
7391 . idxSection - The PetscSection from which to obtain indices (may be local or global)
7392 . point      - The point defining the closure
7393 - useClPerm  - Use the closure point permutation if available
7394 
7395   Output Parameters:
7396 + numIndices - The number of dof indices in the closure of point with the input sections
7397 . indices    - The dof indices
7398 . outOffsets - Array to write the field offsets into, or NULL
7399 - values     - The input values, which may be modified if sign flips are induced by the point symmetries, or NULL
7400 
7401   Notes:
7402   Must call DMPlexRestoreClosureIndices() to free allocated memory
7403 
7404   If idxSection is global, any constrained dofs (see DMAddBoundary(), for example) will get negative indices.  The value
7405   of those indices is not significant.  If idxSection is local, the constrained dofs will yield the involution -(idx+1)
7406   of their index in a local vector.  A caller who does not wish to distinguish those points may recover the nonnegative
7407   indices via involution, -(-(idx+1)+1)==idx.  Local indices are provided when idxSection == section, otherwise global
7408   indices (with the above semantics) are implied.
7409 
7410   Level: advanced
7411 
7412 .seealso DMPlexRestoreClosureIndices(), DMPlexVecGetClosure(), DMPlexMatSetClosure(), DMGetLocalSection(), DMGetGlobalSection()
7413 @*/
7414 PetscErrorCode DMPlexGetClosureIndices(DM dm, PetscSection section, PetscSection idxSection, PetscInt point, PetscBool useClPerm,
7415                                        PetscInt *numIndices, PetscInt *indices[], PetscInt outOffsets[], PetscScalar *values[])
7416 {
7417   /* Closure ordering */
7418   PetscSection        clSection;
7419   IS                  clPoints;
7420   const PetscInt     *clp;
7421   PetscInt           *points;
7422   const PetscInt     *clperm = NULL;
7423   /* Dof permutation and sign flips */
7424   const PetscInt    **perms[32] = {NULL};
7425   const PetscScalar **flips[32] = {NULL};
7426   PetscScalar        *valCopy   = NULL;
7427   /* Hanging node constraints */
7428   PetscInt           *pointsC = NULL;
7429   PetscScalar        *valuesC = NULL;
7430   PetscInt            NclC, NiC;
7431 
7432   PetscInt           *idx;
7433   PetscInt            Nf, Ncl, Ni = 0, offsets[32], p, f;
7434   PetscBool           isLocal = (section == idxSection) ? PETSC_TRUE : PETSC_FALSE;
7435   PetscErrorCode      ierr;
7436 
7437   PetscFunctionBeginHot;
7438   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7439   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
7440   PetscValidHeaderSpecific(idxSection, PETSC_SECTION_CLASSID, 3);
7441   if (numIndices) PetscValidPointer(numIndices, 6);
7442   if (indices)    PetscValidPointer(indices, 7);
7443   if (outOffsets) PetscValidPointer(outOffsets, 8);
7444   if (values)     PetscValidPointer(values, 9);
7445   ierr = PetscSectionGetNumFields(section, &Nf);CHKERRQ(ierr);
7446   PetscCheckFalse(Nf > 31,PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", Nf);
7447   ierr = PetscArrayzero(offsets, 32);CHKERRQ(ierr);
7448   /* 1) Get points in closure */
7449   ierr = DMPlexGetCompressedClosure(dm, section, point, &Ncl, &points, &clSection, &clPoints, &clp);CHKERRQ(ierr);
7450   if (useClPerm) {
7451     PetscInt depth, clsize;
7452     ierr = DMPlexGetPointDepth(dm, point, &depth);CHKERRQ(ierr);
7453     for (clsize=0,p=0; p<Ncl; p++) {
7454       PetscInt dof;
7455       ierr = PetscSectionGetDof(section, points[2*p], &dof);CHKERRQ(ierr);
7456       clsize += dof;
7457     }
7458     ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, depth, clsize, &clperm);CHKERRQ(ierr);
7459   }
7460   /* 2) Get number of indices on these points and field offsets from section */
7461   for (p = 0; p < Ncl*2; p += 2) {
7462     PetscInt dof, fdof;
7463 
7464     ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
7465     for (f = 0; f < Nf; ++f) {
7466       ierr = PetscSectionGetFieldDof(section, points[p], f, &fdof);CHKERRQ(ierr);
7467       offsets[f+1] += fdof;
7468     }
7469     Ni += dof;
7470   }
7471   for (f = 1; f < Nf; ++f) offsets[f+1] += offsets[f];
7472   PetscCheckFalse(Nf && offsets[Nf] != Ni,PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", offsets[Nf], Ni);
7473   /* 3) Get symmetries and sign flips. Apply sign flips to values if passed in (only works for square values matrix) */
7474   for (f = 0; f < PetscMax(1, Nf); ++f) {
7475     if (Nf) {ierr = PetscSectionGetFieldPointSyms(section, f, Ncl, points, &perms[f], &flips[f]);CHKERRQ(ierr);}
7476     else    {ierr = PetscSectionGetPointSyms(section, Ncl, points, &perms[f], &flips[f]);CHKERRQ(ierr);}
7477     /* may need to apply sign changes to the element matrix */
7478     if (values && flips[f]) {
7479       PetscInt foffset = offsets[f];
7480 
7481       for (p = 0; p < Ncl; ++p) {
7482         PetscInt           pnt  = points[2*p], fdof;
7483         const PetscScalar *flip = flips[f] ? flips[f][p] : NULL;
7484 
7485         if (!Nf) {ierr = PetscSectionGetDof(section, pnt, &fdof);CHKERRQ(ierr);}
7486         else     {ierr = PetscSectionGetFieldDof(section, pnt, f, &fdof);CHKERRQ(ierr);}
7487         if (flip) {
7488           PetscInt i, j, k;
7489 
7490           if (!valCopy) {
7491             ierr = DMGetWorkArray(dm, Ni*Ni, MPIU_SCALAR, &valCopy);CHKERRQ(ierr);
7492             for (j = 0; j < Ni * Ni; ++j) valCopy[j] = (*values)[j];
7493             *values = valCopy;
7494           }
7495           for (i = 0; i < fdof; ++i) {
7496             PetscScalar fval = flip[i];
7497 
7498             for (k = 0; k < Ni; ++k) {
7499               valCopy[Ni * (foffset + i) + k] *= fval;
7500               valCopy[Ni * k + (foffset + i)] *= fval;
7501             }
7502           }
7503         }
7504         foffset += fdof;
7505       }
7506     }
7507   }
7508   /* 4) Apply hanging node constraints. Get new symmetries and replace all storage with constrained storage */
7509   ierr = DMPlexAnchorsModifyMat(dm, section, Ncl, Ni, points, perms, values ? *values : NULL, &NclC, &NiC, &pointsC, values ? &valuesC : NULL, offsets, PETSC_TRUE);CHKERRQ(ierr);
7510   if (NclC) {
7511     if (valCopy) {ierr = DMRestoreWorkArray(dm, Ni*Ni, MPIU_SCALAR, &valCopy);CHKERRQ(ierr);}
7512     for (f = 0; f < PetscMax(1, Nf); ++f) {
7513       if (Nf) {ierr = PetscSectionRestoreFieldPointSyms(section, f, Ncl, points, &perms[f], &flips[f]);CHKERRQ(ierr);}
7514       else    {ierr = PetscSectionRestorePointSyms(section, Ncl, points, &perms[f], &flips[f]);CHKERRQ(ierr);}
7515     }
7516     for (f = 0; f < PetscMax(1, Nf); ++f) {
7517       if (Nf) {ierr = PetscSectionGetFieldPointSyms(section, f, NclC, pointsC, &perms[f], &flips[f]);CHKERRQ(ierr);}
7518       else    {ierr = PetscSectionGetPointSyms(section, NclC, pointsC, &perms[f], &flips[f]);CHKERRQ(ierr);}
7519     }
7520     ierr = DMPlexRestoreCompressedClosure(dm, section, point, &Ncl, &points, &clSection, &clPoints, &clp);CHKERRQ(ierr);
7521     Ncl     = NclC;
7522     Ni      = NiC;
7523     points  = pointsC;
7524     if (values) *values = valuesC;
7525   }
7526   /* 5) Calculate indices */
7527   ierr = DMGetWorkArray(dm, Ni, MPIU_INT, &idx);CHKERRQ(ierr);
7528   if (Nf) {
7529     PetscInt  idxOff;
7530     PetscBool useFieldOffsets;
7531 
7532     if (outOffsets) {for (f = 0; f <= Nf; f++) outOffsets[f] = offsets[f];}
7533     ierr = PetscSectionGetUseFieldOffsets(idxSection, &useFieldOffsets);CHKERRQ(ierr);
7534     if (useFieldOffsets) {
7535       for (p = 0; p < Ncl; ++p) {
7536         const PetscInt pnt = points[p*2];
7537 
7538         ierr = DMPlexGetIndicesPointFieldsSplit_Internal(section, idxSection, pnt, offsets, perms, p, clperm, idx);CHKERRQ(ierr);
7539       }
7540     } else {
7541       for (p = 0; p < Ncl; ++p) {
7542         const PetscInt pnt = points[p*2];
7543 
7544         ierr = PetscSectionGetOffset(idxSection, pnt, &idxOff);CHKERRQ(ierr);
7545         /* Note that we pass a local section even though we're using global offsets.  This is because global sections do
7546          * not (at the time of this writing) have fields set. They probably should, in which case we would pass the
7547          * global section. */
7548         ierr = DMPlexGetIndicesPointFields_Internal(section, isLocal, pnt, idxOff < 0 ? -(idxOff+1) : idxOff, offsets, PETSC_FALSE, perms, p, clperm, idx);CHKERRQ(ierr);
7549       }
7550     }
7551   } else {
7552     PetscInt off = 0, idxOff;
7553 
7554     for (p = 0; p < Ncl; ++p) {
7555       const PetscInt  pnt  = points[p*2];
7556       const PetscInt *perm = perms[0] ? perms[0][p] : NULL;
7557 
7558       ierr = PetscSectionGetOffset(idxSection, pnt, &idxOff);CHKERRQ(ierr);
7559       /* Note that we pass a local section even though we're using global offsets.  This is because global sections do
7560        * not (at the time of this writing) have fields set. They probably should, in which case we would pass the global section. */
7561       ierr = DMPlexGetIndicesPoint_Internal(section, isLocal, pnt, idxOff < 0 ? -(idxOff+1) : idxOff, &off, PETSC_FALSE, perm, clperm, idx);CHKERRQ(ierr);
7562     }
7563   }
7564   /* 6) Cleanup */
7565   for (f = 0; f < PetscMax(1, Nf); ++f) {
7566     if (Nf) {ierr = PetscSectionRestoreFieldPointSyms(section, f, Ncl, points, &perms[f], &flips[f]);CHKERRQ(ierr);}
7567     else    {ierr = PetscSectionRestorePointSyms(section, Ncl, points, &perms[f], &flips[f]);CHKERRQ(ierr);}
7568   }
7569   if (NclC) {
7570     ierr = DMRestoreWorkArray(dm, NclC*2, MPIU_INT, &pointsC);CHKERRQ(ierr);
7571   } else {
7572     ierr = DMPlexRestoreCompressedClosure(dm, section, point, &Ncl, &points, &clSection, &clPoints, &clp);CHKERRQ(ierr);
7573   }
7574 
7575   if (numIndices) *numIndices = Ni;
7576   if (indices)    *indices    = idx;
7577   PetscFunctionReturn(0);
7578 }
7579 
7580 /*@C
7581   DMPlexRestoreClosureIndices - Restores the global dof indices associated with the closure of the given point within the provided sections.
7582 
7583   Not collective
7584 
7585   Input Parameters:
7586 + dm         - The DM
7587 . section    - The PetscSection describing the points (a local section)
7588 . idxSection - The PetscSection from which to obtain indices (may be local or global)
7589 . point      - The point defining the closure
7590 - useClPerm  - Use the closure point permutation if available
7591 
7592   Output Parameters:
7593 + numIndices - The number of dof indices in the closure of point with the input sections
7594 . indices    - The dof indices
7595 . outOffsets - Array to write the field offsets into, or NULL
7596 - values     - The input values, which may be modified if sign flips are induced by the point symmetries, or NULL
7597 
7598   Notes:
7599   If values were modified, the user is responsible for calling DMRestoreWorkArray(dm, 0, MPIU_SCALAR, &values).
7600 
7601   If idxSection is global, any constrained dofs (see DMAddBoundary(), for example) will get negative indices.  The value
7602   of those indices is not significant.  If idxSection is local, the constrained dofs will yield the involution -(idx+1)
7603   of their index in a local vector.  A caller who does not wish to distinguish those points may recover the nonnegative
7604   indices via involution, -(-(idx+1)+1)==idx.  Local indices are provided when idxSection == section, otherwise global
7605   indices (with the above semantics) are implied.
7606 
7607   Level: advanced
7608 
7609 .seealso DMPlexGetClosureIndices(), DMPlexVecGetClosure(), DMPlexMatSetClosure(), DMGetLocalSection(), DMGetGlobalSection()
7610 @*/
7611 PetscErrorCode DMPlexRestoreClosureIndices(DM dm, PetscSection section, PetscSection idxSection, PetscInt point, PetscBool useClPerm,
7612                                            PetscInt *numIndices, PetscInt *indices[], PetscInt outOffsets[], PetscScalar *values[])
7613 {
7614   PetscErrorCode ierr;
7615 
7616   PetscFunctionBegin;
7617   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7618   PetscValidPointer(indices, 7);
7619   ierr = DMRestoreWorkArray(dm, 0, MPIU_INT, indices);CHKERRQ(ierr);
7620   PetscFunctionReturn(0);
7621 }
7622 
7623 /*@C
7624   DMPlexMatSetClosure - Set an array of the values on the closure of 'point'
7625 
7626   Not collective
7627 
7628   Input Parameters:
7629 + dm - The DM
7630 . section - The section describing the layout in v, or NULL to use the default section
7631 . globalSection - The section describing the layout in v, or NULL to use the default global section
7632 . A - The matrix
7633 . point - The point in the DM
7634 . values - The array of values
7635 - mode - The insert mode, where INSERT_ALL_VALUES and ADD_ALL_VALUES also overwrite boundary conditions
7636 
7637   Fortran Notes:
7638   This routine is only available in Fortran 90, and you must include petsc.h90 in your code.
7639 
7640   Level: intermediate
7641 
7642 .seealso DMPlexMatSetClosureGeneral(), DMPlexVecGetClosure(), DMPlexVecSetClosure()
7643 @*/
7644 PetscErrorCode DMPlexMatSetClosure(DM dm, PetscSection section, PetscSection globalSection, Mat A, PetscInt point, const PetscScalar values[], InsertMode mode)
7645 {
7646   DM_Plex           *mesh = (DM_Plex*) dm->data;
7647   PetscInt          *indices;
7648   PetscInt           numIndices;
7649   const PetscScalar *valuesOrig = values;
7650   PetscErrorCode     ierr;
7651 
7652   PetscFunctionBegin;
7653   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7654   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
7655   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
7656   if (!globalSection) {ierr = DMGetGlobalSection(dm, &globalSection);CHKERRQ(ierr);}
7657   PetscValidHeaderSpecific(globalSection, PETSC_SECTION_CLASSID, 3);
7658   PetscValidHeaderSpecific(A, MAT_CLASSID, 4);
7659 
7660   ierr = DMPlexGetClosureIndices(dm, section, globalSection, point, PETSC_TRUE, &numIndices, &indices, NULL, (PetscScalar **) &values);CHKERRQ(ierr);
7661 
7662   if (mesh->printSetValues) {ierr = DMPlexPrintMatSetValues(PETSC_VIEWER_STDOUT_SELF, A, point, numIndices, indices, 0, NULL, values);CHKERRQ(ierr);}
7663   ierr = MatSetValues(A, numIndices, indices, numIndices, indices, values, mode);
7664   if (ierr) {
7665     PetscMPIInt    rank;
7666     PetscErrorCode ierr2;
7667 
7668     ierr2 = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRMPI(ierr2);
7669     ierr2 = (*PetscErrorPrintf)("[%d]ERROR in DMPlexMatSetClosure\n", rank);CHKERRQ(ierr2);
7670     ierr2 = DMPlexPrintMatSetValues(PETSC_VIEWER_STDERR_SELF, A, point, numIndices, indices, 0, NULL, values);CHKERRQ(ierr2);
7671     ierr2 = DMPlexRestoreClosureIndices(dm, section, globalSection, point, PETSC_TRUE, &numIndices, &indices, NULL, (PetscScalar **) &values);CHKERRQ(ierr2);
7672     if (values != valuesOrig) {ierr2 = DMRestoreWorkArray(dm, 0, MPIU_SCALAR, &values);CHKERRQ(ierr2);}
7673     SETERRQ(PetscObjectComm((PetscObject)dm),ierr,"Not possible to set matrix values");
7674   }
7675   if (mesh->printFEM > 1) {
7676     PetscInt i;
7677     ierr = PetscPrintf(PETSC_COMM_SELF, "  Indices:");CHKERRQ(ierr);
7678     for (i = 0; i < numIndices; ++i) {ierr = PetscPrintf(PETSC_COMM_SELF, " %D", indices[i]);CHKERRQ(ierr);}
7679     ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
7680   }
7681 
7682   ierr = DMPlexRestoreClosureIndices(dm, section, globalSection, point, PETSC_TRUE, &numIndices, &indices, NULL, (PetscScalar **) &values);CHKERRQ(ierr);
7683   if (values != valuesOrig) {ierr = DMRestoreWorkArray(dm, 0, MPIU_SCALAR, &values);CHKERRQ(ierr);}
7684   PetscFunctionReturn(0);
7685 }
7686 
7687 /*@C
7688   DMPlexMatSetClosure - Set an array of the values on the closure of 'point' using a different row and column section
7689 
7690   Not collective
7691 
7692   Input Parameters:
7693 + dmRow - The DM for the row fields
7694 . sectionRow - The section describing the layout, or NULL to use the default section in dmRow
7695 . globalSectionRow - The section describing the layout, or NULL to use the default global section in dmRow
7696 . dmCol - The DM for the column fields
7697 . sectionCol - The section describing the layout, or NULL to use the default section in dmCol
7698 . globalSectionCol - The section describing the layout, or NULL to use the default global section in dmCol
7699 . A - The matrix
7700 . point - The point in the DMs
7701 . values - The array of values
7702 - mode - The insert mode, where INSERT_ALL_VALUES and ADD_ALL_VALUES also overwrite boundary conditions
7703 
7704   Level: intermediate
7705 
7706 .seealso DMPlexMatSetClosure(), DMPlexVecGetClosure(), DMPlexVecSetClosure()
7707 @*/
7708 PetscErrorCode DMPlexMatSetClosureGeneral(DM dmRow, PetscSection sectionRow, PetscSection globalSectionRow, DM dmCol, PetscSection sectionCol, PetscSection globalSectionCol, Mat A, PetscInt point, const PetscScalar values[], InsertMode mode)
7709 {
7710   DM_Plex           *mesh = (DM_Plex*) dmRow->data;
7711   PetscInt          *indicesRow, *indicesCol;
7712   PetscInt           numIndicesRow, numIndicesCol;
7713   const PetscScalar *valuesOrig = values;
7714   PetscErrorCode     ierr;
7715 
7716   PetscFunctionBegin;
7717   PetscValidHeaderSpecific(dmRow, DM_CLASSID, 1);
7718   if (!sectionRow) {ierr = DMGetLocalSection(dmRow, &sectionRow);CHKERRQ(ierr);}
7719   PetscValidHeaderSpecific(sectionRow, PETSC_SECTION_CLASSID, 2);
7720   if (!globalSectionRow) {ierr = DMGetGlobalSection(dmRow, &globalSectionRow);CHKERRQ(ierr);}
7721   PetscValidHeaderSpecific(globalSectionRow, PETSC_SECTION_CLASSID, 3);
7722   PetscValidHeaderSpecific(dmCol, DM_CLASSID, 4);
7723   if (!sectionCol) {ierr = DMGetLocalSection(dmCol, &sectionCol);CHKERRQ(ierr);}
7724   PetscValidHeaderSpecific(sectionCol, PETSC_SECTION_CLASSID, 5);
7725   if (!globalSectionCol) {ierr = DMGetGlobalSection(dmCol, &globalSectionCol);CHKERRQ(ierr);}
7726   PetscValidHeaderSpecific(globalSectionCol, PETSC_SECTION_CLASSID, 6);
7727   PetscValidHeaderSpecific(A, MAT_CLASSID, 7);
7728 
7729   ierr = DMPlexGetClosureIndices(dmRow, sectionRow, globalSectionRow, point, PETSC_TRUE, &numIndicesRow, &indicesRow, NULL, (PetscScalar **) &values);CHKERRQ(ierr);
7730   ierr = DMPlexGetClosureIndices(dmCol, sectionCol, globalSectionCol, point, PETSC_TRUE, &numIndicesCol, &indicesCol, NULL, (PetscScalar **) &values);CHKERRQ(ierr);
7731 
7732   if (mesh->printSetValues) {ierr = DMPlexPrintMatSetValues(PETSC_VIEWER_STDOUT_SELF, A, point, numIndicesRow, indicesRow, numIndicesCol, indicesCol, values);CHKERRQ(ierr);}
7733   ierr = MatSetValues(A, numIndicesRow, indicesRow, numIndicesCol, indicesCol, values, mode);
7734   if (ierr) {
7735     PetscMPIInt    rank;
7736     PetscErrorCode ierr2;
7737 
7738     ierr2 = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRMPI(ierr2);
7739     ierr2 = (*PetscErrorPrintf)("[%d]ERROR in DMPlexMatSetClosure\n", rank);CHKERRQ(ierr2);
7740     ierr2 = DMPlexPrintMatSetValues(PETSC_VIEWER_STDERR_SELF, A, point, numIndicesRow, indicesRow, numIndicesCol, indicesCol, values);CHKERRQ(ierr2);
7741     ierr2 = DMPlexRestoreClosureIndices(dmRow, sectionRow, globalSectionRow, point, PETSC_TRUE, &numIndicesRow, &indicesRow, NULL, (PetscScalar **) &values);CHKERRQ(ierr2);
7742     ierr2 = DMPlexRestoreClosureIndices(dmCol, sectionCol, globalSectionCol, point, PETSC_TRUE, &numIndicesCol, &indicesRow, NULL, (PetscScalar **) &values);CHKERRQ(ierr2);
7743     if (values != valuesOrig) {ierr2 = DMRestoreWorkArray(dmRow, 0, MPIU_SCALAR, &values);CHKERRQ(ierr2);}
7744     CHKERRQ(ierr);
7745   }
7746 
7747   ierr = DMPlexRestoreClosureIndices(dmRow, sectionRow, globalSectionRow, point, PETSC_TRUE, &numIndicesRow, &indicesRow, NULL, (PetscScalar **) &values);CHKERRQ(ierr);
7748   ierr = DMPlexRestoreClosureIndices(dmCol, sectionCol, globalSectionCol, point, PETSC_TRUE, &numIndicesCol, &indicesCol, NULL, (PetscScalar **) &values);CHKERRQ(ierr);
7749   if (values != valuesOrig) {ierr = DMRestoreWorkArray(dmRow, 0, MPIU_SCALAR, &values);CHKERRQ(ierr);}
7750   PetscFunctionReturn(0);
7751 }
7752 
7753 PetscErrorCode DMPlexMatSetClosureRefined(DM dmf, PetscSection fsection, PetscSection globalFSection, DM dmc, PetscSection csection, PetscSection globalCSection, Mat A, PetscInt point, const PetscScalar values[], InsertMode mode)
7754 {
7755   DM_Plex        *mesh   = (DM_Plex*) dmf->data;
7756   PetscInt       *fpoints = NULL, *ftotpoints = NULL;
7757   PetscInt       *cpoints = NULL;
7758   PetscInt       *findices, *cindices;
7759   const PetscInt *fclperm = NULL, *cclperm = NULL; /* Closure permutations cannot work here */
7760   PetscInt        foffsets[32], coffsets[32];
7761   DMPolytopeType  ct;
7762   PetscInt        numFields, numSubcells, maxFPoints, numFPoints, numCPoints, numFIndices, numCIndices, dof, off, globalOff, pStart, pEnd, p, q, r, s, f;
7763   PetscErrorCode  ierr;
7764 
7765   PetscFunctionBegin;
7766   PetscValidHeaderSpecific(dmf, DM_CLASSID, 1);
7767   PetscValidHeaderSpecific(dmc, DM_CLASSID, 4);
7768   if (!fsection) {ierr = DMGetLocalSection(dmf, &fsection);CHKERRQ(ierr);}
7769   PetscValidHeaderSpecific(fsection, PETSC_SECTION_CLASSID, 2);
7770   if (!csection) {ierr = DMGetLocalSection(dmc, &csection);CHKERRQ(ierr);}
7771   PetscValidHeaderSpecific(csection, PETSC_SECTION_CLASSID, 5);
7772   if (!globalFSection) {ierr = DMGetGlobalSection(dmf, &globalFSection);CHKERRQ(ierr);}
7773   PetscValidHeaderSpecific(globalFSection, PETSC_SECTION_CLASSID, 3);
7774   if (!globalCSection) {ierr = DMGetGlobalSection(dmc, &globalCSection);CHKERRQ(ierr);}
7775   PetscValidHeaderSpecific(globalCSection, PETSC_SECTION_CLASSID, 6);
7776   PetscValidHeaderSpecific(A, MAT_CLASSID, 7);
7777   ierr = PetscSectionGetNumFields(fsection, &numFields);CHKERRQ(ierr);
7778   PetscCheckFalse(numFields > 31,PetscObjectComm((PetscObject)dmf), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", numFields);
7779   ierr = PetscArrayzero(foffsets, 32);CHKERRQ(ierr);
7780   ierr = PetscArrayzero(coffsets, 32);CHKERRQ(ierr);
7781   /* Column indices */
7782   ierr = DMPlexGetTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
7783   maxFPoints = numCPoints;
7784   /* Compress out points not in the section */
7785   /*   TODO: Squeeze out points with 0 dof as well */
7786   ierr = PetscSectionGetChart(csection, &pStart, &pEnd);CHKERRQ(ierr);
7787   for (p = 0, q = 0; p < numCPoints*2; p += 2) {
7788     if ((cpoints[p] >= pStart) && (cpoints[p] < pEnd)) {
7789       cpoints[q*2]   = cpoints[p];
7790       cpoints[q*2+1] = cpoints[p+1];
7791       ++q;
7792     }
7793   }
7794   numCPoints = q;
7795   for (p = 0, numCIndices = 0; p < numCPoints*2; p += 2) {
7796     PetscInt fdof;
7797 
7798     ierr = PetscSectionGetDof(csection, cpoints[p], &dof);CHKERRQ(ierr);
7799     if (!dof) continue;
7800     for (f = 0; f < numFields; ++f) {
7801       ierr           = PetscSectionGetFieldDof(csection, cpoints[p], f, &fdof);CHKERRQ(ierr);
7802       coffsets[f+1] += fdof;
7803     }
7804     numCIndices += dof;
7805   }
7806   for (f = 1; f < numFields; ++f) coffsets[f+1] += coffsets[f];
7807   /* Row indices */
7808   ierr = DMPlexGetCellType(dmc, point, &ct);CHKERRQ(ierr);
7809   {
7810     DMPlexTransform tr;
7811     DMPolytopeType *rct;
7812     PetscInt       *rsize, *rcone, *rornt, Nt;
7813 
7814     ierr = DMPlexTransformCreate(PETSC_COMM_SELF, &tr);CHKERRQ(ierr);
7815     ierr = DMPlexTransformSetType(tr, DMPLEXREFINEREGULAR);CHKERRQ(ierr);
7816     ierr = DMPlexTransformCellTransform(tr, ct, point, NULL, &Nt, &rct, &rsize, &rcone, &rornt);CHKERRQ(ierr);
7817     numSubcells = rsize[Nt-1];
7818     ierr = DMPlexTransformDestroy(&tr);CHKERRQ(ierr);
7819   }
7820   ierr = DMGetWorkArray(dmf, maxFPoints*2*numSubcells, MPIU_INT, &ftotpoints);CHKERRQ(ierr);
7821   for (r = 0, q = 0; r < numSubcells; ++r) {
7822     /* TODO Map from coarse to fine cells */
7823     ierr = DMPlexGetTransitiveClosure(dmf, point*numSubcells + r, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
7824     /* Compress out points not in the section */
7825     ierr = PetscSectionGetChart(fsection, &pStart, &pEnd);CHKERRQ(ierr);
7826     for (p = 0; p < numFPoints*2; p += 2) {
7827       if ((fpoints[p] >= pStart) && (fpoints[p] < pEnd)) {
7828         ierr = PetscSectionGetDof(fsection, fpoints[p], &dof);CHKERRQ(ierr);
7829         if (!dof) continue;
7830         for (s = 0; s < q; ++s) if (fpoints[p] == ftotpoints[s*2]) break;
7831         if (s < q) continue;
7832         ftotpoints[q*2]   = fpoints[p];
7833         ftotpoints[q*2+1] = fpoints[p+1];
7834         ++q;
7835       }
7836     }
7837     ierr = DMPlexRestoreTransitiveClosure(dmf, point, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
7838   }
7839   numFPoints = q;
7840   for (p = 0, numFIndices = 0; p < numFPoints*2; p += 2) {
7841     PetscInt fdof;
7842 
7843     ierr = PetscSectionGetDof(fsection, ftotpoints[p], &dof);CHKERRQ(ierr);
7844     if (!dof) continue;
7845     for (f = 0; f < numFields; ++f) {
7846       ierr           = PetscSectionGetFieldDof(fsection, ftotpoints[p], f, &fdof);CHKERRQ(ierr);
7847       foffsets[f+1] += fdof;
7848     }
7849     numFIndices += dof;
7850   }
7851   for (f = 1; f < numFields; ++f) foffsets[f+1] += foffsets[f];
7852 
7853   PetscCheckFalse(numFields && foffsets[numFields] != numFIndices,PetscObjectComm((PetscObject)dmf), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", foffsets[numFields], numFIndices);
7854   PetscCheckFalse(numFields && coffsets[numFields] != numCIndices,PetscObjectComm((PetscObject)dmc), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", coffsets[numFields], numCIndices);
7855   ierr = DMGetWorkArray(dmf, numFIndices, MPIU_INT, &findices);CHKERRQ(ierr);
7856   ierr = DMGetWorkArray(dmc, numCIndices, MPIU_INT, &cindices);CHKERRQ(ierr);
7857   if (numFields) {
7858     const PetscInt **permsF[32] = {NULL};
7859     const PetscInt **permsC[32] = {NULL};
7860 
7861     for (f = 0; f < numFields; f++) {
7862       ierr = PetscSectionGetFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL);CHKERRQ(ierr);
7863       ierr = PetscSectionGetFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL);CHKERRQ(ierr);
7864     }
7865     for (p = 0; p < numFPoints; p++) {
7866       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff);CHKERRQ(ierr);
7867       ierr = DMPlexGetIndicesPointFields_Internal(fsection, PETSC_FALSE, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, foffsets, PETSC_FALSE, permsF, p, fclperm, findices);CHKERRQ(ierr);
7868     }
7869     for (p = 0; p < numCPoints; p++) {
7870       ierr = PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff);CHKERRQ(ierr);
7871       ierr = DMPlexGetIndicesPointFields_Internal(csection, PETSC_FALSE, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, coffsets, PETSC_FALSE, permsC, p, cclperm, cindices);CHKERRQ(ierr);
7872     }
7873     for (f = 0; f < numFields; f++) {
7874       ierr = PetscSectionRestoreFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL);CHKERRQ(ierr);
7875       ierr = PetscSectionRestoreFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL);CHKERRQ(ierr);
7876     }
7877   } else {
7878     const PetscInt **permsF = NULL;
7879     const PetscInt **permsC = NULL;
7880 
7881     ierr = PetscSectionGetPointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL);CHKERRQ(ierr);
7882     ierr = PetscSectionGetPointSyms(csection,numCPoints,cpoints,&permsC,NULL);CHKERRQ(ierr);
7883     for (p = 0, off = 0; p < numFPoints; p++) {
7884       const PetscInt *perm = permsF ? permsF[p] : NULL;
7885 
7886       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff);CHKERRQ(ierr);
7887       ierr = DMPlexGetIndicesPoint_Internal(fsection, PETSC_FALSE, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, fclperm, findices);CHKERRQ(ierr);
7888     }
7889     for (p = 0, off = 0; p < numCPoints; p++) {
7890       const PetscInt *perm = permsC ? permsC[p] : NULL;
7891 
7892       ierr = PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff);CHKERRQ(ierr);
7893       ierr = DMPlexGetIndicesPoint_Internal(csection, PETSC_FALSE, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, cclperm, cindices);CHKERRQ(ierr);
7894     }
7895     ierr = PetscSectionRestorePointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL);CHKERRQ(ierr);
7896     ierr = PetscSectionRestorePointSyms(csection,numCPoints,cpoints,&permsC,NULL);CHKERRQ(ierr);
7897   }
7898   if (mesh->printSetValues) {ierr = DMPlexPrintMatSetValues(PETSC_VIEWER_STDOUT_SELF, A, point, numFIndices, findices, numCIndices, cindices, values);CHKERRQ(ierr);}
7899   /* TODO: flips */
7900   ierr = MatSetValues(A, numFIndices, findices, numCIndices, cindices, values, mode);
7901   if (ierr) {
7902     PetscMPIInt    rank;
7903     PetscErrorCode ierr2;
7904 
7905     ierr2 = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRMPI(ierr2);
7906     ierr2 = (*PetscErrorPrintf)("[%d]ERROR in DMPlexMatSetClosure\n", rank);CHKERRQ(ierr2);
7907     ierr2 = DMPlexPrintMatSetValues(PETSC_VIEWER_STDERR_SELF, A, point, numFIndices, findices, numCIndices, cindices, values);CHKERRQ(ierr2);
7908     ierr2 = DMRestoreWorkArray(dmf, numFIndices, MPIU_INT, &findices);CHKERRQ(ierr2);
7909     ierr2 = DMRestoreWorkArray(dmc, numCIndices, MPIU_INT, &cindices);CHKERRQ(ierr2);
7910     CHKERRQ(ierr);
7911   }
7912   ierr = DMRestoreWorkArray(dmf, numCPoints*2*4, MPIU_INT, &ftotpoints);CHKERRQ(ierr);
7913   ierr = DMPlexRestoreTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
7914   ierr = DMRestoreWorkArray(dmf, numFIndices, MPIU_INT, &findices);CHKERRQ(ierr);
7915   ierr = DMRestoreWorkArray(dmc, numCIndices, MPIU_INT, &cindices);CHKERRQ(ierr);
7916   PetscFunctionReturn(0);
7917 }
7918 
7919 PetscErrorCode DMPlexMatGetClosureIndicesRefined(DM dmf, PetscSection fsection, PetscSection globalFSection, DM dmc, PetscSection csection, PetscSection globalCSection, PetscInt point, PetscInt cindices[], PetscInt findices[])
7920 {
7921   PetscInt      *fpoints = NULL, *ftotpoints = NULL;
7922   PetscInt      *cpoints = NULL;
7923   PetscInt       foffsets[32], coffsets[32];
7924   const PetscInt *fclperm = NULL, *cclperm = NULL; /* Closure permutations cannot work here */
7925   DMPolytopeType ct;
7926   PetscInt       numFields, numSubcells, maxFPoints, numFPoints, numCPoints, numFIndices, numCIndices, dof, off, globalOff, pStart, pEnd, p, q, r, s, f;
7927   PetscErrorCode ierr;
7928 
7929   PetscFunctionBegin;
7930   PetscValidHeaderSpecific(dmf, DM_CLASSID, 1);
7931   PetscValidHeaderSpecific(dmc, DM_CLASSID, 4);
7932   if (!fsection) {ierr = DMGetLocalSection(dmf, &fsection);CHKERRQ(ierr);}
7933   PetscValidHeaderSpecific(fsection, PETSC_SECTION_CLASSID, 2);
7934   if (!csection) {ierr = DMGetLocalSection(dmc, &csection);CHKERRQ(ierr);}
7935   PetscValidHeaderSpecific(csection, PETSC_SECTION_CLASSID, 5);
7936   if (!globalFSection) {ierr = DMGetGlobalSection(dmf, &globalFSection);CHKERRQ(ierr);}
7937   PetscValidHeaderSpecific(globalFSection, PETSC_SECTION_CLASSID, 3);
7938   if (!globalCSection) {ierr = DMGetGlobalSection(dmc, &globalCSection);CHKERRQ(ierr);}
7939   PetscValidHeaderSpecific(globalCSection, PETSC_SECTION_CLASSID, 6);
7940   ierr = PetscSectionGetNumFields(fsection, &numFields);CHKERRQ(ierr);
7941   PetscCheckFalse(numFields > 31,PetscObjectComm((PetscObject)dmf), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", numFields);
7942   ierr = PetscArrayzero(foffsets, 32);CHKERRQ(ierr);
7943   ierr = PetscArrayzero(coffsets, 32);CHKERRQ(ierr);
7944   /* Column indices */
7945   ierr = DMPlexGetTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
7946   maxFPoints = numCPoints;
7947   /* Compress out points not in the section */
7948   /*   TODO: Squeeze out points with 0 dof as well */
7949   ierr = PetscSectionGetChart(csection, &pStart, &pEnd);CHKERRQ(ierr);
7950   for (p = 0, q = 0; p < numCPoints*2; p += 2) {
7951     if ((cpoints[p] >= pStart) && (cpoints[p] < pEnd)) {
7952       cpoints[q*2]   = cpoints[p];
7953       cpoints[q*2+1] = cpoints[p+1];
7954       ++q;
7955     }
7956   }
7957   numCPoints = q;
7958   for (p = 0, numCIndices = 0; p < numCPoints*2; p += 2) {
7959     PetscInt fdof;
7960 
7961     ierr = PetscSectionGetDof(csection, cpoints[p], &dof);CHKERRQ(ierr);
7962     if (!dof) continue;
7963     for (f = 0; f < numFields; ++f) {
7964       ierr           = PetscSectionGetFieldDof(csection, cpoints[p], f, &fdof);CHKERRQ(ierr);
7965       coffsets[f+1] += fdof;
7966     }
7967     numCIndices += dof;
7968   }
7969   for (f = 1; f < numFields; ++f) coffsets[f+1] += coffsets[f];
7970   /* Row indices */
7971   ierr = DMPlexGetCellType(dmc, point, &ct);CHKERRQ(ierr);
7972   {
7973     DMPlexTransform tr;
7974     DMPolytopeType *rct;
7975     PetscInt       *rsize, *rcone, *rornt, Nt;
7976 
7977     ierr = DMPlexTransformCreate(PETSC_COMM_SELF, &tr);CHKERRQ(ierr);
7978     ierr = DMPlexTransformSetType(tr, DMPLEXREFINEREGULAR);CHKERRQ(ierr);
7979     ierr = DMPlexTransformCellTransform(tr, ct, point, NULL, &Nt, &rct, &rsize, &rcone, &rornt);CHKERRQ(ierr);
7980     numSubcells = rsize[Nt-1];
7981     ierr = DMPlexTransformDestroy(&tr);CHKERRQ(ierr);
7982   }
7983   ierr = DMGetWorkArray(dmf, maxFPoints*2*numSubcells, MPIU_INT, &ftotpoints);CHKERRQ(ierr);
7984   for (r = 0, q = 0; r < numSubcells; ++r) {
7985     /* TODO Map from coarse to fine cells */
7986     ierr = DMPlexGetTransitiveClosure(dmf, point*numSubcells + r, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
7987     /* Compress out points not in the section */
7988     ierr = PetscSectionGetChart(fsection, &pStart, &pEnd);CHKERRQ(ierr);
7989     for (p = 0; p < numFPoints*2; p += 2) {
7990       if ((fpoints[p] >= pStart) && (fpoints[p] < pEnd)) {
7991         ierr = PetscSectionGetDof(fsection, fpoints[p], &dof);CHKERRQ(ierr);
7992         if (!dof) continue;
7993         for (s = 0; s < q; ++s) if (fpoints[p] == ftotpoints[s*2]) break;
7994         if (s < q) continue;
7995         ftotpoints[q*2]   = fpoints[p];
7996         ftotpoints[q*2+1] = fpoints[p+1];
7997         ++q;
7998       }
7999     }
8000     ierr = DMPlexRestoreTransitiveClosure(dmf, point, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
8001   }
8002   numFPoints = q;
8003   for (p = 0, numFIndices = 0; p < numFPoints*2; p += 2) {
8004     PetscInt fdof;
8005 
8006     ierr = PetscSectionGetDof(fsection, ftotpoints[p], &dof);CHKERRQ(ierr);
8007     if (!dof) continue;
8008     for (f = 0; f < numFields; ++f) {
8009       ierr           = PetscSectionGetFieldDof(fsection, ftotpoints[p], f, &fdof);CHKERRQ(ierr);
8010       foffsets[f+1] += fdof;
8011     }
8012     numFIndices += dof;
8013   }
8014   for (f = 1; f < numFields; ++f) foffsets[f+1] += foffsets[f];
8015 
8016   PetscCheckFalse(numFields && foffsets[numFields] != numFIndices,PetscObjectComm((PetscObject)dmf), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", foffsets[numFields], numFIndices);
8017   PetscCheckFalse(numFields && coffsets[numFields] != numCIndices,PetscObjectComm((PetscObject)dmc), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", coffsets[numFields], numCIndices);
8018   if (numFields) {
8019     const PetscInt **permsF[32] = {NULL};
8020     const PetscInt **permsC[32] = {NULL};
8021 
8022     for (f = 0; f < numFields; f++) {
8023       ierr = PetscSectionGetFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL);CHKERRQ(ierr);
8024       ierr = PetscSectionGetFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL);CHKERRQ(ierr);
8025     }
8026     for (p = 0; p < numFPoints; p++) {
8027       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff);CHKERRQ(ierr);
8028       ierr = DMPlexGetIndicesPointFields_Internal(fsection, PETSC_FALSE, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, foffsets, PETSC_FALSE, permsF, p, fclperm, findices);CHKERRQ(ierr);
8029     }
8030     for (p = 0; p < numCPoints; p++) {
8031       ierr = PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff);CHKERRQ(ierr);
8032       ierr = DMPlexGetIndicesPointFields_Internal(csection, PETSC_FALSE, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, coffsets, PETSC_FALSE, permsC, p, cclperm, cindices);CHKERRQ(ierr);
8033     }
8034     for (f = 0; f < numFields; f++) {
8035       ierr = PetscSectionRestoreFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL);CHKERRQ(ierr);
8036       ierr = PetscSectionRestoreFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL);CHKERRQ(ierr);
8037     }
8038   } else {
8039     const PetscInt **permsF = NULL;
8040     const PetscInt **permsC = NULL;
8041 
8042     ierr = PetscSectionGetPointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL);CHKERRQ(ierr);
8043     ierr = PetscSectionGetPointSyms(csection,numCPoints,cpoints,&permsC,NULL);CHKERRQ(ierr);
8044     for (p = 0, off = 0; p < numFPoints; p++) {
8045       const PetscInt *perm = permsF ? permsF[p] : NULL;
8046 
8047       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff);CHKERRQ(ierr);
8048       ierr = DMPlexGetIndicesPoint_Internal(fsection, PETSC_FALSE, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, fclperm, findices);CHKERRQ(ierr);
8049     }
8050     for (p = 0, off = 0; p < numCPoints; p++) {
8051       const PetscInt *perm = permsC ? permsC[p] : NULL;
8052 
8053       ierr = PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff);CHKERRQ(ierr);
8054       ierr = DMPlexGetIndicesPoint_Internal(csection, PETSC_FALSE, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, cclperm, cindices);CHKERRQ(ierr);
8055     }
8056     ierr = PetscSectionRestorePointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL);CHKERRQ(ierr);
8057     ierr = PetscSectionRestorePointSyms(csection,numCPoints,cpoints,&permsC,NULL);CHKERRQ(ierr);
8058   }
8059   ierr = DMRestoreWorkArray(dmf, numCPoints*2*4, MPIU_INT, &ftotpoints);CHKERRQ(ierr);
8060   ierr = DMPlexRestoreTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
8061   PetscFunctionReturn(0);
8062 }
8063 
8064 /*@C
8065   DMPlexGetVTKCellHeight - Returns the height in the DAG used to determine which points are cells (normally 0)
8066 
8067   Input Parameter:
8068 . dm   - The DMPlex object
8069 
8070   Output Parameter:
8071 . cellHeight - The height of a cell
8072 
8073   Level: developer
8074 
8075 .seealso DMPlexSetVTKCellHeight()
8076 @*/
8077 PetscErrorCode DMPlexGetVTKCellHeight(DM dm, PetscInt *cellHeight)
8078 {
8079   DM_Plex *mesh = (DM_Plex*) dm->data;
8080 
8081   PetscFunctionBegin;
8082   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8083   PetscValidPointer(cellHeight, 2);
8084   *cellHeight = mesh->vtkCellHeight;
8085   PetscFunctionReturn(0);
8086 }
8087 
8088 /*@C
8089   DMPlexSetVTKCellHeight - Sets the height in the DAG used to determine which points are cells (normally 0)
8090 
8091   Input Parameters:
8092 + dm   - The DMPlex object
8093 - cellHeight - The height of a cell
8094 
8095   Level: developer
8096 
8097 .seealso DMPlexGetVTKCellHeight()
8098 @*/
8099 PetscErrorCode DMPlexSetVTKCellHeight(DM dm, PetscInt cellHeight)
8100 {
8101   DM_Plex *mesh = (DM_Plex*) dm->data;
8102 
8103   PetscFunctionBegin;
8104   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8105   mesh->vtkCellHeight = cellHeight;
8106   PetscFunctionReturn(0);
8107 }
8108 
8109 /*@
8110   DMPlexGetGhostCellStratum - Get the range of cells which are used to enforce FV boundary conditions
8111 
8112   Input Parameter:
8113 . dm - The DMPlex object
8114 
8115   Output Parameters:
8116 + gcStart - The first ghost cell, or NULL
8117 - gcEnd   - The upper bound on ghost cells, or NULL
8118 
8119   Level: advanced
8120 
8121 .seealso DMPlexConstructGhostCells(), DMPlexGetGhostCellStratum()
8122 @*/
8123 PetscErrorCode DMPlexGetGhostCellStratum(DM dm, PetscInt *gcStart, PetscInt *gcEnd)
8124 {
8125   DMLabel        ctLabel;
8126   PetscErrorCode ierr;
8127 
8128   PetscFunctionBegin;
8129   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8130   ierr = DMPlexGetCellTypeLabel(dm, &ctLabel);CHKERRQ(ierr);
8131   ierr = DMLabelGetStratumBounds(ctLabel, DM_POLYTOPE_FV_GHOST, gcStart, gcEnd);CHKERRQ(ierr);
8132   PetscFunctionReturn(0);
8133 }
8134 
8135 PetscErrorCode DMPlexCreateNumbering_Plex(DM dm, PetscInt pStart, PetscInt pEnd, PetscInt shift, PetscInt *globalSize, PetscSF sf, IS *numbering)
8136 {
8137   PetscSection   section, globalSection;
8138   PetscInt      *numbers, p;
8139   PetscErrorCode ierr;
8140 
8141   PetscFunctionBegin;
8142   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &section);CHKERRQ(ierr);
8143   ierr = PetscSectionSetChart(section, pStart, pEnd);CHKERRQ(ierr);
8144   for (p = pStart; p < pEnd; ++p) {
8145     ierr = PetscSectionSetDof(section, p, 1);CHKERRQ(ierr);
8146   }
8147   ierr = PetscSectionSetUp(section);CHKERRQ(ierr);
8148   ierr = PetscSectionCreateGlobalSection(section, sf, PETSC_FALSE, PETSC_FALSE, &globalSection);CHKERRQ(ierr);
8149   ierr = PetscMalloc1(pEnd - pStart, &numbers);CHKERRQ(ierr);
8150   for (p = pStart; p < pEnd; ++p) {
8151     ierr = PetscSectionGetOffset(globalSection, p, &numbers[p-pStart]);CHKERRQ(ierr);
8152     if (numbers[p-pStart] < 0) numbers[p-pStart] -= shift;
8153     else                       numbers[p-pStart] += shift;
8154   }
8155   ierr = ISCreateGeneral(PetscObjectComm((PetscObject) dm), pEnd - pStart, numbers, PETSC_OWN_POINTER, numbering);CHKERRQ(ierr);
8156   if (globalSize) {
8157     PetscLayout layout;
8158     ierr = PetscSectionGetPointLayout(PetscObjectComm((PetscObject) dm), globalSection, &layout);CHKERRQ(ierr);
8159     ierr = PetscLayoutGetSize(layout, globalSize);CHKERRQ(ierr);
8160     ierr = PetscLayoutDestroy(&layout);CHKERRQ(ierr);
8161   }
8162   ierr = PetscSectionDestroy(&section);CHKERRQ(ierr);
8163   ierr = PetscSectionDestroy(&globalSection);CHKERRQ(ierr);
8164   PetscFunctionReturn(0);
8165 }
8166 
8167 PetscErrorCode DMPlexCreateCellNumbering_Internal(DM dm, PetscBool includeHybrid, IS *globalCellNumbers)
8168 {
8169   PetscInt       cellHeight, cStart, cEnd;
8170   PetscErrorCode ierr;
8171 
8172   PetscFunctionBegin;
8173   ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
8174   if (includeHybrid) {ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);}
8175   else               {ierr = DMPlexGetSimplexOrBoxCells(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);}
8176   ierr = DMPlexCreateNumbering_Plex(dm, cStart, cEnd, 0, NULL, dm->sf, globalCellNumbers);CHKERRQ(ierr);
8177   PetscFunctionReturn(0);
8178 }
8179 
8180 /*@
8181   DMPlexGetCellNumbering - Get a global cell numbering for all cells on this process
8182 
8183   Input Parameter:
8184 . dm   - The DMPlex object
8185 
8186   Output Parameter:
8187 . globalCellNumbers - Global cell numbers for all cells on this process
8188 
8189   Level: developer
8190 
8191 .seealso DMPlexGetVertexNumbering()
8192 @*/
8193 PetscErrorCode DMPlexGetCellNumbering(DM dm, IS *globalCellNumbers)
8194 {
8195   DM_Plex       *mesh = (DM_Plex*) dm->data;
8196   PetscErrorCode ierr;
8197 
8198   PetscFunctionBegin;
8199   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8200   if (!mesh->globalCellNumbers) {ierr = DMPlexCreateCellNumbering_Internal(dm, PETSC_FALSE, &mesh->globalCellNumbers);CHKERRQ(ierr);}
8201   *globalCellNumbers = mesh->globalCellNumbers;
8202   PetscFunctionReturn(0);
8203 }
8204 
8205 PetscErrorCode DMPlexCreateVertexNumbering_Internal(DM dm, PetscBool includeHybrid, IS *globalVertexNumbers)
8206 {
8207   PetscInt       vStart, vEnd;
8208   PetscErrorCode ierr;
8209 
8210   PetscFunctionBegin;
8211   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8212   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
8213   ierr = DMPlexCreateNumbering_Plex(dm, vStart, vEnd, 0, NULL, dm->sf, globalVertexNumbers);CHKERRQ(ierr);
8214   PetscFunctionReturn(0);
8215 }
8216 
8217 /*@
8218   DMPlexGetVertexNumbering - Get a global vertex numbering for all vertices on this process
8219 
8220   Input Parameter:
8221 . dm   - The DMPlex object
8222 
8223   Output Parameter:
8224 . globalVertexNumbers - Global vertex numbers for all vertices on this process
8225 
8226   Level: developer
8227 
8228 .seealso DMPlexGetCellNumbering()
8229 @*/
8230 PetscErrorCode DMPlexGetVertexNumbering(DM dm, IS *globalVertexNumbers)
8231 {
8232   DM_Plex       *mesh = (DM_Plex*) dm->data;
8233   PetscErrorCode ierr;
8234 
8235   PetscFunctionBegin;
8236   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8237   if (!mesh->globalVertexNumbers) {ierr = DMPlexCreateVertexNumbering_Internal(dm, PETSC_FALSE, &mesh->globalVertexNumbers);CHKERRQ(ierr);}
8238   *globalVertexNumbers = mesh->globalVertexNumbers;
8239   PetscFunctionReturn(0);
8240 }
8241 
8242 /*@
8243   DMPlexCreatePointNumbering - Create a global numbering for all points on this process
8244 
8245   Input Parameter:
8246 . dm   - The DMPlex object
8247 
8248   Output Parameter:
8249 . globalPointNumbers - Global numbers for all points on this process
8250 
8251   Level: developer
8252 
8253 .seealso DMPlexGetCellNumbering()
8254 @*/
8255 PetscErrorCode DMPlexCreatePointNumbering(DM dm, IS *globalPointNumbers)
8256 {
8257   IS             nums[4];
8258   PetscInt       depths[4], gdepths[4], starts[4];
8259   PetscInt       depth, d, shift = 0;
8260   PetscErrorCode ierr;
8261 
8262   PetscFunctionBegin;
8263   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8264   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
8265   /* For unstratified meshes use dim instead of depth */
8266   if (depth < 0) {ierr = DMGetDimension(dm, &depth);CHKERRQ(ierr);}
8267   for (d = 0; d <= depth; ++d) {
8268     PetscInt end;
8269 
8270     depths[d] = depth-d;
8271     ierr = DMPlexGetDepthStratum(dm, depths[d], &starts[d], &end);CHKERRQ(ierr);
8272     if (!(starts[d]-end)) { starts[d] = depths[d] = -1; }
8273   }
8274   ierr = PetscSortIntWithArray(depth+1, starts, depths);CHKERRQ(ierr);
8275   ierr = MPIU_Allreduce(depths, gdepths, depth+1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject) dm));CHKERRMPI(ierr);
8276   for (d = 0; d <= depth; ++d) {
8277     PetscCheckFalse(starts[d] >= 0 && depths[d] != gdepths[d],PETSC_COMM_SELF,PETSC_ERR_PLIB,"Expected depth %D, found %D",depths[d],gdepths[d]);
8278   }
8279   for (d = 0; d <= depth; ++d) {
8280     PetscInt pStart, pEnd, gsize;
8281 
8282     ierr = DMPlexGetDepthStratum(dm, gdepths[d], &pStart, &pEnd);CHKERRQ(ierr);
8283     ierr = DMPlexCreateNumbering_Plex(dm, pStart, pEnd, shift, &gsize, dm->sf, &nums[d]);CHKERRQ(ierr);
8284     shift += gsize;
8285   }
8286   ierr = ISConcatenate(PetscObjectComm((PetscObject) dm), depth+1, nums, globalPointNumbers);CHKERRQ(ierr);
8287   for (d = 0; d <= depth; ++d) {ierr = ISDestroy(&nums[d]);CHKERRQ(ierr);}
8288   PetscFunctionReturn(0);
8289 }
8290 
8291 /*@
8292   DMPlexCreateRankField - Create a cell field whose value is the rank of the owner
8293 
8294   Input Parameter:
8295 . dm - The DMPlex object
8296 
8297   Output Parameter:
8298 . ranks - The rank field
8299 
8300   Options Database Keys:
8301 . -dm_partition_view - Adds the rank field into the DM output from -dm_view using the same viewer
8302 
8303   Level: intermediate
8304 
8305 .seealso: DMView()
8306 @*/
8307 PetscErrorCode DMPlexCreateRankField(DM dm, Vec *ranks)
8308 {
8309   DM             rdm;
8310   PetscFE        fe;
8311   PetscScalar   *r;
8312   PetscMPIInt    rank;
8313   DMPolytopeType ct;
8314   PetscInt       dim, cStart, cEnd, c;
8315   PetscBool      simplex;
8316   PetscErrorCode ierr;
8317 
8318   PetscFunctionBeginUser;
8319   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8320   PetscValidPointer(ranks, 2);
8321   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRMPI(ierr);
8322   ierr = DMClone(dm, &rdm);CHKERRQ(ierr);
8323   ierr = DMGetDimension(rdm, &dim);CHKERRQ(ierr);
8324   ierr = DMPlexGetHeightStratum(rdm, 0, &cStart, &cEnd);CHKERRQ(ierr);
8325   ierr = DMPlexGetCellType(dm, cStart, &ct);CHKERRQ(ierr);
8326   simplex = DMPolytopeTypeGetNumVertices(ct) == DMPolytopeTypeGetDim(ct)+1 ? PETSC_TRUE : PETSC_FALSE;
8327   ierr = PetscFECreateDefault(PETSC_COMM_SELF, dim, 1, simplex, "PETSc___rank_", -1, &fe);CHKERRQ(ierr);
8328   ierr = PetscObjectSetName((PetscObject) fe, "rank");CHKERRQ(ierr);
8329   ierr = DMSetField(rdm, 0, NULL, (PetscObject) fe);CHKERRQ(ierr);
8330   ierr = PetscFEDestroy(&fe);CHKERRQ(ierr);
8331   ierr = DMCreateDS(rdm);CHKERRQ(ierr);
8332   ierr = DMCreateGlobalVector(rdm, ranks);CHKERRQ(ierr);
8333   ierr = PetscObjectSetName((PetscObject) *ranks, "partition");CHKERRQ(ierr);
8334   ierr = VecGetArray(*ranks, &r);CHKERRQ(ierr);
8335   for (c = cStart; c < cEnd; ++c) {
8336     PetscScalar *lr;
8337 
8338     ierr = DMPlexPointGlobalRef(rdm, c, r, &lr);CHKERRQ(ierr);
8339     if (lr) *lr = rank;
8340   }
8341   ierr = VecRestoreArray(*ranks, &r);CHKERRQ(ierr);
8342   ierr = DMDestroy(&rdm);CHKERRQ(ierr);
8343   PetscFunctionReturn(0);
8344 }
8345 
8346 /*@
8347   DMPlexCreateLabelField - Create a cell field whose value is the label value for that cell
8348 
8349   Input Parameters:
8350 + dm    - The DMPlex
8351 - label - The DMLabel
8352 
8353   Output Parameter:
8354 . val - The label value field
8355 
8356   Options Database Keys:
8357 . -dm_label_view - Adds the label value field into the DM output from -dm_view using the same viewer
8358 
8359   Level: intermediate
8360 
8361 .seealso: DMView()
8362 @*/
8363 PetscErrorCode DMPlexCreateLabelField(DM dm, DMLabel label, Vec *val)
8364 {
8365   DM             rdm;
8366   PetscFE        fe;
8367   PetscScalar   *v;
8368   PetscInt       dim, cStart, cEnd, c;
8369   PetscErrorCode ierr;
8370 
8371   PetscFunctionBeginUser;
8372   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8373   PetscValidPointer(label, 2);
8374   PetscValidPointer(val, 3);
8375   ierr = DMClone(dm, &rdm);CHKERRQ(ierr);
8376   ierr = DMGetDimension(rdm, &dim);CHKERRQ(ierr);
8377   ierr = PetscFECreateDefault(PetscObjectComm((PetscObject) rdm), dim, 1, PETSC_TRUE, "PETSc___label_value_", -1, &fe);CHKERRQ(ierr);
8378   ierr = PetscObjectSetName((PetscObject) fe, "label_value");CHKERRQ(ierr);
8379   ierr = DMSetField(rdm, 0, NULL, (PetscObject) fe);CHKERRQ(ierr);
8380   ierr = PetscFEDestroy(&fe);CHKERRQ(ierr);
8381   ierr = DMCreateDS(rdm);CHKERRQ(ierr);
8382   ierr = DMPlexGetHeightStratum(rdm, 0, &cStart, &cEnd);CHKERRQ(ierr);
8383   ierr = DMCreateGlobalVector(rdm, val);CHKERRQ(ierr);
8384   ierr = PetscObjectSetName((PetscObject) *val, "label_value");CHKERRQ(ierr);
8385   ierr = VecGetArray(*val, &v);CHKERRQ(ierr);
8386   for (c = cStart; c < cEnd; ++c) {
8387     PetscScalar *lv;
8388     PetscInt     cval;
8389 
8390     ierr = DMPlexPointGlobalRef(rdm, c, v, &lv);CHKERRQ(ierr);
8391     ierr = DMLabelGetValue(label, c, &cval);CHKERRQ(ierr);
8392     *lv = cval;
8393   }
8394   ierr = VecRestoreArray(*val, &v);CHKERRQ(ierr);
8395   ierr = DMDestroy(&rdm);CHKERRQ(ierr);
8396   PetscFunctionReturn(0);
8397 }
8398 
8399 /*@
8400   DMPlexCheckSymmetry - Check that the adjacency information in the mesh is symmetric.
8401 
8402   Input Parameter:
8403 . dm - The DMPlex object
8404 
8405   Notes:
8406   This is a useful diagnostic when creating meshes programmatically.
8407 
8408   For the complete list of DMPlexCheck* functions, see DMSetFromOptions().
8409 
8410   Level: developer
8411 
8412 .seealso: DMCreate(), DMSetFromOptions()
8413 @*/
8414 PetscErrorCode DMPlexCheckSymmetry(DM dm)
8415 {
8416   PetscSection    coneSection, supportSection;
8417   const PetscInt *cone, *support;
8418   PetscInt        coneSize, c, supportSize, s;
8419   PetscInt        pStart, pEnd, p, pp, csize, ssize;
8420   PetscBool       storagecheck = PETSC_TRUE;
8421   PetscErrorCode  ierr;
8422 
8423   PetscFunctionBegin;
8424   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8425   ierr = DMViewFromOptions(dm, NULL, "-sym_dm_view");CHKERRQ(ierr);
8426   ierr = DMPlexGetConeSection(dm, &coneSection);CHKERRQ(ierr);
8427   ierr = DMPlexGetSupportSection(dm, &supportSection);CHKERRQ(ierr);
8428   /* Check that point p is found in the support of its cone points, and vice versa */
8429   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
8430   for (p = pStart; p < pEnd; ++p) {
8431     ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
8432     ierr = DMPlexGetCone(dm, p, &cone);CHKERRQ(ierr);
8433     for (c = 0; c < coneSize; ++c) {
8434       PetscBool dup = PETSC_FALSE;
8435       PetscInt  d;
8436       for (d = c-1; d >= 0; --d) {
8437         if (cone[c] == cone[d]) {dup = PETSC_TRUE; break;}
8438       }
8439       ierr = DMPlexGetSupportSize(dm, cone[c], &supportSize);CHKERRQ(ierr);
8440       ierr = DMPlexGetSupport(dm, cone[c], &support);CHKERRQ(ierr);
8441       for (s = 0; s < supportSize; ++s) {
8442         if (support[s] == p) break;
8443       }
8444       if ((s >= supportSize) || (dup && (support[s+1] != p))) {
8445         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %D cone: ", p);CHKERRQ(ierr);
8446         for (s = 0; s < coneSize; ++s) {
8447           ierr = PetscPrintf(PETSC_COMM_SELF, "%D, ", cone[s]);CHKERRQ(ierr);
8448         }
8449         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
8450         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %D support: ", cone[c]);CHKERRQ(ierr);
8451         for (s = 0; s < supportSize; ++s) {
8452           ierr = PetscPrintf(PETSC_COMM_SELF, "%D, ", support[s]);CHKERRQ(ierr);
8453         }
8454         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
8455         PetscCheckFalse(dup,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %D not repeatedly found in support of repeated cone point %D", p, cone[c]);
8456         else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %D not found in support of cone point %D", p, cone[c]);
8457       }
8458     }
8459     ierr = DMPlexGetTreeParent(dm, p, &pp, NULL);CHKERRQ(ierr);
8460     if (p != pp) { storagecheck = PETSC_FALSE; continue; }
8461     ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
8462     ierr = DMPlexGetSupport(dm, p, &support);CHKERRQ(ierr);
8463     for (s = 0; s < supportSize; ++s) {
8464       ierr = DMPlexGetConeSize(dm, support[s], &coneSize);CHKERRQ(ierr);
8465       ierr = DMPlexGetCone(dm, support[s], &cone);CHKERRQ(ierr);
8466       for (c = 0; c < coneSize; ++c) {
8467         ierr = DMPlexGetTreeParent(dm, cone[c], &pp, NULL);CHKERRQ(ierr);
8468         if (cone[c] != pp) { c = 0; break; }
8469         if (cone[c] == p) break;
8470       }
8471       if (c >= coneSize) {
8472         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %D support: ", p);CHKERRQ(ierr);
8473         for (c = 0; c < supportSize; ++c) {
8474           ierr = PetscPrintf(PETSC_COMM_SELF, "%D, ", support[c]);CHKERRQ(ierr);
8475         }
8476         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
8477         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %D cone: ", support[s]);CHKERRQ(ierr);
8478         for (c = 0; c < coneSize; ++c) {
8479           ierr = PetscPrintf(PETSC_COMM_SELF, "%D, ", cone[c]);CHKERRQ(ierr);
8480         }
8481         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
8482         SETERRQ(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %D not found in cone of support point %D", p, support[s]);
8483       }
8484     }
8485   }
8486   if (storagecheck) {
8487     ierr = PetscSectionGetStorageSize(coneSection, &csize);CHKERRQ(ierr);
8488     ierr = PetscSectionGetStorageSize(supportSection, &ssize);CHKERRQ(ierr);
8489     PetscCheckFalse(csize != ssize,PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "Total cone size %D != Total support size %D", csize, ssize);
8490   }
8491   PetscFunctionReturn(0);
8492 }
8493 
8494 /*
8495   For submeshes with cohesive cells (see DMPlexConstructCohesiveCells()), we allow a special case where some of the boundary of a face (edges and vertices) are not duplicated. We call these special boundary points "unsplit", since the same edge or vertex appears in both copies of the face. These unsplit points throw off our counting, so we have to explicitly account for them here.
8496 */
8497 static PetscErrorCode DMPlexCellUnsplitVertices_Private(DM dm, PetscInt c, DMPolytopeType ct, PetscInt *unsplit)
8498 {
8499   DMPolytopeType  cct;
8500   PetscInt        ptpoints[4];
8501   const PetscInt *cone, *ccone, *ptcone;
8502   PetscInt        coneSize, cp, cconeSize, ccp, npt = 0, pt;
8503   PetscErrorCode  ierr;
8504 
8505   PetscFunctionBegin;
8506   *unsplit = 0;
8507   switch (ct) {
8508     case DM_POLYTOPE_POINT_PRISM_TENSOR:
8509       ptpoints[npt++] = c;
8510       break;
8511     case DM_POLYTOPE_SEG_PRISM_TENSOR:
8512       ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
8513       ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
8514       for (cp = 0; cp < coneSize; ++cp) {
8515         ierr = DMPlexGetCellType(dm, cone[cp], &cct);CHKERRQ(ierr);
8516         if (cct == DM_POLYTOPE_POINT_PRISM_TENSOR) ptpoints[npt++] = cone[cp];
8517       }
8518       break;
8519     case DM_POLYTOPE_TRI_PRISM_TENSOR:
8520     case DM_POLYTOPE_QUAD_PRISM_TENSOR:
8521       ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
8522       ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
8523       for (cp = 0; cp < coneSize; ++cp) {
8524         ierr = DMPlexGetCone(dm, cone[cp], &ccone);CHKERRQ(ierr);
8525         ierr = DMPlexGetConeSize(dm, cone[cp], &cconeSize);CHKERRQ(ierr);
8526         for (ccp = 0; ccp < cconeSize; ++ccp) {
8527           ierr = DMPlexGetCellType(dm, ccone[ccp], &cct);CHKERRQ(ierr);
8528           if (cct == DM_POLYTOPE_POINT_PRISM_TENSOR) {
8529             PetscInt p;
8530             for (p = 0; p < npt; ++p) if (ptpoints[p] == ccone[ccp]) break;
8531             if (p == npt) ptpoints[npt++] = ccone[ccp];
8532           }
8533         }
8534       }
8535       break;
8536     default: break;
8537   }
8538   for (pt = 0; pt < npt; ++pt) {
8539     ierr = DMPlexGetCone(dm, ptpoints[pt], &ptcone);CHKERRQ(ierr);
8540     if (ptcone[0] == ptcone[1]) ++(*unsplit);
8541   }
8542   PetscFunctionReturn(0);
8543 }
8544 
8545 /*@
8546   DMPlexCheckSkeleton - Check that each cell has the correct number of vertices
8547 
8548   Input Parameters:
8549 + dm - The DMPlex object
8550 - cellHeight - Normally 0
8551 
8552   Notes:
8553   This is a useful diagnostic when creating meshes programmatically.
8554   Currently applicable only to homogeneous simplex or tensor meshes.
8555 
8556   For the complete list of DMPlexCheck* functions, see DMSetFromOptions().
8557 
8558   Level: developer
8559 
8560 .seealso: DMCreate(), DMSetFromOptions()
8561 @*/
8562 PetscErrorCode DMPlexCheckSkeleton(DM dm, PetscInt cellHeight)
8563 {
8564   DMPlexInterpolatedFlag interp;
8565   DMPolytopeType         ct;
8566   PetscInt               vStart, vEnd, cStart, cEnd, c;
8567   PetscErrorCode         ierr;
8568 
8569   PetscFunctionBegin;
8570   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8571   ierr = DMPlexIsInterpolated(dm, &interp);CHKERRQ(ierr);
8572   ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
8573   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
8574   for (c = cStart; c < cEnd; ++c) {
8575     PetscInt *closure = NULL;
8576     PetscInt  coneSize, closureSize, cl, Nv = 0;
8577 
8578     ierr = DMPlexGetCellType(dm, c, &ct);CHKERRQ(ierr);
8579     PetscCheckFalse((PetscInt) ct < 0,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %D has no cell type", c);
8580     if (ct == DM_POLYTOPE_UNKNOWN) continue;
8581     if (interp == DMPLEX_INTERPOLATED_FULL) {
8582       ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
8583       PetscCheckFalse(coneSize != DMPolytopeTypeGetConeSize(ct),PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %D of type %s has cone size %D != %D", c, DMPolytopeTypes[ct], coneSize, DMPolytopeTypeGetConeSize(ct));
8584     }
8585     ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
8586     for (cl = 0; cl < closureSize*2; cl += 2) {
8587       const PetscInt p = closure[cl];
8588       if ((p >= vStart) && (p < vEnd)) ++Nv;
8589     }
8590     ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
8591     /* Special Case: Tensor faces with identified vertices */
8592     if (Nv < DMPolytopeTypeGetNumVertices(ct)) {
8593       PetscInt unsplit;
8594 
8595       ierr = DMPlexCellUnsplitVertices_Private(dm, c, ct, &unsplit);CHKERRQ(ierr);
8596       if (Nv + unsplit == DMPolytopeTypeGetNumVertices(ct)) continue;
8597     }
8598     PetscCheckFalse(Nv != DMPolytopeTypeGetNumVertices(ct),PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %D of type %s has %D vertices != %D", c, DMPolytopeTypes[ct], Nv, DMPolytopeTypeGetNumVertices(ct));
8599   }
8600   PetscFunctionReturn(0);
8601 }
8602 
8603 /*@
8604   DMPlexCheckFaces - Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type
8605 
8606   Not Collective
8607 
8608   Input Parameters:
8609 + dm - The DMPlex object
8610 - cellHeight - Normally 0
8611 
8612   Notes:
8613   This is a useful diagnostic when creating meshes programmatically.
8614   This routine is only relevant for meshes that are fully interpolated across all ranks.
8615   It will error out if a partially interpolated mesh is given on some rank.
8616   It will do nothing for locally uninterpolated mesh (as there is nothing to check).
8617 
8618   For the complete list of DMPlexCheck* functions, see DMSetFromOptions().
8619 
8620   Level: developer
8621 
8622 .seealso: DMCreate(), DMPlexGetVTKCellHeight(), DMSetFromOptions()
8623 @*/
8624 PetscErrorCode DMPlexCheckFaces(DM dm, PetscInt cellHeight)
8625 {
8626   PetscInt       dim, depth, vStart, vEnd, cStart, cEnd, c, h;
8627   PetscErrorCode ierr;
8628   DMPlexInterpolatedFlag interpEnum;
8629 
8630   PetscFunctionBegin;
8631   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8632   ierr = DMPlexIsInterpolated(dm, &interpEnum);CHKERRQ(ierr);
8633   if (interpEnum == DMPLEX_INTERPOLATED_NONE) PetscFunctionReturn(0);
8634   if (interpEnum == DMPLEX_INTERPOLATED_PARTIAL) {
8635     PetscMPIInt rank;
8636     MPI_Comm    comm;
8637 
8638     ierr = PetscObjectGetComm((PetscObject) dm, &comm);CHKERRQ(ierr);
8639     ierr = MPI_Comm_rank(comm, &rank);CHKERRMPI(ierr);
8640     SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Mesh is only partially interpolated on rank %d, this is currently not supported", rank);
8641   }
8642 
8643   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
8644   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
8645   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
8646   for (h = cellHeight; h < PetscMin(depth, dim); ++h) {
8647     ierr = DMPlexGetHeightStratum(dm, h, &cStart, &cEnd);CHKERRQ(ierr);
8648     for (c = cStart; c < cEnd; ++c) {
8649       const PetscInt      *cone, *ornt, *faceSizes, *faces;
8650       const DMPolytopeType *faceTypes;
8651       DMPolytopeType        ct;
8652       PetscInt              numFaces, coneSize, f;
8653       PetscInt             *closure = NULL, closureSize, cl, numCorners = 0, fOff = 0, unsplit;
8654 
8655       ierr = DMPlexGetCellType(dm, c, &ct);CHKERRQ(ierr);
8656       ierr = DMPlexCellUnsplitVertices_Private(dm, c, ct, &unsplit);CHKERRQ(ierr);
8657       if (unsplit) continue;
8658       ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
8659       ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
8660       ierr = DMPlexGetConeOrientation(dm, c, &ornt);CHKERRQ(ierr);
8661       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
8662       for (cl = 0; cl < closureSize*2; cl += 2) {
8663         const PetscInt p = closure[cl];
8664         if ((p >= vStart) && (p < vEnd)) closure[numCorners++] = p;
8665       }
8666       ierr = DMPlexGetRawFaces_Internal(dm, ct, closure, &numFaces, &faceTypes, &faceSizes, &faces);CHKERRQ(ierr);
8667       PetscCheckFalse(coneSize != numFaces,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %D of type %s has %D faces but should have %D", c, DMPolytopeTypes[ct], coneSize, numFaces);
8668       for (f = 0; f < numFaces; ++f) {
8669         DMPolytopeType fct;
8670         PetscInt       *fclosure = NULL, fclosureSize, cl, fnumCorners = 0, v;
8671 
8672         ierr = DMPlexGetCellType(dm, cone[f], &fct);CHKERRQ(ierr);
8673         ierr = DMPlexGetTransitiveClosure_Internal(dm, cone[f], ornt[f], PETSC_TRUE, &fclosureSize, &fclosure);CHKERRQ(ierr);
8674         for (cl = 0; cl < fclosureSize*2; cl += 2) {
8675           const PetscInt p = fclosure[cl];
8676           if ((p >= vStart) && (p < vEnd)) fclosure[fnumCorners++] = p;
8677         }
8678         PetscCheckFalse(fnumCorners != faceSizes[f],PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %D of type %s (cone idx %D) of cell %D of type %s has %D vertices but should have %D", cone[f], DMPolytopeTypes[fct], f, c, DMPolytopeTypes[ct], fnumCorners, faceSizes[f]);
8679         for (v = 0; v < fnumCorners; ++v) {
8680           if (fclosure[v] != faces[fOff+v]) {
8681             PetscInt v1;
8682 
8683             ierr = PetscPrintf(PETSC_COMM_SELF, "face closure:");CHKERRQ(ierr);
8684             for (v1 = 0; v1 < fnumCorners; ++v1) {ierr = PetscPrintf(PETSC_COMM_SELF, " %D", fclosure[v1]);CHKERRQ(ierr);}
8685             ierr = PetscPrintf(PETSC_COMM_SELF, "\ncell face:");CHKERRQ(ierr);
8686             for (v1 = 0; v1 < fnumCorners; ++v1) {ierr = PetscPrintf(PETSC_COMM_SELF, " %D", faces[fOff+v1]);CHKERRQ(ierr);}
8687             ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
8688             SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %D of type %s (cone idx %d, ornt %D) of cell %D of type %s vertex %D, %D != %D", cone[f], DMPolytopeTypes[fct], f, ornt[f], c, DMPolytopeTypes[ct], v, fclosure[v], faces[fOff+v]);
8689           }
8690         }
8691         ierr = DMPlexRestoreTransitiveClosure(dm, cone[f], PETSC_TRUE, &fclosureSize, &fclosure);CHKERRQ(ierr);
8692         fOff += faceSizes[f];
8693       }
8694       ierr = DMPlexRestoreRawFaces_Internal(dm, ct, closure, &numFaces, &faceTypes, &faceSizes, &faces);CHKERRQ(ierr);
8695       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
8696     }
8697   }
8698   PetscFunctionReturn(0);
8699 }
8700 
8701 /*@
8702   DMPlexCheckGeometry - Check the geometry of mesh cells
8703 
8704   Input Parameter:
8705 . dm - The DMPlex object
8706 
8707   Notes:
8708   This is a useful diagnostic when creating meshes programmatically.
8709 
8710   For the complete list of DMPlexCheck* functions, see DMSetFromOptions().
8711 
8712   Level: developer
8713 
8714 .seealso: DMCreate(), DMSetFromOptions()
8715 @*/
8716 PetscErrorCode DMPlexCheckGeometry(DM dm)
8717 {
8718   Vec            coordinates;
8719   PetscReal      detJ, J[9], refVol = 1.0;
8720   PetscReal      vol;
8721   PetscBool      periodic;
8722   PetscInt       dim, depth, dE, d, cStart, cEnd, c;
8723   PetscErrorCode ierr;
8724 
8725   PetscFunctionBegin;
8726   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
8727   ierr = DMGetCoordinateDim(dm, &dE);CHKERRQ(ierr);
8728   if (dim != dE) PetscFunctionReturn(0);
8729   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
8730   ierr = DMGetPeriodicity(dm, &periodic, NULL, NULL, NULL);CHKERRQ(ierr);
8731   for (d = 0; d < dim; ++d) refVol *= 2.0;
8732   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
8733   /* Make sure local coordinates are created, because that step is collective */
8734   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
8735   for (c = cStart; c < cEnd; ++c) {
8736     DMPolytopeType ct;
8737     PetscInt       unsplit;
8738     PetscBool      ignoreZeroVol = PETSC_FALSE;
8739 
8740     ierr = DMPlexGetCellType(dm, c, &ct);CHKERRQ(ierr);
8741     switch (ct) {
8742       case DM_POLYTOPE_SEG_PRISM_TENSOR:
8743       case DM_POLYTOPE_TRI_PRISM_TENSOR:
8744       case DM_POLYTOPE_QUAD_PRISM_TENSOR:
8745         ignoreZeroVol = PETSC_TRUE; break;
8746       default: break;
8747     }
8748     switch (ct) {
8749       case DM_POLYTOPE_TRI_PRISM:
8750       case DM_POLYTOPE_TRI_PRISM_TENSOR:
8751       case DM_POLYTOPE_QUAD_PRISM_TENSOR:
8752       case DM_POLYTOPE_PYRAMID:
8753         continue;
8754       default: break;
8755     }
8756     ierr = DMPlexCellUnsplitVertices_Private(dm, c, ct, &unsplit);CHKERRQ(ierr);
8757     if (unsplit) continue;
8758     ierr = DMPlexComputeCellGeometryFEM(dm, c, NULL, NULL, J, NULL, &detJ);CHKERRQ(ierr);
8759     PetscCheckFalse(detJ < -PETSC_SMALL || (detJ <= 0.0 && !ignoreZeroVol),PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Mesh cell %D of type %s is inverted, |J| = %g", c, DMPolytopeTypes[ct], (double) detJ);
8760     ierr = PetscInfo(dm, "Cell %D FEM Volume %g\n", c, (double) detJ*refVol);CHKERRQ(ierr);
8761     if (depth > 1 && !periodic) {
8762       ierr = DMPlexComputeCellGeometryFVM(dm, c, &vol, NULL, NULL);CHKERRQ(ierr);
8763       PetscCheckFalse(vol < -PETSC_SMALL || (vol <= 0.0 && !ignoreZeroVol),PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Mesh cell %D of type %s is inverted, vol = %g", c, DMPolytopeTypes[ct], (double) vol);
8764       ierr = PetscInfo(dm, "Cell %D FVM Volume %g\n", c, (double) vol);CHKERRQ(ierr);
8765     }
8766   }
8767   PetscFunctionReturn(0);
8768 }
8769 
8770 /*@
8771   DMPlexCheckPointSF - Check that several necessary conditions are met for the point SF of this plex.
8772 
8773   Input Parameters:
8774 . dm - The DMPlex object
8775 
8776   Notes:
8777   This is mainly intended for debugging/testing purposes.
8778   It currently checks only meshes with no partition overlapping.
8779 
8780   For the complete list of DMPlexCheck* functions, see DMSetFromOptions().
8781 
8782   Level: developer
8783 
8784 .seealso: DMGetPointSF(), DMSetFromOptions()
8785 @*/
8786 PetscErrorCode DMPlexCheckPointSF(DM dm)
8787 {
8788   PetscSF         pointSF;
8789   PetscInt        cellHeight, cStart, cEnd, l, nleaves, nroots, overlap;
8790   const PetscInt *locals, *rootdegree;
8791   PetscBool       distributed;
8792   PetscErrorCode  ierr;
8793 
8794   PetscFunctionBegin;
8795   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8796   ierr = DMGetPointSF(dm, &pointSF);CHKERRQ(ierr);
8797   ierr = DMPlexIsDistributed(dm, &distributed);CHKERRQ(ierr);
8798   if (!distributed) PetscFunctionReturn(0);
8799   ierr = DMPlexGetOverlap(dm, &overlap);CHKERRQ(ierr);
8800   if (overlap) {
8801     ierr = PetscPrintf(PetscObjectComm((PetscObject)dm), "Warning: DMPlexCheckPointSF() is currently not implemented for meshes with partition overlapping");CHKERRQ(ierr);
8802     PetscFunctionReturn(0);
8803   }
8804   PetscCheckFalse(!pointSF,PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONGSTATE, "This DMPlex is distributed but does not have PointSF attached");
8805   ierr = PetscSFGetGraph(pointSF, &nroots, &nleaves, &locals, NULL);CHKERRQ(ierr);
8806   PetscCheckFalse(nroots < 0,PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONGSTATE, "This DMPlex is distributed but its PointSF has no graph set");
8807   ierr = PetscSFComputeDegreeBegin(pointSF, &rootdegree);CHKERRQ(ierr);
8808   ierr = PetscSFComputeDegreeEnd(pointSF, &rootdegree);CHKERRQ(ierr);
8809 
8810   /* 1) check there are no faces in 2D, cells in 3D, in interface */
8811   ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
8812   ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
8813   for (l = 0; l < nleaves; ++l) {
8814     const PetscInt point = locals[l];
8815 
8816     PetscCheckFalse(point >= cStart && point < cEnd,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point SF contains %D which is a cell", point);
8817   }
8818 
8819   /* 2) if some point is in interface, then all its cone points must be also in interface (either as leaves or roots) */
8820   for (l = 0; l < nleaves; ++l) {
8821     const PetscInt  point = locals[l];
8822     const PetscInt *cone;
8823     PetscInt        coneSize, c, idx;
8824 
8825     ierr = DMPlexGetConeSize(dm, point, &coneSize);CHKERRQ(ierr);
8826     ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
8827     for (c = 0; c < coneSize; ++c) {
8828       if (!rootdegree[cone[c]]) {
8829         ierr = PetscFindInt(cone[c], nleaves, locals, &idx);CHKERRQ(ierr);
8830         PetscCheckFalse(idx < 0,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point SF contains %D but not %D from its cone", point, cone[c]);
8831       }
8832     }
8833   }
8834   PetscFunctionReturn(0);
8835 }
8836 
8837 PetscErrorCode DMPlexCheckAll_Internal(DM dm, PetscInt cellHeight)
8838 {
8839   PetscErrorCode ierr;
8840 
8841   PetscFunctionBegin;
8842   ierr = DMPlexCheckSymmetry(dm);CHKERRQ(ierr);
8843   ierr = DMPlexCheckSkeleton(dm, cellHeight);CHKERRQ(ierr);
8844   ierr = DMPlexCheckFaces(dm, cellHeight);CHKERRQ(ierr);
8845   ierr = DMPlexCheckGeometry(dm);CHKERRQ(ierr);
8846   ierr = DMPlexCheckPointSF(dm);CHKERRQ(ierr);
8847   ierr = DMPlexCheckInterfaceCones(dm);CHKERRQ(ierr);
8848   PetscFunctionReturn(0);
8849 }
8850 
8851 typedef struct cell_stats
8852 {
8853   PetscReal min, max, sum, squaresum;
8854   PetscInt  count;
8855 } cell_stats_t;
8856 
8857 static void MPIAPI cell_stats_reduce(void *a, void *b, int * len, MPI_Datatype *datatype)
8858 {
8859   PetscInt i, N = *len;
8860 
8861   for (i = 0; i < N; i++) {
8862     cell_stats_t *A = (cell_stats_t *) a;
8863     cell_stats_t *B = (cell_stats_t *) b;
8864 
8865     B->min = PetscMin(A->min,B->min);
8866     B->max = PetscMax(A->max,B->max);
8867     B->sum += A->sum;
8868     B->squaresum += A->squaresum;
8869     B->count += A->count;
8870   }
8871 }
8872 
8873 /*@
8874   DMPlexCheckCellShape - Checks the Jacobian of the mapping from reference to real cells and computes some minimal statistics.
8875 
8876   Collective on dm
8877 
8878   Input Parameters:
8879 + dm        - The DMPlex object
8880 . output    - If true, statistics will be displayed on stdout
8881 - condLimit - Display all cells above this condition number, or PETSC_DETERMINE for no cell output
8882 
8883   Notes:
8884   This is mainly intended for debugging/testing purposes.
8885 
8886   For the complete list of DMPlexCheck* functions, see DMSetFromOptions().
8887 
8888   Level: developer
8889 
8890 .seealso: DMSetFromOptions(), DMPlexComputeOrthogonalQuality()
8891 @*/
8892 PetscErrorCode DMPlexCheckCellShape(DM dm, PetscBool output, PetscReal condLimit)
8893 {
8894   DM             dmCoarse;
8895   cell_stats_t   stats, globalStats;
8896   MPI_Comm       comm = PetscObjectComm((PetscObject)dm);
8897   PetscReal      *J, *invJ, min = 0, max = 0, mean = 0, stdev = 0;
8898   PetscReal      limit = condLimit > 0 ? condLimit : PETSC_MAX_REAL;
8899   PetscInt       cdim, cStart, cEnd, c, eStart, eEnd, count = 0;
8900   PetscMPIInt    rank,size;
8901   PetscErrorCode ierr;
8902 
8903   PetscFunctionBegin;
8904   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8905   stats.min   = PETSC_MAX_REAL;
8906   stats.max   = PETSC_MIN_REAL;
8907   stats.sum   = stats.squaresum = 0.;
8908   stats.count = 0;
8909 
8910   ierr = MPI_Comm_size(comm, &size);CHKERRMPI(ierr);
8911   ierr = MPI_Comm_rank(comm, &rank);CHKERRMPI(ierr);
8912   ierr = DMGetCoordinateDim(dm,&cdim);CHKERRQ(ierr);
8913   ierr = PetscMalloc2(PetscSqr(cdim), &J, PetscSqr(cdim), &invJ);CHKERRQ(ierr);
8914   ierr = DMPlexGetSimplexOrBoxCells(dm,0,&cStart,&cEnd);CHKERRQ(ierr);
8915   ierr = DMPlexGetDepthStratum(dm,1,&eStart,&eEnd);CHKERRQ(ierr);
8916   for (c = cStart; c < cEnd; c++) {
8917     PetscInt  i;
8918     PetscReal frobJ = 0., frobInvJ = 0., cond2, cond, detJ;
8919 
8920     ierr = DMPlexComputeCellGeometryAffineFEM(dm,c,NULL,J,invJ,&detJ);CHKERRQ(ierr);
8921     PetscCheckFalse(detJ < 0.0,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Mesh cell %D is inverted", c);
8922     for (i = 0; i < PetscSqr(cdim); ++i) {
8923       frobJ    += J[i] * J[i];
8924       frobInvJ += invJ[i] * invJ[i];
8925     }
8926     cond2 = frobJ * frobInvJ;
8927     cond  = PetscSqrtReal(cond2);
8928 
8929     stats.min        = PetscMin(stats.min,cond);
8930     stats.max        = PetscMax(stats.max,cond);
8931     stats.sum       += cond;
8932     stats.squaresum += cond2;
8933     stats.count++;
8934     if (output && cond > limit) {
8935       PetscSection coordSection;
8936       Vec          coordsLocal;
8937       PetscScalar *coords = NULL;
8938       PetscInt     Nv, d, clSize, cl, *closure = NULL;
8939 
8940       ierr = DMGetCoordinatesLocal(dm, &coordsLocal);CHKERRQ(ierr);
8941       ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
8942       ierr = DMPlexVecGetClosure(dm, coordSection, coordsLocal, c, &Nv, &coords);CHKERRQ(ierr);
8943       ierr = PetscSynchronizedPrintf(comm, "[%d] Cell %D cond %g\n", rank, c, (double) cond);CHKERRQ(ierr);
8944       for (i = 0; i < Nv/cdim; ++i) {
8945         ierr = PetscSynchronizedPrintf(comm, "  Vertex %D: (", i);CHKERRQ(ierr);
8946         for (d = 0; d < cdim; ++d) {
8947           if (d > 0) {ierr = PetscSynchronizedPrintf(comm, ", ");CHKERRQ(ierr);}
8948           ierr = PetscSynchronizedPrintf(comm, "%g", (double) PetscRealPart(coords[i*cdim+d]));CHKERRQ(ierr);
8949         }
8950         ierr = PetscSynchronizedPrintf(comm, ")\n");CHKERRQ(ierr);
8951       }
8952       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &clSize, &closure);CHKERRQ(ierr);
8953       for (cl = 0; cl < clSize*2; cl += 2) {
8954         const PetscInt edge = closure[cl];
8955 
8956         if ((edge >= eStart) && (edge < eEnd)) {
8957           PetscReal len;
8958 
8959           ierr = DMPlexComputeCellGeometryFVM(dm, edge, &len, NULL, NULL);CHKERRQ(ierr);
8960           ierr = PetscSynchronizedPrintf(comm, "  Edge %D: length %g\n", edge, (double) len);CHKERRQ(ierr);
8961         }
8962       }
8963       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &clSize, &closure);CHKERRQ(ierr);
8964       ierr = DMPlexVecRestoreClosure(dm, coordSection, coordsLocal, c, &Nv, &coords);CHKERRQ(ierr);
8965     }
8966   }
8967   if (output) {ierr = PetscSynchronizedFlush(comm, NULL);CHKERRQ(ierr);}
8968 
8969   if (size > 1) {
8970     PetscMPIInt   blockLengths[2] = {4,1};
8971     MPI_Aint      blockOffsets[2] = {offsetof(cell_stats_t,min),offsetof(cell_stats_t,count)};
8972     MPI_Datatype  blockTypes[2]   = {MPIU_REAL,MPIU_INT}, statType;
8973     MPI_Op        statReduce;
8974 
8975     ierr = MPI_Type_create_struct(2,blockLengths,blockOffsets,blockTypes,&statType);CHKERRMPI(ierr);
8976     ierr = MPI_Type_commit(&statType);CHKERRMPI(ierr);
8977     ierr = MPI_Op_create(cell_stats_reduce, PETSC_TRUE, &statReduce);CHKERRMPI(ierr);
8978     ierr = MPI_Reduce(&stats,&globalStats,1,statType,statReduce,0,comm);CHKERRMPI(ierr);
8979     ierr = MPI_Op_free(&statReduce);CHKERRMPI(ierr);
8980     ierr = MPI_Type_free(&statType);CHKERRMPI(ierr);
8981   } else {
8982     ierr = PetscArraycpy(&globalStats,&stats,1);CHKERRQ(ierr);
8983   }
8984   if (rank == 0) {
8985     count = globalStats.count;
8986     min   = globalStats.min;
8987     max   = globalStats.max;
8988     mean  = globalStats.sum / globalStats.count;
8989     stdev = globalStats.count > 1 ? PetscSqrtReal(PetscMax((globalStats.squaresum - globalStats.count * mean * mean) / (globalStats.count - 1),0)) : 0.0;
8990   }
8991 
8992   if (output) {
8993     ierr = PetscPrintf(comm,"Mesh with %D cells, shape condition numbers: min = %g, max = %g, mean = %g, stddev = %g\n", count, (double) min, (double) max, (double) mean, (double) stdev);CHKERRQ(ierr);
8994   }
8995   ierr = PetscFree2(J,invJ);CHKERRQ(ierr);
8996 
8997   ierr = DMGetCoarseDM(dm,&dmCoarse);CHKERRQ(ierr);
8998   if (dmCoarse) {
8999     PetscBool isplex;
9000 
9001     ierr = PetscObjectTypeCompare((PetscObject)dmCoarse,DMPLEX,&isplex);CHKERRQ(ierr);
9002     if (isplex) {
9003       ierr = DMPlexCheckCellShape(dmCoarse,output,condLimit);CHKERRQ(ierr);
9004     }
9005   }
9006   PetscFunctionReturn(0);
9007 }
9008 
9009 /*@
9010   DMPlexComputeOrthogonalQuality - Compute cell-wise orthogonal quality mesh statistic. Optionally tags all cells with
9011   orthogonal quality below given tolerance.
9012 
9013   Collective on dm
9014 
9015   Input Parameters:
9016 + dm   - The DMPlex object
9017 . fv   - Optional PetscFV object for pre-computed cell/face centroid information
9018 - atol - [0, 1] Absolute tolerance for tagging cells.
9019 
9020   Output Parameters:
9021 + OrthQual      - Vec containing orthogonal quality per cell
9022 - OrthQualLabel - DMLabel tagging cells below atol with DM_ADAPT_REFINE
9023 
9024   Options Database Keys:
9025 + -dm_plex_orthogonal_quality_label_view - view OrthQualLabel if label is requested. Currently only PETSCVIEWERASCII is
9026 supported.
9027 - -dm_plex_orthogonal_quality_vec_view - view OrthQual vector.
9028 
9029   Notes:
9030   Orthogonal quality is given by the following formula:
9031 
9032   \min \left[ \frac{A_i \cdot f_i}{\|A_i\| \|f_i\|} , \frac{A_i \cdot c_i}{\|A_i\| \|c_i\|} \right]
9033 
9034   Where A_i is the i'th face-normal vector, f_i is the vector from the cell centroid to the i'th face centroid, and c_i
9035   is the vector from the current cells centroid to the centroid of its i'th neighbor (which shares a face with the
9036   current cell). This computes the vector similarity between each cell face and its corresponding neighbor centroid by
9037   calculating the cosine of the angle between these vectors.
9038 
9039   Orthogonal quality ranges from 1 (best) to 0 (worst).
9040 
9041   This routine is mainly useful for FVM, however is not restricted to only FVM. The PetscFV object is optionally used to check for
9042   pre-computed FVM cell data, but if it is not passed in then this data will be computed.
9043 
9044   Cells are tagged if they have an orthogonal quality less than or equal to the absolute tolerance.
9045 
9046   Level: intermediate
9047 
9048 .seealso: DMPlexCheckCellShape(), DMCreateLabel()
9049 @*/
9050 PetscErrorCode DMPlexComputeOrthogonalQuality(DM dm, PetscFV fv, PetscReal atol, Vec *OrthQual, DMLabel *OrthQualLabel)
9051 {
9052   PetscInt                nc, cellHeight, cStart, cEnd, cell, cellIter = 0;
9053   PetscInt                *idx;
9054   PetscScalar             *oqVals;
9055   const PetscScalar       *cellGeomArr, *faceGeomArr;
9056   PetscReal               *ci, *fi, *Ai;
9057   MPI_Comm                comm;
9058   Vec                     cellgeom, facegeom;
9059   DM                      dmFace, dmCell;
9060   IS                      glob;
9061   ISLocalToGlobalMapping  ltog;
9062   PetscViewer             vwr;
9063   PetscErrorCode          ierr;
9064 
9065   PetscFunctionBegin;
9066   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9067   if (fv) {PetscValidHeaderSpecific(fv, PETSCFV_CLASSID, 2);}
9068   PetscValidPointer(OrthQual, 4);
9069   PetscCheck(atol >= 0.0 && atol <= 1.0,PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"Absolute tolerance %g not in [0,1]",(double)atol);
9070   ierr = PetscObjectGetComm((PetscObject) dm, &comm);CHKERRQ(ierr);
9071   ierr = DMGetDimension(dm, &nc);CHKERRQ(ierr);
9072   PetscCheckFalse(nc < 2,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONGSTATE, "DM must have dimension >= 2 (current %D)", nc);
9073   {
9074     DMPlexInterpolatedFlag interpFlag;
9075 
9076     ierr = DMPlexIsInterpolated(dm, &interpFlag);CHKERRQ(ierr);
9077     if (interpFlag != DMPLEX_INTERPOLATED_FULL) {
9078       PetscMPIInt rank;
9079 
9080       ierr = MPI_Comm_rank(comm, &rank);CHKERRMPI(ierr);
9081       SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONGSTATE, "DM must be fully interpolated, DM on rank %d is not fully interpolated", rank);
9082     }
9083   }
9084   if (OrthQualLabel) {
9085     PetscValidPointer(OrthQualLabel, 5);
9086     ierr = DMCreateLabel(dm, "Orthogonal_Quality");CHKERRQ(ierr);
9087     ierr = DMGetLabel(dm, "Orthogonal_Quality", OrthQualLabel);CHKERRQ(ierr);
9088   } else {*OrthQualLabel = NULL;}
9089   ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
9090   ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
9091   ierr = DMPlexCreateCellNumbering_Internal(dm, PETSC_TRUE, &glob);CHKERRQ(ierr);
9092   ierr = ISLocalToGlobalMappingCreateIS(glob, &ltog);CHKERRQ(ierr);
9093   ierr = ISLocalToGlobalMappingSetType(ltog, ISLOCALTOGLOBALMAPPINGHASH);CHKERRQ(ierr);
9094   ierr = VecCreate(comm, OrthQual);CHKERRQ(ierr);
9095   ierr = VecSetType(*OrthQual, VECSTANDARD);CHKERRQ(ierr);
9096   ierr = VecSetSizes(*OrthQual, cEnd-cStart, PETSC_DETERMINE);CHKERRQ(ierr);
9097   ierr = VecSetLocalToGlobalMapping(*OrthQual, ltog);CHKERRQ(ierr);
9098   ierr = VecSetUp(*OrthQual);CHKERRQ(ierr);
9099   ierr = ISDestroy(&glob);CHKERRQ(ierr);
9100   ierr = ISLocalToGlobalMappingDestroy(&ltog);CHKERRQ(ierr);
9101   ierr = DMPlexGetDataFVM(dm, fv, &cellgeom, &facegeom, NULL);CHKERRQ(ierr);
9102   ierr = VecGetArrayRead(cellgeom, &cellGeomArr);CHKERRQ(ierr);
9103   ierr = VecGetArrayRead(facegeom, &faceGeomArr);CHKERRQ(ierr);
9104   ierr = VecGetDM(cellgeom, &dmCell);CHKERRQ(ierr);
9105   ierr = VecGetDM(facegeom, &dmFace);CHKERRQ(ierr);
9106   ierr = PetscMalloc5(cEnd-cStart, &idx, cEnd-cStart, &oqVals, nc, &ci, nc, &fi, nc, &Ai);CHKERRQ(ierr);
9107   for (cell = cStart; cell < cEnd; cellIter++,cell++) {
9108     PetscInt           cellneigh, cellneighiter = 0, adjSize = PETSC_DETERMINE;
9109     PetscInt           cellarr[2], *adj = NULL;
9110     PetscScalar        *cArr, *fArr;
9111     PetscReal          minvalc = 1.0, minvalf = 1.0;
9112     PetscFVCellGeom    *cg;
9113 
9114     idx[cellIter] = cell-cStart;
9115     cellarr[0] = cell;
9116     /* Make indexing into cellGeom easier */
9117     ierr = DMPlexPointLocalRead(dmCell, cell, cellGeomArr, &cg);CHKERRQ(ierr);
9118     ierr = DMPlexGetAdjacency_Internal(dm, cell, PETSC_TRUE, PETSC_FALSE, PETSC_FALSE, &adjSize, &adj);CHKERRQ(ierr);
9119     /* Technically 1 too big, but easier than fiddling with empty adjacency array */
9120     ierr = PetscCalloc2(adjSize, &cArr, adjSize, &fArr);CHKERRQ(ierr);
9121     for (cellneigh = 0; cellneigh < adjSize; cellneighiter++,cellneigh++) {
9122       PetscInt         i;
9123       const PetscInt   neigh = adj[cellneigh];
9124       PetscReal        normci = 0, normfi = 0, normai = 0;
9125       PetscFVCellGeom  *cgneigh;
9126       PetscFVFaceGeom  *fg;
9127 
9128       /* Don't count ourselves in the neighbor list */
9129       if (neigh == cell) continue;
9130       ierr = DMPlexPointLocalRead(dmCell, neigh, cellGeomArr, &cgneigh);CHKERRQ(ierr);
9131       cellarr[1] = neigh;
9132       {
9133         PetscInt       numcovpts;
9134         const PetscInt *covpts;
9135 
9136         ierr = DMPlexGetMeet(dm, 2, cellarr, &numcovpts, &covpts);CHKERRQ(ierr);
9137         ierr = DMPlexPointLocalRead(dmFace, covpts[0], faceGeomArr, &fg);CHKERRQ(ierr);
9138         ierr = DMPlexRestoreMeet(dm, 2, cellarr, &numcovpts, &covpts);CHKERRQ(ierr);
9139       }
9140 
9141       /* Compute c_i, f_i and their norms */
9142       for (i = 0; i < nc; i++) {
9143         ci[i] = cgneigh->centroid[i] - cg->centroid[i];
9144         fi[i] = fg->centroid[i] - cg->centroid[i];
9145         Ai[i] = fg->normal[i];
9146         normci += PetscPowReal(ci[i], 2);
9147         normfi += PetscPowReal(fi[i], 2);
9148         normai += PetscPowReal(Ai[i], 2);
9149       }
9150       normci = PetscSqrtReal(normci);
9151       normfi = PetscSqrtReal(normfi);
9152       normai = PetscSqrtReal(normai);
9153 
9154       /* Normalize and compute for each face-cell-normal pair */
9155       for (i = 0; i < nc; i++) {
9156         ci[i] = ci[i]/normci;
9157         fi[i] = fi[i]/normfi;
9158         Ai[i] = Ai[i]/normai;
9159         /* PetscAbs because I don't know if normals are guaranteed to point out */
9160         cArr[cellneighiter] += PetscAbs(Ai[i]*ci[i]);
9161         fArr[cellneighiter] += PetscAbs(Ai[i]*fi[i]);
9162       }
9163       if (PetscRealPart(cArr[cellneighiter]) < minvalc) {
9164         minvalc = PetscRealPart(cArr[cellneighiter]);
9165       }
9166       if (PetscRealPart(fArr[cellneighiter]) < minvalf) {
9167         minvalf = PetscRealPart(fArr[cellneighiter]);
9168       }
9169     }
9170     ierr = PetscFree(adj);CHKERRQ(ierr);
9171     ierr = PetscFree2(cArr, fArr);CHKERRQ(ierr);
9172     /* Defer to cell if they're equal */
9173     oqVals[cellIter] = PetscMin(minvalf, minvalc);
9174     if (OrthQualLabel) {
9175       if (PetscRealPart(oqVals[cellIter]) <= atol) {ierr = DMLabelSetValue(*OrthQualLabel, cell, DM_ADAPT_REFINE);CHKERRQ(ierr);}
9176     }
9177   }
9178   ierr = VecSetValuesLocal(*OrthQual, cEnd-cStart, idx, oqVals, INSERT_VALUES);CHKERRQ(ierr);
9179   ierr = VecAssemblyBegin(*OrthQual);CHKERRQ(ierr);
9180   ierr = VecAssemblyEnd(*OrthQual);CHKERRQ(ierr);
9181   ierr = VecRestoreArrayRead(cellgeom, &cellGeomArr);CHKERRQ(ierr);
9182   ierr = VecRestoreArrayRead(facegeom, &faceGeomArr);CHKERRQ(ierr);
9183   ierr = PetscOptionsGetViewer(comm, NULL, NULL, "-dm_plex_orthogonal_quality_label_view", &vwr, NULL, NULL);CHKERRQ(ierr);
9184   if (OrthQualLabel) {
9185     if (vwr) {ierr = DMLabelView(*OrthQualLabel, vwr);CHKERRQ(ierr);}
9186   }
9187   ierr = PetscFree5(idx, oqVals, ci, fi, Ai);CHKERRQ(ierr);
9188   ierr = PetscViewerDestroy(&vwr);CHKERRQ(ierr);
9189   ierr = VecViewFromOptions(*OrthQual, NULL, "-dm_plex_orthogonal_quality_vec_view");CHKERRQ(ierr);
9190   PetscFunctionReturn(0);
9191 }
9192 
9193 /* this is here insead of DMGetOutputDM because output DM still has constraints in the local indices that affect
9194  * interpolator construction */
9195 static PetscErrorCode DMGetFullDM(DM dm, DM *odm)
9196 {
9197   PetscSection   section, newSection, gsection;
9198   PetscSF        sf;
9199   PetscBool      hasConstraints, ghasConstraints;
9200   PetscErrorCode ierr;
9201 
9202   PetscFunctionBegin;
9203   PetscValidHeaderSpecific(dm,DM_CLASSID,1);
9204   PetscValidPointer(odm,2);
9205   ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);
9206   ierr = PetscSectionHasConstraints(section, &hasConstraints);CHKERRQ(ierr);
9207   ierr = MPI_Allreduce(&hasConstraints, &ghasConstraints, 1, MPIU_BOOL, MPI_LOR, PetscObjectComm((PetscObject) dm));CHKERRMPI(ierr);
9208   if (!ghasConstraints) {
9209     ierr = PetscObjectReference((PetscObject)dm);CHKERRQ(ierr);
9210     *odm = dm;
9211     PetscFunctionReturn(0);
9212   }
9213   ierr = DMClone(dm, odm);CHKERRQ(ierr);
9214   ierr = DMCopyFields(dm, *odm);CHKERRQ(ierr);
9215   ierr = DMGetLocalSection(*odm, &newSection);CHKERRQ(ierr);
9216   ierr = DMGetPointSF(*odm, &sf);CHKERRQ(ierr);
9217   ierr = PetscSectionCreateGlobalSection(newSection, sf, PETSC_TRUE, PETSC_FALSE, &gsection);CHKERRQ(ierr);
9218   ierr = DMSetGlobalSection(*odm, gsection);CHKERRQ(ierr);
9219   ierr = PetscSectionDestroy(&gsection);CHKERRQ(ierr);
9220   PetscFunctionReturn(0);
9221 }
9222 
9223 static PetscErrorCode DMCreateAffineInterpolationCorrection_Plex(DM dmc, DM dmf, Vec *shift)
9224 {
9225   DM             dmco, dmfo;
9226   Mat            interpo;
9227   Vec            rscale;
9228   Vec            cglobalo, clocal;
9229   Vec            fglobal, fglobalo, flocal;
9230   PetscBool      regular;
9231   PetscErrorCode ierr;
9232 
9233   PetscFunctionBegin;
9234   ierr = DMGetFullDM(dmc, &dmco);CHKERRQ(ierr);
9235   ierr = DMGetFullDM(dmf, &dmfo);CHKERRQ(ierr);
9236   ierr = DMSetCoarseDM(dmfo, dmco);CHKERRQ(ierr);
9237   ierr = DMPlexGetRegularRefinement(dmf, &regular);CHKERRQ(ierr);
9238   ierr = DMPlexSetRegularRefinement(dmfo, regular);CHKERRQ(ierr);
9239   ierr = DMCreateInterpolation(dmco, dmfo, &interpo, &rscale);CHKERRQ(ierr);
9240   ierr = DMCreateGlobalVector(dmco, &cglobalo);CHKERRQ(ierr);
9241   ierr = DMCreateLocalVector(dmc, &clocal);CHKERRQ(ierr);
9242   ierr = VecSet(cglobalo, 0.);CHKERRQ(ierr);
9243   ierr = VecSet(clocal, 0.);CHKERRQ(ierr);
9244   ierr = DMCreateGlobalVector(dmf, &fglobal);CHKERRQ(ierr);
9245   ierr = DMCreateGlobalVector(dmfo, &fglobalo);CHKERRQ(ierr);
9246   ierr = DMCreateLocalVector(dmf, &flocal);CHKERRQ(ierr);
9247   ierr = VecSet(fglobal, 0.);CHKERRQ(ierr);
9248   ierr = VecSet(fglobalo, 0.);CHKERRQ(ierr);
9249   ierr = VecSet(flocal, 0.);CHKERRQ(ierr);
9250   ierr = DMPlexInsertBoundaryValues(dmc, PETSC_TRUE, clocal, 0., NULL, NULL, NULL);CHKERRQ(ierr);
9251   ierr = DMLocalToGlobalBegin(dmco, clocal, INSERT_VALUES, cglobalo);CHKERRQ(ierr);
9252   ierr = DMLocalToGlobalEnd(dmco, clocal, INSERT_VALUES, cglobalo);CHKERRQ(ierr);
9253   ierr = MatMult(interpo, cglobalo, fglobalo);CHKERRQ(ierr);
9254   ierr = DMGlobalToLocalBegin(dmfo, fglobalo, INSERT_VALUES, flocal);CHKERRQ(ierr);
9255   ierr = DMGlobalToLocalEnd(dmfo, fglobalo, INSERT_VALUES, flocal);CHKERRQ(ierr);
9256   ierr = DMLocalToGlobalBegin(dmf, flocal, INSERT_VALUES, fglobal);CHKERRQ(ierr);
9257   ierr = DMLocalToGlobalEnd(dmf, flocal, INSERT_VALUES, fglobal);CHKERRQ(ierr);
9258   *shift = fglobal;
9259   ierr = VecDestroy(&flocal);CHKERRQ(ierr);
9260   ierr = VecDestroy(&fglobalo);CHKERRQ(ierr);
9261   ierr = VecDestroy(&clocal);CHKERRQ(ierr);
9262   ierr = VecDestroy(&cglobalo);CHKERRQ(ierr);
9263   ierr = VecDestroy(&rscale);CHKERRQ(ierr);
9264   ierr = MatDestroy(&interpo);CHKERRQ(ierr);
9265   ierr = DMDestroy(&dmfo);CHKERRQ(ierr);
9266   ierr = DMDestroy(&dmco);CHKERRQ(ierr);
9267   PetscFunctionReturn(0);
9268 }
9269 
9270 PETSC_INTERN PetscErrorCode DMInterpolateSolution_Plex(DM coarse, DM fine, Mat interp, Vec coarseSol, Vec fineSol)
9271 {
9272   PetscObject    shifto;
9273   Vec            shift;
9274 
9275   PetscErrorCode ierr;
9276 
9277   PetscFunctionBegin;
9278   if (!interp) {
9279     Vec rscale;
9280 
9281     ierr = DMCreateInterpolation(coarse, fine, &interp, &rscale);CHKERRQ(ierr);
9282     ierr = VecDestroy(&rscale);CHKERRQ(ierr);
9283   } else {
9284     ierr = PetscObjectReference((PetscObject)interp);CHKERRQ(ierr);
9285   }
9286   ierr = PetscObjectQuery((PetscObject)interp, "_DMInterpolateSolution_Plex_Vec", &shifto);CHKERRQ(ierr);
9287   if (!shifto) {
9288     ierr = DMCreateAffineInterpolationCorrection_Plex(coarse, fine, &shift);CHKERRQ(ierr);
9289     ierr = PetscObjectCompose((PetscObject)interp, "_DMInterpolateSolution_Plex_Vec", (PetscObject) shift);CHKERRQ(ierr);
9290     shifto = (PetscObject) shift;
9291     ierr = VecDestroy(&shift);CHKERRQ(ierr);
9292   }
9293   shift = (Vec) shifto;
9294   ierr = MatInterpolate(interp, coarseSol, fineSol);CHKERRQ(ierr);
9295   ierr = VecAXPY(fineSol, 1.0, shift);CHKERRQ(ierr);
9296   ierr = MatDestroy(&interp);CHKERRQ(ierr);
9297   PetscFunctionReturn(0);
9298 }
9299 
9300 /* Pointwise interpolation
9301      Just code FEM for now
9302      u^f = I u^c
9303      sum_k u^f_k phi^f_k = I sum_j u^c_j phi^c_j
9304      u^f_i = sum_j psi^f_i I phi^c_j u^c_j
9305      I_{ij} = psi^f_i phi^c_j
9306 */
9307 PetscErrorCode DMCreateInterpolation_Plex(DM dmCoarse, DM dmFine, Mat *interpolation, Vec *scaling)
9308 {
9309   PetscSection   gsc, gsf;
9310   PetscInt       m, n;
9311   void          *ctx;
9312   DM             cdm;
9313   PetscBool      regular, ismatis, isRefined = dmCoarse->data == dmFine->data ? PETSC_FALSE : PETSC_TRUE;
9314   PetscErrorCode ierr;
9315 
9316   PetscFunctionBegin;
9317   ierr = DMGetGlobalSection(dmFine, &gsf);CHKERRQ(ierr);
9318   ierr = PetscSectionGetConstrainedStorageSize(gsf, &m);CHKERRQ(ierr);
9319   ierr = DMGetGlobalSection(dmCoarse, &gsc);CHKERRQ(ierr);
9320   ierr = PetscSectionGetConstrainedStorageSize(gsc, &n);CHKERRQ(ierr);
9321 
9322   ierr = PetscStrcmp(dmCoarse->mattype, MATIS, &ismatis);CHKERRQ(ierr);
9323   ierr = MatCreate(PetscObjectComm((PetscObject) dmCoarse), interpolation);CHKERRQ(ierr);
9324   ierr = MatSetSizes(*interpolation, m, n, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
9325   ierr = MatSetType(*interpolation, ismatis ? MATAIJ : dmCoarse->mattype);CHKERRQ(ierr);
9326   ierr = DMGetApplicationContext(dmFine, &ctx);CHKERRQ(ierr);
9327 
9328   ierr = DMGetCoarseDM(dmFine, &cdm);CHKERRQ(ierr);
9329   ierr = DMPlexGetRegularRefinement(dmFine, &regular);CHKERRQ(ierr);
9330   if (!isRefined || (regular && cdm == dmCoarse)) {ierr = DMPlexComputeInterpolatorNested(dmCoarse, dmFine, isRefined, *interpolation, ctx);CHKERRQ(ierr);}
9331   else                                            {ierr = DMPlexComputeInterpolatorGeneral(dmCoarse, dmFine, *interpolation, ctx);CHKERRQ(ierr);}
9332   ierr = MatViewFromOptions(*interpolation, NULL, "-interp_mat_view");CHKERRQ(ierr);
9333   if (scaling) {
9334     /* Use naive scaling */
9335     ierr = DMCreateInterpolationScale(dmCoarse, dmFine, *interpolation, scaling);CHKERRQ(ierr);
9336   }
9337   PetscFunctionReturn(0);
9338 }
9339 
9340 PetscErrorCode DMCreateInjection_Plex(DM dmCoarse, DM dmFine, Mat *mat)
9341 {
9342   PetscErrorCode ierr;
9343   VecScatter     ctx;
9344 
9345   PetscFunctionBegin;
9346   ierr = DMPlexComputeInjectorFEM(dmCoarse, dmFine, &ctx, NULL);CHKERRQ(ierr);
9347   ierr = MatCreateScatter(PetscObjectComm((PetscObject)ctx), ctx, mat);CHKERRQ(ierr);
9348   ierr = VecScatterDestroy(&ctx);CHKERRQ(ierr);
9349   PetscFunctionReturn(0);
9350 }
9351 
9352 static void g0_identity_private(PetscInt dim, PetscInt Nf, PetscInt NfAux,
9353                                 const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[],
9354                                 const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[],
9355                                 PetscReal t, PetscReal u_tShift, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar g0[])
9356 {
9357   const PetscInt Nc = uOff[1] - uOff[0];
9358   PetscInt       c;
9359   for (c = 0; c < Nc; ++c) g0[c*Nc+c] = 1.0;
9360 }
9361 
9362 PetscErrorCode DMCreateMassMatrixLumped_Plex(DM dm, Vec *mass)
9363 {
9364   DM             dmc;
9365   PetscDS        ds;
9366   Vec            ones, locmass;
9367   IS             cellIS;
9368   PetscFormKey   key;
9369   PetscInt       depth;
9370   PetscErrorCode ierr;
9371 
9372   PetscFunctionBegin;
9373   ierr = DMClone(dm, &dmc);CHKERRQ(ierr);
9374   ierr = DMCopyDisc(dm, dmc);CHKERRQ(ierr);
9375   ierr = DMGetDS(dmc, &ds);CHKERRQ(ierr);
9376   ierr = PetscDSSetJacobian(ds, 0, 0, g0_identity_private, NULL, NULL, NULL);CHKERRQ(ierr);
9377   ierr = DMCreateGlobalVector(dmc, mass);CHKERRQ(ierr);
9378   ierr = DMGetLocalVector(dmc, &ones);CHKERRQ(ierr);
9379   ierr = DMGetLocalVector(dmc, &locmass);CHKERRQ(ierr);
9380   ierr = DMPlexGetDepth(dmc, &depth);CHKERRQ(ierr);
9381   ierr = DMGetStratumIS(dmc, "depth", depth, &cellIS);CHKERRQ(ierr);
9382   ierr = VecSet(locmass, 0.0);CHKERRQ(ierr);
9383   ierr = VecSet(ones, 1.0);CHKERRQ(ierr);
9384   key.label = NULL;
9385   key.value = 0;
9386   key.field = 0;
9387   key.part  = 0;
9388   ierr = DMPlexComputeJacobian_Action_Internal(dmc, key, cellIS, 0.0, 0.0, ones, NULL, ones, locmass, NULL);CHKERRQ(ierr);
9389   ierr = ISDestroy(&cellIS);CHKERRQ(ierr);
9390   ierr = VecSet(*mass, 0.0);CHKERRQ(ierr);
9391   ierr = DMLocalToGlobalBegin(dmc, locmass, ADD_VALUES, *mass);CHKERRQ(ierr);
9392   ierr = DMLocalToGlobalEnd(dmc, locmass, ADD_VALUES, *mass);CHKERRQ(ierr);
9393   ierr = DMRestoreLocalVector(dmc, &ones);CHKERRQ(ierr);
9394   ierr = DMRestoreLocalVector(dmc, &locmass);CHKERRQ(ierr);
9395   ierr = DMDestroy(&dmc);CHKERRQ(ierr);
9396   PetscFunctionReturn(0);
9397 }
9398 
9399 PetscErrorCode DMCreateMassMatrix_Plex(DM dmCoarse, DM dmFine, Mat *mass)
9400 {
9401   PetscSection   gsc, gsf;
9402   PetscInt       m, n;
9403   void          *ctx;
9404   DM             cdm;
9405   PetscBool      regular;
9406   PetscErrorCode ierr;
9407 
9408   PetscFunctionBegin;
9409   if (dmFine == dmCoarse) {
9410     DM            dmc;
9411     PetscDS       ds;
9412     PetscWeakForm wf;
9413     Vec           u;
9414     IS            cellIS;
9415     PetscFormKey  key;
9416     PetscInt      depth;
9417 
9418     ierr = DMClone(dmFine, &dmc);CHKERRQ(ierr);
9419     ierr = DMCopyDisc(dmFine, dmc);CHKERRQ(ierr);
9420     ierr = DMGetDS(dmc, &ds);CHKERRQ(ierr);
9421     ierr = PetscDSGetWeakForm(ds, &wf);CHKERRQ(ierr);
9422     ierr = PetscWeakFormClear(wf);CHKERRQ(ierr);
9423     ierr = PetscDSSetJacobian(ds, 0, 0, g0_identity_private, NULL, NULL, NULL);CHKERRQ(ierr);
9424     ierr = DMCreateMatrix(dmc, mass);CHKERRQ(ierr);
9425     ierr = DMGetGlobalVector(dmc, &u);CHKERRQ(ierr);
9426     ierr = DMPlexGetDepth(dmc, &depth);CHKERRQ(ierr);
9427     ierr = DMGetStratumIS(dmc, "depth", depth, &cellIS);CHKERRQ(ierr);
9428     ierr = MatZeroEntries(*mass);CHKERRQ(ierr);
9429     key.label = NULL;
9430     key.value = 0;
9431     key.field = 0;
9432     key.part  = 0;
9433     ierr = DMPlexComputeJacobian_Internal(dmc, key, cellIS, 0.0, 0.0, u, NULL, *mass, *mass, NULL);CHKERRQ(ierr);
9434     ierr = ISDestroy(&cellIS);CHKERRQ(ierr);
9435     ierr = DMRestoreGlobalVector(dmc, &u);CHKERRQ(ierr);
9436     ierr = DMDestroy(&dmc);CHKERRQ(ierr);
9437   } else {
9438     ierr = DMGetGlobalSection(dmFine, &gsf);CHKERRQ(ierr);
9439     ierr = PetscSectionGetConstrainedStorageSize(gsf, &m);CHKERRQ(ierr);
9440     ierr = DMGetGlobalSection(dmCoarse, &gsc);CHKERRQ(ierr);
9441     ierr = PetscSectionGetConstrainedStorageSize(gsc, &n);CHKERRQ(ierr);
9442 
9443     ierr = MatCreate(PetscObjectComm((PetscObject) dmCoarse), mass);CHKERRQ(ierr);
9444     ierr = MatSetSizes(*mass, m, n, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
9445     ierr = MatSetType(*mass, dmCoarse->mattype);CHKERRQ(ierr);
9446     ierr = DMGetApplicationContext(dmFine, &ctx);CHKERRQ(ierr);
9447 
9448     ierr = DMGetCoarseDM(dmFine, &cdm);CHKERRQ(ierr);
9449     ierr = DMPlexGetRegularRefinement(dmFine, &regular);CHKERRQ(ierr);
9450     if (regular && cdm == dmCoarse) {ierr = DMPlexComputeMassMatrixNested(dmCoarse, dmFine, *mass, ctx);CHKERRQ(ierr);}
9451     else                            {ierr = DMPlexComputeMassMatrixGeneral(dmCoarse, dmFine, *mass, ctx);CHKERRQ(ierr);}
9452   }
9453   ierr = MatViewFromOptions(*mass, NULL, "-mass_mat_view");CHKERRQ(ierr);
9454   PetscFunctionReturn(0);
9455 }
9456 
9457 /*@
9458   DMPlexGetRegularRefinement - Get the flag indicating that this mesh was obtained by regular refinement from its coarse mesh
9459 
9460   Input Parameter:
9461 . dm - The DMPlex object
9462 
9463   Output Parameter:
9464 . regular - The flag
9465 
9466   Level: intermediate
9467 
9468 .seealso: DMPlexSetRegularRefinement()
9469 @*/
9470 PetscErrorCode DMPlexGetRegularRefinement(DM dm, PetscBool *regular)
9471 {
9472   PetscFunctionBegin;
9473   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9474   PetscValidPointer(regular, 2);
9475   *regular = ((DM_Plex *) dm->data)->regularRefinement;
9476   PetscFunctionReturn(0);
9477 }
9478 
9479 /*@
9480   DMPlexSetRegularRefinement - Set the flag indicating that this mesh was obtained by regular refinement from its coarse mesh
9481 
9482   Input Parameters:
9483 + dm - The DMPlex object
9484 - regular - The flag
9485 
9486   Level: intermediate
9487 
9488 .seealso: DMPlexGetRegularRefinement()
9489 @*/
9490 PetscErrorCode DMPlexSetRegularRefinement(DM dm, PetscBool regular)
9491 {
9492   PetscFunctionBegin;
9493   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9494   ((DM_Plex *) dm->data)->regularRefinement = regular;
9495   PetscFunctionReturn(0);
9496 }
9497 
9498 /* anchors */
9499 /*@
9500   DMPlexGetAnchors - Get the layout of the anchor (point-to-point) constraints.  Typically, the user will not have to
9501   call DMPlexGetAnchors() directly: if there are anchors, then DMPlexGetAnchors() is called during DMGetDefaultConstraints().
9502 
9503   not collective
9504 
9505   Input Parameter:
9506 . dm - The DMPlex object
9507 
9508   Output Parameters:
9509 + anchorSection - If not NULL, set to the section describing which points anchor the constrained points.
9510 - anchorIS - If not NULL, set to the list of anchors indexed by anchorSection
9511 
9512   Level: intermediate
9513 
9514 .seealso: DMPlexSetAnchors(), DMGetDefaultConstraints(), DMSetDefaultConstraints()
9515 @*/
9516 PetscErrorCode DMPlexGetAnchors(DM dm, PetscSection *anchorSection, IS *anchorIS)
9517 {
9518   DM_Plex *plex = (DM_Plex *)dm->data;
9519   PetscErrorCode ierr;
9520 
9521   PetscFunctionBegin;
9522   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9523   if (!plex->anchorSection && !plex->anchorIS && plex->createanchors) {ierr = (*plex->createanchors)(dm);CHKERRQ(ierr);}
9524   if (anchorSection) *anchorSection = plex->anchorSection;
9525   if (anchorIS) *anchorIS = plex->anchorIS;
9526   PetscFunctionReturn(0);
9527 }
9528 
9529 /*@
9530   DMPlexSetAnchors - Set the layout of the local anchor (point-to-point) constraints.  Unlike boundary conditions,
9531   when a point's degrees of freedom in a section are constrained to an outside value, the anchor constraints set a
9532   point's degrees of freedom to be a linear combination of other points' degrees of freedom.
9533 
9534   After specifying the layout of constraints with DMPlexSetAnchors(), one specifies the constraints by calling
9535   DMGetDefaultConstraints() and filling in the entries in the constraint matrix.
9536 
9537   collective on dm
9538 
9539   Input Parameters:
9540 + dm - The DMPlex object
9541 . anchorSection - The section that describes the mapping from constrained points to the anchor points listed in anchorIS.  Must have a local communicator (PETSC_COMM_SELF or derivative).
9542 - anchorIS - The list of all anchor points.  Must have a local communicator (PETSC_COMM_SELF or derivative).
9543 
9544   The reference counts of anchorSection and anchorIS are incremented.
9545 
9546   Level: intermediate
9547 
9548 .seealso: DMPlexGetAnchors(), DMGetDefaultConstraints(), DMSetDefaultConstraints()
9549 @*/
9550 PetscErrorCode DMPlexSetAnchors(DM dm, PetscSection anchorSection, IS anchorIS)
9551 {
9552   DM_Plex        *plex = (DM_Plex *)dm->data;
9553   PetscMPIInt    result;
9554   PetscErrorCode ierr;
9555 
9556   PetscFunctionBegin;
9557   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9558   if (anchorSection) {
9559     PetscValidHeaderSpecific(anchorSection,PETSC_SECTION_CLASSID,2);
9560     ierr = MPI_Comm_compare(PETSC_COMM_SELF,PetscObjectComm((PetscObject)anchorSection),&result);CHKERRMPI(ierr);
9561     PetscCheckFalse(result != MPI_CONGRUENT && result != MPI_IDENT,PETSC_COMM_SELF,PETSC_ERR_ARG_NOTSAMECOMM,"anchor section must have local communicator");
9562   }
9563   if (anchorIS) {
9564     PetscValidHeaderSpecific(anchorIS,IS_CLASSID,3);
9565     ierr = MPI_Comm_compare(PETSC_COMM_SELF,PetscObjectComm((PetscObject)anchorIS),&result);CHKERRMPI(ierr);
9566     PetscCheckFalse(result != MPI_CONGRUENT && result != MPI_IDENT,PETSC_COMM_SELF,PETSC_ERR_ARG_NOTSAMECOMM,"anchor IS must have local communicator");
9567   }
9568 
9569   ierr = PetscObjectReference((PetscObject)anchorSection);CHKERRQ(ierr);
9570   ierr = PetscSectionDestroy(&plex->anchorSection);CHKERRQ(ierr);
9571   plex->anchorSection = anchorSection;
9572 
9573   ierr = PetscObjectReference((PetscObject)anchorIS);CHKERRQ(ierr);
9574   ierr = ISDestroy(&plex->anchorIS);CHKERRQ(ierr);
9575   plex->anchorIS = anchorIS;
9576 
9577   if (PetscUnlikelyDebug(anchorIS && anchorSection)) {
9578     PetscInt size, a, pStart, pEnd;
9579     const PetscInt *anchors;
9580 
9581     ierr = PetscSectionGetChart(anchorSection,&pStart,&pEnd);CHKERRQ(ierr);
9582     ierr = ISGetLocalSize(anchorIS,&size);CHKERRQ(ierr);
9583     ierr = ISGetIndices(anchorIS,&anchors);CHKERRQ(ierr);
9584     for (a = 0; a < size; a++) {
9585       PetscInt p;
9586 
9587       p = anchors[a];
9588       if (p >= pStart && p < pEnd) {
9589         PetscInt dof;
9590 
9591         ierr = PetscSectionGetDof(anchorSection,p,&dof);CHKERRQ(ierr);
9592         if (dof) {
9593           PetscErrorCode ierr2;
9594 
9595           ierr2 = ISRestoreIndices(anchorIS,&anchors);CHKERRQ(ierr2);
9596           SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_INCOMP,"Point %D cannot be constrained and an anchor",p);
9597         }
9598       }
9599     }
9600     ierr = ISRestoreIndices(anchorIS,&anchors);CHKERRQ(ierr);
9601   }
9602   /* reset the generic constraints */
9603   ierr = DMSetDefaultConstraints(dm,NULL,NULL,NULL);CHKERRQ(ierr);
9604   PetscFunctionReturn(0);
9605 }
9606 
9607 static PetscErrorCode DMPlexCreateConstraintSection_Anchors(DM dm, PetscSection section, PetscSection *cSec)
9608 {
9609   PetscSection anchorSection;
9610   PetscInt pStart, pEnd, sStart, sEnd, p, dof, numFields, f;
9611   PetscErrorCode ierr;
9612 
9613   PetscFunctionBegin;
9614   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9615   ierr = DMPlexGetAnchors(dm,&anchorSection,NULL);CHKERRQ(ierr);
9616   ierr = PetscSectionCreate(PETSC_COMM_SELF,cSec);CHKERRQ(ierr);
9617   ierr = PetscSectionGetNumFields(section,&numFields);CHKERRQ(ierr);
9618   if (numFields) {
9619     PetscInt f;
9620     ierr = PetscSectionSetNumFields(*cSec,numFields);CHKERRQ(ierr);
9621 
9622     for (f = 0; f < numFields; f++) {
9623       PetscInt numComp;
9624 
9625       ierr = PetscSectionGetFieldComponents(section,f,&numComp);CHKERRQ(ierr);
9626       ierr = PetscSectionSetFieldComponents(*cSec,f,numComp);CHKERRQ(ierr);
9627     }
9628   }
9629   ierr = PetscSectionGetChart(anchorSection,&pStart,&pEnd);CHKERRQ(ierr);
9630   ierr = PetscSectionGetChart(section,&sStart,&sEnd);CHKERRQ(ierr);
9631   pStart = PetscMax(pStart,sStart);
9632   pEnd   = PetscMin(pEnd,sEnd);
9633   pEnd   = PetscMax(pStart,pEnd);
9634   ierr = PetscSectionSetChart(*cSec,pStart,pEnd);CHKERRQ(ierr);
9635   for (p = pStart; p < pEnd; p++) {
9636     ierr = PetscSectionGetDof(anchorSection,p,&dof);CHKERRQ(ierr);
9637     if (dof) {
9638       ierr = PetscSectionGetDof(section,p,&dof);CHKERRQ(ierr);
9639       ierr = PetscSectionSetDof(*cSec,p,dof);CHKERRQ(ierr);
9640       for (f = 0; f < numFields; f++) {
9641         ierr = PetscSectionGetFieldDof(section,p,f,&dof);CHKERRQ(ierr);
9642         ierr = PetscSectionSetFieldDof(*cSec,p,f,dof);CHKERRQ(ierr);
9643       }
9644     }
9645   }
9646   ierr = PetscSectionSetUp(*cSec);CHKERRQ(ierr);
9647   ierr = PetscObjectSetName((PetscObject) *cSec, "Constraint Section");CHKERRQ(ierr);
9648   PetscFunctionReturn(0);
9649 }
9650 
9651 static PetscErrorCode DMPlexCreateConstraintMatrix_Anchors(DM dm, PetscSection section, PetscSection cSec, Mat *cMat)
9652 {
9653   PetscSection   aSec;
9654   PetscInt       pStart, pEnd, p, sStart, sEnd, dof, aDof, aOff, off, nnz, annz, m, n, q, a, offset, *i, *j;
9655   const PetscInt *anchors;
9656   PetscInt       numFields, f;
9657   IS             aIS;
9658   PetscErrorCode ierr;
9659   MatType        mtype;
9660   PetscBool      iscuda,iskokkos;
9661 
9662   PetscFunctionBegin;
9663   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9664   ierr = PetscSectionGetStorageSize(cSec, &m);CHKERRQ(ierr);
9665   ierr = PetscSectionGetStorageSize(section, &n);CHKERRQ(ierr);
9666   ierr = MatCreate(PETSC_COMM_SELF,cMat);CHKERRQ(ierr);
9667   ierr = MatSetSizes(*cMat,m,n,m,n);CHKERRQ(ierr);
9668   ierr = PetscStrcmp(dm->mattype,MATSEQAIJCUSPARSE,&iscuda);CHKERRQ(ierr);
9669   if (!iscuda) { ierr = PetscStrcmp(dm->mattype,MATMPIAIJCUSPARSE,&iscuda);CHKERRQ(ierr); }
9670   ierr = PetscStrcmp(dm->mattype,MATSEQAIJKOKKOS,&iskokkos);CHKERRQ(ierr);
9671   if (!iskokkos) { ierr = PetscStrcmp(dm->mattype,MATMPIAIJKOKKOS,&iskokkos);CHKERRQ(ierr); }
9672   if (iscuda) mtype = MATSEQAIJCUSPARSE;
9673   else if (iskokkos) mtype = MATSEQAIJKOKKOS;
9674   else mtype = MATSEQAIJ;
9675   ierr = MatSetType(*cMat,mtype);CHKERRQ(ierr);
9676   ierr = DMPlexGetAnchors(dm,&aSec,&aIS);CHKERRQ(ierr);
9677   ierr = ISGetIndices(aIS,&anchors);CHKERRQ(ierr);
9678   /* cSec will be a subset of aSec and section */
9679   ierr = PetscSectionGetChart(cSec,&pStart,&pEnd);CHKERRQ(ierr);
9680   ierr = PetscSectionGetChart(section,&sStart,&sEnd);CHKERRQ(ierr);
9681   ierr = PetscMalloc1(m+1,&i);CHKERRQ(ierr);
9682   i[0] = 0;
9683   ierr = PetscSectionGetNumFields(section,&numFields);CHKERRQ(ierr);
9684   for (p = pStart; p < pEnd; p++) {
9685     PetscInt rDof, rOff, r;
9686 
9687     ierr = PetscSectionGetDof(aSec,p,&rDof);CHKERRQ(ierr);
9688     if (!rDof) continue;
9689     ierr = PetscSectionGetOffset(aSec,p,&rOff);CHKERRQ(ierr);
9690     if (numFields) {
9691       for (f = 0; f < numFields; f++) {
9692         annz = 0;
9693         for (r = 0; r < rDof; r++) {
9694           a = anchors[rOff + r];
9695           if (a < sStart || a >= sEnd) continue;
9696           ierr = PetscSectionGetFieldDof(section,a,f,&aDof);CHKERRQ(ierr);
9697           annz += aDof;
9698         }
9699         ierr = PetscSectionGetFieldDof(cSec,p,f,&dof);CHKERRQ(ierr);
9700         ierr = PetscSectionGetFieldOffset(cSec,p,f,&off);CHKERRQ(ierr);
9701         for (q = 0; q < dof; q++) {
9702           i[off + q + 1] = i[off + q] + annz;
9703         }
9704       }
9705     } else {
9706       annz = 0;
9707       ierr = PetscSectionGetDof(cSec,p,&dof);CHKERRQ(ierr);
9708       for (q = 0; q < dof; q++) {
9709         a = anchors[rOff + q];
9710         if (a < sStart || a >= sEnd) continue;
9711         ierr = PetscSectionGetDof(section,a,&aDof);CHKERRQ(ierr);
9712         annz += aDof;
9713       }
9714       ierr = PetscSectionGetDof(cSec,p,&dof);CHKERRQ(ierr);
9715       ierr = PetscSectionGetOffset(cSec,p,&off);CHKERRQ(ierr);
9716       for (q = 0; q < dof; q++) {
9717         i[off + q + 1] = i[off + q] + annz;
9718       }
9719     }
9720   }
9721   nnz = i[m];
9722   ierr = PetscMalloc1(nnz,&j);CHKERRQ(ierr);
9723   offset = 0;
9724   for (p = pStart; p < pEnd; p++) {
9725     if (numFields) {
9726       for (f = 0; f < numFields; f++) {
9727         ierr = PetscSectionGetFieldDof(cSec,p,f,&dof);CHKERRQ(ierr);
9728         for (q = 0; q < dof; q++) {
9729           PetscInt rDof, rOff, r;
9730           ierr = PetscSectionGetDof(aSec,p,&rDof);CHKERRQ(ierr);
9731           ierr = PetscSectionGetOffset(aSec,p,&rOff);CHKERRQ(ierr);
9732           for (r = 0; r < rDof; r++) {
9733             PetscInt s;
9734 
9735             a = anchors[rOff + r];
9736             if (a < sStart || a >= sEnd) continue;
9737             ierr = PetscSectionGetFieldDof(section,a,f,&aDof);CHKERRQ(ierr);
9738             ierr = PetscSectionGetFieldOffset(section,a,f,&aOff);CHKERRQ(ierr);
9739             for (s = 0; s < aDof; s++) {
9740               j[offset++] = aOff + s;
9741             }
9742           }
9743         }
9744       }
9745     } else {
9746       ierr = PetscSectionGetDof(cSec,p,&dof);CHKERRQ(ierr);
9747       for (q = 0; q < dof; q++) {
9748         PetscInt rDof, rOff, r;
9749         ierr = PetscSectionGetDof(aSec,p,&rDof);CHKERRQ(ierr);
9750         ierr = PetscSectionGetOffset(aSec,p,&rOff);CHKERRQ(ierr);
9751         for (r = 0; r < rDof; r++) {
9752           PetscInt s;
9753 
9754           a = anchors[rOff + r];
9755           if (a < sStart || a >= sEnd) continue;
9756           ierr = PetscSectionGetDof(section,a,&aDof);CHKERRQ(ierr);
9757           ierr = PetscSectionGetOffset(section,a,&aOff);CHKERRQ(ierr);
9758           for (s = 0; s < aDof; s++) {
9759             j[offset++] = aOff + s;
9760           }
9761         }
9762       }
9763     }
9764   }
9765   ierr = MatSeqAIJSetPreallocationCSR(*cMat,i,j,NULL);CHKERRQ(ierr);
9766   ierr = PetscFree(i);CHKERRQ(ierr);
9767   ierr = PetscFree(j);CHKERRQ(ierr);
9768   ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);
9769   PetscFunctionReturn(0);
9770 }
9771 
9772 PetscErrorCode DMCreateDefaultConstraints_Plex(DM dm)
9773 {
9774   DM_Plex        *plex = (DM_Plex *)dm->data;
9775   PetscSection   anchorSection, section, cSec;
9776   Mat            cMat;
9777   PetscErrorCode ierr;
9778 
9779   PetscFunctionBegin;
9780   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9781   ierr = DMPlexGetAnchors(dm,&anchorSection,NULL);CHKERRQ(ierr);
9782   if (anchorSection) {
9783     PetscInt Nf;
9784 
9785     ierr = DMGetLocalSection(dm,&section);CHKERRQ(ierr);
9786     ierr = DMPlexCreateConstraintSection_Anchors(dm,section,&cSec);CHKERRQ(ierr);
9787     ierr = DMPlexCreateConstraintMatrix_Anchors(dm,section,cSec,&cMat);CHKERRQ(ierr);
9788     ierr = DMGetNumFields(dm,&Nf);CHKERRQ(ierr);
9789     if (Nf && plex->computeanchormatrix) {ierr = (*plex->computeanchormatrix)(dm,section,cSec,cMat);CHKERRQ(ierr);}
9790     ierr = DMSetDefaultConstraints(dm,cSec,cMat,NULL);CHKERRQ(ierr);
9791     ierr = PetscSectionDestroy(&cSec);CHKERRQ(ierr);
9792     ierr = MatDestroy(&cMat);CHKERRQ(ierr);
9793   }
9794   PetscFunctionReturn(0);
9795 }
9796 
9797 PetscErrorCode DMCreateSubDomainDM_Plex(DM dm, DMLabel label, PetscInt value, IS *is, DM *subdm)
9798 {
9799   IS             subis;
9800   PetscSection   section, subsection;
9801   PetscErrorCode ierr;
9802 
9803   PetscFunctionBegin;
9804   ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);
9805   PetscCheckFalse(!section,PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Must set default section for DM before splitting subdomain");
9806   PetscCheckFalse(!subdm,PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Must set output subDM for splitting subdomain");
9807   /* Create subdomain */
9808   ierr = DMPlexFilter(dm, label, value, subdm);CHKERRQ(ierr);
9809   /* Create submodel */
9810   ierr = DMPlexGetSubpointIS(*subdm, &subis);CHKERRQ(ierr);
9811   ierr = PetscSectionCreateSubmeshSection(section, subis, &subsection);CHKERRQ(ierr);
9812   ierr = DMSetLocalSection(*subdm, subsection);CHKERRQ(ierr);
9813   ierr = PetscSectionDestroy(&subsection);CHKERRQ(ierr);
9814   ierr = DMCopyDisc(dm, *subdm);CHKERRQ(ierr);
9815   /* Create map from submodel to global model */
9816   if (is) {
9817     PetscSection    sectionGlobal, subsectionGlobal;
9818     IS              spIS;
9819     const PetscInt *spmap;
9820     PetscInt       *subIndices;
9821     PetscInt        subSize = 0, subOff = 0, pStart, pEnd, p;
9822     PetscInt        Nf, f, bs = -1, bsLocal[2], bsMinMax[2];
9823 
9824     ierr = DMPlexGetSubpointIS(*subdm, &spIS);CHKERRQ(ierr);
9825     ierr = ISGetIndices(spIS, &spmap);CHKERRQ(ierr);
9826     ierr = PetscSectionGetNumFields(section, &Nf);CHKERRQ(ierr);
9827     ierr = DMGetGlobalSection(dm, &sectionGlobal);CHKERRQ(ierr);
9828     ierr = DMGetGlobalSection(*subdm, &subsectionGlobal);CHKERRQ(ierr);
9829     ierr = PetscSectionGetChart(subsection, &pStart, &pEnd);CHKERRQ(ierr);
9830     for (p = pStart; p < pEnd; ++p) {
9831       PetscInt gdof, pSubSize  = 0;
9832 
9833       ierr = PetscSectionGetDof(sectionGlobal, p, &gdof);CHKERRQ(ierr);
9834       if (gdof > 0) {
9835         for (f = 0; f < Nf; ++f) {
9836           PetscInt fdof, fcdof;
9837 
9838           ierr     = PetscSectionGetFieldDof(subsection, p, f, &fdof);CHKERRQ(ierr);
9839           ierr     = PetscSectionGetFieldConstraintDof(subsection, p, f, &fcdof);CHKERRQ(ierr);
9840           pSubSize += fdof-fcdof;
9841         }
9842         subSize += pSubSize;
9843         if (pSubSize) {
9844           if (bs < 0) {
9845             bs = pSubSize;
9846           } else if (bs != pSubSize) {
9847             /* Layout does not admit a pointwise block size */
9848             bs = 1;
9849           }
9850         }
9851       }
9852     }
9853     /* Must have same blocksize on all procs (some might have no points) */
9854     bsLocal[0] = bs < 0 ? PETSC_MAX_INT : bs; bsLocal[1] = bs;
9855     ierr = PetscGlobalMinMaxInt(PetscObjectComm((PetscObject) dm), bsLocal, bsMinMax);CHKERRQ(ierr);
9856     if (bsMinMax[0] != bsMinMax[1]) {bs = 1;}
9857     else                            {bs = bsMinMax[0];}
9858     ierr = PetscMalloc1(subSize, &subIndices);CHKERRQ(ierr);
9859     for (p = pStart; p < pEnd; ++p) {
9860       PetscInt gdof, goff;
9861 
9862       ierr = PetscSectionGetDof(subsectionGlobal, p, &gdof);CHKERRQ(ierr);
9863       if (gdof > 0) {
9864         const PetscInt point = spmap[p];
9865 
9866         ierr = PetscSectionGetOffset(sectionGlobal, point, &goff);CHKERRQ(ierr);
9867         for (f = 0; f < Nf; ++f) {
9868           PetscInt fdof, fcdof, fc, f2, poff = 0;
9869 
9870           /* Can get rid of this loop by storing field information in the global section */
9871           for (f2 = 0; f2 < f; ++f2) {
9872             ierr  = PetscSectionGetFieldDof(section, p, f2, &fdof);CHKERRQ(ierr);
9873             ierr  = PetscSectionGetFieldConstraintDof(section, p, f2, &fcdof);CHKERRQ(ierr);
9874             poff += fdof-fcdof;
9875           }
9876           ierr = PetscSectionGetFieldDof(section, p, f, &fdof);CHKERRQ(ierr);
9877           ierr = PetscSectionGetFieldConstraintDof(section, p, f, &fcdof);CHKERRQ(ierr);
9878           for (fc = 0; fc < fdof-fcdof; ++fc, ++subOff) {
9879             subIndices[subOff] = goff+poff+fc;
9880           }
9881         }
9882       }
9883     }
9884     ierr = ISRestoreIndices(spIS, &spmap);CHKERRQ(ierr);
9885     ierr = ISCreateGeneral(PetscObjectComm((PetscObject)dm), subSize, subIndices, PETSC_OWN_POINTER, is);CHKERRQ(ierr);
9886     if (bs > 1) {
9887       /* We need to check that the block size does not come from non-contiguous fields */
9888       PetscInt i, j, set = 1;
9889       for (i = 0; i < subSize; i += bs) {
9890         for (j = 0; j < bs; ++j) {
9891           if (subIndices[i+j] != subIndices[i]+j) {set = 0; break;}
9892         }
9893       }
9894       if (set) {ierr = ISSetBlockSize(*is, bs);CHKERRQ(ierr);}
9895     }
9896     /* Attach nullspace */
9897     for (f = 0; f < Nf; ++f) {
9898       (*subdm)->nullspaceConstructors[f] = dm->nullspaceConstructors[f];
9899       if ((*subdm)->nullspaceConstructors[f]) break;
9900     }
9901     if (f < Nf) {
9902       MatNullSpace nullSpace;
9903       ierr = (*(*subdm)->nullspaceConstructors[f])(*subdm, f, f, &nullSpace);CHKERRQ(ierr);
9904 
9905       ierr = PetscObjectCompose((PetscObject) *is, "nullspace", (PetscObject) nullSpace);CHKERRQ(ierr);
9906       ierr = MatNullSpaceDestroy(&nullSpace);CHKERRQ(ierr);
9907     }
9908   }
9909   PetscFunctionReturn(0);
9910 }
9911 
9912 /*@
9913   DMPlexMonitorThroughput - Report the cell throughput of FE integration
9914 
9915   Input Parameter:
9916 - dm - The DM
9917 
9918   Level: developer
9919 
9920   Options Database Keys:
9921 . -dm_plex_monitor_throughput - Activate the monitor
9922 
9923 .seealso: DMSetFromOptions(), DMPlexCreate()
9924 @*/
9925 PetscErrorCode DMPlexMonitorThroughput(DM dm, void *dummy)
9926 {
9927 #if defined(PETSC_USE_LOG)
9928   PetscStageLog      stageLog;
9929   PetscLogEvent      event;
9930   PetscLogStage      stage;
9931   PetscEventPerfInfo eventInfo;
9932   PetscReal          cellRate, flopRate;
9933   PetscInt           cStart, cEnd, Nf, N;
9934   const char        *name;
9935   PetscErrorCode     ierr;
9936 #endif
9937 
9938   PetscFunctionBegin;
9939   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9940 #if defined(PETSC_USE_LOG)
9941   ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
9942   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
9943   ierr = DMGetNumFields(dm, &Nf);CHKERRQ(ierr);
9944   ierr = PetscLogGetStageLog(&stageLog);CHKERRQ(ierr);
9945   ierr = PetscStageLogGetCurrent(stageLog, &stage);CHKERRQ(ierr);
9946   ierr = PetscLogEventGetId("DMPlexResidualFE", &event);CHKERRQ(ierr);
9947   ierr = PetscLogEventGetPerfInfo(stage, event, &eventInfo);CHKERRQ(ierr);
9948   N        = (cEnd - cStart)*Nf*eventInfo.count;
9949   flopRate = eventInfo.flops/eventInfo.time;
9950   cellRate = N/eventInfo.time;
9951   ierr = PetscPrintf(PetscObjectComm((PetscObject) dm), "DM (%s) FE Residual Integration: %D integrals %D reps\n  Cell rate: %.2g/s flop rate: %.2g MF/s\n", name ? name : "unknown", N, eventInfo.count, (double) cellRate, (double) (flopRate/1.e6));CHKERRQ(ierr);
9952 #else
9953   SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Plex Throughput Monitor is not supported if logging is turned off. Reconfigure using --with-log.");
9954 #endif
9955   PetscFunctionReturn(0);
9956 }
9957