xref: /petsc/src/dm/impls/plex/plex.c (revision 1e1ea65d8de51fde77ce8a787efbef25e407badc) 
1 #include <petsc/private/dmpleximpl.h>   /*I      "petscdmplex.h"   I*/
2 #include <petsc/private/isimpl.h>
3 #include <petsc/private/vecimpl.h>
4 #include <petsc/private/glvisvecimpl.h>
5 #include <petscsf.h>
6 #include <petscds.h>
7 #include <petscdraw.h>
8 #include <petscdmfield.h>
9 
10 /* Logging support */
11 PetscLogEvent DMPLEX_Interpolate, DMPLEX_Partition, DMPLEX_Distribute, DMPLEX_DistributeCones, DMPLEX_DistributeLabels, DMPLEX_DistributeSF, DMPLEX_DistributeOverlap, DMPLEX_DistributeField, DMPLEX_DistributeData, DMPLEX_Migrate, DMPLEX_InterpolateSF, DMPLEX_GlobalToNaturalBegin, DMPLEX_GlobalToNaturalEnd, DMPLEX_NaturalToGlobalBegin, DMPLEX_NaturalToGlobalEnd, DMPLEX_Stratify, DMPLEX_Symmetrize, DMPLEX_Preallocate, DMPLEX_ResidualFEM, DMPLEX_JacobianFEM, DMPLEX_InterpolatorFEM, DMPLEX_InjectorFEM, DMPLEX_IntegralFEM, DMPLEX_CreateGmsh, DMPLEX_RebalanceSharedPoints, DMPLEX_PartSelf, DMPLEX_PartLabelInvert, DMPLEX_PartLabelCreateSF, DMPLEX_PartStratSF, DMPLEX_CreatePointSF,DMPLEX_LocatePoints;
12 
13 PETSC_EXTERN PetscErrorCode VecView_MPI(Vec, PetscViewer);
14 
15 /*@
16   DMPlexIsSimplex - Is the first cell in this mesh a simplex?
17 
18   Input Parameter:
19 . dm      - The DMPlex object
20 
21   Output Parameter:
22 . simplex - Flag checking for a simplex
23 
24   Note: This just gives the first range of cells found. If the mesh has several cell types, it will only give the first.
25   If the mesh has no cells, this returns PETSC_FALSE.
26 
27   Level: intermediate
28 
29 .seealso DMPlexGetSimplexOrBoxCells(), DMPlexGetCellType(), DMPlexGetHeightStratum(), DMPolytopeTypeGetNumVertices()
30 @*/
31 PetscErrorCode DMPlexIsSimplex(DM dm, PetscBool *simplex)
32 {
33   DMPolytopeType ct;
34   PetscInt       cStart, cEnd;
35   PetscErrorCode ierr;
36 
37   PetscFunctionBegin;
38   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
39   if (cEnd <= cStart) {*simplex = PETSC_FALSE; PetscFunctionReturn(0);}
40   ierr = DMPlexGetCellType(dm, cStart, &ct);CHKERRQ(ierr);
41   *simplex = DMPolytopeTypeGetNumVertices(ct) == DMPolytopeTypeGetDim(ct)+1 ? PETSC_TRUE : PETSC_FALSE;
42   PetscFunctionReturn(0);
43 }
44 
45 /*@
46   DMPlexGetSimplexOrBoxCells - Get the range of cells which are neither prisms nor ghost FV cells
47 
48   Input Parameter:
49 + dm     - The DMPlex object
50 - height - The cell height in the Plex, 0 is the default
51 
52   Output Parameters:
53 + cStart - The first "normal" cell
54 - cEnd   - The upper bound on "normal"" cells
55 
56   Note: This just gives the first range of cells found. If the mesh has several cell types, it will only give the first.
57 
58   Level: developer
59 
60 .seealso DMPlexConstructGhostCells(), DMPlexGetGhostCellStratum()
61 @*/
62 PetscErrorCode DMPlexGetSimplexOrBoxCells(DM dm, PetscInt height, PetscInt *cStart, PetscInt *cEnd)
63 {
64   DMPolytopeType ct = DM_POLYTOPE_UNKNOWN;
65   PetscInt       cS, cE, c;
66   PetscErrorCode ierr;
67 
68   PetscFunctionBegin;
69   ierr = DMPlexGetHeightStratum(dm, PetscMax(height, 0), &cS, &cE);CHKERRQ(ierr);
70   for (c = cS; c < cE; ++c) {
71     DMPolytopeType cct;
72 
73     ierr = DMPlexGetCellType(dm, c, &cct);CHKERRQ(ierr);
74     if ((PetscInt) cct < 0) break;
75     switch (cct) {
76       case DM_POLYTOPE_POINT:
77       case DM_POLYTOPE_SEGMENT:
78       case DM_POLYTOPE_TRIANGLE:
79       case DM_POLYTOPE_QUADRILATERAL:
80       case DM_POLYTOPE_TETRAHEDRON:
81       case DM_POLYTOPE_HEXAHEDRON:
82         ct = cct;
83         break;
84       default: break;
85     }
86     if (ct != DM_POLYTOPE_UNKNOWN) break;
87   }
88   if (ct != DM_POLYTOPE_UNKNOWN) {
89     DMLabel ctLabel;
90 
91     ierr = DMPlexGetCellTypeLabel(dm, &ctLabel);CHKERRQ(ierr);
92     ierr = DMLabelGetStratumBounds(ctLabel, ct, &cS, &cE);CHKERRQ(ierr);
93   }
94   if (cStart) *cStart = cS;
95   if (cEnd)   *cEnd   = cE;
96   PetscFunctionReturn(0);
97 }
98 
99 PetscErrorCode DMPlexGetFieldType_Internal(DM dm, PetscSection section, PetscInt field, PetscInt *sStart, PetscInt *sEnd, PetscViewerVTKFieldType *ft)
100 {
101   PetscInt       cdim, pStart, pEnd, vStart, vEnd, cStart, cEnd;
102   PetscInt       vcdof[2] = {0,0}, globalvcdof[2];
103   PetscErrorCode ierr;
104 
105   PetscFunctionBegin;
106   *ft  = PETSC_VTK_INVALID;
107   ierr = DMGetCoordinateDim(dm, &cdim);CHKERRQ(ierr);
108   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
109   ierr = DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
110   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
111   if (field >= 0) {
112     if ((vStart >= pStart) && (vStart < pEnd)) {ierr = PetscSectionGetFieldDof(section, vStart, field, &vcdof[0]);CHKERRQ(ierr);}
113     if ((cStart >= pStart) && (cStart < pEnd)) {ierr = PetscSectionGetFieldDof(section, cStart, field, &vcdof[1]);CHKERRQ(ierr);}
114   } else {
115     if ((vStart >= pStart) && (vStart < pEnd)) {ierr = PetscSectionGetDof(section, vStart, &vcdof[0]);CHKERRQ(ierr);}
116     if ((cStart >= pStart) && (cStart < pEnd)) {ierr = PetscSectionGetDof(section, cStart, &vcdof[1]);CHKERRQ(ierr);}
117   }
118   ierr = MPI_Allreduce(vcdof, globalvcdof, 2, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject)dm));CHKERRMPI(ierr);
119   if (globalvcdof[0]) {
120     *sStart = vStart;
121     *sEnd   = vEnd;
122     if (globalvcdof[0] == cdim) *ft = PETSC_VTK_POINT_VECTOR_FIELD;
123     else                        *ft = PETSC_VTK_POINT_FIELD;
124   } else if (globalvcdof[1]) {
125     *sStart = cStart;
126     *sEnd   = cEnd;
127     if (globalvcdof[1] == cdim) *ft = PETSC_VTK_CELL_VECTOR_FIELD;
128     else                        *ft = PETSC_VTK_CELL_FIELD;
129   } else {
130     if (field >= 0) {
131       const char *fieldname;
132 
133       ierr = PetscSectionGetFieldName(section, field, &fieldname);CHKERRQ(ierr);
134       ierr = PetscInfo2((PetscObject) dm, "Could not classify VTK output type of section field %D \"%s\"\n", field, fieldname);CHKERRQ(ierr);
135     } else {
136       ierr = PetscInfo((PetscObject) dm, "Could not classify VTK output type of section\"%s\"\n");CHKERRQ(ierr);
137     }
138   }
139   PetscFunctionReturn(0);
140 }
141 
142 static PetscErrorCode VecView_Plex_Local_Draw(Vec v, PetscViewer viewer)
143 {
144   DM                 dm;
145   PetscSection       s;
146   PetscDraw          draw, popup;
147   DM                 cdm;
148   PetscSection       coordSection;
149   Vec                coordinates;
150   const PetscScalar *coords, *array;
151   PetscReal          bound[4] = {PETSC_MAX_REAL, PETSC_MAX_REAL, PETSC_MIN_REAL, PETSC_MIN_REAL};
152   PetscReal          vbound[2], time;
153   PetscBool          isnull, flg;
154   PetscInt           dim, Nf, f, Nc, comp, vStart, vEnd, cStart, cEnd, c, N, level, step, w = 0;
155   const char        *name;
156   char               title[PETSC_MAX_PATH_LEN];
157   PetscErrorCode     ierr;
158 
159   PetscFunctionBegin;
160   ierr = PetscViewerDrawGetDraw(viewer, 0, &draw);CHKERRQ(ierr);
161   ierr = PetscDrawIsNull(draw, &isnull);CHKERRQ(ierr);
162   if (isnull) PetscFunctionReturn(0);
163 
164   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
165   ierr = DMGetCoordinateDim(dm, &dim);CHKERRQ(ierr);
166   if (dim != 2) SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Cannot draw meshes of dimension %D. Use PETSCVIEWERGLVIS", dim);
167   ierr = DMGetLocalSection(dm, &s);CHKERRQ(ierr);
168   ierr = PetscSectionGetNumFields(s, &Nf);CHKERRQ(ierr);
169   ierr = DMGetCoarsenLevel(dm, &level);CHKERRQ(ierr);
170   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
171   ierr = DMGetLocalSection(cdm, &coordSection);CHKERRQ(ierr);
172   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
173   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
174   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
175 
176   ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
177   ierr = DMGetOutputSequenceNumber(dm, &step, &time);CHKERRQ(ierr);
178 
179   ierr = VecGetLocalSize(coordinates, &N);CHKERRQ(ierr);
180   ierr = VecGetArrayRead(coordinates, &coords);CHKERRQ(ierr);
181   for (c = 0; c < N; c += dim) {
182     bound[0] = PetscMin(bound[0], PetscRealPart(coords[c]));   bound[2] = PetscMax(bound[2], PetscRealPart(coords[c]));
183     bound[1] = PetscMin(bound[1], PetscRealPart(coords[c+1])); bound[3] = PetscMax(bound[3], PetscRealPart(coords[c+1]));
184   }
185   ierr = VecRestoreArrayRead(coordinates, &coords);CHKERRQ(ierr);
186   ierr = PetscDrawClear(draw);CHKERRQ(ierr);
187 
188   /* Could implement something like DMDASelectFields() */
189   for (f = 0; f < Nf; ++f) {
190     DM   fdm = dm;
191     Vec  fv  = v;
192     IS   fis;
193     char prefix[PETSC_MAX_PATH_LEN];
194     const char *fname;
195 
196     ierr = PetscSectionGetFieldComponents(s, f, &Nc);CHKERRQ(ierr);
197     ierr = PetscSectionGetFieldName(s, f, &fname);CHKERRQ(ierr);
198 
199     if (v->hdr.prefix) {ierr = PetscStrncpy(prefix, v->hdr.prefix,sizeof(prefix));CHKERRQ(ierr);}
200     else               {prefix[0] = '\0';}
201     if (Nf > 1) {
202       ierr = DMCreateSubDM(dm, 1, &f, &fis, &fdm);CHKERRQ(ierr);
203       ierr = VecGetSubVector(v, fis, &fv);CHKERRQ(ierr);
204       ierr = PetscStrlcat(prefix, fname,sizeof(prefix));CHKERRQ(ierr);
205       ierr = PetscStrlcat(prefix, "_",sizeof(prefix));CHKERRQ(ierr);
206     }
207     for (comp = 0; comp < Nc; ++comp, ++w) {
208       PetscInt nmax = 2;
209 
210       ierr = PetscViewerDrawGetDraw(viewer, w, &draw);CHKERRQ(ierr);
211       if (Nc > 1) {ierr = PetscSNPrintf(title, sizeof(title), "%s:%s_%D Step: %D Time: %.4g", name, fname, comp, step, time);CHKERRQ(ierr);}
212       else        {ierr = PetscSNPrintf(title, sizeof(title), "%s:%s Step: %D Time: %.4g", name, fname, step, time);CHKERRQ(ierr);}
213       ierr = PetscDrawSetTitle(draw, title);CHKERRQ(ierr);
214 
215       /* TODO Get max and min only for this component */
216       ierr = PetscOptionsGetRealArray(NULL, prefix, "-vec_view_bounds", vbound, &nmax, &flg);CHKERRQ(ierr);
217       if (!flg) {
218         ierr = VecMin(fv, NULL, &vbound[0]);CHKERRQ(ierr);
219         ierr = VecMax(fv, NULL, &vbound[1]);CHKERRQ(ierr);
220         if (vbound[1] <= vbound[0]) vbound[1] = vbound[0] + 1.0;
221       }
222       ierr = PetscDrawGetPopup(draw, &popup);CHKERRQ(ierr);
223       ierr = PetscDrawScalePopup(popup, vbound[0], vbound[1]);CHKERRQ(ierr);
224       ierr = PetscDrawSetCoordinates(draw, bound[0], bound[1], bound[2], bound[3]);CHKERRQ(ierr);
225 
226       ierr = VecGetArrayRead(fv, &array);CHKERRQ(ierr);
227       for (c = cStart; c < cEnd; ++c) {
228         PetscScalar *coords = NULL, *a = NULL;
229         PetscInt     numCoords, color[4] = {-1,-1,-1,-1};
230 
231         ierr = DMPlexPointLocalRead(fdm, c, array, &a);CHKERRQ(ierr);
232         if (a) {
233           color[0] = PetscDrawRealToColor(PetscRealPart(a[comp]), vbound[0], vbound[1]);
234           color[1] = color[2] = color[3] = color[0];
235         } else {
236           PetscScalar *vals = NULL;
237           PetscInt     numVals, va;
238 
239           ierr = DMPlexVecGetClosure(fdm, NULL, fv, c, &numVals, &vals);CHKERRQ(ierr);
240           if (numVals % Nc) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "The number of components %D does not divide the number of values in the closure %D", Nc, numVals);
241           switch (numVals/Nc) {
242           case 3: /* P1 Triangle */
243           case 4: /* P1 Quadrangle */
244             for (va = 0; va < numVals/Nc; ++va) color[va] = PetscDrawRealToColor(PetscRealPart(vals[va*Nc+comp]), vbound[0], vbound[1]);
245             break;
246           case 6: /* P2 Triangle */
247           case 8: /* P2 Quadrangle */
248             for (va = 0; va < numVals/(Nc*2); ++va) color[va] = PetscDrawRealToColor(PetscRealPart(vals[va*Nc+comp + numVals/(Nc*2)]), vbound[0], vbound[1]);
249             break;
250           default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of values for cell closure %D cannot be handled", numVals/Nc);
251           }
252           ierr = DMPlexVecRestoreClosure(fdm, NULL, fv, c, &numVals, &vals);CHKERRQ(ierr);
253         }
254         ierr = DMPlexVecGetClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
255         switch (numCoords) {
256         case 6:
257           ierr = PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), color[0], color[1], color[2]);CHKERRQ(ierr);
258           break;
259         case 8:
260           ierr = PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), color[0], color[1], color[2]);CHKERRQ(ierr);
261           ierr = PetscDrawTriangle(draw, PetscRealPart(coords[4]), PetscRealPart(coords[5]), PetscRealPart(coords[6]), PetscRealPart(coords[7]), PetscRealPart(coords[0]), PetscRealPart(coords[1]), color[2], color[3], color[0]);CHKERRQ(ierr);
262           break;
263         default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_SUP, "Cannot draw cells with %D coordinates", numCoords);
264         }
265         ierr = DMPlexVecRestoreClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
266       }
267       ierr = VecRestoreArrayRead(fv, &array);CHKERRQ(ierr);
268       ierr = PetscDrawFlush(draw);CHKERRQ(ierr);
269       ierr = PetscDrawPause(draw);CHKERRQ(ierr);
270       ierr = PetscDrawSave(draw);CHKERRQ(ierr);
271     }
272     if (Nf > 1) {
273       ierr = VecRestoreSubVector(v, fis, &fv);CHKERRQ(ierr);
274       ierr = ISDestroy(&fis);CHKERRQ(ierr);
275       ierr = DMDestroy(&fdm);CHKERRQ(ierr);
276     }
277   }
278   PetscFunctionReturn(0);
279 }
280 
281 static PetscErrorCode VecView_Plex_Local_VTK(Vec v, PetscViewer viewer)
282 {
283   DM                      dm;
284   Vec                     locv;
285   const char              *name;
286   PetscSection            section;
287   PetscInt                pStart, pEnd;
288   PetscInt                numFields;
289   PetscViewerVTKFieldType ft;
290   PetscErrorCode          ierr;
291 
292   PetscFunctionBegin;
293   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
294   ierr = DMCreateLocalVector(dm, &locv);CHKERRQ(ierr); /* VTK viewer requires exclusive ownership of the vector */
295   ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
296   ierr = PetscObjectSetName((PetscObject) locv, name);CHKERRQ(ierr);
297   ierr = VecCopy(v, locv);CHKERRQ(ierr);
298   ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);
299   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
300   if (!numFields) {
301     ierr = DMPlexGetFieldType_Internal(dm, section, PETSC_DETERMINE, &pStart, &pEnd, &ft);CHKERRQ(ierr);
302     ierr = PetscViewerVTKAddField(viewer, (PetscObject) dm, DMPlexVTKWriteAll, PETSC_DEFAULT, ft, PETSC_TRUE,(PetscObject) locv);CHKERRQ(ierr);
303   } else {
304     PetscInt f;
305 
306     for (f = 0; f < numFields; f++) {
307       ierr = DMPlexGetFieldType_Internal(dm, section, f, &pStart, &pEnd, &ft);CHKERRQ(ierr);
308       if (ft == PETSC_VTK_INVALID) continue;
309       ierr = PetscObjectReference((PetscObject)locv);CHKERRQ(ierr);
310       ierr = PetscViewerVTKAddField(viewer, (PetscObject) dm, DMPlexVTKWriteAll, f, ft, PETSC_TRUE,(PetscObject) locv);CHKERRQ(ierr);
311     }
312     ierr = VecDestroy(&locv);CHKERRQ(ierr);
313   }
314   PetscFunctionReturn(0);
315 }
316 
317 PetscErrorCode VecView_Plex_Local(Vec v, PetscViewer viewer)
318 {
319   DM             dm;
320   PetscBool      isvtk, ishdf5, isdraw, isglvis;
321   PetscErrorCode ierr;
322 
323   PetscFunctionBegin;
324   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
325   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
326   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,   &isvtk);CHKERRQ(ierr);
327   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,  &ishdf5);CHKERRQ(ierr);
328   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERDRAW,  &isdraw);CHKERRQ(ierr);
329   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERGLVIS, &isglvis);CHKERRQ(ierr);
330   if (isvtk || ishdf5 || isdraw || isglvis) {
331     PetscInt    i,numFields;
332     PetscObject fe;
333     PetscBool   fem = PETSC_FALSE;
334     Vec         locv = v;
335     const char  *name;
336     PetscInt    step;
337     PetscReal   time;
338 
339     ierr = DMGetNumFields(dm, &numFields);CHKERRQ(ierr);
340     for (i=0; i<numFields; i++) {
341       ierr = DMGetField(dm, i, NULL, &fe);CHKERRQ(ierr);
342       if (fe->classid == PETSCFE_CLASSID) { fem = PETSC_TRUE; break; }
343     }
344     if (fem) {
345       PetscObject isZero;
346 
347       ierr = DMGetLocalVector(dm, &locv);CHKERRQ(ierr);
348       ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
349       ierr = PetscObjectSetName((PetscObject) locv, name);CHKERRQ(ierr);
350       ierr = PetscObjectQuery((PetscObject) v, "__Vec_bc_zero__", &isZero);CHKERRQ(ierr);
351       ierr = PetscObjectCompose((PetscObject) locv, "__Vec_bc_zero__", isZero);CHKERRQ(ierr);
352       ierr = VecCopy(v, locv);CHKERRQ(ierr);
353       ierr = DMGetOutputSequenceNumber(dm, NULL, &time);CHKERRQ(ierr);
354       ierr = DMPlexInsertBoundaryValues(dm, PETSC_TRUE, locv, time, NULL, NULL, NULL);CHKERRQ(ierr);
355     }
356     if (isvtk) {
357       ierr = VecView_Plex_Local_VTK(locv, viewer);CHKERRQ(ierr);
358     } else if (ishdf5) {
359 #if defined(PETSC_HAVE_HDF5)
360       ierr = VecView_Plex_Local_HDF5_Internal(locv, viewer);CHKERRQ(ierr);
361 #else
362       SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
363 #endif
364     } else if (isdraw) {
365       ierr = VecView_Plex_Local_Draw(locv, viewer);CHKERRQ(ierr);
366     } else if (isglvis) {
367       ierr = DMGetOutputSequenceNumber(dm, &step, NULL);CHKERRQ(ierr);
368       ierr = PetscViewerGLVisSetSnapId(viewer, step);CHKERRQ(ierr);
369       ierr = VecView_GLVis(locv, viewer);CHKERRQ(ierr);
370     }
371     if (fem) {
372       ierr = PetscObjectCompose((PetscObject) locv, "__Vec_bc_zero__", NULL);CHKERRQ(ierr);
373       ierr = DMRestoreLocalVector(dm, &locv);CHKERRQ(ierr);
374     }
375   } else {
376     PetscBool isseq;
377 
378     ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
379     if (isseq) {ierr = VecView_Seq(v, viewer);CHKERRQ(ierr);}
380     else       {ierr = VecView_MPI(v, viewer);CHKERRQ(ierr);}
381   }
382   PetscFunctionReturn(0);
383 }
384 
385 PetscErrorCode VecView_Plex(Vec v, PetscViewer viewer)
386 {
387   DM             dm;
388   PetscBool      isvtk, ishdf5, isdraw, isglvis, isexodusii;
389   PetscErrorCode ierr;
390 
391   PetscFunctionBegin;
392   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
393   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
394   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,      &isvtk);CHKERRQ(ierr);
395   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,     &ishdf5);CHKERRQ(ierr);
396   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERDRAW,     &isdraw);CHKERRQ(ierr);
397   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERGLVIS,    &isglvis);CHKERRQ(ierr);
398   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWEREXODUSII, &isexodusii);CHKERRQ(ierr);
399   if (isvtk || isdraw || isglvis) {
400     Vec         locv;
401     PetscObject isZero;
402     const char *name;
403 
404     ierr = DMGetLocalVector(dm, &locv);CHKERRQ(ierr);
405     ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
406     ierr = PetscObjectSetName((PetscObject) locv, name);CHKERRQ(ierr);
407     ierr = DMGlobalToLocalBegin(dm, v, INSERT_VALUES, locv);CHKERRQ(ierr);
408     ierr = DMGlobalToLocalEnd(dm, v, INSERT_VALUES, locv);CHKERRQ(ierr);
409     ierr = PetscObjectQuery((PetscObject) v, "__Vec_bc_zero__", &isZero);CHKERRQ(ierr);
410     ierr = PetscObjectCompose((PetscObject) locv, "__Vec_bc_zero__", isZero);CHKERRQ(ierr);
411     ierr = VecView_Plex_Local(locv, viewer);CHKERRQ(ierr);
412     ierr = PetscObjectCompose((PetscObject) locv, "__Vec_bc_zero__", NULL);CHKERRQ(ierr);
413     ierr = DMRestoreLocalVector(dm, &locv);CHKERRQ(ierr);
414   } else if (ishdf5) {
415 #if defined(PETSC_HAVE_HDF5)
416     ierr = VecView_Plex_HDF5_Internal(v, viewer);CHKERRQ(ierr);
417 #else
418     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
419 #endif
420   } else if (isexodusii) {
421 #if defined(PETSC_HAVE_EXODUSII)
422     ierr = VecView_PlexExodusII_Internal(v, viewer);CHKERRQ(ierr);
423 #else
424     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "ExodusII not supported in this build.\nPlease reconfigure using --download-exodusii");
425 #endif
426   } else {
427     PetscBool isseq;
428 
429     ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
430     if (isseq) {ierr = VecView_Seq(v, viewer);CHKERRQ(ierr);}
431     else       {ierr = VecView_MPI(v, viewer);CHKERRQ(ierr);}
432   }
433   PetscFunctionReturn(0);
434 }
435 
436 PetscErrorCode VecView_Plex_Native(Vec originalv, PetscViewer viewer)
437 {
438   DM                dm;
439   MPI_Comm          comm;
440   PetscViewerFormat format;
441   Vec               v;
442   PetscBool         isvtk, ishdf5;
443   PetscErrorCode    ierr;
444 
445   PetscFunctionBegin;
446   ierr = VecGetDM(originalv, &dm);CHKERRQ(ierr);
447   ierr = PetscObjectGetComm((PetscObject) originalv, &comm);CHKERRQ(ierr);
448   if (!dm) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
449   ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
450   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
451   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,  &isvtk);CHKERRQ(ierr);
452   if (format == PETSC_VIEWER_NATIVE) {
453     /* Natural ordering is the common case for DMDA, NATIVE means plain vector, for PLEX is the opposite */
454     /* this need a better fix */
455     if (dm->useNatural) {
456       if (dm->sfNatural) {
457         const char *vecname;
458         PetscInt    n, nroots;
459 
460         ierr = VecGetLocalSize(originalv, &n);CHKERRQ(ierr);
461         ierr = PetscSFGetGraph(dm->sfNatural, &nroots, NULL, NULL, NULL);CHKERRQ(ierr);
462         if (n == nroots) {
463           ierr = DMGetGlobalVector(dm, &v);CHKERRQ(ierr);
464           ierr = DMPlexGlobalToNaturalBegin(dm, originalv, v);CHKERRQ(ierr);
465           ierr = DMPlexGlobalToNaturalEnd(dm, originalv, v);CHKERRQ(ierr);
466           ierr = PetscObjectGetName((PetscObject) originalv, &vecname);CHKERRQ(ierr);
467           ierr = PetscObjectSetName((PetscObject) v, vecname);CHKERRQ(ierr);
468         } else SETERRQ(comm, PETSC_ERR_ARG_WRONG, "DM global to natural SF only handles global vectors");
469       } else SETERRQ(comm, PETSC_ERR_ARG_WRONGSTATE, "DM global to natural SF was not created");
470     } else v = originalv;
471   } else v = originalv;
472 
473   if (ishdf5) {
474 #if defined(PETSC_HAVE_HDF5)
475     ierr = VecView_Plex_HDF5_Native_Internal(v, viewer);CHKERRQ(ierr);
476 #else
477     SETERRQ(comm, PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
478 #endif
479   } else if (isvtk) {
480     SETERRQ(comm, PETSC_ERR_SUP, "VTK format does not support viewing in natural order. Please switch to HDF5.");
481   } else {
482     PetscBool isseq;
483 
484     ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
485     if (isseq) {ierr = VecView_Seq(v, viewer);CHKERRQ(ierr);}
486     else       {ierr = VecView_MPI(v, viewer);CHKERRQ(ierr);}
487   }
488   if (v != originalv) {ierr = DMRestoreGlobalVector(dm, &v);CHKERRQ(ierr);}
489   PetscFunctionReturn(0);
490 }
491 
492 PetscErrorCode VecLoad_Plex_Local(Vec v, PetscViewer viewer)
493 {
494   DM             dm;
495   PetscBool      ishdf5;
496   PetscErrorCode ierr;
497 
498   PetscFunctionBegin;
499   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
500   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
501   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
502   if (ishdf5) {
503     DM          dmBC;
504     Vec         gv;
505     const char *name;
506 
507     ierr = DMGetOutputDM(dm, &dmBC);CHKERRQ(ierr);
508     ierr = DMGetGlobalVector(dmBC, &gv);CHKERRQ(ierr);
509     ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
510     ierr = PetscObjectSetName((PetscObject) gv, name);CHKERRQ(ierr);
511     ierr = VecLoad_Default(gv, viewer);CHKERRQ(ierr);
512     ierr = DMGlobalToLocalBegin(dmBC, gv, INSERT_VALUES, v);CHKERRQ(ierr);
513     ierr = DMGlobalToLocalEnd(dmBC, gv, INSERT_VALUES, v);CHKERRQ(ierr);
514     ierr = DMRestoreGlobalVector(dmBC, &gv);CHKERRQ(ierr);
515   } else {
516     ierr = VecLoad_Default(v, viewer);CHKERRQ(ierr);
517   }
518   PetscFunctionReturn(0);
519 }
520 
521 PetscErrorCode VecLoad_Plex(Vec v, PetscViewer viewer)
522 {
523   DM             dm;
524   PetscBool      ishdf5,isexodusii;
525   PetscErrorCode ierr;
526 
527   PetscFunctionBegin;
528   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
529   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
530   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,     &ishdf5);CHKERRQ(ierr);
531   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWEREXODUSII, &isexodusii);CHKERRQ(ierr);
532   if (ishdf5) {
533 #if defined(PETSC_HAVE_HDF5)
534     ierr = VecLoad_Plex_HDF5_Internal(v, viewer);CHKERRQ(ierr);
535 #else
536     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
537 #endif
538   } else if (isexodusii) {
539 #if defined(PETSC_HAVE_EXODUSII)
540     ierr = VecLoad_PlexExodusII_Internal(v, viewer);CHKERRQ(ierr);
541 #else
542     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "ExodusII not supported in this build.\nPlease reconfigure using --download-exodusii");
543 #endif
544   } else {
545     ierr = VecLoad_Default(v, viewer);CHKERRQ(ierr);
546   }
547   PetscFunctionReturn(0);
548 }
549 
550 PetscErrorCode VecLoad_Plex_Native(Vec originalv, PetscViewer viewer)
551 {
552   DM                dm;
553   PetscViewerFormat format;
554   PetscBool         ishdf5;
555   PetscErrorCode    ierr;
556 
557   PetscFunctionBegin;
558   ierr = VecGetDM(originalv, &dm);CHKERRQ(ierr);
559   if (!dm) SETERRQ(PetscObjectComm((PetscObject) originalv), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
560   ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
561   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
562   if (format == PETSC_VIEWER_NATIVE) {
563     if (dm->useNatural) {
564       if (dm->sfNatural) {
565         if (ishdf5) {
566 #if defined(PETSC_HAVE_HDF5)
567           Vec         v;
568           const char *vecname;
569 
570           ierr = DMGetGlobalVector(dm, &v);CHKERRQ(ierr);
571           ierr = PetscObjectGetName((PetscObject) originalv, &vecname);CHKERRQ(ierr);
572           ierr = PetscObjectSetName((PetscObject) v, vecname);CHKERRQ(ierr);
573           ierr = VecLoad_Plex_HDF5_Native_Internal(v, viewer);CHKERRQ(ierr);
574           ierr = DMPlexNaturalToGlobalBegin(dm, v, originalv);CHKERRQ(ierr);
575           ierr = DMPlexNaturalToGlobalEnd(dm, v, originalv);CHKERRQ(ierr);
576           ierr = DMRestoreGlobalVector(dm, &v);CHKERRQ(ierr);
577 #else
578           SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
579 #endif
580         } else SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Reading in natural order is not supported for anything but HDF5.");
581       }
582     } else {
583       ierr = VecLoad_Default(originalv, viewer);CHKERRQ(ierr);
584     }
585   }
586   PetscFunctionReturn(0);
587 }
588 
589 PETSC_UNUSED static PetscErrorCode DMPlexView_Ascii_Geometry(DM dm, PetscViewer viewer)
590 {
591   PetscSection       coordSection;
592   Vec                coordinates;
593   DMLabel            depthLabel, celltypeLabel;
594   const char        *name[4];
595   const PetscScalar *a;
596   PetscInt           dim, pStart, pEnd, cStart, cEnd, c;
597   PetscErrorCode     ierr;
598 
599   PetscFunctionBegin;
600   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
601   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
602   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
603   ierr = DMPlexGetDepthLabel(dm, &depthLabel);CHKERRQ(ierr);
604   ierr = DMPlexGetCellTypeLabel(dm, &celltypeLabel);CHKERRQ(ierr);
605   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
606   ierr = PetscSectionGetChart(coordSection, &pStart, &pEnd);CHKERRQ(ierr);
607   ierr = VecGetArrayRead(coordinates, &a);CHKERRQ(ierr);
608   name[0]     = "vertex";
609   name[1]     = "edge";
610   name[dim-1] = "face";
611   name[dim]   = "cell";
612   for (c = cStart; c < cEnd; ++c) {
613     PetscInt *closure = NULL;
614     PetscInt  closureSize, cl, ct;
615 
616     ierr = DMLabelGetValue(celltypeLabel, c, &ct);CHKERRQ(ierr);
617     ierr = PetscViewerASCIIPrintf(viewer, "Geometry for cell %D polytope type %s:\n", c, DMPolytopeTypes[ct]);CHKERRQ(ierr);
618     ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
619     ierr = PetscViewerASCIIPushTab(viewer);CHKERRQ(ierr);
620     for (cl = 0; cl < closureSize*2; cl += 2) {
621       PetscInt point = closure[cl], depth, dof, off, d, p;
622 
623       if ((point < pStart) || (point >= pEnd)) continue;
624       ierr = PetscSectionGetDof(coordSection, point, &dof);CHKERRQ(ierr);
625       if (!dof) continue;
626       ierr = DMLabelGetValue(depthLabel, point, &depth);CHKERRQ(ierr);
627       ierr = PetscSectionGetOffset(coordSection, point, &off);CHKERRQ(ierr);
628       ierr = PetscViewerASCIIPrintf(viewer, "%s %D coords:", name[depth], point);CHKERRQ(ierr);
629       for (p = 0; p < dof/dim; ++p) {
630         ierr = PetscViewerASCIIPrintf(viewer, " (");CHKERRQ(ierr);
631         for (d = 0; d < dim; ++d) {
632           if (d > 0) {ierr = PetscViewerASCIIPrintf(viewer, ", ");CHKERRQ(ierr);}
633           ierr = PetscViewerASCIIPrintf(viewer, "%g", (double) PetscRealPart(a[off+p*dim+d]));CHKERRQ(ierr);
634         }
635         ierr = PetscViewerASCIIPrintf(viewer, ")");CHKERRQ(ierr);
636       }
637       ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
638     }
639     ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
640     ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr);
641   }
642   ierr = VecRestoreArrayRead(coordinates, &a);CHKERRQ(ierr);
643   PetscFunctionReturn(0);
644 }
645 
646 static PetscErrorCode DMPlexView_Ascii(DM dm, PetscViewer viewer)
647 {
648   DM_Plex          *mesh = (DM_Plex*) dm->data;
649   DM                cdm;
650   PetscSection      coordSection;
651   Vec               coordinates;
652   PetscViewerFormat format;
653   PetscErrorCode    ierr;
654 
655   PetscFunctionBegin;
656   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
657   ierr = DMGetLocalSection(cdm, &coordSection);CHKERRQ(ierr);
658   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
659   ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
660   if (format == PETSC_VIEWER_ASCII_INFO_DETAIL) {
661     const char *name;
662     PetscInt    dim, cellHeight, maxConeSize, maxSupportSize;
663     PetscInt    pStart, pEnd, p, numLabels, l;
664     PetscMPIInt rank, size;
665 
666     ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRMPI(ierr);
667     ierr = MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size);CHKERRMPI(ierr);
668     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
669     ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
670     ierr = DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize);CHKERRQ(ierr);
671     ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
672     ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
673     if (name) {ierr = PetscViewerASCIIPrintf(viewer, "%s in %D dimension%s:\n", name, dim, dim == 1 ? "" : "s");CHKERRQ(ierr);}
674     else      {ierr = PetscViewerASCIIPrintf(viewer, "Mesh in %D dimension%s:\n", dim, dim == 1 ? "" : "s");CHKERRQ(ierr);}
675     if (cellHeight) {ierr = PetscViewerASCIIPrintf(viewer, "  Cells are at height %D\n", cellHeight);CHKERRQ(ierr);}
676     ierr = PetscViewerASCIIPrintf(viewer, "Supports:\n", name);CHKERRQ(ierr);
677     ierr = PetscViewerASCIIPushSynchronized(viewer);CHKERRQ(ierr);
678     ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%d] Max support size: %D\n", rank, maxSupportSize);CHKERRQ(ierr);
679     for (p = pStart; p < pEnd; ++p) {
680       PetscInt dof, off, s;
681 
682       ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
683       ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
684       for (s = off; s < off+dof; ++s) {
685         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%d]: %D ----> %D\n", rank, p, mesh->supports[s]);CHKERRQ(ierr);
686       }
687     }
688     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
689     ierr = PetscViewerASCIIPrintf(viewer, "Cones:\n", name);CHKERRQ(ierr);
690     ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%d] Max cone size: %D\n", rank, maxConeSize);CHKERRQ(ierr);
691     for (p = pStart; p < pEnd; ++p) {
692       PetscInt dof, off, c;
693 
694       ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
695       ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
696       for (c = off; c < off+dof; ++c) {
697         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%d]: %D <---- %D (%D)\n", rank, p, mesh->cones[c], mesh->coneOrientations[c]);CHKERRQ(ierr);
698       }
699     }
700     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
701     ierr = PetscViewerASCIIPopSynchronized(viewer);CHKERRQ(ierr);
702     if (coordSection && coordinates) {
703       ierr = PetscSectionVecView(coordSection, coordinates, viewer);CHKERRQ(ierr);
704     }
705     ierr = DMGetNumLabels(dm, &numLabels);CHKERRQ(ierr);
706     if (numLabels) {ierr = PetscViewerASCIIPrintf(viewer, "Labels:\n");CHKERRQ(ierr);}
707     for (l = 0; l < numLabels; ++l) {
708       DMLabel     label;
709       PetscBool   isdepth;
710       const char *name;
711 
712       ierr = DMGetLabelName(dm, l, &name);CHKERRQ(ierr);
713       ierr = PetscStrcmp(name, "depth", &isdepth);CHKERRQ(ierr);
714       if (isdepth) continue;
715       ierr = DMGetLabel(dm, name, &label);CHKERRQ(ierr);
716       ierr = DMLabelView(label, viewer);CHKERRQ(ierr);
717     }
718     if (size > 1) {
719       PetscSF sf;
720 
721       ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr);
722       ierr = PetscSFView(sf, viewer);CHKERRQ(ierr);
723     }
724     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
725   } else if (format == PETSC_VIEWER_ASCII_LATEX) {
726     const char  *name, *color;
727     const char  *defcolors[3]  = {"gray", "orange", "green"};
728     const char  *deflcolors[4] = {"blue", "cyan", "red", "magenta"};
729     char         lname[PETSC_MAX_PATH_LEN];
730     PetscReal    scale         = 2.0;
731     PetscReal    tikzscale     = 1.0;
732     PetscBool    useNumbers    = PETSC_TRUE, useLabels, useColors;
733     double       tcoords[3];
734     PetscScalar *coords;
735     PetscInt     numLabels, l, numColors, numLColors, dim, depth, cStart, cEnd, c, vStart, vEnd, v, eStart = 0, eEnd = 0, e, p;
736     PetscMPIInt  rank, size;
737     char         **names, **colors, **lcolors;
738     PetscBool    plotEdges, flg, lflg;
739     PetscBT      wp = NULL;
740     PetscInt     pEnd, pStart;
741 
742     ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
743     ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
744     ierr = DMGetNumLabels(dm, &numLabels);CHKERRQ(ierr);
745     numLabels  = PetscMax(numLabels, 10);
746     numColors  = 10;
747     numLColors = 10;
748     ierr = PetscCalloc3(numLabels, &names, numColors, &colors, numLColors, &lcolors);CHKERRQ(ierr);
749     ierr = PetscOptionsGetReal(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_scale", &scale, NULL);CHKERRQ(ierr);
750     ierr = PetscOptionsGetReal(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_tikzscale", &tikzscale, NULL);CHKERRQ(ierr);
751     ierr = PetscOptionsGetBool(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_numbers", &useNumbers, NULL);CHKERRQ(ierr);
752     ierr = PetscOptionsGetStringArray(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_labels", names, &numLabels, &useLabels);CHKERRQ(ierr);
753     if (!useLabels) numLabels = 0;
754     ierr = PetscOptionsGetStringArray(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_colors", colors, &numColors, &useColors);CHKERRQ(ierr);
755     if (!useColors) {
756       numColors = 3;
757       for (c = 0; c < numColors; ++c) {ierr = PetscStrallocpy(defcolors[c], &colors[c]);CHKERRQ(ierr);}
758     }
759     ierr = PetscOptionsGetStringArray(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_lcolors", lcolors, &numLColors, &useColors);CHKERRQ(ierr);
760     if (!useColors) {
761       numLColors = 4;
762       for (c = 0; c < numLColors; ++c) {ierr = PetscStrallocpy(deflcolors[c], &lcolors[c]);CHKERRQ(ierr);}
763     }
764     ierr = PetscOptionsGetString(((PetscObject) viewer)->options, ((PetscObject) viewer)->prefix, "-dm_plex_view_label_filter", lname, sizeof(lname), &lflg);CHKERRQ(ierr);
765     plotEdges = (PetscBool)(depth > 1 && useNumbers && dim < 3);
766     ierr = PetscOptionsGetBool(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_edges", &plotEdges, &flg);CHKERRQ(ierr);
767     if (flg && plotEdges && depth < dim) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Mesh must be interpolated");
768     if (depth < dim) plotEdges = PETSC_FALSE;
769 
770     /* filter points with labelvalue != labeldefaultvalue */
771     ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
772     if (lflg) {
773       DMLabel lbl;
774 
775       ierr = DMGetLabel(dm, lname, &lbl);CHKERRQ(ierr);
776       if (lbl) {
777         PetscInt val, defval;
778 
779         ierr = DMLabelGetDefaultValue(lbl, &defval);CHKERRQ(ierr);
780         ierr = PetscBTCreate(pEnd-pStart, &wp);CHKERRQ(ierr);
781         for (c = pStart;  c < pEnd; c++) {
782           PetscInt *closure = NULL;
783           PetscInt  closureSize;
784 
785           ierr = DMLabelGetValue(lbl, c, &val);CHKERRQ(ierr);
786           if (val == defval) continue;
787 
788           ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
789           for (p = 0; p < closureSize*2; p += 2) {
790             ierr = PetscBTSet(wp, closure[p] - pStart);CHKERRQ(ierr);
791           }
792           ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
793         }
794       }
795     }
796 
797     ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRMPI(ierr);
798     ierr = MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size);CHKERRMPI(ierr);
799     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
800     ierr = PetscViewerASCIIPrintf(viewer, "\
801 \\documentclass[tikz]{standalone}\n\n\
802 \\usepackage{pgflibraryshapes}\n\
803 \\usetikzlibrary{backgrounds}\n\
804 \\usetikzlibrary{arrows}\n\
805 \\begin{document}\n");CHKERRQ(ierr);
806     if (size > 1) {
807       ierr = PetscViewerASCIIPrintf(viewer, "%s for process ", name);CHKERRQ(ierr);
808       for (p = 0; p < size; ++p) {
809         if (p > 0 && p == size-1) {
810           ierr = PetscViewerASCIIPrintf(viewer, ", and ", colors[p%numColors], p);CHKERRQ(ierr);
811         } else if (p > 0) {
812           ierr = PetscViewerASCIIPrintf(viewer, ", ", colors[p%numColors], p);CHKERRQ(ierr);
813         }
814         ierr = PetscViewerASCIIPrintf(viewer, "{\\textcolor{%s}%D}", colors[p%numColors], p);CHKERRQ(ierr);
815       }
816       ierr = PetscViewerASCIIPrintf(viewer, ".\n\n\n");CHKERRQ(ierr);
817     }
818     ierr = PetscViewerASCIIPrintf(viewer, "\\begin{tikzpicture}[scale = %g,font=\\fontsize{8}{8}\\selectfont]\n", (double) tikzscale);CHKERRQ(ierr);
819 
820     /* Plot vertices */
821     ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
822     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
823     ierr = PetscViewerASCIIPushSynchronized(viewer);CHKERRQ(ierr);
824     for (v = vStart; v < vEnd; ++v) {
825       PetscInt  off, dof, d;
826       PetscBool isLabeled = PETSC_FALSE;
827 
828       if (wp && !PetscBTLookup(wp,v - pStart)) continue;
829       ierr = PetscSectionGetDof(coordSection, v, &dof);CHKERRQ(ierr);
830       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
831       ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\path (");CHKERRQ(ierr);
832       if (PetscUnlikely(dof > 3)) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"coordSection vertex %D has dof %D > 3",v,dof);
833       for (d = 0; d < dof; ++d) {
834         tcoords[d] = (double) (scale*PetscRealPart(coords[off+d]));
835         tcoords[d] = PetscAbs(tcoords[d]) < 1e-10 ? 0.0 : tcoords[d];
836       }
837       /* Rotate coordinates since PGF makes z point out of the page instead of up */
838       if (dim == 3) {PetscReal tmp = tcoords[1]; tcoords[1] = tcoords[2]; tcoords[2] = -tmp;}
839       for (d = 0; d < dof; ++d) {
840         if (d > 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, ",");CHKERRQ(ierr);}
841         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "%g", (double) tcoords[d]);CHKERRQ(ierr);
842       }
843       color = colors[rank%numColors];
844       for (l = 0; l < numLabels; ++l) {
845         PetscInt val;
846         ierr = DMGetLabelValue(dm, names[l], v, &val);CHKERRQ(ierr);
847         if (val >= 0) {color = lcolors[l%numLColors]; isLabeled = PETSC_TRUE; break;}
848       }
849       if (useNumbers) {
850         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [draw,shape=circle,color=%s] {%D};\n", v, rank, color, v);CHKERRQ(ierr);
851       } else {
852         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [fill,inner sep=%dpt,shape=circle,color=%s] {};\n", v, rank, !isLabeled ? 1 : 2, color);CHKERRQ(ierr);
853       }
854     }
855     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
856     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
857     /* Plot cells */
858     ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
859     ierr = DMPlexGetDepthStratum(dm, 1, &eStart, &eEnd);CHKERRQ(ierr);
860     if (dim == 3 || !useNumbers) {
861       for (e = eStart; e < eEnd; ++e) {
862         const PetscInt *cone;
863 
864         if (wp && !PetscBTLookup(wp,e - pStart)) continue;
865         color = colors[rank%numColors];
866         for (l = 0; l < numLabels; ++l) {
867           PetscInt val;
868           ierr = DMGetLabelValue(dm, names[l], e, &val);CHKERRQ(ierr);
869           if (val >= 0) {color = lcolors[l%numLColors]; break;}
870         }
871         ierr = DMPlexGetCone(dm, e, &cone);CHKERRQ(ierr);
872         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\draw[color=%s] (%D_%d) -- (%D_%d);\n", color, cone[0], rank, cone[1], rank);CHKERRQ(ierr);
873       }
874     } else {
875       for (c = cStart; c < cEnd; ++c) {
876         PetscInt *closure = NULL;
877         PetscInt  closureSize, firstPoint = -1;
878 
879         if (wp && !PetscBTLookup(wp,c - pStart)) continue;
880         ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
881         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\draw[color=%s] ", colors[rank%numColors]);CHKERRQ(ierr);
882         for (p = 0; p < closureSize*2; p += 2) {
883           const PetscInt point = closure[p];
884 
885           if ((point < vStart) || (point >= vEnd)) continue;
886           if (firstPoint >= 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- ");CHKERRQ(ierr);}
887           ierr = PetscViewerASCIISynchronizedPrintf(viewer, "(%D_%d)", point, rank);CHKERRQ(ierr);
888           if (firstPoint < 0) firstPoint = point;
889         }
890         /* Why doesn't this work? ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- cycle;\n");CHKERRQ(ierr); */
891         ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- (%D_%d);\n", firstPoint, rank);CHKERRQ(ierr);
892         ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
893       }
894     }
895     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
896     for (c = cStart; c < cEnd; ++c) {
897       double    ccoords[3] = {0.0, 0.0, 0.0};
898       PetscBool isLabeled  = PETSC_FALSE;
899       PetscInt *closure    = NULL;
900       PetscInt  closureSize, dof, d, n = 0;
901 
902       if (wp && !PetscBTLookup(wp,c - pStart)) continue;
903       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
904       ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\path (");CHKERRQ(ierr);
905       for (p = 0; p < closureSize*2; p += 2) {
906         const PetscInt point = closure[p];
907         PetscInt       off;
908 
909         if ((point < vStart) || (point >= vEnd)) continue;
910         ierr = PetscSectionGetDof(coordSection, point, &dof);CHKERRQ(ierr);
911         ierr = PetscSectionGetOffset(coordSection, point, &off);CHKERRQ(ierr);
912         for (d = 0; d < dof; ++d) {
913           tcoords[d] = (double) (scale*PetscRealPart(coords[off+d]));
914           tcoords[d] = PetscAbs(tcoords[d]) < 1e-10 ? 0.0 : tcoords[d];
915         }
916         /* Rotate coordinates since PGF makes z point out of the page instead of up */
917         if (dof == 3) {PetscReal tmp = tcoords[1]; tcoords[1] = tcoords[2]; tcoords[2] = -tmp;}
918         for (d = 0; d < dof; ++d) {ccoords[d] += tcoords[d];}
919         ++n;
920       }
921       for (d = 0; d < dof; ++d) {ccoords[d] /= n;}
922       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
923       for (d = 0; d < dof; ++d) {
924         if (d > 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, ",");CHKERRQ(ierr);}
925         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "%g", (double) ccoords[d]);CHKERRQ(ierr);
926       }
927       color = colors[rank%numColors];
928       for (l = 0; l < numLabels; ++l) {
929         PetscInt val;
930         ierr = DMGetLabelValue(dm, names[l], c, &val);CHKERRQ(ierr);
931         if (val >= 0) {color = lcolors[l%numLColors]; isLabeled = PETSC_TRUE; break;}
932       }
933       if (useNumbers) {
934         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [draw,shape=circle,color=%s] {%D};\n", c, rank, color, c);CHKERRQ(ierr);
935       } else {
936         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [fill,inner sep=%dpt,shape=circle,color=%s] {};\n", c, rank, !isLabeled ? 1 : 2, color);CHKERRQ(ierr);
937       }
938     }
939     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
940     /* Plot edges */
941     if (plotEdges) {
942       ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
943       ierr = PetscViewerASCIIPrintf(viewer, "\\path\n");CHKERRQ(ierr);
944       for (e = eStart; e < eEnd; ++e) {
945         const PetscInt *cone;
946         PetscInt        coneSize, offA, offB, dof, d;
947 
948         if (wp && !PetscBTLookup(wp,e - pStart)) continue;
949         ierr = DMPlexGetConeSize(dm, e, &coneSize);CHKERRQ(ierr);
950         if (coneSize != 2) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Edge %D cone should have two vertices, not %D", e, coneSize);
951         ierr = DMPlexGetCone(dm, e, &cone);CHKERRQ(ierr);
952         ierr = PetscSectionGetDof(coordSection, cone[0], &dof);CHKERRQ(ierr);
953         ierr = PetscSectionGetOffset(coordSection, cone[0], &offA);CHKERRQ(ierr);
954         ierr = PetscSectionGetOffset(coordSection, cone[1], &offB);CHKERRQ(ierr);
955         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "(");CHKERRQ(ierr);
956         for (d = 0; d < dof; ++d) {
957           tcoords[d] = (double) (0.5*scale*PetscRealPart(coords[offA+d]+coords[offB+d]));
958           tcoords[d] = PetscAbs(tcoords[d]) < 1e-10 ? 0.0 : tcoords[d];
959         }
960         /* Rotate coordinates since PGF makes z point out of the page instead of up */
961         if (dim == 3) {PetscReal tmp = tcoords[1]; tcoords[1] = tcoords[2]; tcoords[2] = -tmp;}
962         for (d = 0; d < dof; ++d) {
963           if (d > 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, ",");CHKERRQ(ierr);}
964           ierr = PetscViewerASCIISynchronizedPrintf(viewer, "%g", (double)tcoords[d]);CHKERRQ(ierr);
965         }
966         color = colors[rank%numColors];
967         for (l = 0; l < numLabels; ++l) {
968           PetscInt val;
969           ierr = DMGetLabelValue(dm, names[l], v, &val);CHKERRQ(ierr);
970           if (val >= 0) {color = lcolors[l%numLColors]; break;}
971         }
972         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [draw,shape=circle,color=%s] {%D} --\n", e, rank, color, e);CHKERRQ(ierr);
973       }
974       ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
975       ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
976       ierr = PetscViewerASCIIPrintf(viewer, "(0,0);\n");CHKERRQ(ierr);
977     }
978     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
979     ierr = PetscViewerASCIIPopSynchronized(viewer);CHKERRQ(ierr);
980     ierr = PetscViewerASCIIPrintf(viewer, "\\end{tikzpicture}\n");CHKERRQ(ierr);
981     ierr = PetscViewerASCIIPrintf(viewer, "\\end{document}\n", name);CHKERRQ(ierr);
982     for (l = 0; l < numLabels;  ++l) {ierr = PetscFree(names[l]);CHKERRQ(ierr);}
983     for (c = 0; c < numColors;  ++c) {ierr = PetscFree(colors[c]);CHKERRQ(ierr);}
984     for (c = 0; c < numLColors; ++c) {ierr = PetscFree(lcolors[c]);CHKERRQ(ierr);}
985     ierr = PetscFree3(names, colors, lcolors);CHKERRQ(ierr);
986     ierr = PetscBTDestroy(&wp);CHKERRQ(ierr);
987   } else if (format == PETSC_VIEWER_LOAD_BALANCE) {
988     Vec                    cown,acown;
989     VecScatter             sct;
990     ISLocalToGlobalMapping g2l;
991     IS                     gid,acis;
992     MPI_Comm               comm,ncomm = MPI_COMM_NULL;
993     MPI_Group              ggroup,ngroup;
994     PetscScalar            *array,nid;
995     const PetscInt         *idxs;
996     PetscInt               *idxs2,*start,*adjacency,*work;
997     PetscInt64             lm[3],gm[3];
998     PetscInt               i,c,cStart,cEnd,cum,numVertices,ect,ectn,cellHeight;
999     PetscMPIInt            d1,d2,rank;
1000 
1001     ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
1002     ierr = MPI_Comm_rank(comm,&rank);CHKERRMPI(ierr);
1003 #if defined(PETSC_HAVE_MPI_PROCESS_SHARED_MEMORY)
1004     ierr = MPI_Comm_split_type(comm,MPI_COMM_TYPE_SHARED,rank,MPI_INFO_NULL,&ncomm);CHKERRMPI(ierr);
1005 #endif
1006     if (ncomm != MPI_COMM_NULL) {
1007       ierr = MPI_Comm_group(comm,&ggroup);CHKERRMPI(ierr);
1008       ierr = MPI_Comm_group(ncomm,&ngroup);CHKERRMPI(ierr);
1009       d1   = 0;
1010       ierr = MPI_Group_translate_ranks(ngroup,1,&d1,ggroup,&d2);CHKERRMPI(ierr);
1011       nid  = d2;
1012       ierr = MPI_Group_free(&ggroup);CHKERRMPI(ierr);
1013       ierr = MPI_Group_free(&ngroup);CHKERRMPI(ierr);
1014       ierr = MPI_Comm_free(&ncomm);CHKERRMPI(ierr);
1015     } else nid = 0.0;
1016 
1017     /* Get connectivity */
1018     ierr = DMPlexGetVTKCellHeight(dm,&cellHeight);CHKERRQ(ierr);
1019     ierr = DMPlexCreatePartitionerGraph(dm,cellHeight,&numVertices,&start,&adjacency,&gid);CHKERRQ(ierr);
1020 
1021     /* filter overlapped local cells */
1022     ierr = DMPlexGetHeightStratum(dm,cellHeight,&cStart,&cEnd);CHKERRQ(ierr);
1023     ierr = ISGetIndices(gid,&idxs);CHKERRQ(ierr);
1024     ierr = ISGetLocalSize(gid,&cum);CHKERRQ(ierr);
1025     ierr = PetscMalloc1(cum,&idxs2);CHKERRQ(ierr);
1026     for (c = cStart, cum = 0; c < cEnd; c++) {
1027       if (idxs[c-cStart] < 0) continue;
1028       idxs2[cum++] = idxs[c-cStart];
1029     }
1030     ierr = ISRestoreIndices(gid,&idxs);CHKERRQ(ierr);
1031     if (numVertices != cum) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Unexpected %D != %D",numVertices,cum);
1032     ierr = ISDestroy(&gid);CHKERRQ(ierr);
1033     ierr = ISCreateGeneral(comm,numVertices,idxs2,PETSC_OWN_POINTER,&gid);CHKERRQ(ierr);
1034 
1035     /* support for node-aware cell locality */
1036     ierr = ISCreateGeneral(comm,start[numVertices],adjacency,PETSC_USE_POINTER,&acis);CHKERRQ(ierr);
1037     ierr = VecCreateSeq(PETSC_COMM_SELF,start[numVertices],&acown);CHKERRQ(ierr);
1038     ierr = VecCreateMPI(comm,numVertices,PETSC_DECIDE,&cown);CHKERRQ(ierr);
1039     ierr = VecGetArray(cown,&array);CHKERRQ(ierr);
1040     for (c = 0; c < numVertices; c++) array[c] = nid;
1041     ierr = VecRestoreArray(cown,&array);CHKERRQ(ierr);
1042     ierr = VecScatterCreate(cown,acis,acown,NULL,&sct);CHKERRQ(ierr);
1043     ierr = VecScatterBegin(sct,cown,acown,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
1044     ierr = VecScatterEnd(sct,cown,acown,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
1045     ierr = ISDestroy(&acis);CHKERRQ(ierr);
1046     ierr = VecScatterDestroy(&sct);CHKERRQ(ierr);
1047     ierr = VecDestroy(&cown);CHKERRQ(ierr);
1048 
1049     /* compute edgeCut */
1050     for (c = 0, cum = 0; c < numVertices; c++) cum = PetscMax(cum,start[c+1]-start[c]);
1051     ierr = PetscMalloc1(cum,&work);CHKERRQ(ierr);
1052     ierr = ISLocalToGlobalMappingCreateIS(gid,&g2l);CHKERRQ(ierr);
1053     ierr = ISLocalToGlobalMappingSetType(g2l,ISLOCALTOGLOBALMAPPINGHASH);CHKERRQ(ierr);
1054     ierr = ISDestroy(&gid);CHKERRQ(ierr);
1055     ierr = VecGetArray(acown,&array);CHKERRQ(ierr);
1056     for (c = 0, ect = 0, ectn = 0; c < numVertices; c++) {
1057       PetscInt totl;
1058 
1059       totl = start[c+1]-start[c];
1060       ierr = ISGlobalToLocalMappingApply(g2l,IS_GTOLM_MASK,totl,adjacency+start[c],NULL,work);CHKERRQ(ierr);
1061       for (i = 0; i < totl; i++) {
1062         if (work[i] < 0) {
1063           ect  += 1;
1064           ectn += (array[i + start[c]] != nid) ? 0 : 1;
1065         }
1066       }
1067     }
1068     ierr  = PetscFree(work);CHKERRQ(ierr);
1069     ierr  = VecRestoreArray(acown,&array);CHKERRQ(ierr);
1070     lm[0] = numVertices > 0 ?  numVertices : PETSC_MAX_INT;
1071     lm[1] = -numVertices;
1072     ierr  = MPIU_Allreduce(lm,gm,2,MPIU_INT64,MPI_MIN,comm);CHKERRMPI(ierr);
1073     ierr  = PetscViewerASCIIPrintf(viewer,"  Cell balance: %.2f (max %D, min %D",-((double)gm[1])/((double)gm[0]),-(PetscInt)gm[1],(PetscInt)gm[0]);CHKERRQ(ierr);
1074     lm[0] = ect; /* edgeCut */
1075     lm[1] = ectn; /* node-aware edgeCut */
1076     lm[2] = numVertices > 0 ? 0 : 1; /* empty processes */
1077     ierr  = MPIU_Allreduce(lm,gm,3,MPIU_INT64,MPI_SUM,comm);CHKERRMPI(ierr);
1078     ierr  = PetscViewerASCIIPrintf(viewer,", empty %D)\n",(PetscInt)gm[2]);CHKERRQ(ierr);
1079 #if defined(PETSC_HAVE_MPI_PROCESS_SHARED_MEMORY)
1080     ierr  = PetscViewerASCIIPrintf(viewer,"  Edge Cut: %D (on node %.3f)\n",(PetscInt)(gm[0]/2),gm[0] ? ((double)(gm[1]))/((double)gm[0]) : 1.);CHKERRQ(ierr);
1081 #else
1082     ierr  = PetscViewerASCIIPrintf(viewer,"  Edge Cut: %D (on node %.3f)\n",(PetscInt)(gm[0]/2),0.0);CHKERRQ(ierr);
1083 #endif
1084     ierr  = ISLocalToGlobalMappingDestroy(&g2l);CHKERRQ(ierr);
1085     ierr  = PetscFree(start);CHKERRQ(ierr);
1086     ierr  = PetscFree(adjacency);CHKERRQ(ierr);
1087     ierr  = VecDestroy(&acown);CHKERRQ(ierr);
1088   } else {
1089     const char    *name;
1090     PetscInt      *sizes, *hybsizes, *ghostsizes;
1091     PetscInt       locDepth, depth, cellHeight, dim, d;
1092     PetscInt       pStart, pEnd, p, gcStart, gcEnd, gcNum;
1093     PetscInt       numLabels, l;
1094     DMPolytopeType ct0 = DM_POLYTOPE_UNKNOWN;
1095     MPI_Comm       comm;
1096     PetscMPIInt    size, rank;
1097 
1098     ierr = PetscObjectGetComm((PetscObject) dm, &comm);CHKERRQ(ierr);
1099     ierr = MPI_Comm_size(comm, &size);CHKERRMPI(ierr);
1100     ierr = MPI_Comm_rank(comm, &rank);CHKERRMPI(ierr);
1101     ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
1102     ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
1103     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
1104     if (name) {ierr = PetscViewerASCIIPrintf(viewer, "%s in %D dimension%s:\n", name, dim, dim == 1 ? "" : "s");CHKERRQ(ierr);}
1105     else      {ierr = PetscViewerASCIIPrintf(viewer, "Mesh in %D dimension%s:\n", dim, dim == 1 ? "" : "s");CHKERRQ(ierr);}
1106     if (cellHeight) {ierr = PetscViewerASCIIPrintf(viewer, "  Cells are at height %D\n", cellHeight);CHKERRQ(ierr);}
1107     ierr = DMPlexGetDepth(dm, &locDepth);CHKERRQ(ierr);
1108     ierr = MPIU_Allreduce(&locDepth, &depth, 1, MPIU_INT, MPI_MAX, comm);CHKERRMPI(ierr);
1109     ierr = DMPlexGetGhostCellStratum(dm, &gcStart, &gcEnd);CHKERRQ(ierr);
1110     gcNum = gcEnd - gcStart;
1111     ierr = PetscCalloc3(size,&sizes,size,&hybsizes,size,&ghostsizes);CHKERRQ(ierr);
1112     for (d = 0; d <= depth; d++) {
1113       PetscInt Nc[2] = {0, 0}, ict;
1114 
1115       ierr = DMPlexGetDepthStratum(dm, d, &pStart, &pEnd);CHKERRQ(ierr);
1116       if (pStart < pEnd) {ierr = DMPlexGetCellType(dm, pStart, &ct0);CHKERRQ(ierr);}
1117       ict  = ct0;
1118       ierr = MPI_Bcast(&ict, 1, MPIU_INT, 0, comm);CHKERRMPI(ierr);
1119       ct0  = (DMPolytopeType) ict;
1120       for (p = pStart; p < pEnd; ++p) {
1121         DMPolytopeType ct;
1122 
1123         ierr = DMPlexGetCellType(dm, p, &ct);CHKERRQ(ierr);
1124         if (ct == ct0) ++Nc[0];
1125         else           ++Nc[1];
1126       }
1127       ierr = MPI_Gather(&Nc[0], 1, MPIU_INT, sizes,    1, MPIU_INT, 0, comm);CHKERRMPI(ierr);
1128       ierr = MPI_Gather(&Nc[1], 1, MPIU_INT, hybsizes, 1, MPIU_INT, 0, comm);CHKERRMPI(ierr);
1129       if (d == depth) {ierr = MPI_Gather(&gcNum, 1, MPIU_INT, ghostsizes, 1, MPIU_INT, 0, comm);CHKERRMPI(ierr);}
1130       ierr = PetscViewerASCIIPrintf(viewer, "  %D-cells:", (depth == 1) && d ? dim : d);CHKERRQ(ierr);
1131       for (p = 0; p < size; ++p) {
1132         if (!rank) {
1133           ierr = PetscViewerASCIIPrintf(viewer, " %D", sizes[p]+hybsizes[p]);CHKERRQ(ierr);
1134           if (hybsizes[p]   > 0) {ierr = PetscViewerASCIIPrintf(viewer, " (%D)", hybsizes[p]);CHKERRQ(ierr);}
1135           if (ghostsizes[p] > 0) {ierr = PetscViewerASCIIPrintf(viewer, " [%D]", ghostsizes[p]);CHKERRQ(ierr);}
1136         }
1137       }
1138       ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
1139     }
1140     ierr = PetscFree3(sizes,hybsizes,ghostsizes);CHKERRQ(ierr);
1141     {
1142       const PetscReal      *maxCell;
1143       const PetscReal      *L;
1144       const DMBoundaryType *bd;
1145       PetscBool             per, localized;
1146 
1147       ierr = DMGetPeriodicity(dm, &per, &maxCell, &L, &bd);CHKERRQ(ierr);
1148       ierr = DMGetCoordinatesLocalized(dm, &localized);CHKERRQ(ierr);
1149       if (per) {
1150         ierr = PetscViewerASCIIPrintf(viewer, "Periodic mesh (");CHKERRQ(ierr);
1151         ierr = PetscViewerASCIIUseTabs(viewer, PETSC_FALSE);CHKERRQ(ierr);
1152         for (d = 0; d < dim; ++d) {
1153           if (bd && d > 0) {ierr = PetscViewerASCIIPrintf(viewer, ", ");CHKERRQ(ierr);}
1154           if (bd)    {ierr = PetscViewerASCIIPrintf(viewer, "%s", DMBoundaryTypes[bd[d]]);CHKERRQ(ierr);}
1155         }
1156         ierr = PetscViewerASCIIPrintf(viewer, ") coordinates %s\n", localized ? "localized" : "not localized");CHKERRQ(ierr);
1157         ierr = PetscViewerASCIIUseTabs(viewer, PETSC_TRUE);CHKERRQ(ierr);
1158       }
1159     }
1160     ierr = DMGetNumLabels(dm, &numLabels);CHKERRQ(ierr);
1161     if (numLabels) {ierr = PetscViewerASCIIPrintf(viewer, "Labels:\n");CHKERRQ(ierr);}
1162     for (l = 0; l < numLabels; ++l) {
1163       DMLabel         label;
1164       const char     *name;
1165       IS              valueIS;
1166       const PetscInt *values;
1167       PetscInt        numValues, v;
1168 
1169       ierr = DMGetLabelName(dm, l, &name);CHKERRQ(ierr);
1170       ierr = DMGetLabel(dm, name, &label);CHKERRQ(ierr);
1171       ierr = DMLabelGetNumValues(label, &numValues);CHKERRQ(ierr);
1172       ierr = PetscViewerASCIIPrintf(viewer, "  %s: %D strata with value/size (", name, numValues);CHKERRQ(ierr);
1173       ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
1174       ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
1175       ierr = PetscViewerASCIIUseTabs(viewer, PETSC_FALSE);CHKERRQ(ierr);
1176       for (v = 0; v < numValues; ++v) {
1177         PetscInt size;
1178 
1179         ierr = DMLabelGetStratumSize(label, values[v], &size);CHKERRQ(ierr);
1180         if (v > 0) {ierr = PetscViewerASCIIPrintf(viewer, ", ");CHKERRQ(ierr);}
1181         ierr = PetscViewerASCIIPrintf(viewer, "%D (%D)", values[v], size);CHKERRQ(ierr);
1182       }
1183       ierr = PetscViewerASCIIPrintf(viewer, ")\n");CHKERRQ(ierr);
1184       ierr = PetscViewerASCIIUseTabs(viewer, PETSC_TRUE);CHKERRQ(ierr);
1185       ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
1186       ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
1187     }
1188     /* If no fields are specified, people do not want to see adjacency */
1189     if (dm->Nf) {
1190       PetscInt f;
1191 
1192       for (f = 0; f < dm->Nf; ++f) {
1193         const char *name;
1194 
1195         ierr = PetscObjectGetName(dm->fields[f].disc, &name);CHKERRQ(ierr);
1196         if (numLabels) {ierr = PetscViewerASCIIPrintf(viewer, "Field %s:\n", name);CHKERRQ(ierr);}
1197         ierr = PetscViewerASCIIPushTab(viewer);CHKERRQ(ierr);
1198         if (dm->fields[f].label) {ierr = DMLabelView(dm->fields[f].label, viewer);CHKERRQ(ierr);}
1199         if (dm->fields[f].adjacency[0]) {
1200           if (dm->fields[f].adjacency[1]) {ierr = PetscViewerASCIIPrintf(viewer, "adjacency FVM++\n");CHKERRQ(ierr);}
1201           else                            {ierr = PetscViewerASCIIPrintf(viewer, "adjacency FVM\n");CHKERRQ(ierr);}
1202         } else {
1203           if (dm->fields[f].adjacency[1]) {ierr = PetscViewerASCIIPrintf(viewer, "adjacency FEM\n");CHKERRQ(ierr);}
1204           else                            {ierr = PetscViewerASCIIPrintf(viewer, "adjacency FUNKY\n");CHKERRQ(ierr);}
1205         }
1206         ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr);
1207       }
1208     }
1209     ierr = DMGetCoarseDM(dm, &cdm);CHKERRQ(ierr);
1210     if (cdm) {
1211       ierr = PetscViewerASCIIPushTab(viewer);CHKERRQ(ierr);
1212       ierr = DMPlexView_Ascii(cdm, viewer);CHKERRQ(ierr);
1213       ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr);
1214     }
1215   }
1216   PetscFunctionReturn(0);
1217 }
1218 
1219 static PetscErrorCode DMPlexDrawCell(DM dm, PetscDraw draw, PetscInt cell, const PetscScalar coords[])
1220 {
1221   DMPolytopeType ct;
1222   PetscMPIInt    rank;
1223   PetscErrorCode ierr;
1224 
1225   PetscFunctionBegin;
1226   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRMPI(ierr);
1227   ierr = DMPlexGetCellType(dm, cell, &ct);CHKERRQ(ierr);
1228   switch (ct) {
1229   case DM_POLYTOPE_TRIANGLE:
1230     ierr = PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]),
1231                              PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1232                              PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1233                              PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2);CHKERRQ(ierr);
1234     ierr = PetscDrawLine(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1235     ierr = PetscDrawLine(draw, PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1236     ierr = PetscDrawLine(draw, PetscRealPart(coords[4]), PetscRealPart(coords[5]), PetscRealPart(coords[0]), PetscRealPart(coords[1]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1237     break;
1238   case DM_POLYTOPE_QUADRILATERAL:
1239     ierr = PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]),
1240                               PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1241                               PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1242                               PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2);CHKERRQ(ierr);
1243     ierr = PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), PetscRealPart(coords[6]), PetscRealPart(coords[7]),
1244                               PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1245                               PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1246                               PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2);CHKERRQ(ierr);
1247     ierr = PetscDrawLine(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1248     ierr = PetscDrawLine(draw, PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1249     ierr = PetscDrawLine(draw, PetscRealPart(coords[4]), PetscRealPart(coords[5]), PetscRealPart(coords[6]), PetscRealPart(coords[7]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1250     ierr = PetscDrawLine(draw, PetscRealPart(coords[6]), PetscRealPart(coords[7]), PetscRealPart(coords[0]), PetscRealPart(coords[1]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1251     break;
1252   default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_SUP, "Cannot draw cells of type %s", DMPolytopeTypes[ct]);
1253   }
1254   PetscFunctionReturn(0);
1255 }
1256 
1257 static PetscErrorCode DMPlexDrawCellHighOrder(DM dm, PetscDraw draw, PetscInt cell, const PetscScalar coords[], PetscInt edgeDiv, PetscReal refCoords[], PetscReal edgeCoords[])
1258 {
1259   DMPolytopeType ct;
1260   PetscReal      centroid[2] = {0., 0.};
1261   PetscMPIInt    rank;
1262   PetscInt       fillColor, v, e, d;
1263   PetscErrorCode ierr;
1264 
1265   PetscFunctionBegin;
1266   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRMPI(ierr);
1267   ierr = DMPlexGetCellType(dm, cell, &ct);CHKERRQ(ierr);
1268   fillColor = PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2;
1269   switch (ct) {
1270   case DM_POLYTOPE_TRIANGLE:
1271     {
1272       PetscReal refVertices[6] = {-1., -1., 1., -1., -1., 1.};
1273 
1274       for (v = 0; v < 3; ++v) {centroid[0] += PetscRealPart(coords[v*2+0])/3.;centroid[1] += PetscRealPart(coords[v*2+1])/3.;}
1275       for (e = 0; e < 3; ++e) {
1276         refCoords[0] = refVertices[e*2+0];
1277         refCoords[1] = refVertices[e*2+1];
1278         for (d = 1; d <= edgeDiv; ++d) {
1279           refCoords[d*2+0] = refCoords[0] + (refVertices[(e+1)%3 * 2 + 0] - refCoords[0])*d/edgeDiv;
1280           refCoords[d*2+1] = refCoords[1] + (refVertices[(e+1)%3 * 2 + 1] - refCoords[1])*d/edgeDiv;
1281         }
1282         ierr = DMPlexReferenceToCoordinates(dm, cell, edgeDiv+1, refCoords, edgeCoords);CHKERRQ(ierr);
1283         for (d = 0; d < edgeDiv; ++d) {
1284           ierr = PetscDrawTriangle(draw, centroid[0], centroid[1], edgeCoords[d*2+0], edgeCoords[d*2+1], edgeCoords[(d+1)*2+0], edgeCoords[(d+1)*2+1], fillColor, fillColor, fillColor);CHKERRQ(ierr);
1285           ierr = PetscDrawLine(draw, edgeCoords[d*2+0], edgeCoords[d*2+1], edgeCoords[(d+1)*2+0], edgeCoords[(d+1)*2+1], PETSC_DRAW_BLACK);CHKERRQ(ierr);
1286         }
1287       }
1288     }
1289     break;
1290   default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_SUP, "Cannot draw cells of type %s", DMPolytopeTypes[ct]);
1291   }
1292   PetscFunctionReturn(0);
1293 }
1294 
1295 static PetscErrorCode DMPlexView_Draw(DM dm, PetscViewer viewer)
1296 {
1297   PetscDraw          draw;
1298   DM                 cdm;
1299   PetscSection       coordSection;
1300   Vec                coordinates;
1301   const PetscScalar *coords;
1302   PetscReal          xyl[2],xyr[2],bound[4] = {PETSC_MAX_REAL, PETSC_MAX_REAL, PETSC_MIN_REAL, PETSC_MIN_REAL};
1303   PetscReal         *refCoords, *edgeCoords;
1304   PetscBool          isnull, drawAffine = PETSC_TRUE;
1305   PetscInt           dim, vStart, vEnd, cStart, cEnd, c, N, edgeDiv = 4;
1306   PetscErrorCode     ierr;
1307 
1308   PetscFunctionBegin;
1309   ierr = DMGetCoordinateDim(dm, &dim);CHKERRQ(ierr);
1310   if (dim != 2) SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Cannot draw meshes of dimension %D", dim);
1311   ierr = PetscOptionsGetBool(((PetscObject) dm)->options, ((PetscObject) dm)->prefix, "-dm_view_draw_affine", &drawAffine, NULL);CHKERRQ(ierr);
1312   if (!drawAffine) {ierr = PetscMalloc2((edgeDiv+1)*dim, &refCoords, (edgeDiv+1)*dim, &edgeCoords);CHKERRQ(ierr);}
1313   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
1314   ierr = DMGetLocalSection(cdm, &coordSection);CHKERRQ(ierr);
1315   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
1316   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
1317   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
1318 
1319   ierr = PetscViewerDrawGetDraw(viewer, 0, &draw);CHKERRQ(ierr);
1320   ierr = PetscDrawIsNull(draw, &isnull);CHKERRQ(ierr);
1321   if (isnull) PetscFunctionReturn(0);
1322   ierr = PetscDrawSetTitle(draw, "Mesh");CHKERRQ(ierr);
1323 
1324   ierr = VecGetLocalSize(coordinates, &N);CHKERRQ(ierr);
1325   ierr = VecGetArrayRead(coordinates, &coords);CHKERRQ(ierr);
1326   for (c = 0; c < N; c += dim) {
1327     bound[0] = PetscMin(bound[0], PetscRealPart(coords[c]));   bound[2] = PetscMax(bound[2], PetscRealPart(coords[c]));
1328     bound[1] = PetscMin(bound[1], PetscRealPart(coords[c+1])); bound[3] = PetscMax(bound[3], PetscRealPart(coords[c+1]));
1329   }
1330   ierr = VecRestoreArrayRead(coordinates, &coords);CHKERRQ(ierr);
1331   ierr = MPIU_Allreduce(&bound[0],xyl,2,MPIU_REAL,MPIU_MIN,PetscObjectComm((PetscObject)dm));CHKERRMPI(ierr);
1332   ierr = MPIU_Allreduce(&bound[2],xyr,2,MPIU_REAL,MPIU_MAX,PetscObjectComm((PetscObject)dm));CHKERRMPI(ierr);
1333   ierr = PetscDrawSetCoordinates(draw, xyl[0], xyl[1], xyr[0], xyr[1]);CHKERRQ(ierr);
1334   ierr = PetscDrawClear(draw);CHKERRQ(ierr);
1335 
1336   for (c = cStart; c < cEnd; ++c) {
1337     PetscScalar *coords = NULL;
1338     PetscInt     numCoords;
1339 
1340     ierr = DMPlexVecGetClosureAtDepth_Internal(dm, coordSection, coordinates, c, 0, &numCoords, &coords);CHKERRQ(ierr);
1341     if (drawAffine) {
1342       ierr = DMPlexDrawCell(dm, draw, c, coords);CHKERRQ(ierr);
1343     } else {
1344       ierr = DMPlexDrawCellHighOrder(dm, draw, c, coords, edgeDiv, refCoords, edgeCoords);CHKERRQ(ierr);
1345     }
1346     ierr = DMPlexVecRestoreClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
1347   }
1348   if (!drawAffine) {ierr = PetscFree2(refCoords, edgeCoords);CHKERRQ(ierr);}
1349   ierr = PetscDrawFlush(draw);CHKERRQ(ierr);
1350   ierr = PetscDrawPause(draw);CHKERRQ(ierr);
1351   ierr = PetscDrawSave(draw);CHKERRQ(ierr);
1352   PetscFunctionReturn(0);
1353 }
1354 
1355 #if defined(PETSC_HAVE_EXODUSII)
1356 #include <exodusII.h>
1357 #include <petscviewerexodusii.h>
1358 #endif
1359 
1360 PetscErrorCode DMView_Plex(DM dm, PetscViewer viewer)
1361 {
1362   PetscBool      iascii, ishdf5, isvtk, isdraw, flg, isglvis, isexodus;
1363   char           name[PETSC_MAX_PATH_LEN];
1364   PetscErrorCode ierr;
1365 
1366   PetscFunctionBegin;
1367   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1368   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1369   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERASCII,    &iascii);CHKERRQ(ierr);
1370   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,      &isvtk);CHKERRQ(ierr);
1371   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,     &ishdf5);CHKERRQ(ierr);
1372   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERDRAW,     &isdraw);CHKERRQ(ierr);
1373   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERGLVIS,    &isglvis);CHKERRQ(ierr);
1374   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWEREXODUSII, &isexodus);CHKERRQ(ierr);
1375   if (iascii) {
1376     PetscViewerFormat format;
1377     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
1378     if (format == PETSC_VIEWER_ASCII_GLVIS) {
1379       ierr = DMPlexView_GLVis(dm, viewer);CHKERRQ(ierr);
1380     } else {
1381       ierr = DMPlexView_Ascii(dm, viewer);CHKERRQ(ierr);
1382     }
1383   } else if (ishdf5) {
1384 #if defined(PETSC_HAVE_HDF5)
1385     ierr = DMPlexView_HDF5_Internal(dm, viewer);CHKERRQ(ierr);
1386 #else
1387     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1388 #endif
1389   } else if (isvtk) {
1390     ierr = DMPlexVTKWriteAll((PetscObject) dm,viewer);CHKERRQ(ierr);
1391   } else if (isdraw) {
1392     ierr = DMPlexView_Draw(dm, viewer);CHKERRQ(ierr);
1393   } else if (isglvis) {
1394     ierr = DMPlexView_GLVis(dm, viewer);CHKERRQ(ierr);
1395 #if defined(PETSC_HAVE_EXODUSII)
1396   } else if (isexodus) {
1397 /*
1398       exodusII requires that all sets be part of exactly one cell set.
1399       If the dm does not have a "Cell Sets" label defined, we create one
1400       with ID 1, containig all cells.
1401       Note that if the Cell Sets label is defined but does not cover all cells,
1402       we may still have a problem. This should probably be checked here or in the viewer;
1403     */
1404     PetscInt numCS;
1405     ierr = DMGetLabelSize(dm,"Cell Sets",&numCS);CHKERRQ(ierr);
1406     if (!numCS) {
1407       PetscInt cStart, cEnd, c;
1408       ierr = DMCreateLabel(dm, "Cell Sets");CHKERRQ(ierr);
1409       ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
1410       for (c = cStart; c < cEnd; ++c) {ierr = DMSetLabelValue(dm, "Cell Sets", c, 1);CHKERRQ(ierr);}
1411     }
1412     ierr = DMView_PlexExodusII(dm, viewer);CHKERRQ(ierr);
1413 #endif
1414   } else {
1415     SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Viewer type %s not yet supported for DMPlex writing", ((PetscObject)viewer)->type_name);
1416   }
1417   /* Optionally view the partition */
1418   ierr = PetscOptionsHasName(((PetscObject) dm)->options, ((PetscObject) dm)->prefix, "-dm_partition_view", &flg);CHKERRQ(ierr);
1419   if (flg) {
1420     Vec ranks;
1421     ierr = DMPlexCreateRankField(dm, &ranks);CHKERRQ(ierr);
1422     ierr = VecView(ranks, viewer);CHKERRQ(ierr);
1423     ierr = VecDestroy(&ranks);CHKERRQ(ierr);
1424   }
1425   /* Optionally view a label */
1426   ierr = PetscOptionsGetString(((PetscObject) dm)->options, ((PetscObject) dm)->prefix, "-dm_label_view", name, sizeof(name), &flg);CHKERRQ(ierr);
1427   if (flg) {
1428     DMLabel label;
1429     Vec     val;
1430 
1431     ierr = DMGetLabel(dm, name, &label);CHKERRQ(ierr);
1432     if (!label) SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Label %s provided to -dm_label_view does not exist in this DM", name);
1433     ierr = DMPlexCreateLabelField(dm, label, &val);CHKERRQ(ierr);
1434     ierr = VecView(val, viewer);CHKERRQ(ierr);
1435     ierr = VecDestroy(&val);CHKERRQ(ierr);
1436   }
1437   PetscFunctionReturn(0);
1438 }
1439 
1440 /*@
1441   DMPlexTopologyView - Saves a DMPlex topology into a file
1442 
1443   Collective on DM
1444 
1445   Input Parameters:
1446 + dm     - The DM whose topology is to be saved
1447 - viewer - The PetscViewer for saving
1448 
1449   Level: advanced
1450 
1451 .seealso: DMView(), DMPlexCoordinatesView(), DMPlexLabelsView(), DMPlexTopologyLoad()
1452 @*/
1453 PetscErrorCode DMPlexTopologyView(DM dm, PetscViewer viewer)
1454 {
1455   PetscBool      ishdf5;
1456   PetscErrorCode ierr;
1457 
1458   PetscFunctionBegin;
1459   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1460   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1461   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
1462   if (ishdf5) {
1463 #if defined(PETSC_HAVE_HDF5)
1464     PetscViewerFormat format;
1465     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
1466     if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
1467       IS globalPointNumbering;
1468 
1469       ierr = DMPlexCreatePointNumbering(dm, &globalPointNumbering);CHKERRQ(ierr);
1470       ierr = DMPlexTopologyView_HDF5_Internal(dm, globalPointNumbering, viewer);CHKERRQ(ierr);
1471       ierr = ISDestroy(&globalPointNumbering);CHKERRQ(ierr);
1472     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 output.", PetscViewerFormats[format]);
1473 #else
1474     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1475 #endif
1476   }
1477   PetscFunctionReturn(0);
1478 }
1479 
1480 /*@
1481   DMPlexCoordinatesView - Saves DMPlex coordinates into a file
1482 
1483   Collective on DM
1484 
1485   Input Parameters:
1486 + dm     - The DM whose coordinates are to be saved
1487 - viewer - The PetscViewer for saving
1488 
1489   Level: advanced
1490 
1491 .seealso: DMView(), DMPlexTopologyView(), DMPlexLabelsView(), DMPlexCoordinatesLoad()
1492 @*/
1493 PetscErrorCode DMPlexCoordinatesView(DM dm, PetscViewer viewer)
1494 {
1495   PetscBool      ishdf5;
1496   PetscErrorCode ierr;
1497 
1498   PetscFunctionBegin;
1499   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1500   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1501   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
1502   if (ishdf5) {
1503 #if defined(PETSC_HAVE_HDF5)
1504     PetscViewerFormat format;
1505     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
1506     if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
1507       ierr = DMPlexCoordinatesView_HDF5_Internal(dm, viewer);CHKERRQ(ierr);
1508     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 input.", PetscViewerFormats[format]);
1509 #else
1510     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1511 #endif
1512   }
1513   PetscFunctionReturn(0);
1514 }
1515 
1516 /*@
1517   DMPlexLabelsView - Saves DMPlex labels into a file
1518 
1519   Collective on DM
1520 
1521   Input Parameters:
1522 + dm     - The DM whose labels are to be saved
1523 - viewer - The PetscViewer for saving
1524 
1525   Level: advanced
1526 
1527 .seealso: DMView(), DMPlexTopologyView(), DMPlexCoordinatesView(), DMPlexLabelsLoad()
1528 @*/
1529 PetscErrorCode DMPlexLabelsView(DM dm, PetscViewer viewer)
1530 {
1531   PetscBool      ishdf5;
1532   PetscErrorCode ierr;
1533 
1534   PetscFunctionBegin;
1535   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1536   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1537   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
1538   if (ishdf5) {
1539 #if defined(PETSC_HAVE_HDF5)
1540     IS                globalPointNumbering;
1541     PetscViewerFormat format;
1542 
1543     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
1544     if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
1545       ierr = DMPlexCreatePointNumbering(dm, &globalPointNumbering);CHKERRQ(ierr);
1546       ierr = DMPlexLabelsView_HDF5_Internal(dm, globalPointNumbering, viewer);CHKERRQ(ierr);
1547       ierr = ISDestroy(&globalPointNumbering);CHKERRQ(ierr);
1548     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 input.", PetscViewerFormats[format]);
1549 #else
1550     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1551 #endif
1552   }
1553   PetscFunctionReturn(0);
1554 }
1555 
1556 /*@
1557   DMPlexSectionView - Saves a section associated with a DMPlex
1558 
1559   Collective on DM
1560 
1561   Input Parameters:
1562 + dm         - The DM that contains the topology on which the section to be saved is defined
1563 . viewer     - The PetscViewer for saving
1564 - sectiondm  - The DM that contains the section to be saved
1565 
1566   Level: advanced
1567 
1568   Notes:
1569   This function is a wrapper around PetscSectionView(); in addition to the raw section, it saves information that associates the section points to the topology (dm) points. When the topology (dm) and the section are later loaded with DMPlexTopologyLoad() and DMPlexSectionLoad(), respectively, this information is used to match section points with topology points.
1570 
1571   In general dm and sectiondm are two different objects, the former carrying the topology and the latter carrying the section, and have been given a topology name and a section name, respectively, with PetscObjectSetName(). In practice, however, they can be the same object if it carries both topology and section; in that case the name of the object is used as both the topology name and the section name.
1572 
1573 .seealso: DMView(), DMPlexTopologyView(), DMPlexCoordinatesView(), DMPlexLabelsView(), DMPlexGlobalVectorView(), DMPlexLocalVectorView(), PetscSectionView(), DMPlexSectionLoad()
1574 @*/
1575 PetscErrorCode DMPlexSectionView(DM dm, PetscViewer viewer, DM sectiondm)
1576 {
1577   PetscBool      ishdf5;
1578   PetscErrorCode ierr;
1579 
1580   PetscFunctionBegin;
1581   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1582   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1583   PetscValidHeaderSpecific(sectiondm, DM_CLASSID, 3);
1584   ierr = PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERHDF5,&ishdf5);CHKERRQ(ierr);
1585   if (ishdf5) {
1586 #if defined(PETSC_HAVE_HDF5)
1587     ierr = DMPlexSectionView_HDF5_Internal(dm, viewer, sectiondm);CHKERRQ(ierr);
1588 #else
1589     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1590 #endif
1591   }
1592   PetscFunctionReturn(0);
1593 }
1594 
1595 /*@
1596   DMPlexGlobalVectorView - Saves a global vector
1597 
1598   Collective on DM
1599 
1600   Input Parameters:
1601 + dm        - The DM that represents the topology
1602 . viewer    - The PetscViewer to save data with
1603 . sectiondm - The DM that contains the global section on which vec is defined
1604 - vec       - The global vector to be saved
1605 
1606   Level: advanced
1607 
1608   Notes:
1609   In general dm and sectiondm are two different objects, the former carrying the topology and the latter carrying the section, and have been given a topology name and a section name, respectively, with PetscObjectSetName(). In practice, however, they can be the same object if it carries both topology and section; in that case the name of the object is used as both the topology name and the section name.
1610 
1611   Typical calling sequence
1612 $       DMCreate(PETSC_COMM_WORLD, &dm);
1613 $       DMSetType(dm, DMPLEX);
1614 $       PetscObjectSetName((PetscObject)dm, "topologydm_name");
1615 $       DMClone(dm, &sectiondm);
1616 $       PetscObjectSetName((PetscObject)sectiondm, "sectiondm_name");
1617 $       PetscSectionCreate(PETSC_COMM_WORLD, &section);
1618 $       DMPlexGetChart(sectiondm, &pStart, &pEnd);
1619 $       PetscSectionSetChart(section, pStart, pEnd);
1620 $       PetscSectionSetUp(section);
1621 $       DMSetLocalSection(sectiondm, section);
1622 $       PetscSectionDestroy(&section);
1623 $       DMGetGlobalVector(sectiondm, &vec);
1624 $       PetscObjectSetName((PetscObject)vec, "vec_name");
1625 $       DMPlexTopologyView(dm, viewer);
1626 $       DMPlexSectionView(dm, viewer, sectiondm);
1627 $       DMPlexGlobalVectorView(dm, viewer, sectiondm, vec);
1628 $       DMRestoreGlobalVector(sectiondm, &vec);
1629 $       DMDestroy(&sectiondm);
1630 $       DMDestroy(&dm);
1631 
1632 .seealso: DMPlexTopologyView(), DMPlexSectionView(), DMPlexLocalVectorView(), DMPlexGlobalVectorLoad(), DMPlexLocalVectorLoad()
1633 @*/
1634 PetscErrorCode DMPlexGlobalVectorView(DM dm, PetscViewer viewer, DM sectiondm, Vec vec)
1635 {
1636   PetscBool       ishdf5;
1637   PetscErrorCode  ierr;
1638 
1639   PetscFunctionBegin;
1640   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1641   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1642   PetscValidHeaderSpecific(sectiondm, DM_CLASSID, 3);
1643   PetscValidHeaderSpecific(vec, VEC_CLASSID, 4);
1644   /* Check consistency */
1645   {
1646     PetscSection  section;
1647     PetscBool     includesConstraints;
1648     PetscInt      m, m1;
1649 
1650     ierr = VecGetLocalSize(vec, &m1);CHKERRQ(ierr);
1651     ierr = DMGetGlobalSection(sectiondm, &section);CHKERRQ(ierr);
1652     ierr = PetscSectionGetIncludesConstraints(section, &includesConstraints);CHKERRQ(ierr);
1653     if (includesConstraints) {ierr = PetscSectionGetStorageSize(section, &m);CHKERRQ(ierr);}
1654     else {ierr = PetscSectionGetConstrainedStorageSize(section, &m);CHKERRQ(ierr);}
1655     if (m1 != m) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Global vector size (%D) != global section storage size (%D)", m1, m);
1656   }
1657   ierr = PetscObjectTypeCompare((PetscObject)viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
1658   if (ishdf5) {
1659 #if defined(PETSC_HAVE_HDF5)
1660     ierr = DMPlexGlobalVectorView_HDF5_Internal(dm, viewer, sectiondm, vec);CHKERRQ(ierr);
1661 #else
1662     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1663 #endif
1664   }
1665   PetscFunctionReturn(0);
1666 }
1667 
1668 /*@
1669   DMPlexLocalVectorView - Saves a local vector
1670 
1671   Collective on DM
1672 
1673   Input Parameters:
1674 + dm        - The DM that represents the topology
1675 . viewer    - The PetscViewer to save data with
1676 . sectiondm - The DM that contains the local section on which vec is defined; may be the same as dm
1677 - vec       - The local vector to be saved
1678 
1679   Level: advanced
1680 
1681   Notes:
1682   In general dm and sectiondm are two different objects, the former carrying the topology and the latter carrying the section, and have been given a topology name and a section name, respectively, with PetscObjectSetName(). In practice, however, they can be the same object if it carries both topology and section; in that case the name of the object is used as both the topology name and the section name.
1683 
1684   Typical calling sequence
1685 $       DMCreate(PETSC_COMM_WORLD, &dm);
1686 $       DMSetType(dm, DMPLEX);
1687 $       PetscObjectSetName((PetscObject)dm, "topologydm_name");
1688 $       DMClone(dm, &sectiondm);
1689 $       PetscObjectSetName((PetscObject)sectiondm, "sectiondm_name");
1690 $       PetscSectionCreate(PETSC_COMM_WORLD, &section);
1691 $       DMPlexGetChart(sectiondm, &pStart, &pEnd);
1692 $       PetscSectionSetChart(section, pStart, pEnd);
1693 $       PetscSectionSetUp(section);
1694 $       DMSetLocalSection(sectiondm, section);
1695 $       DMGetLocalVector(sectiondm, &vec);
1696 $       PetscObjectSetName((PetscObject)vec, "vec_name");
1697 $       DMPlexTopologyView(dm, viewer);
1698 $       DMPlexSectionView(dm, viewer, sectiondm);
1699 $       DMPlexLocalVectorView(dm, viewer, sectiondm, vec);
1700 $       DMRestoreLocalVector(sectiondm, &vec);
1701 $       DMDestroy(&sectiondm);
1702 $       DMDestroy(&dm);
1703 
1704 .seealso: DMPlexTopologyView(), DMPlexSectionView(), DMPlexGlobalVectorView(), DMPlexGlobalVectorLoad(), DMPlexLocalVectorLoad()
1705 @*/
1706 PetscErrorCode DMPlexLocalVectorView(DM dm, PetscViewer viewer, DM sectiondm, Vec vec)
1707 {
1708   PetscBool       ishdf5;
1709   PetscErrorCode  ierr;
1710 
1711   PetscFunctionBegin;
1712   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1713   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1714   PetscValidHeaderSpecific(sectiondm, DM_CLASSID, 3);
1715   PetscValidHeaderSpecific(vec, VEC_CLASSID, 4);
1716   /* Check consistency */
1717   {
1718     PetscSection  section;
1719     PetscBool     includesConstraints;
1720     PetscInt      m, m1;
1721 
1722     ierr = VecGetLocalSize(vec, &m1);CHKERRQ(ierr);
1723     ierr = DMGetLocalSection(sectiondm, &section);CHKERRQ(ierr);
1724     ierr = PetscSectionGetIncludesConstraints(section, &includesConstraints);CHKERRQ(ierr);
1725     if (includesConstraints) {ierr = PetscSectionGetStorageSize(section, &m);CHKERRQ(ierr);}
1726     else {ierr = PetscSectionGetConstrainedStorageSize(section, &m);CHKERRQ(ierr);}
1727     if (m1 != m) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Local vector size (%D) != local section storage size (%D)", m1, m);
1728   }
1729   ierr = PetscObjectTypeCompare((PetscObject)viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
1730   if (ishdf5) {
1731 #if defined(PETSC_HAVE_HDF5)
1732     ierr = DMPlexLocalVectorView_HDF5_Internal(dm, viewer, sectiondm, vec);CHKERRQ(ierr);
1733 #else
1734     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1735 #endif
1736   }
1737   PetscFunctionReturn(0);
1738 }
1739 
1740 PetscErrorCode DMLoad_Plex(DM dm, PetscViewer viewer)
1741 {
1742   PetscBool      ishdf5;
1743   PetscErrorCode ierr;
1744 
1745   PetscFunctionBegin;
1746   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1747   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1748   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,   &ishdf5);CHKERRQ(ierr);
1749   if (ishdf5) {
1750 #if defined(PETSC_HAVE_HDF5)
1751     PetscViewerFormat format;
1752     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
1753     if (format == PETSC_VIEWER_HDF5_XDMF || format == PETSC_VIEWER_HDF5_VIZ) {
1754       ierr = DMPlexLoad_HDF5_Xdmf_Internal(dm, viewer);CHKERRQ(ierr);
1755     } else if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
1756       ierr = DMPlexLoad_HDF5_Internal(dm, viewer);CHKERRQ(ierr);
1757     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 input.", PetscViewerFormats[format]);
1758     PetscFunctionReturn(0);
1759 #else
1760     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1761 #endif
1762   } else SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Viewer type %s not yet supported for DMPlex loading", ((PetscObject)viewer)->type_name);
1763 }
1764 
1765 /*@
1766   DMPlexTopologyLoad - Loads a topology into a DMPlex
1767 
1768   Collective on DM
1769 
1770   Input Parameters:
1771 + dm     - The DM into which the topology is loaded
1772 - viewer - The PetscViewer for the saved topology
1773 
1774   Output Parameters:
1775 . globalToLocalPointSF - The PetscSF that pushes points in [0, N) to the associated points in the loaded plex, where N is the global number of points; NULL if unneeded
1776 
1777   Level: advanced
1778 
1779 .seealso: DMLoad(), DMPlexCoordinatesLoad(), DMPlexLabelsLoad(), DMView(), PetscViewerHDF5Open(), PetscViewerPushFormat()
1780 @*/
1781 PetscErrorCode DMPlexTopologyLoad(DM dm, PetscViewer viewer, PetscSF *globalToLocalPointSF)
1782 {
1783   PetscBool      ishdf5;
1784   PetscErrorCode ierr;
1785 
1786   PetscFunctionBegin;
1787   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1788   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1789   if (globalToLocalPointSF) PetscValidPointer(globalToLocalPointSF, 3);
1790   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
1791   if (ishdf5) {
1792 #if defined(PETSC_HAVE_HDF5)
1793     PetscViewerFormat format;
1794     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
1795     if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
1796       ierr = DMPlexTopologyLoad_HDF5_Internal(dm, viewer, globalToLocalPointSF);CHKERRQ(ierr);
1797     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 input.", PetscViewerFormats[format]);
1798 #else
1799     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1800 #endif
1801   }
1802   PetscFunctionReturn(0);
1803 }
1804 
1805 /*@
1806   DMPlexCoordinatesLoad - Loads coordinates into a DMPlex
1807 
1808   Collective on DM
1809 
1810   Input Parameters:
1811 + dm     - The DM into which the coordinates are loaded
1812 - viewer - The PetscViewer for the saved coordinates
1813 
1814   Level: advanced
1815 
1816 .seealso: DMLoad(), DMPlexTopologyLoad(), DMPlexLabelsLoad(), DMView(), PetscViewerHDF5Open(), PetscViewerPushFormat()
1817 @*/
1818 PetscErrorCode DMPlexCoordinatesLoad(DM dm, PetscViewer viewer)
1819 {
1820   PetscBool      ishdf5;
1821   PetscErrorCode ierr;
1822 
1823   PetscFunctionBegin;
1824   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1825   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1826   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
1827   if (ishdf5) {
1828 #if defined(PETSC_HAVE_HDF5)
1829     PetscViewerFormat format;
1830     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
1831     if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
1832       ierr = DMPlexCoordinatesLoad_HDF5_Internal(dm, viewer);CHKERRQ(ierr);
1833     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 input.", PetscViewerFormats[format]);
1834 #else
1835     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1836 #endif
1837   }
1838   PetscFunctionReturn(0);
1839 }
1840 
1841 /*@
1842   DMPlexLabelsLoad - Loads labels into a DMPlex
1843 
1844   Collective on DM
1845 
1846   Input Parameters:
1847 + dm     - The DM into which the labels are loaded
1848 - viewer - The PetscViewer for the saved labels
1849 
1850   Level: advanced
1851 
1852 .seealso: DMLoad(), DMPlexTopologyLoad(), DMPlexCoordinatesLoad(), DMView(), PetscViewerHDF5Open(), PetscViewerPushFormat()
1853 @*/
1854 PetscErrorCode DMPlexLabelsLoad(DM dm, PetscViewer viewer)
1855 {
1856   PetscBool      ishdf5;
1857   PetscErrorCode ierr;
1858 
1859   PetscFunctionBegin;
1860   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1861   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1862   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
1863   if (ishdf5) {
1864 #if defined(PETSC_HAVE_HDF5)
1865     PetscViewerFormat format;
1866 
1867     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
1868     if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
1869       ierr = DMPlexLabelsLoad_HDF5_Internal(dm, viewer);CHKERRQ(ierr);
1870     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 input.", PetscViewerFormats[format]);
1871 #else
1872     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1873 #endif
1874   }
1875   PetscFunctionReturn(0);
1876 }
1877 
1878 /*@
1879   DMPlexSectionLoad - Loads section into a DMPlex
1880 
1881   Collective on DM
1882 
1883   Input Parameters:
1884 + dm          - The DM that represents the topology
1885 . viewer      - The PetscViewer that represents the on-disk section (sectionA)
1886 . sectiondm   - The DM into which the on-disk section (sectionA) is migrated
1887 - globalToLocalPointSF - The SF returned by DMPlexTopologyLoad() when loading dm from viewer
1888 
1889   Output Parameters
1890 + globalDofSF - The SF that migrates any on-disk Vec data associated with sectionA into a global Vec associated with the sectiondm's global section (NULL if not needed)
1891 - localDofSF  - The SF that migrates any on-disk Vec data associated with sectionA into a local Vec associated with the sectiondm's local section (NULL if not needed)
1892 
1893   Level: advanced
1894 
1895   Notes:
1896   This function is a wrapper around PetscSectionLoad(); it loads, in addition to the raw section, a list of global point numbers that associates each on-disk section point with a global point number in [0, NX), where NX is the number of topology points in dm. Noting that globalToLocalPointSF associates each topology point in dm with a global number in [0, NX), one can readily establish an association of the on-disk section points with the topology points.
1897 
1898   In general dm and sectiondm are two different objects, the former carrying the topology and the latter carrying the section, and have been given a topology name and a section name, respectively, with PetscObjectSetName(). In practice, however, they can be the same object if it carries both topology and section; in that case the name of the object is used as both the topology name and the section name.
1899 
1900   The output parameter, globalDofSF (localDofSF), can later be used with DMPlexGlobalVectorLoad() (DMPlexLocalVectorLoad()) to load on-disk vectors into global (local) vectors associated with sectiondm's global (local) section.
1901 
1902   Example using 2 processes:
1903 $  NX (number of points on dm): 4
1904 $  sectionA                   : the on-disk section
1905 $  vecA                       : a vector associated with sectionA
1906 $  sectionB                   : sectiondm's local section constructed in this function
1907 $  vecB (local)               : a vector associated with sectiondm's local section
1908 $  vecB (global)              : a vector associated with sectiondm's global section
1909 $
1910 $                                     rank 0    rank 1
1911 $  vecA (global)                  : [.0 .4 .1 | .2 .3]        <- to be loaded in DMPlexGlobalVectorLoad() or DMPlexLocalVectorLoad()
1912 $  sectionA->atlasOff             :       0 2 | 1             <- loaded in PetscSectionLoad()
1913 $  sectionA->atlasDof             :       1 3 | 1             <- loaded in PetscSectionLoad()
1914 $  sectionA's global point numbers:       0 2 | 3             <- loaded in DMPlexSectionLoad()
1915 $  [0, NX)                        :       0 1 | 2 3           <- conceptual partition used in globalToLocalPointSF
1916 $  sectionB's global point numbers:     0 1 3 | 3 2           <- associated with [0, NX) by globalToLocalPointSF
1917 $  sectionB->atlasDof             :     1 0 1 | 1 3
1918 $  sectionB->atlasOff (no perm)   :     0 1 1 | 0 1
1919 $  vecB (local)                   :   [.0 .4] | [.4 .1 .2 .3] <- to be constructed by calling DMPlexLocalVectorLoad() with localDofSF
1920 $  vecB (global)                  :    [.0 .4 | .1 .2 .3]     <- to be constructed by calling DMPlexGlobalVectorLoad() with globalDofSF
1921 $
1922 $  where "|" represents a partition of loaded data, and global point 3 is assumed to be owned by rank 0.
1923 
1924 .seealso: DMLoad(), DMPlexTopologyLoad(), DMPlexCoordinatesLoad(), DMPlexLabelsLoad(), DMPlexGlobalVectorLoad(), DMPlexLocalVectorLoad(), PetscSectionLoad(), DMPlexSectionView()
1925 @*/
1926 PetscErrorCode DMPlexSectionLoad(DM dm, PetscViewer viewer, DM sectiondm, PetscSF globalToLocalPointSF, PetscSF *globalDofSF, PetscSF *localDofSF)
1927 {
1928   PetscBool      ishdf5;
1929   PetscErrorCode ierr;
1930 
1931   PetscFunctionBegin;
1932   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1933   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1934   PetscValidHeaderSpecific(sectiondm, DM_CLASSID, 3);
1935   PetscValidHeaderSpecific(globalToLocalPointSF, PETSCSF_CLASSID, 4);
1936   if (globalDofSF) PetscValidPointer(globalDofSF, 5);
1937   if (localDofSF) PetscValidPointer(localDofSF, 6);
1938   ierr = PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERHDF5,&ishdf5);CHKERRQ(ierr);
1939   if (ishdf5) {
1940 #if defined(PETSC_HAVE_HDF5)
1941     ierr = DMPlexSectionLoad_HDF5_Internal(dm, viewer, sectiondm, globalToLocalPointSF, globalDofSF, localDofSF);CHKERRQ(ierr);
1942 #else
1943     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1944 #endif
1945   }
1946   PetscFunctionReturn(0);
1947 }
1948 
1949 /*@
1950   DMPlexGlobalVectorLoad - Loads on-disk vector data into a global vector
1951 
1952   Collective on DM
1953 
1954   Input Parameters:
1955 + dm        - The DM that represents the topology
1956 . viewer    - The PetscViewer that represents the on-disk vector data
1957 . sectiondm - The DM that contains the global section on which vec is defined
1958 . sf        - The SF that migrates the on-disk vector data into vec
1959 - vec       - The global vector to set values of
1960 
1961   Level: advanced
1962 
1963   Notes:
1964   In general dm and sectiondm are two different objects, the former carrying the topology and the latter carrying the section, and have been given a topology name and a section name, respectively, with PetscObjectSetName(). In practice, however, they can be the same object if it carries both topology and section; in that case the name of the object is used as both the topology name and the section name.
1965 
1966   Typical calling sequence
1967 $       DMCreate(PETSC_COMM_WORLD, &dm);
1968 $       DMSetType(dm, DMPLEX);
1969 $       PetscObjectSetName((PetscObject)dm, "topologydm_name");
1970 $       DMPlexTopologyLoad(dm, viewer, &sfX);
1971 $       DMClone(dm, &sectiondm);
1972 $       PetscObjectSetName((PetscObject)sectiondm, "sectiondm_name");
1973 $       DMPlexSectionLoad(dm, viewer, sectiondm, sfX, &gsf, NULL);
1974 $       DMGetGlobalVector(sectiondm, &vec);
1975 $       PetscObjectSetName((PetscObject)vec, "vec_name");
1976 $       DMPlexGlobalVectorLoad(dm, viewer, sectiondm, gsf, vec);
1977 $       DMRestoreGlobalVector(sectiondm, &vec);
1978 $       PetscSFDestroy(&gsf);
1979 $       PetscSFDestroy(&sfX);
1980 $       DMDestroy(&sectiondm);
1981 $       DMDestroy(&dm);
1982 
1983 .seealso: DMPlexTopologyLoad(), DMPlexSectionLoad(), DMPlexLocalVectorLoad(), DMPlexGlobalVectorView(), DMPlexLocalVectorView()
1984 @*/
1985 PetscErrorCode DMPlexGlobalVectorLoad(DM dm, PetscViewer viewer, DM sectiondm, PetscSF sf, Vec vec)
1986 {
1987   PetscBool       ishdf5;
1988   PetscErrorCode  ierr;
1989 
1990   PetscFunctionBegin;
1991   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1992   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1993   PetscValidHeaderSpecific(sectiondm, DM_CLASSID, 3);
1994   PetscValidHeaderSpecific(sf, PETSCSF_CLASSID, 4);
1995   PetscValidHeaderSpecific(vec, VEC_CLASSID, 5);
1996   /* Check consistency */
1997   {
1998     PetscSection  section;
1999     PetscBool     includesConstraints;
2000     PetscInt      m, m1;
2001 
2002     ierr = VecGetLocalSize(vec, &m1);CHKERRQ(ierr);
2003     ierr = DMGetGlobalSection(sectiondm, &section);CHKERRQ(ierr);
2004     ierr = PetscSectionGetIncludesConstraints(section, &includesConstraints);CHKERRQ(ierr);
2005     if (includesConstraints) {ierr = PetscSectionGetStorageSize(section, &m);CHKERRQ(ierr);}
2006     else {ierr = PetscSectionGetConstrainedStorageSize(section, &m);CHKERRQ(ierr);}
2007     if (m1 != m) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Global vector size (%D) != global section storage size (%D)", m1, m);
2008   }
2009   ierr = PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERHDF5,&ishdf5);CHKERRQ(ierr);
2010   if (ishdf5) {
2011 #if defined(PETSC_HAVE_HDF5)
2012     ierr = DMPlexVecLoad_HDF5_Internal(dm, viewer, sectiondm, sf, vec);CHKERRQ(ierr);
2013 #else
2014     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
2015 #endif
2016   }
2017   PetscFunctionReturn(0);
2018 }
2019 
2020 /*@
2021   DMPlexLocalVectorLoad - Loads on-disk vector data into a local vector
2022 
2023   Collective on DM
2024 
2025   Input Parameters:
2026 + dm        - The DM that represents the topology
2027 . viewer    - The PetscViewer that represents the on-disk vector data
2028 . sectiondm - The DM that contains the local section on which vec is defined
2029 . sf        - The SF that migrates the on-disk vector data into vec
2030 - vec       - The local vector to set values of
2031 
2032   Level: advanced
2033 
2034   Notes:
2035   In general dm and sectiondm are two different objects, the former carrying the topology and the latter carrying the section, and have been given a topology name and a section name, respectively, with PetscObjectSetName(). In practice, however, they can be the same object if it carries both topology and section; in that case the name of the object is used as both the topology name and the section name.
2036 
2037   Typical calling sequence
2038 $       DMCreate(PETSC_COMM_WORLD, &dm);
2039 $       DMSetType(dm, DMPLEX);
2040 $       PetscObjectSetName((PetscObject)dm, "topologydm_name");
2041 $       DMPlexTopologyLoad(dm, viewer, &sfX);
2042 $       DMClone(dm, &sectiondm);
2043 $       PetscObjectSetName((PetscObject)sectiondm, "sectiondm_name");
2044 $       DMPlexSectionLoad(dm, viewer, sectiondm, sfX, NULL, &lsf);
2045 $       DMGetLocalVector(sectiondm, &vec);
2046 $       PetscObjectSetName((PetscObject)vec, "vec_name");
2047 $       DMPlexLocalVectorLoad(dm, viewer, sectiondm, lsf, vec);
2048 $       DMRestoreLocalVector(sectiondm, &vec);
2049 $       PetscSFDestroy(&lsf);
2050 $       PetscSFDestroy(&sfX);
2051 $       DMDestroy(&sectiondm);
2052 $       DMDestroy(&dm);
2053 
2054 .seealso: DMPlexTopologyLoad(), DMPlexSectionLoad(), DMPlexGlobalVectorLoad(), DMPlexGlobalVectorView(), DMPlexLocalVectorView()
2055 @*/
2056 PetscErrorCode DMPlexLocalVectorLoad(DM dm, PetscViewer viewer, DM sectiondm, PetscSF sf, Vec vec)
2057 {
2058   PetscBool       ishdf5;
2059   PetscErrorCode  ierr;
2060 
2061   PetscFunctionBegin;
2062   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2063   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
2064   PetscValidHeaderSpecific(sectiondm, DM_CLASSID, 3);
2065   PetscValidHeaderSpecific(sf, PETSCSF_CLASSID, 4);
2066   PetscValidHeaderSpecific(vec, VEC_CLASSID, 5);
2067   /* Check consistency */
2068   {
2069     PetscSection  section;
2070     PetscBool     includesConstraints;
2071     PetscInt      m, m1;
2072 
2073     ierr = VecGetLocalSize(vec, &m1);CHKERRQ(ierr);
2074     ierr = DMGetLocalSection(sectiondm, &section);CHKERRQ(ierr);
2075     ierr = PetscSectionGetIncludesConstraints(section, &includesConstraints);CHKERRQ(ierr);
2076     if (includesConstraints) {ierr = PetscSectionGetStorageSize(section, &m);CHKERRQ(ierr);}
2077     else {ierr = PetscSectionGetConstrainedStorageSize(section, &m);CHKERRQ(ierr);}
2078     if (m1 != m) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Local vector size (%D) != local section storage size (%D)", m1, m);
2079   }
2080   ierr = PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERHDF5,&ishdf5);CHKERRQ(ierr);
2081   if (ishdf5) {
2082 #if defined(PETSC_HAVE_HDF5)
2083     ierr = DMPlexVecLoad_HDF5_Internal(dm, viewer, sectiondm, sf, vec);CHKERRQ(ierr);
2084 #else
2085     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
2086 #endif
2087   }
2088   PetscFunctionReturn(0);
2089 }
2090 
2091 PetscErrorCode DMDestroy_Plex(DM dm)
2092 {
2093   DM_Plex       *mesh = (DM_Plex*) dm->data;
2094   PetscErrorCode ierr;
2095 
2096   PetscFunctionBegin;
2097   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMSetUpGLVisViewer_C",NULL);CHKERRQ(ierr);
2098   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMPlexInsertBoundaryValues_C", NULL);CHKERRQ(ierr);
2099   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMCreateNeumannOverlap_C", NULL);CHKERRQ(ierr);
2100   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMInterpolateSolution_C", NULL);CHKERRQ(ierr);
2101   if (--mesh->refct > 0) PetscFunctionReturn(0);
2102   ierr = PetscSectionDestroy(&mesh->coneSection);CHKERRQ(ierr);
2103   ierr = PetscFree(mesh->cones);CHKERRQ(ierr);
2104   ierr = PetscFree(mesh->coneOrientations);CHKERRQ(ierr);
2105   ierr = PetscSectionDestroy(&mesh->supportSection);CHKERRQ(ierr);
2106   ierr = PetscSectionDestroy(&mesh->subdomainSection);CHKERRQ(ierr);
2107   ierr = PetscFree(mesh->supports);CHKERRQ(ierr);
2108   ierr = PetscFree(mesh->facesTmp);CHKERRQ(ierr);
2109   ierr = PetscFree(mesh->tetgenOpts);CHKERRQ(ierr);
2110   ierr = PetscFree(mesh->triangleOpts);CHKERRQ(ierr);
2111   ierr = PetscPartitionerDestroy(&mesh->partitioner);CHKERRQ(ierr);
2112   ierr = DMLabelDestroy(&mesh->subpointMap);CHKERRQ(ierr);
2113   ierr = ISDestroy(&mesh->subpointIS);CHKERRQ(ierr);
2114   ierr = ISDestroy(&mesh->globalVertexNumbers);CHKERRQ(ierr);
2115   ierr = ISDestroy(&mesh->globalCellNumbers);CHKERRQ(ierr);
2116   ierr = PetscSectionDestroy(&mesh->anchorSection);CHKERRQ(ierr);
2117   ierr = ISDestroy(&mesh->anchorIS);CHKERRQ(ierr);
2118   ierr = PetscSectionDestroy(&mesh->parentSection);CHKERRQ(ierr);
2119   ierr = PetscFree(mesh->parents);CHKERRQ(ierr);
2120   ierr = PetscFree(mesh->childIDs);CHKERRQ(ierr);
2121   ierr = PetscSectionDestroy(&mesh->childSection);CHKERRQ(ierr);
2122   ierr = PetscFree(mesh->children);CHKERRQ(ierr);
2123   ierr = DMDestroy(&mesh->referenceTree);CHKERRQ(ierr);
2124   ierr = PetscGridHashDestroy(&mesh->lbox);CHKERRQ(ierr);
2125   ierr = PetscFree(mesh->neighbors);CHKERRQ(ierr);
2126   /* This was originally freed in DMDestroy(), but that prevents reference counting of backend objects */
2127   ierr = PetscFree(mesh);CHKERRQ(ierr);
2128   PetscFunctionReturn(0);
2129 }
2130 
2131 PetscErrorCode DMCreateMatrix_Plex(DM dm, Mat *J)
2132 {
2133   PetscSection           sectionGlobal;
2134   PetscInt               bs = -1, mbs;
2135   PetscInt               localSize;
2136   PetscBool              isShell, isBlock, isSeqBlock, isMPIBlock, isSymBlock, isSymSeqBlock, isSymMPIBlock, isMatIS;
2137   PetscErrorCode         ierr;
2138   MatType                mtype;
2139   ISLocalToGlobalMapping ltog;
2140 
2141   PetscFunctionBegin;
2142   ierr = MatInitializePackage();CHKERRQ(ierr);
2143   mtype = dm->mattype;
2144   ierr = DMGetGlobalSection(dm, &sectionGlobal);CHKERRQ(ierr);
2145   /* ierr = PetscSectionGetStorageSize(sectionGlobal, &localSize);CHKERRQ(ierr); */
2146   ierr = PetscSectionGetConstrainedStorageSize(sectionGlobal, &localSize);CHKERRQ(ierr);
2147   ierr = MatCreate(PetscObjectComm((PetscObject)dm), J);CHKERRQ(ierr);
2148   ierr = MatSetSizes(*J, localSize, localSize, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
2149   ierr = MatSetType(*J, mtype);CHKERRQ(ierr);
2150   ierr = MatSetFromOptions(*J);CHKERRQ(ierr);
2151   ierr = MatGetBlockSize(*J, &mbs);CHKERRQ(ierr);
2152   if (mbs > 1) bs = mbs;
2153   ierr = PetscStrcmp(mtype, MATSHELL, &isShell);CHKERRQ(ierr);
2154   ierr = PetscStrcmp(mtype, MATBAIJ, &isBlock);CHKERRQ(ierr);
2155   ierr = PetscStrcmp(mtype, MATSEQBAIJ, &isSeqBlock);CHKERRQ(ierr);
2156   ierr = PetscStrcmp(mtype, MATMPIBAIJ, &isMPIBlock);CHKERRQ(ierr);
2157   ierr = PetscStrcmp(mtype, MATSBAIJ, &isSymBlock);CHKERRQ(ierr);
2158   ierr = PetscStrcmp(mtype, MATSEQSBAIJ, &isSymSeqBlock);CHKERRQ(ierr);
2159   ierr = PetscStrcmp(mtype, MATMPISBAIJ, &isSymMPIBlock);CHKERRQ(ierr);
2160   ierr = PetscStrcmp(mtype, MATIS, &isMatIS);CHKERRQ(ierr);
2161   if (!isShell) {
2162     PetscSection subSection;
2163     PetscBool    fillMatrix = (PetscBool)(!dm->prealloc_only && !isMatIS);
2164     PetscInt    *dnz, *onz, *dnzu, *onzu, bsLocal[2], bsMinMax[2], *ltogidx, lsize;
2165     PetscInt     pStart, pEnd, p, dof, cdof;
2166 
2167     /* Set localtoglobalmapping on the matrix for MatSetValuesLocal() to work (it also creates the local matrices in case of MATIS) */
2168     if (isMatIS) { /* need a different l2g map than the one computed by DMGetLocalToGlobalMapping */
2169       PetscSection section;
2170       PetscInt     size;
2171 
2172       ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);
2173       ierr = PetscSectionGetStorageSize(section, &size);CHKERRQ(ierr);
2174       ierr = PetscMalloc1(size,&ltogidx);CHKERRQ(ierr);
2175       ierr = DMPlexGetSubdomainSection(dm, &subSection);CHKERRQ(ierr);
2176     } else {
2177       ierr = DMGetLocalToGlobalMapping(dm,&ltog);CHKERRQ(ierr);
2178     }
2179     ierr = PetscSectionGetChart(sectionGlobal, &pStart, &pEnd);CHKERRQ(ierr);
2180     for (p = pStart, lsize = 0; p < pEnd; ++p) {
2181       PetscInt bdof;
2182 
2183       ierr = PetscSectionGetDof(sectionGlobal, p, &dof);CHKERRQ(ierr);
2184       ierr = PetscSectionGetConstraintDof(sectionGlobal, p, &cdof);CHKERRQ(ierr);
2185       dof  = dof < 0 ? -(dof+1) : dof;
2186       bdof = cdof && (dof-cdof) ? 1 : dof;
2187       if (dof) {
2188         if (bs < 0)          {bs = bdof;}
2189         else if (bs != bdof) {bs = 1; if (!isMatIS) break;}
2190       }
2191       if (isMatIS) {
2192         PetscInt loff,c,off;
2193         ierr = PetscSectionGetOffset(subSection, p, &loff);CHKERRQ(ierr);
2194         ierr = PetscSectionGetOffset(sectionGlobal, p, &off);CHKERRQ(ierr);
2195         for (c = 0; c < dof-cdof; ++c, ++lsize) ltogidx[loff+c] = off > -1 ? off+c : -(off+1)+c;
2196       }
2197     }
2198     /* Must have same blocksize on all procs (some might have no points) */
2199     bsLocal[0] = bs < 0 ? PETSC_MAX_INT : bs; bsLocal[1] = bs;
2200     ierr = PetscGlobalMinMaxInt(PetscObjectComm((PetscObject) dm), bsLocal, bsMinMax);CHKERRQ(ierr);
2201     if (bsMinMax[0] != bsMinMax[1]) {bs = 1;}
2202     else                            {bs = bsMinMax[0];}
2203     bs = PetscMax(1,bs);
2204     if (isMatIS) { /* Must reduce indices by blocksize */
2205       PetscInt l;
2206 
2207       lsize = lsize/bs;
2208       if (bs > 1) for (l = 0; l < lsize; ++l) ltogidx[l] = ltogidx[l*bs]/bs;
2209       ierr = ISLocalToGlobalMappingCreate(PetscObjectComm((PetscObject)dm), bs, lsize, ltogidx, PETSC_OWN_POINTER, &ltog);CHKERRQ(ierr);
2210     }
2211     ierr = MatSetLocalToGlobalMapping(*J,ltog,ltog);CHKERRQ(ierr);
2212     if (isMatIS) {
2213       ierr = ISLocalToGlobalMappingDestroy(&ltog);CHKERRQ(ierr);
2214     }
2215     ierr = PetscCalloc4(localSize/bs, &dnz, localSize/bs, &onz, localSize/bs, &dnzu, localSize/bs, &onzu);CHKERRQ(ierr);
2216     ierr = DMPlexPreallocateOperator(dm, bs, dnz, onz, dnzu, onzu, *J, fillMatrix);CHKERRQ(ierr);
2217     ierr = PetscFree4(dnz, onz, dnzu, onzu);CHKERRQ(ierr);
2218   }
2219   ierr = MatSetDM(*J, dm);CHKERRQ(ierr);
2220   PetscFunctionReturn(0);
2221 }
2222 
2223 /*@
2224   DMPlexGetSubdomainSection - Returns the section associated with the subdomain
2225 
2226   Not collective
2227 
2228   Input Parameter:
2229 . mesh - The DMPlex
2230 
2231   Output Parameters:
2232 . subsection - The subdomain section
2233 
2234   Level: developer
2235 
2236 .seealso:
2237 @*/
2238 PetscErrorCode DMPlexGetSubdomainSection(DM dm, PetscSection *subsection)
2239 {
2240   DM_Plex       *mesh = (DM_Plex*) dm->data;
2241   PetscErrorCode ierr;
2242 
2243   PetscFunctionBegin;
2244   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2245   if (!mesh->subdomainSection) {
2246     PetscSection section;
2247     PetscSF      sf;
2248 
2249     ierr = PetscSFCreate(PETSC_COMM_SELF,&sf);CHKERRQ(ierr);
2250     ierr = DMGetLocalSection(dm,&section);CHKERRQ(ierr);
2251     ierr = PetscSectionCreateGlobalSection(section,sf,PETSC_FALSE,PETSC_TRUE,&mesh->subdomainSection);CHKERRQ(ierr);
2252     ierr = PetscSFDestroy(&sf);CHKERRQ(ierr);
2253   }
2254   *subsection = mesh->subdomainSection;
2255   PetscFunctionReturn(0);
2256 }
2257 
2258 /*@
2259   DMPlexGetChart - Return the interval for all mesh points [pStart, pEnd)
2260 
2261   Not collective
2262 
2263   Input Parameter:
2264 . mesh - The DMPlex
2265 
2266   Output Parameters:
2267 + pStart - The first mesh point
2268 - pEnd   - The upper bound for mesh points
2269 
2270   Level: beginner
2271 
2272 .seealso: DMPlexCreate(), DMPlexSetChart()
2273 @*/
2274 PetscErrorCode DMPlexGetChart(DM dm, PetscInt *pStart, PetscInt *pEnd)
2275 {
2276   DM_Plex       *mesh = (DM_Plex*) dm->data;
2277   PetscErrorCode ierr;
2278 
2279   PetscFunctionBegin;
2280   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2281   ierr = PetscSectionGetChart(mesh->coneSection, pStart, pEnd);CHKERRQ(ierr);
2282   PetscFunctionReturn(0);
2283 }
2284 
2285 /*@
2286   DMPlexSetChart - Set the interval for all mesh points [pStart, pEnd)
2287 
2288   Not collective
2289 
2290   Input Parameters:
2291 + mesh - The DMPlex
2292 . pStart - The first mesh point
2293 - pEnd   - The upper bound for mesh points
2294 
2295   Output Parameters:
2296 
2297   Level: beginner
2298 
2299 .seealso: DMPlexCreate(), DMPlexGetChart()
2300 @*/
2301 PetscErrorCode DMPlexSetChart(DM dm, PetscInt pStart, PetscInt pEnd)
2302 {
2303   DM_Plex       *mesh = (DM_Plex*) dm->data;
2304   PetscErrorCode ierr;
2305 
2306   PetscFunctionBegin;
2307   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2308   ierr = PetscSectionSetChart(mesh->coneSection, pStart, pEnd);CHKERRQ(ierr);
2309   ierr = PetscSectionSetChart(mesh->supportSection, pStart, pEnd);CHKERRQ(ierr);
2310   PetscFunctionReturn(0);
2311 }
2312 
2313 /*@
2314   DMPlexGetConeSize - Return the number of in-edges for this point in the DAG
2315 
2316   Not collective
2317 
2318   Input Parameters:
2319 + mesh - The DMPlex
2320 - p - The point, which must lie in the chart set with DMPlexSetChart()
2321 
2322   Output Parameter:
2323 . size - The cone size for point p
2324 
2325   Level: beginner
2326 
2327 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart()
2328 @*/
2329 PetscErrorCode DMPlexGetConeSize(DM dm, PetscInt p, PetscInt *size)
2330 {
2331   DM_Plex       *mesh = (DM_Plex*) dm->data;
2332   PetscErrorCode ierr;
2333 
2334   PetscFunctionBegin;
2335   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2336   PetscValidPointer(size, 3);
2337   ierr = PetscSectionGetDof(mesh->coneSection, p, size);CHKERRQ(ierr);
2338   PetscFunctionReturn(0);
2339 }
2340 
2341 /*@
2342   DMPlexSetConeSize - Set the number of in-edges for this point in the DAG
2343 
2344   Not collective
2345 
2346   Input Parameters:
2347 + mesh - The DMPlex
2348 . p - The point, which must lie in the chart set with DMPlexSetChart()
2349 - size - The cone size for point p
2350 
2351   Output Parameter:
2352 
2353   Note:
2354   This should be called after DMPlexSetChart().
2355 
2356   Level: beginner
2357 
2358 .seealso: DMPlexCreate(), DMPlexGetConeSize(), DMPlexSetChart()
2359 @*/
2360 PetscErrorCode DMPlexSetConeSize(DM dm, PetscInt p, PetscInt size)
2361 {
2362   DM_Plex       *mesh = (DM_Plex*) dm->data;
2363   PetscErrorCode ierr;
2364 
2365   PetscFunctionBegin;
2366   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2367   ierr = PetscSectionSetDof(mesh->coneSection, p, size);CHKERRQ(ierr);
2368 
2369   mesh->maxConeSize = PetscMax(mesh->maxConeSize, size);
2370   PetscFunctionReturn(0);
2371 }
2372 
2373 /*@
2374   DMPlexAddConeSize - Add the given number of in-edges to this point in the DAG
2375 
2376   Not collective
2377 
2378   Input Parameters:
2379 + mesh - The DMPlex
2380 . p - The point, which must lie in the chart set with DMPlexSetChart()
2381 - size - The additional cone size for point p
2382 
2383   Output Parameter:
2384 
2385   Note:
2386   This should be called after DMPlexSetChart().
2387 
2388   Level: beginner
2389 
2390 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexGetConeSize(), DMPlexSetChart()
2391 @*/
2392 PetscErrorCode DMPlexAddConeSize(DM dm, PetscInt p, PetscInt size)
2393 {
2394   DM_Plex       *mesh = (DM_Plex*) dm->data;
2395   PetscInt       csize;
2396   PetscErrorCode ierr;
2397 
2398   PetscFunctionBegin;
2399   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2400   ierr = PetscSectionAddDof(mesh->coneSection, p, size);CHKERRQ(ierr);
2401   ierr = PetscSectionGetDof(mesh->coneSection, p, &csize);CHKERRQ(ierr);
2402 
2403   mesh->maxConeSize = PetscMax(mesh->maxConeSize, csize);
2404   PetscFunctionReturn(0);
2405 }
2406 
2407 /*@C
2408   DMPlexGetCone - Return the points on the in-edges for this point in the DAG
2409 
2410   Not collective
2411 
2412   Input Parameters:
2413 + dm - The DMPlex
2414 - p - The point, which must lie in the chart set with DMPlexSetChart()
2415 
2416   Output Parameter:
2417 . cone - An array of points which are on the in-edges for point p
2418 
2419   Level: beginner
2420 
2421   Fortran Notes:
2422   Since it returns an array, this routine is only available in Fortran 90, and you must
2423   include petsc.h90 in your code.
2424   You must also call DMPlexRestoreCone() after you finish using the returned array.
2425   DMPlexRestoreCone() is not needed/available in C.
2426 
2427 .seealso: DMPlexGetConeSize(), DMPlexSetCone(), DMPlexGetConeTuple(), DMPlexSetChart()
2428 @*/
2429 PetscErrorCode DMPlexGetCone(DM dm, PetscInt p, const PetscInt *cone[])
2430 {
2431   DM_Plex       *mesh = (DM_Plex*) dm->data;
2432   PetscInt       off;
2433   PetscErrorCode ierr;
2434 
2435   PetscFunctionBegin;
2436   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2437   PetscValidPointer(cone, 3);
2438   ierr  = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2439   *cone = &mesh->cones[off];
2440   PetscFunctionReturn(0);
2441 }
2442 
2443 /*@C
2444   DMPlexGetConeTuple - Return the points on the in-edges of several points in the DAG
2445 
2446   Not collective
2447 
2448   Input Parameters:
2449 + dm - The DMPlex
2450 - p - The IS of points, which must lie in the chart set with DMPlexSetChart()
2451 
2452   Output Parameter:
2453 + pConesSection - PetscSection describing the layout of pCones
2454 - pCones - An array of points which are on the in-edges for the point set p
2455 
2456   Level: intermediate
2457 
2458 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexGetConeRecursive(), DMPlexSetChart()
2459 @*/
2460 PetscErrorCode DMPlexGetConeTuple(DM dm, IS p, PetscSection *pConesSection, IS *pCones)
2461 {
2462   PetscSection        cs, newcs;
2463   PetscInt            *cones;
2464   PetscInt            *newarr=NULL;
2465   PetscInt            n;
2466   PetscErrorCode      ierr;
2467 
2468   PetscFunctionBegin;
2469   ierr = DMPlexGetCones(dm, &cones);CHKERRQ(ierr);
2470   ierr = DMPlexGetConeSection(dm, &cs);CHKERRQ(ierr);
2471   ierr = PetscSectionExtractDofsFromArray(cs, MPIU_INT, cones, p, &newcs, pCones ? ((void**)&newarr) : NULL);CHKERRQ(ierr);
2472   if (pConesSection) *pConesSection = newcs;
2473   if (pCones) {
2474     ierr = PetscSectionGetStorageSize(newcs, &n);CHKERRQ(ierr);
2475     ierr = ISCreateGeneral(PetscObjectComm((PetscObject)p), n, newarr, PETSC_OWN_POINTER, pCones);CHKERRQ(ierr);
2476   }
2477   PetscFunctionReturn(0);
2478 }
2479 
2480 /*@
2481   DMPlexGetConeRecursiveVertices - Expand each given point into its cone points and do that recursively until we end up just with vertices.
2482 
2483   Not collective
2484 
2485   Input Parameters:
2486 + dm - The DMPlex
2487 - points - The IS of points, which must lie in the chart set with DMPlexSetChart()
2488 
2489   Output Parameter:
2490 . expandedPoints - An array of vertices recursively expanded from input points
2491 
2492   Level: advanced
2493 
2494   Notes:
2495   Like DMPlexGetConeRecursive but returns only the 0-depth IS (i.e. vertices only) and no sections.
2496   There is no corresponding Restore function, just call ISDestroy() on the returned IS to deallocate.
2497 
2498 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexGetConeTuple(), DMPlexGetConeRecursive(), DMPlexRestoreConeRecursive(), DMPlexGetDepth()
2499 @*/
2500 PetscErrorCode DMPlexGetConeRecursiveVertices(DM dm, IS points, IS *expandedPoints)
2501 {
2502   IS                  *expandedPointsAll;
2503   PetscInt            depth;
2504   PetscErrorCode      ierr;
2505 
2506   PetscFunctionBegin;
2507   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2508   PetscValidHeaderSpecific(points, IS_CLASSID, 2);
2509   PetscValidPointer(expandedPoints, 3);
2510   ierr = DMPlexGetConeRecursive(dm, points, &depth, &expandedPointsAll, NULL);CHKERRQ(ierr);
2511   *expandedPoints = expandedPointsAll[0];
2512   ierr = PetscObjectReference((PetscObject)expandedPointsAll[0]);CHKERRQ(ierr);
2513   ierr = DMPlexRestoreConeRecursive(dm, points, &depth, &expandedPointsAll, NULL);CHKERRQ(ierr);
2514   PetscFunctionReturn(0);
2515 }
2516 
2517 /*@
2518   DMPlexGetConeRecursive - Expand each given point into its cone points and do that recursively until we end up just with vertices (DAG points of depth 0, i.e. without cones).
2519 
2520   Not collective
2521 
2522   Input Parameters:
2523 + dm - The DMPlex
2524 - points - The IS of points, which must lie in the chart set with DMPlexSetChart()
2525 
2526   Output Parameter:
2527 + depth - (optional) Size of the output arrays, equal to DMPlex depth, returned by DMPlexGetDepth()
2528 . expandedPoints - (optional) An array of index sets with recursively expanded cones
2529 - sections - (optional) An array of sections which describe mappings from points to their cone points
2530 
2531   Level: advanced
2532 
2533   Notes:
2534   Like DMPlexGetConeTuple() but recursive.
2535 
2536   Array expandedPoints has size equal to depth. Each expandedPoints[d] contains DAG points with maximum depth d, recursively cone-wise expanded from the input points.
2537   For example, for d=0 it contains only vertices, for d=1 it can contain vertices and edges, etc.
2538 
2539   Array section has size equal to depth.  Each PetscSection sections[d] realizes mapping from expandedPoints[d+1] (section points) to expandedPoints[d] (section dofs) as follows:
2540   (1) DAG points in expandedPoints[d+1] with depth d+1 to their cone points in expandedPoints[d];
2541   (2) DAG points in expandedPoints[d+1] with depth in [0,d] to the same points in expandedPoints[d].
2542 
2543 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexGetConeTuple(), DMPlexRestoreConeRecursive(), DMPlexGetConeRecursiveVertices(), DMPlexGetDepth()
2544 @*/
2545 PetscErrorCode DMPlexGetConeRecursive(DM dm, IS points, PetscInt *depth, IS *expandedPoints[], PetscSection *sections[])
2546 {
2547   const PetscInt      *arr0=NULL, *cone=NULL;
2548   PetscInt            *arr=NULL, *newarr=NULL;
2549   PetscInt            d, depth_, i, n, newn, cn, co, start, end;
2550   IS                  *expandedPoints_;
2551   PetscSection        *sections_;
2552   PetscErrorCode      ierr;
2553 
2554   PetscFunctionBegin;
2555   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2556   PetscValidHeaderSpecific(points, IS_CLASSID, 2);
2557   if (depth) PetscValidIntPointer(depth, 3);
2558   if (expandedPoints) PetscValidPointer(expandedPoints, 4);
2559   if (sections) PetscValidPointer(sections, 5);
2560   ierr = ISGetLocalSize(points, &n);CHKERRQ(ierr);
2561   ierr = ISGetIndices(points, &arr0);CHKERRQ(ierr);
2562   ierr = DMPlexGetDepth(dm, &depth_);CHKERRQ(ierr);
2563   ierr = PetscCalloc1(depth_, &expandedPoints_);CHKERRQ(ierr);
2564   ierr = PetscCalloc1(depth_, &sections_);CHKERRQ(ierr);
2565   arr = (PetscInt*) arr0; /* this is ok because first generation of arr is not modified */
2566   for (d=depth_-1; d>=0; d--) {
2567     ierr = PetscSectionCreate(PETSC_COMM_SELF, &sections_[d]);CHKERRQ(ierr);
2568     ierr = PetscSectionSetChart(sections_[d], 0, n);CHKERRQ(ierr);
2569     for (i=0; i<n; i++) {
2570       ierr = DMPlexGetDepthStratum(dm, d+1, &start, &end);CHKERRQ(ierr);
2571       if (arr[i] >= start && arr[i] < end) {
2572         ierr = DMPlexGetConeSize(dm, arr[i], &cn);CHKERRQ(ierr);
2573         ierr = PetscSectionSetDof(sections_[d], i, cn);CHKERRQ(ierr);
2574       } else {
2575         ierr = PetscSectionSetDof(sections_[d], i, 1);CHKERRQ(ierr);
2576       }
2577     }
2578     ierr = PetscSectionSetUp(sections_[d]);CHKERRQ(ierr);
2579     ierr = PetscSectionGetStorageSize(sections_[d], &newn);CHKERRQ(ierr);
2580     ierr = PetscMalloc1(newn, &newarr);CHKERRQ(ierr);
2581     for (i=0; i<n; i++) {
2582       ierr = PetscSectionGetDof(sections_[d], i, &cn);CHKERRQ(ierr);
2583       ierr = PetscSectionGetOffset(sections_[d], i, &co);CHKERRQ(ierr);
2584       if (cn > 1) {
2585         ierr = DMPlexGetCone(dm, arr[i], &cone);CHKERRQ(ierr);
2586         ierr = PetscMemcpy(&newarr[co], cone, cn*sizeof(PetscInt));CHKERRQ(ierr);
2587       } else {
2588         newarr[co] = arr[i];
2589       }
2590     }
2591     ierr = ISCreateGeneral(PETSC_COMM_SELF, newn, newarr, PETSC_OWN_POINTER, &expandedPoints_[d]);CHKERRQ(ierr);
2592     arr = newarr;
2593     n = newn;
2594   }
2595   ierr = ISRestoreIndices(points, &arr0);CHKERRQ(ierr);
2596   *depth = depth_;
2597   if (expandedPoints) *expandedPoints = expandedPoints_;
2598   else {
2599     for (d=0; d<depth_; d++) {ierr = ISDestroy(&expandedPoints_[d]);CHKERRQ(ierr);}
2600     ierr = PetscFree(expandedPoints_);CHKERRQ(ierr);
2601   }
2602   if (sections) *sections = sections_;
2603   else {
2604     for (d=0; d<depth_; d++) {ierr = PetscSectionDestroy(&sections_[d]);CHKERRQ(ierr);}
2605     ierr = PetscFree(sections_);CHKERRQ(ierr);
2606   }
2607   PetscFunctionReturn(0);
2608 }
2609 
2610 /*@
2611   DMPlexRestoreConeRecursive - Deallocates arrays created by DMPlexGetConeRecursive
2612 
2613   Not collective
2614 
2615   Input Parameters:
2616 + dm - The DMPlex
2617 - points - The IS of points, which must lie in the chart set with DMPlexSetChart()
2618 
2619   Output Parameter:
2620 + depth - (optional) Size of the output arrays, equal to DMPlex depth, returned by DMPlexGetDepth()
2621 . expandedPoints - (optional) An array of recursively expanded cones
2622 - sections - (optional) An array of sections which describe mappings from points to their cone points
2623 
2624   Level: advanced
2625 
2626   Notes:
2627   See DMPlexGetConeRecursive() for details.
2628 
2629 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexGetConeTuple(), DMPlexGetConeRecursive(), DMPlexGetConeRecursiveVertices(), DMPlexGetDepth()
2630 @*/
2631 PetscErrorCode DMPlexRestoreConeRecursive(DM dm, IS points, PetscInt *depth, IS *expandedPoints[], PetscSection *sections[])
2632 {
2633   PetscInt            d, depth_;
2634   PetscErrorCode      ierr;
2635 
2636   PetscFunctionBegin;
2637   ierr = DMPlexGetDepth(dm, &depth_);CHKERRQ(ierr);
2638   if (depth && *depth != depth_) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "depth changed since last call to DMPlexGetConeRecursive");
2639   if (depth) *depth = 0;
2640   if (expandedPoints) {
2641     for (d=0; d<depth_; d++) {ierr = ISDestroy(&((*expandedPoints)[d]));CHKERRQ(ierr);}
2642     ierr = PetscFree(*expandedPoints);CHKERRQ(ierr);
2643   }
2644   if (sections)  {
2645     for (d=0; d<depth_; d++) {ierr = PetscSectionDestroy(&((*sections)[d]));CHKERRQ(ierr);}
2646     ierr = PetscFree(*sections);CHKERRQ(ierr);
2647   }
2648   PetscFunctionReturn(0);
2649 }
2650 
2651 /*@
2652   DMPlexSetCone - Set the points on the in-edges for this point in the DAG; that is these are the points that cover the specific point
2653 
2654   Not collective
2655 
2656   Input Parameters:
2657 + mesh - The DMPlex
2658 . p - The point, which must lie in the chart set with DMPlexSetChart()
2659 - cone - An array of points which are on the in-edges for point p
2660 
2661   Output Parameter:
2662 
2663   Note:
2664   This should be called after all calls to DMPlexSetConeSize() and DMSetUp().
2665 
2666   Developer Note: Why not call this DMPlexSetCover()
2667 
2668   Level: beginner
2669 
2670 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp(), DMPlexSetSupport(), DMPlexSetSupportSize()
2671 @*/
2672 PetscErrorCode DMPlexSetCone(DM dm, PetscInt p, const PetscInt cone[])
2673 {
2674   DM_Plex       *mesh = (DM_Plex*) dm->data;
2675   PetscInt       pStart, pEnd;
2676   PetscInt       dof, off, c;
2677   PetscErrorCode ierr;
2678 
2679   PetscFunctionBegin;
2680   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2681   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
2682   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2683   if (dof) PetscValidPointer(cone, 3);
2684   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2685   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
2686   for (c = 0; c < dof; ++c) {
2687     if ((cone[c] < pStart) || (cone[c] >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone point %D is not in the valid range [%D, %D)", cone[c], pStart, pEnd);
2688     mesh->cones[off+c] = cone[c];
2689   }
2690   PetscFunctionReturn(0);
2691 }
2692 
2693 /*@C
2694   DMPlexGetConeOrientation - Return the orientations on the in-edges for this point in the DAG
2695 
2696   Not collective
2697 
2698   Input Parameters:
2699 + mesh - The DMPlex
2700 - p - The point, which must lie in the chart set with DMPlexSetChart()
2701 
2702   Output Parameter:
2703 . coneOrientation - An array of orientations which are on the in-edges for point p. An orientation is an
2704                     integer giving the prescription for cone traversal. If it is negative, the cone is
2705                     traversed in the opposite direction. Its value 'o', or if negative '-(o+1)', gives
2706                     the index of the cone point on which to start.
2707 
2708   Level: beginner
2709 
2710   Fortran Notes:
2711   Since it returns an array, this routine is only available in Fortran 90, and you must
2712   include petsc.h90 in your code.
2713   You must also call DMPlexRestoreConeOrientation() after you finish using the returned array.
2714   DMPlexRestoreConeOrientation() is not needed/available in C.
2715 
2716 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetCone(), DMPlexSetChart()
2717 @*/
2718 PetscErrorCode DMPlexGetConeOrientation(DM dm, PetscInt p, const PetscInt *coneOrientation[])
2719 {
2720   DM_Plex       *mesh = (DM_Plex*) dm->data;
2721   PetscInt       off;
2722   PetscErrorCode ierr;
2723 
2724   PetscFunctionBegin;
2725   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2726   if (PetscDefined(USE_DEBUG)) {
2727     PetscInt dof;
2728     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2729     if (dof) PetscValidPointer(coneOrientation, 3);
2730   }
2731   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2732 
2733   *coneOrientation = &mesh->coneOrientations[off];
2734   PetscFunctionReturn(0);
2735 }
2736 
2737 /*@
2738   DMPlexSetConeOrientation - Set the orientations on the in-edges for this point in the DAG
2739 
2740   Not collective
2741 
2742   Input Parameters:
2743 + mesh - The DMPlex
2744 . p - The point, which must lie in the chart set with DMPlexSetChart()
2745 - coneOrientation - An array of orientations which are on the in-edges for point p. An orientation is an
2746                     integer giving the prescription for cone traversal. If it is negative, the cone is
2747                     traversed in the opposite direction. Its value 'o', or if negative '-(o+1)', gives
2748                     the index of the cone point on which to start.
2749 
2750   Output Parameter:
2751 
2752   Note:
2753   This should be called after all calls to DMPlexSetConeSize() and DMSetUp().
2754 
2755   Level: beginner
2756 
2757 .seealso: DMPlexCreate(), DMPlexGetConeOrientation(), DMPlexSetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
2758 @*/
2759 PetscErrorCode DMPlexSetConeOrientation(DM dm, PetscInt p, const PetscInt coneOrientation[])
2760 {
2761   DM_Plex       *mesh = (DM_Plex*) dm->data;
2762   PetscInt       pStart, pEnd;
2763   PetscInt       dof, off, c;
2764   PetscErrorCode ierr;
2765 
2766   PetscFunctionBegin;
2767   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2768   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
2769   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2770   if (dof) PetscValidPointer(coneOrientation, 3);
2771   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2772   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
2773   for (c = 0; c < dof; ++c) {
2774     PetscInt cdof, o = coneOrientation[c];
2775 
2776     ierr = PetscSectionGetDof(mesh->coneSection, mesh->cones[off+c], &cdof);CHKERRQ(ierr);
2777     if (o && ((o < -(cdof+1)) || (o >= cdof))) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone orientation %D is not in the valid range [%D. %D)", o, -(cdof+1), cdof);
2778     mesh->coneOrientations[off+c] = o;
2779   }
2780   PetscFunctionReturn(0);
2781 }
2782 
2783 /*@
2784   DMPlexInsertCone - Insert a point into the in-edges for the point p in the DAG
2785 
2786   Not collective
2787 
2788   Input Parameters:
2789 + mesh - The DMPlex
2790 . p - The point, which must lie in the chart set with DMPlexSetChart()
2791 . conePos - The local index in the cone where the point should be put
2792 - conePoint - The mesh point to insert
2793 
2794   Level: beginner
2795 
2796 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
2797 @*/
2798 PetscErrorCode DMPlexInsertCone(DM dm, PetscInt p, PetscInt conePos, PetscInt conePoint)
2799 {
2800   DM_Plex       *mesh = (DM_Plex*) dm->data;
2801   PetscInt       pStart, pEnd;
2802   PetscInt       dof, off;
2803   PetscErrorCode ierr;
2804 
2805   PetscFunctionBegin;
2806   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2807   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
2808   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
2809   if ((conePoint < pStart) || (conePoint >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone point %D is not in the valid range [%D, %D)", conePoint, pStart, pEnd);
2810   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2811   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2812   if ((conePos < 0) || (conePos >= dof)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone position %D of point %D is not in the valid range [0, %D)", conePos, p, dof);
2813   mesh->cones[off+conePos] = conePoint;
2814   PetscFunctionReturn(0);
2815 }
2816 
2817 /*@
2818   DMPlexInsertConeOrientation - Insert a point orientation for the in-edge for the point p in the DAG
2819 
2820   Not collective
2821 
2822   Input Parameters:
2823 + mesh - The DMPlex
2824 . p - The point, which must lie in the chart set with DMPlexSetChart()
2825 . conePos - The local index in the cone where the point should be put
2826 - coneOrientation - The point orientation to insert
2827 
2828   Level: beginner
2829 
2830 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
2831 @*/
2832 PetscErrorCode DMPlexInsertConeOrientation(DM dm, PetscInt p, PetscInt conePos, PetscInt coneOrientation)
2833 {
2834   DM_Plex       *mesh = (DM_Plex*) dm->data;
2835   PetscInt       pStart, pEnd;
2836   PetscInt       dof, off;
2837   PetscErrorCode ierr;
2838 
2839   PetscFunctionBegin;
2840   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2841   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
2842   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
2843   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2844   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2845   if ((conePos < 0) || (conePos >= dof)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone position %D of point %D is not in the valid range [0, %D)", conePos, p, dof);
2846   mesh->coneOrientations[off+conePos] = coneOrientation;
2847   PetscFunctionReturn(0);
2848 }
2849 
2850 /*@
2851   DMPlexGetSupportSize - Return the number of out-edges for this point in the DAG
2852 
2853   Not collective
2854 
2855   Input Parameters:
2856 + mesh - The DMPlex
2857 - p - The point, which must lie in the chart set with DMPlexSetChart()
2858 
2859   Output Parameter:
2860 . size - The support size for point p
2861 
2862   Level: beginner
2863 
2864 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart(), DMPlexGetConeSize()
2865 @*/
2866 PetscErrorCode DMPlexGetSupportSize(DM dm, PetscInt p, PetscInt *size)
2867 {
2868   DM_Plex       *mesh = (DM_Plex*) dm->data;
2869   PetscErrorCode ierr;
2870 
2871   PetscFunctionBegin;
2872   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2873   PetscValidPointer(size, 3);
2874   ierr = PetscSectionGetDof(mesh->supportSection, p, size);CHKERRQ(ierr);
2875   PetscFunctionReturn(0);
2876 }
2877 
2878 /*@
2879   DMPlexSetSupportSize - Set the number of out-edges for this point in the DAG
2880 
2881   Not collective
2882 
2883   Input Parameters:
2884 + mesh - The DMPlex
2885 . p - The point, which must lie in the chart set with DMPlexSetChart()
2886 - size - The support size for point p
2887 
2888   Output Parameter:
2889 
2890   Note:
2891   This should be called after DMPlexSetChart().
2892 
2893   Level: beginner
2894 
2895 .seealso: DMPlexCreate(), DMPlexGetSupportSize(), DMPlexSetChart()
2896 @*/
2897 PetscErrorCode DMPlexSetSupportSize(DM dm, PetscInt p, PetscInt size)
2898 {
2899   DM_Plex       *mesh = (DM_Plex*) dm->data;
2900   PetscErrorCode ierr;
2901 
2902   PetscFunctionBegin;
2903   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2904   ierr = PetscSectionSetDof(mesh->supportSection, p, size);CHKERRQ(ierr);
2905 
2906   mesh->maxSupportSize = PetscMax(mesh->maxSupportSize, size);
2907   PetscFunctionReturn(0);
2908 }
2909 
2910 /*@C
2911   DMPlexGetSupport - Return the points on the out-edges for this point in the DAG
2912 
2913   Not collective
2914 
2915   Input Parameters:
2916 + mesh - The DMPlex
2917 - p - The point, which must lie in the chart set with DMPlexSetChart()
2918 
2919   Output Parameter:
2920 . support - An array of points which are on the out-edges for point p
2921 
2922   Level: beginner
2923 
2924   Fortran Notes:
2925   Since it returns an array, this routine is only available in Fortran 90, and you must
2926   include petsc.h90 in your code.
2927   You must also call DMPlexRestoreSupport() after you finish using the returned array.
2928   DMPlexRestoreSupport() is not needed/available in C.
2929 
2930 .seealso: DMPlexGetSupportSize(), DMPlexSetSupport(), DMPlexGetCone(), DMPlexSetChart()
2931 @*/
2932 PetscErrorCode DMPlexGetSupport(DM dm, PetscInt p, const PetscInt *support[])
2933 {
2934   DM_Plex       *mesh = (DM_Plex*) dm->data;
2935   PetscInt       off;
2936   PetscErrorCode ierr;
2937 
2938   PetscFunctionBegin;
2939   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2940   PetscValidPointer(support, 3);
2941   ierr     = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
2942   *support = &mesh->supports[off];
2943   PetscFunctionReturn(0);
2944 }
2945 
2946 /*@
2947   DMPlexSetSupport - Set the points on the out-edges for this point in the DAG, that is the list of points that this point covers
2948 
2949   Not collective
2950 
2951   Input Parameters:
2952 + mesh - The DMPlex
2953 . p - The point, which must lie in the chart set with DMPlexSetChart()
2954 - support - An array of points which are on the out-edges for point p
2955 
2956   Output Parameter:
2957 
2958   Note:
2959   This should be called after all calls to DMPlexSetSupportSize() and DMSetUp().
2960 
2961   Level: beginner
2962 
2963 .seealso: DMPlexSetCone(), DMPlexSetConeSize(), DMPlexCreate(), DMPlexGetSupport(), DMPlexSetChart(), DMPlexSetSupportSize(), DMSetUp()
2964 @*/
2965 PetscErrorCode DMPlexSetSupport(DM dm, PetscInt p, const PetscInt support[])
2966 {
2967   DM_Plex       *mesh = (DM_Plex*) dm->data;
2968   PetscInt       pStart, pEnd;
2969   PetscInt       dof, off, c;
2970   PetscErrorCode ierr;
2971 
2972   PetscFunctionBegin;
2973   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2974   ierr = PetscSectionGetChart(mesh->supportSection, &pStart, &pEnd);CHKERRQ(ierr);
2975   ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
2976   if (dof) PetscValidPointer(support, 3);
2977   ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
2978   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
2979   for (c = 0; c < dof; ++c) {
2980     if ((support[c] < pStart) || (support[c] >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support point %D is not in the valid range [%D, %D)", support[c], pStart, pEnd);
2981     mesh->supports[off+c] = support[c];
2982   }
2983   PetscFunctionReturn(0);
2984 }
2985 
2986 /*@
2987   DMPlexInsertSupport - Insert a point into the out-edges for the point p in the DAG
2988 
2989   Not collective
2990 
2991   Input Parameters:
2992 + mesh - The DMPlex
2993 . p - The point, which must lie in the chart set with DMPlexSetChart()
2994 . supportPos - The local index in the cone where the point should be put
2995 - supportPoint - The mesh point to insert
2996 
2997   Level: beginner
2998 
2999 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
3000 @*/
3001 PetscErrorCode DMPlexInsertSupport(DM dm, PetscInt p, PetscInt supportPos, PetscInt supportPoint)
3002 {
3003   DM_Plex       *mesh = (DM_Plex*) dm->data;
3004   PetscInt       pStart, pEnd;
3005   PetscInt       dof, off;
3006   PetscErrorCode ierr;
3007 
3008   PetscFunctionBegin;
3009   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3010   ierr = PetscSectionGetChart(mesh->supportSection, &pStart, &pEnd);CHKERRQ(ierr);
3011   ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
3012   ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
3013   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
3014   if ((supportPoint < pStart) || (supportPoint >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support point %D is not in the valid range [%D, %D)", supportPoint, pStart, pEnd);
3015   if (supportPos >= dof) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support position %D of point %D is not in the valid range [0, %D)", supportPos, p, dof);
3016   mesh->supports[off+supportPos] = supportPoint;
3017   PetscFunctionReturn(0);
3018 }
3019 
3020 /*@C
3021   DMPlexGetTransitiveClosure - Return the points on the transitive closure of the in-edges or out-edges for this point in the DAG
3022 
3023   Not collective
3024 
3025   Input Parameters:
3026 + mesh - The DMPlex
3027 . p - The point, which must lie in the chart set with DMPlexSetChart()
3028 . useCone - PETSC_TRUE for in-edges,  otherwise use out-edges
3029 - points - If points is NULL on input, internal storage will be returned, otherwise the provided array is used
3030 
3031   Output Parameters:
3032 + numPoints - The number of points in the closure, so points[] is of size 2*numPoints
3033 - points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...]
3034 
3035   Note:
3036   If using internal storage (points is NULL on input), each call overwrites the last output.
3037 
3038   Fortran Notes:
3039   Since it returns an array, this routine is only available in Fortran 90, and you must
3040   include petsc.h90 in your code.
3041 
3042   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3043 
3044   Level: beginner
3045 
3046 .seealso: DMPlexRestoreTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
3047 @*/
3048 PetscErrorCode DMPlexGetTransitiveClosure(DM dm, PetscInt p, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
3049 {
3050   DM_Plex        *mesh = (DM_Plex*) dm->data;
3051   PetscInt       *closure, *fifo;
3052   const PetscInt *tmp = NULL, *tmpO = NULL;
3053   PetscInt        tmpSize, t;
3054   PetscInt        depth       = 0, maxSize;
3055   PetscInt        closureSize = 2, fifoSize = 0, fifoStart = 0;
3056   PetscErrorCode  ierr;
3057 
3058   PetscFunctionBegin;
3059   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3060   ierr    = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3061   /* This is only 1-level */
3062   if (useCone) {
3063     ierr = DMPlexGetConeSize(dm, p, &tmpSize);CHKERRQ(ierr);
3064     ierr = DMPlexGetCone(dm, p, &tmp);CHKERRQ(ierr);
3065     ierr = DMPlexGetConeOrientation(dm, p, &tmpO);CHKERRQ(ierr);
3066   } else {
3067     ierr = DMPlexGetSupportSize(dm, p, &tmpSize);CHKERRQ(ierr);
3068     ierr = DMPlexGetSupport(dm, p, &tmp);CHKERRQ(ierr);
3069   }
3070   if (depth == 1) {
3071     if (*points) {
3072       closure = *points;
3073     } else {
3074       maxSize = 2*(PetscMax(mesh->maxConeSize, mesh->maxSupportSize)+1);
3075       ierr = DMGetWorkArray(dm, maxSize, MPIU_INT, &closure);CHKERRQ(ierr);
3076     }
3077     closure[0] = p; closure[1] = 0;
3078     for (t = 0; t < tmpSize; ++t, closureSize += 2) {
3079       closure[closureSize]   = tmp[t];
3080       closure[closureSize+1] = tmpO ? tmpO[t] : 0;
3081     }
3082     if (numPoints) *numPoints = closureSize/2;
3083     if (points)    *points    = closure;
3084     PetscFunctionReturn(0);
3085   }
3086   {
3087     PetscInt c, coneSeries, s,supportSeries;
3088 
3089     c = mesh->maxConeSize;
3090     coneSeries = (c > 1) ? ((PetscPowInt(c,depth+1)-1)/(c-1)) : depth+1;
3091     s = mesh->maxSupportSize;
3092     supportSeries = (s > 1) ? ((PetscPowInt(s,depth+1)-1)/(s-1)) : depth+1;
3093     maxSize = 2*PetscMax(coneSeries,supportSeries);
3094   }
3095   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &fifo);CHKERRQ(ierr);
3096   if (*points) {
3097     closure = *points;
3098   } else {
3099     ierr = DMGetWorkArray(dm, maxSize, MPIU_INT, &closure);CHKERRQ(ierr);
3100   }
3101   closure[0] = p; closure[1] = 0;
3102   for (t = 0; t < tmpSize; ++t, closureSize += 2, fifoSize += 2) {
3103     const PetscInt cp = tmp[t];
3104     const PetscInt co = tmpO ? tmpO[t] : 0;
3105 
3106     closure[closureSize]   = cp;
3107     closure[closureSize+1] = co;
3108     fifo[fifoSize]         = cp;
3109     fifo[fifoSize+1]       = co;
3110   }
3111   /* Should kick out early when depth is reached, rather than checking all vertices for empty cones */
3112   while (fifoSize - fifoStart) {
3113     const PetscInt q   = fifo[fifoStart];
3114     const PetscInt o   = fifo[fifoStart+1];
3115     const PetscInt rev = o >= 0 ? 0 : 1;
3116     const PetscInt off = rev ? -(o+1) : o;
3117 
3118     if (useCone) {
3119       ierr = DMPlexGetConeSize(dm, q, &tmpSize);CHKERRQ(ierr);
3120       ierr = DMPlexGetCone(dm, q, &tmp);CHKERRQ(ierr);
3121       ierr = DMPlexGetConeOrientation(dm, q, &tmpO);CHKERRQ(ierr);
3122     } else {
3123       ierr = DMPlexGetSupportSize(dm, q, &tmpSize);CHKERRQ(ierr);
3124       ierr = DMPlexGetSupport(dm, q, &tmp);CHKERRQ(ierr);
3125       tmpO = NULL;
3126     }
3127     for (t = 0; t < tmpSize; ++t) {
3128       const PetscInt i  = ((rev ? tmpSize-t : t) + off)%tmpSize;
3129       const PetscInt cp = tmp[i];
3130       /* Must propogate orientation: When we reverse orientation, we both reverse the direction of iteration and start at the other end of the chain. */
3131       /* HACK: It is worse to get the size here, than to change the interpretation of -(*+1)
3132        const PetscInt co = tmpO ? (rev ? -(tmpO[i]+1) : tmpO[i]) : 0; */
3133       PetscInt       co = tmpO ? tmpO[i] : 0;
3134       PetscInt       c;
3135 
3136       if (rev) {
3137         PetscInt childSize, coff;
3138         ierr = DMPlexGetConeSize(dm, cp, &childSize);CHKERRQ(ierr);
3139         coff = tmpO[i] < 0 ? -(tmpO[i]+1) : tmpO[i];
3140         co   = childSize ? -(((coff+childSize-1)%childSize)+1) : 0;
3141       }
3142       /* Check for duplicate */
3143       for (c = 0; c < closureSize; c += 2) {
3144         if (closure[c] == cp) break;
3145       }
3146       if (c == closureSize) {
3147         closure[closureSize]   = cp;
3148         closure[closureSize+1] = co;
3149         fifo[fifoSize]         = cp;
3150         fifo[fifoSize+1]       = co;
3151         closureSize           += 2;
3152         fifoSize              += 2;
3153       }
3154     }
3155     fifoStart += 2;
3156   }
3157   if (numPoints) *numPoints = closureSize/2;
3158   if (points)    *points    = closure;
3159   ierr = DMRestoreWorkArray(dm, maxSize, MPIU_INT, &fifo);CHKERRQ(ierr);
3160   PetscFunctionReturn(0);
3161 }
3162 
3163 /*@C
3164   DMPlexGetTransitiveClosure_Internal - Return the points on the transitive closure of the in-edges or out-edges for this point in the DAG with a specified initial orientation
3165 
3166   Not collective
3167 
3168   Input Parameters:
3169 + mesh - The DMPlex
3170 . p - The point, which must lie in the chart set with DMPlexSetChart()
3171 . orientation - The orientation of the point
3172 . useCone - PETSC_TRUE for in-edges,  otherwise use out-edges
3173 - points - If points is NULL on input, internal storage will be returned, otherwise the provided array is used
3174 
3175   Output Parameters:
3176 + numPoints - The number of points in the closure, so points[] is of size 2*numPoints
3177 - points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...]
3178 
3179   Note:
3180   If using internal storage (points is NULL on input), each call overwrites the last output.
3181 
3182   Fortran Notes:
3183   Since it returns an array, this routine is only available in Fortran 90, and you must
3184   include petsc.h90 in your code.
3185 
3186   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3187 
3188   Level: beginner
3189 
3190 .seealso: DMPlexRestoreTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
3191 @*/
3192 PetscErrorCode DMPlexGetTransitiveClosure_Internal(DM dm, PetscInt p, PetscInt ornt, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
3193 {
3194   DM_Plex        *mesh = (DM_Plex*) dm->data;
3195   PetscInt       *closure, *fifo;
3196   const PetscInt *tmp = NULL, *tmpO = NULL;
3197   PetscInt        tmpSize, t;
3198   PetscInt        depth       = 0, maxSize;
3199   PetscInt        closureSize = 2, fifoSize = 0, fifoStart = 0;
3200   PetscErrorCode  ierr;
3201 
3202   PetscFunctionBegin;
3203   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3204   ierr    = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3205   /* This is only 1-level */
3206   if (useCone) {
3207     ierr = DMPlexGetConeSize(dm, p, &tmpSize);CHKERRQ(ierr);
3208     ierr = DMPlexGetCone(dm, p, &tmp);CHKERRQ(ierr);
3209     ierr = DMPlexGetConeOrientation(dm, p, &tmpO);CHKERRQ(ierr);
3210   } else {
3211     ierr = DMPlexGetSupportSize(dm, p, &tmpSize);CHKERRQ(ierr);
3212     ierr = DMPlexGetSupport(dm, p, &tmp);CHKERRQ(ierr);
3213   }
3214   if (depth == 1) {
3215     if (*points) {
3216       closure = *points;
3217     } else {
3218       maxSize = 2*(PetscMax(mesh->maxConeSize, mesh->maxSupportSize)+1);
3219       ierr = DMGetWorkArray(dm, maxSize, MPIU_INT, &closure);CHKERRQ(ierr);
3220     }
3221     closure[0] = p; closure[1] = ornt;
3222     for (t = 0; t < tmpSize; ++t, closureSize += 2) {
3223       const PetscInt i = ornt >= 0 ? (t+ornt)%tmpSize : (-(ornt+1) + tmpSize-t)%tmpSize;
3224       closure[closureSize]   = tmp[i];
3225       closure[closureSize+1] = tmpO ? tmpO[i] : 0;
3226     }
3227     if (numPoints) *numPoints = closureSize/2;
3228     if (points)    *points    = closure;
3229     PetscFunctionReturn(0);
3230   }
3231   {
3232     PetscInt c, coneSeries, s,supportSeries;
3233 
3234     c = mesh->maxConeSize;
3235     coneSeries = (c > 1) ? ((PetscPowInt(c,depth+1)-1)/(c-1)) : depth+1;
3236     s = mesh->maxSupportSize;
3237     supportSeries = (s > 1) ? ((PetscPowInt(s,depth+1)-1)/(s-1)) : depth+1;
3238     maxSize = 2*PetscMax(coneSeries,supportSeries);
3239   }
3240   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &fifo);CHKERRQ(ierr);
3241   if (*points) {
3242     closure = *points;
3243   } else {
3244     ierr = DMGetWorkArray(dm, maxSize, MPIU_INT, &closure);CHKERRQ(ierr);
3245   }
3246   closure[0] = p; closure[1] = ornt;
3247   for (t = 0; t < tmpSize; ++t, closureSize += 2, fifoSize += 2) {
3248     const PetscInt i  = ornt >= 0 ? (t+ornt)%tmpSize : (-(ornt+1) + tmpSize-t)%tmpSize;
3249     const PetscInt cp = tmp[i];
3250     PetscInt       co = tmpO ? tmpO[i] : 0;
3251 
3252     if (ornt < 0) {
3253       PetscInt childSize, coff;
3254       ierr = DMPlexGetConeSize(dm, cp, &childSize);CHKERRQ(ierr);
3255       coff = co < 0 ? -(tmpO[i]+1) : tmpO[i];
3256       co   = childSize ? -(((coff+childSize-1)%childSize)+1) : 0;
3257     }
3258     closure[closureSize]   = cp;
3259     closure[closureSize+1] = co;
3260     fifo[fifoSize]         = cp;
3261     fifo[fifoSize+1]       = co;
3262   }
3263   /* Should kick out early when depth is reached, rather than checking all vertices for empty cones */
3264   while (fifoSize - fifoStart) {
3265     const PetscInt q   = fifo[fifoStart];
3266     const PetscInt o   = fifo[fifoStart+1];
3267     const PetscInt rev = o >= 0 ? 0 : 1;
3268     const PetscInt off = rev ? -(o+1) : o;
3269 
3270     if (useCone) {
3271       ierr = DMPlexGetConeSize(dm, q, &tmpSize);CHKERRQ(ierr);
3272       ierr = DMPlexGetCone(dm, q, &tmp);CHKERRQ(ierr);
3273       ierr = DMPlexGetConeOrientation(dm, q, &tmpO);CHKERRQ(ierr);
3274     } else {
3275       ierr = DMPlexGetSupportSize(dm, q, &tmpSize);CHKERRQ(ierr);
3276       ierr = DMPlexGetSupport(dm, q, &tmp);CHKERRQ(ierr);
3277       tmpO = NULL;
3278     }
3279     for (t = 0; t < tmpSize; ++t) {
3280       const PetscInt i  = ((rev ? tmpSize-t : t) + off)%tmpSize;
3281       const PetscInt cp = tmp[i];
3282       /* Must propogate orientation: When we reverse orientation, we both reverse the direction of iteration and start at the other end of the chain. */
3283       /* HACK: It is worse to get the size here, than to change the interpretation of -(*+1)
3284        const PetscInt co = tmpO ? (rev ? -(tmpO[i]+1) : tmpO[i]) : 0; */
3285       PetscInt       co = tmpO ? tmpO[i] : 0;
3286       PetscInt       c;
3287 
3288       if (rev) {
3289         PetscInt childSize, coff;
3290         ierr = DMPlexGetConeSize(dm, cp, &childSize);CHKERRQ(ierr);
3291         coff = tmpO[i] < 0 ? -(tmpO[i]+1) : tmpO[i];
3292         co   = childSize ? -(((coff+childSize-1)%childSize)+1) : 0;
3293       }
3294       /* Check for duplicate */
3295       for (c = 0; c < closureSize; c += 2) {
3296         if (closure[c] == cp) break;
3297       }
3298       if (c == closureSize) {
3299         closure[closureSize]   = cp;
3300         closure[closureSize+1] = co;
3301         fifo[fifoSize]         = cp;
3302         fifo[fifoSize+1]       = co;
3303         closureSize           += 2;
3304         fifoSize              += 2;
3305       }
3306     }
3307     fifoStart += 2;
3308   }
3309   if (numPoints) *numPoints = closureSize/2;
3310   if (points)    *points    = closure;
3311   ierr = DMRestoreWorkArray(dm, maxSize, MPIU_INT, &fifo);CHKERRQ(ierr);
3312   PetscFunctionReturn(0);
3313 }
3314 
3315 /*@C
3316   DMPlexRestoreTransitiveClosure - Restore the array of points on the transitive closure of the in-edges or out-edges for this point in the DAG
3317 
3318   Not collective
3319 
3320   Input Parameters:
3321 + mesh - The DMPlex
3322 . p - The point, which must lie in the chart set with DMPlexSetChart()
3323 . useCone - PETSC_TRUE for in-edges,  otherwise use out-edges
3324 . numPoints - The number of points in the closure, so points[] is of size 2*numPoints, zeroed on exit
3325 - points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...], zeroed on exit
3326 
3327   Note:
3328   If not using internal storage (points is not NULL on input), this call is unnecessary
3329 
3330   Fortran Notes:
3331   Since it returns an array, this routine is only available in Fortran 90, and you must
3332   include petsc.h90 in your code.
3333 
3334   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3335 
3336   Level: beginner
3337 
3338 .seealso: DMPlexGetTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
3339 @*/
3340 PetscErrorCode DMPlexRestoreTransitiveClosure(DM dm, PetscInt p, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
3341 {
3342   PetscErrorCode ierr;
3343 
3344   PetscFunctionBegin;
3345   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3346   if (numPoints) PetscValidIntPointer(numPoints,4);
3347   if (points) PetscValidPointer(points,5);
3348   ierr = DMRestoreWorkArray(dm, 0, MPIU_INT, points);CHKERRQ(ierr);
3349   if (numPoints) *numPoints = 0;
3350   PetscFunctionReturn(0);
3351 }
3352 
3353 /*@
3354   DMPlexGetMaxSizes - Return the maximum number of in-edges (cone) and out-edges (support) for any point in the DAG
3355 
3356   Not collective
3357 
3358   Input Parameter:
3359 . mesh - The DMPlex
3360 
3361   Output Parameters:
3362 + maxConeSize - The maximum number of in-edges
3363 - maxSupportSize - The maximum number of out-edges
3364 
3365   Level: beginner
3366 
3367 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart()
3368 @*/
3369 PetscErrorCode DMPlexGetMaxSizes(DM dm, PetscInt *maxConeSize, PetscInt *maxSupportSize)
3370 {
3371   DM_Plex *mesh = (DM_Plex*) dm->data;
3372 
3373   PetscFunctionBegin;
3374   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3375   if (maxConeSize)    *maxConeSize    = mesh->maxConeSize;
3376   if (maxSupportSize) *maxSupportSize = mesh->maxSupportSize;
3377   PetscFunctionReturn(0);
3378 }
3379 
3380 PetscErrorCode DMSetUp_Plex(DM dm)
3381 {
3382   DM_Plex       *mesh = (DM_Plex*) dm->data;
3383   PetscInt       size;
3384   PetscErrorCode ierr;
3385 
3386   PetscFunctionBegin;
3387   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3388   ierr = PetscSectionSetUp(mesh->coneSection);CHKERRQ(ierr);
3389   ierr = PetscSectionGetStorageSize(mesh->coneSection, &size);CHKERRQ(ierr);
3390   ierr = PetscMalloc1(size, &mesh->cones);CHKERRQ(ierr);
3391   ierr = PetscCalloc1(size, &mesh->coneOrientations);CHKERRQ(ierr);
3392   ierr = PetscLogObjectMemory((PetscObject) dm, size*2*sizeof(PetscInt));CHKERRQ(ierr);
3393   if (mesh->maxSupportSize) {
3394     ierr = PetscSectionSetUp(mesh->supportSection);CHKERRQ(ierr);
3395     ierr = PetscSectionGetStorageSize(mesh->supportSection, &size);CHKERRQ(ierr);
3396     ierr = PetscMalloc1(size, &mesh->supports);CHKERRQ(ierr);
3397     ierr = PetscLogObjectMemory((PetscObject) dm, size*sizeof(PetscInt));CHKERRQ(ierr);
3398   }
3399   PetscFunctionReturn(0);
3400 }
3401 
3402 PetscErrorCode DMCreateSubDM_Plex(DM dm, PetscInt numFields, const PetscInt fields[], IS *is, DM *subdm)
3403 {
3404   PetscErrorCode ierr;
3405 
3406   PetscFunctionBegin;
3407   if (subdm) {ierr = DMClone(dm, subdm);CHKERRQ(ierr);}
3408   ierr = DMCreateSectionSubDM(dm, numFields, fields, is, subdm);CHKERRQ(ierr);
3409   if (subdm) {(*subdm)->useNatural = dm->useNatural;}
3410   if (dm->useNatural && dm->sfMigration) {
3411     PetscSF        sfMigrationInv,sfNatural;
3412     PetscSection   section, sectionSeq;
3413 
3414     (*subdm)->sfMigration = dm->sfMigration;
3415     ierr = PetscObjectReference((PetscObject) dm->sfMigration);CHKERRQ(ierr);
3416     ierr = DMGetLocalSection((*subdm), &section);CHKERRQ(ierr);
3417     ierr = PetscSFCreateInverseSF((*subdm)->sfMigration, &sfMigrationInv);CHKERRQ(ierr);
3418     ierr = PetscSectionCreate(PetscObjectComm((PetscObject) (*subdm)), &sectionSeq);CHKERRQ(ierr);
3419     ierr = PetscSFDistributeSection(sfMigrationInv, section, NULL, sectionSeq);CHKERRQ(ierr);
3420 
3421     ierr = DMPlexCreateGlobalToNaturalSF(*subdm, sectionSeq, (*subdm)->sfMigration, &sfNatural);CHKERRQ(ierr);
3422     (*subdm)->sfNatural = sfNatural;
3423     ierr = PetscSectionDestroy(&sectionSeq);CHKERRQ(ierr);
3424     ierr = PetscSFDestroy(&sfMigrationInv);CHKERRQ(ierr);
3425   }
3426   PetscFunctionReturn(0);
3427 }
3428 
3429 PetscErrorCode DMCreateSuperDM_Plex(DM dms[], PetscInt len, IS **is, DM *superdm)
3430 {
3431   PetscErrorCode ierr;
3432   PetscInt       i = 0;
3433 
3434   PetscFunctionBegin;
3435   ierr = DMClone(dms[0], superdm);CHKERRQ(ierr);
3436   ierr = DMCreateSectionSuperDM(dms, len, is, superdm);CHKERRQ(ierr);
3437   (*superdm)->useNatural = PETSC_FALSE;
3438   for (i = 0; i < len; i++) {
3439     if (dms[i]->useNatural && dms[i]->sfMigration) {
3440       PetscSF        sfMigrationInv,sfNatural;
3441       PetscSection   section, sectionSeq;
3442 
3443       (*superdm)->sfMigration = dms[i]->sfMigration;
3444       ierr = PetscObjectReference((PetscObject) dms[i]->sfMigration);CHKERRQ(ierr);
3445       (*superdm)->useNatural = PETSC_TRUE;
3446       ierr = DMGetLocalSection((*superdm), &section);CHKERRQ(ierr);
3447       ierr = PetscSFCreateInverseSF((*superdm)->sfMigration, &sfMigrationInv);CHKERRQ(ierr);
3448       ierr = PetscSectionCreate(PetscObjectComm((PetscObject) (*superdm)), &sectionSeq);CHKERRQ(ierr);
3449       ierr = PetscSFDistributeSection(sfMigrationInv, section, NULL, sectionSeq);CHKERRQ(ierr);
3450 
3451       ierr = DMPlexCreateGlobalToNaturalSF(*superdm, sectionSeq, (*superdm)->sfMigration, &sfNatural);CHKERRQ(ierr);
3452       (*superdm)->sfNatural = sfNatural;
3453       ierr = PetscSectionDestroy(&sectionSeq);CHKERRQ(ierr);
3454       ierr = PetscSFDestroy(&sfMigrationInv);CHKERRQ(ierr);
3455       break;
3456     }
3457   }
3458   PetscFunctionReturn(0);
3459 }
3460 
3461 /*@
3462   DMPlexSymmetrize - Create support (out-edge) information from cone (in-edge) information
3463 
3464   Not collective
3465 
3466   Input Parameter:
3467 . mesh - The DMPlex
3468 
3469   Output Parameter:
3470 
3471   Note:
3472   This should be called after all calls to DMPlexSetCone()
3473 
3474   Level: beginner
3475 
3476 .seealso: DMPlexCreate(), DMPlexSetChart(), DMPlexSetConeSize(), DMPlexSetCone()
3477 @*/
3478 PetscErrorCode DMPlexSymmetrize(DM dm)
3479 {
3480   DM_Plex       *mesh = (DM_Plex*) dm->data;
3481   PetscInt      *offsets;
3482   PetscInt       supportSize;
3483   PetscInt       pStart, pEnd, p;
3484   PetscErrorCode ierr;
3485 
3486   PetscFunctionBegin;
3487   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3488   if (mesh->supports) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONGSTATE, "Supports were already setup in this DMPlex");
3489   ierr = PetscLogEventBegin(DMPLEX_Symmetrize,dm,0,0,0);CHKERRQ(ierr);
3490   /* Calculate support sizes */
3491   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
3492   for (p = pStart; p < pEnd; ++p) {
3493     PetscInt dof, off, c;
3494 
3495     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
3496     ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
3497     for (c = off; c < off+dof; ++c) {
3498       ierr = PetscSectionAddDof(mesh->supportSection, mesh->cones[c], 1);CHKERRQ(ierr);
3499     }
3500   }
3501   for (p = pStart; p < pEnd; ++p) {
3502     PetscInt dof;
3503 
3504     ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
3505 
3506     mesh->maxSupportSize = PetscMax(mesh->maxSupportSize, dof);
3507   }
3508   ierr = PetscSectionSetUp(mesh->supportSection);CHKERRQ(ierr);
3509   /* Calculate supports */
3510   ierr = PetscSectionGetStorageSize(mesh->supportSection, &supportSize);CHKERRQ(ierr);
3511   ierr = PetscMalloc1(supportSize, &mesh->supports);CHKERRQ(ierr);
3512   ierr = PetscCalloc1(pEnd - pStart, &offsets);CHKERRQ(ierr);
3513   for (p = pStart; p < pEnd; ++p) {
3514     PetscInt dof, off, c;
3515 
3516     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
3517     ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
3518     for (c = off; c < off+dof; ++c) {
3519       const PetscInt q = mesh->cones[c];
3520       PetscInt       offS;
3521 
3522       ierr = PetscSectionGetOffset(mesh->supportSection, q, &offS);CHKERRQ(ierr);
3523 
3524       mesh->supports[offS+offsets[q]] = p;
3525       ++offsets[q];
3526     }
3527   }
3528   ierr = PetscFree(offsets);CHKERRQ(ierr);
3529   ierr = PetscLogEventEnd(DMPLEX_Symmetrize,dm,0,0,0);CHKERRQ(ierr);
3530   PetscFunctionReturn(0);
3531 }
3532 
3533 static PetscErrorCode DMPlexCreateDepthStratum(DM dm, DMLabel label, PetscInt depth, PetscInt pStart, PetscInt pEnd)
3534 {
3535   IS             stratumIS;
3536   PetscErrorCode ierr;
3537 
3538   PetscFunctionBegin;
3539   if (pStart >= pEnd) PetscFunctionReturn(0);
3540   if (PetscDefined(USE_DEBUG)) {
3541     PetscInt  qStart, qEnd, numLevels, level;
3542     PetscBool overlap = PETSC_FALSE;
3543     ierr = DMLabelGetNumValues(label, &numLevels);CHKERRQ(ierr);
3544     for (level = 0; level < numLevels; level++) {
3545       ierr = DMLabelGetStratumBounds(label, level, &qStart, &qEnd);CHKERRQ(ierr);
3546       if ((pStart >= qStart && pStart < qEnd) || (pEnd > qStart && pEnd <= qEnd)) {overlap = PETSC_TRUE; break;}
3547     }
3548     if (overlap) SETERRQ6(PETSC_COMM_SELF, PETSC_ERR_PLIB, "New depth %D range [%D,%D) overlaps with depth %D range [%D,%D)", depth, pStart, pEnd, level, qStart, qEnd);
3549   }
3550   ierr = ISCreateStride(PETSC_COMM_SELF, pEnd-pStart, pStart, 1, &stratumIS);CHKERRQ(ierr);
3551   ierr = DMLabelSetStratumIS(label, depth, stratumIS);CHKERRQ(ierr);
3552   ierr = ISDestroy(&stratumIS);CHKERRQ(ierr);
3553   PetscFunctionReturn(0);
3554 }
3555 
3556 /*@
3557   DMPlexStratify - The DAG for most topologies is a graded poset (https://en.wikipedia.org/wiki/Graded_poset), and
3558   can be illustrated by a Hasse Diagram (https://en.wikipedia.org/wiki/Hasse_diagram). The strata group all points of the
3559   same grade, and this function calculates the strata. This grade can be seen as the height (or depth) of the point in
3560   the DAG.
3561 
3562   Collective on dm
3563 
3564   Input Parameter:
3565 . mesh - The DMPlex
3566 
3567   Output Parameter:
3568 
3569   Notes:
3570   Concretely, DMPlexStratify() creates a new label named "depth" containing the depth in the DAG of each point. For cell-vertex
3571   meshes, vertices are depth 0 and cells are depth 1. For fully interpolated meshes, depth 0 for vertices, 1 for edges, and so on
3572   until cells have depth equal to the dimension of the mesh. The depth label can be accessed through DMPlexGetDepthLabel() or DMPlexGetDepthStratum(), or
3573   manually via DMGetLabel().  The height is defined implicitly by height = maxDimension - depth, and can be accessed
3574   via DMPlexGetHeightStratum().  For example, cells have height 0 and faces have height 1.
3575 
3576   The depth of a point is calculated by executing a breadth-first search (BFS) on the DAG. This could produce surprising results
3577   if run on a partially interpolated mesh, meaning one that had some edges and faces, but not others. For example, suppose that
3578   we had a mesh consisting of one triangle (c0) and three vertices (v0, v1, v2), and only one edge is on the boundary so we choose
3579   to interpolate only that one (e0), so that
3580 $  cone(c0) = {e0, v2}
3581 $  cone(e0) = {v0, v1}
3582   If DMPlexStratify() is run on this mesh, it will give depths
3583 $  depth 0 = {v0, v1, v2}
3584 $  depth 1 = {e0, c0}
3585   where the triangle has been given depth 1, instead of 2, because it is reachable from vertex v2.
3586 
3587   DMPlexStratify() should be called after all calls to DMPlexSymmetrize()
3588 
3589   Level: beginner
3590 
3591 .seealso: DMPlexCreate(), DMPlexSymmetrize(), DMPlexComputeCellTypes()
3592 @*/
3593 PetscErrorCode DMPlexStratify(DM dm)
3594 {
3595   DM_Plex       *mesh = (DM_Plex*) dm->data;
3596   DMLabel        label;
3597   PetscInt       pStart, pEnd, p;
3598   PetscInt       numRoots = 0, numLeaves = 0;
3599   PetscErrorCode ierr;
3600 
3601   PetscFunctionBegin;
3602   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3603   ierr = PetscLogEventBegin(DMPLEX_Stratify,dm,0,0,0);CHKERRQ(ierr);
3604 
3605   /* Create depth label */
3606   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
3607   ierr = DMCreateLabel(dm, "depth");CHKERRQ(ierr);
3608   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
3609 
3610   {
3611     /* Initialize roots and count leaves */
3612     PetscInt sMin = PETSC_MAX_INT;
3613     PetscInt sMax = PETSC_MIN_INT;
3614     PetscInt coneSize, supportSize;
3615 
3616     for (p = pStart; p < pEnd; ++p) {
3617       ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
3618       ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
3619       if (!coneSize && supportSize) {
3620         sMin = PetscMin(p, sMin);
3621         sMax = PetscMax(p, sMax);
3622         ++numRoots;
3623       } else if (!supportSize && coneSize) {
3624         ++numLeaves;
3625       } else if (!supportSize && !coneSize) {
3626         /* Isolated points */
3627         sMin = PetscMin(p, sMin);
3628         sMax = PetscMax(p, sMax);
3629       }
3630     }
3631     ierr = DMPlexCreateDepthStratum(dm, label, 0, sMin, sMax+1);CHKERRQ(ierr);
3632   }
3633 
3634   if (numRoots + numLeaves == (pEnd - pStart)) {
3635     PetscInt sMin = PETSC_MAX_INT;
3636     PetscInt sMax = PETSC_MIN_INT;
3637     PetscInt coneSize, supportSize;
3638 
3639     for (p = pStart; p < pEnd; ++p) {
3640       ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
3641       ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
3642       if (!supportSize && coneSize) {
3643         sMin = PetscMin(p, sMin);
3644         sMax = PetscMax(p, sMax);
3645       }
3646     }
3647     ierr = DMPlexCreateDepthStratum(dm, label, 1, sMin, sMax+1);CHKERRQ(ierr);
3648   } else {
3649     PetscInt level = 0;
3650     PetscInt qStart, qEnd, q;
3651 
3652     ierr = DMLabelGetStratumBounds(label, level, &qStart, &qEnd);CHKERRQ(ierr);
3653     while (qEnd > qStart) {
3654       PetscInt sMin = PETSC_MAX_INT;
3655       PetscInt sMax = PETSC_MIN_INT;
3656 
3657       for (q = qStart; q < qEnd; ++q) {
3658         const PetscInt *support;
3659         PetscInt        supportSize, s;
3660 
3661         ierr = DMPlexGetSupportSize(dm, q, &supportSize);CHKERRQ(ierr);
3662         ierr = DMPlexGetSupport(dm, q, &support);CHKERRQ(ierr);
3663         for (s = 0; s < supportSize; ++s) {
3664           sMin = PetscMin(support[s], sMin);
3665           sMax = PetscMax(support[s], sMax);
3666         }
3667       }
3668       ierr = DMLabelGetNumValues(label, &level);CHKERRQ(ierr);
3669       ierr = DMPlexCreateDepthStratum(dm, label, level, sMin, sMax+1);CHKERRQ(ierr);
3670       ierr = DMLabelGetStratumBounds(label, level, &qStart, &qEnd);CHKERRQ(ierr);
3671     }
3672   }
3673   { /* just in case there is an empty process */
3674     PetscInt numValues, maxValues = 0, v;
3675 
3676     ierr = DMLabelGetNumValues(label, &numValues);CHKERRQ(ierr);
3677     ierr = MPI_Allreduce(&numValues,&maxValues,1,MPIU_INT,MPI_MAX,PetscObjectComm((PetscObject)dm));CHKERRMPI(ierr);
3678     for (v = numValues; v < maxValues; v++) {
3679       ierr = DMLabelAddStratum(label, v);CHKERRQ(ierr);
3680     }
3681   }
3682   ierr = PetscObjectStateGet((PetscObject) label, &mesh->depthState);CHKERRQ(ierr);
3683   ierr = PetscLogEventEnd(DMPLEX_Stratify,dm,0,0,0);CHKERRQ(ierr);
3684   PetscFunctionReturn(0);
3685 }
3686 
3687 PetscErrorCode DMPlexComputeCellType_Internal(DM dm, PetscInt p, PetscInt pdepth, DMPolytopeType *pt)
3688 {
3689   DMPolytopeType ct = DM_POLYTOPE_UNKNOWN;
3690   PetscInt       dim, depth, pheight, coneSize;
3691   PetscErrorCode ierr;
3692 
3693   PetscFunctionBeginHot;
3694   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
3695   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3696   ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
3697   pheight = depth - pdepth;
3698   if (depth <= 1) {
3699     switch (pdepth) {
3700       case 0: ct = DM_POLYTOPE_POINT;break;
3701       case 1:
3702         switch (coneSize) {
3703           case 2: ct = DM_POLYTOPE_SEGMENT;break;
3704           case 3: ct = DM_POLYTOPE_TRIANGLE;break;
3705           case 4:
3706           switch (dim) {
3707             case 2: ct = DM_POLYTOPE_QUADRILATERAL;break;
3708             case 3: ct = DM_POLYTOPE_TETRAHEDRON;break;
3709             default: break;
3710           }
3711           break;
3712         case 5: ct = DM_POLYTOPE_PYRAMID;break;
3713         case 6: ct = DM_POLYTOPE_TRI_PRISM_TENSOR;break;
3714         case 8: ct = DM_POLYTOPE_HEXAHEDRON;break;
3715         default: break;
3716       }
3717     }
3718   } else {
3719     if (pdepth == 0) {
3720       ct = DM_POLYTOPE_POINT;
3721     } else if (pheight == 0) {
3722       switch (dim) {
3723         case 1:
3724           switch (coneSize) {
3725             case 2: ct = DM_POLYTOPE_SEGMENT;break;
3726             default: break;
3727           }
3728           break;
3729         case 2:
3730           switch (coneSize) {
3731             case 3: ct = DM_POLYTOPE_TRIANGLE;break;
3732             case 4: ct = DM_POLYTOPE_QUADRILATERAL;break;
3733             default: break;
3734           }
3735           break;
3736         case 3:
3737           switch (coneSize) {
3738             case 4: ct = DM_POLYTOPE_TETRAHEDRON;break;
3739             case 5:
3740             {
3741               const PetscInt *cone;
3742               PetscInt        faceConeSize;
3743 
3744               ierr = DMPlexGetCone(dm, p, &cone);CHKERRQ(ierr);
3745               ierr = DMPlexGetConeSize(dm, cone[0], &faceConeSize);CHKERRQ(ierr);
3746               switch (faceConeSize) {
3747                 case 3: ct = DM_POLYTOPE_TRI_PRISM_TENSOR;break;
3748                 case 4: ct = DM_POLYTOPE_PYRAMID;break;
3749               }
3750             }
3751             break;
3752             case 6: ct = DM_POLYTOPE_HEXAHEDRON;break;
3753             default: break;
3754           }
3755           break;
3756         default: break;
3757       }
3758     } else if (pheight > 0) {
3759       switch (coneSize) {
3760         case 2: ct = DM_POLYTOPE_SEGMENT;break;
3761         case 3: ct = DM_POLYTOPE_TRIANGLE;break;
3762         case 4: ct = DM_POLYTOPE_QUADRILATERAL;break;
3763         default: break;
3764       }
3765     }
3766   }
3767   *pt = ct;
3768   PetscFunctionReturn(0);
3769 }
3770 
3771 /*@
3772   DMPlexComputeCellTypes - Infer the polytope type of every cell using its dimension and cone size.
3773 
3774   Collective on dm
3775 
3776   Input Parameter:
3777 . mesh - The DMPlex
3778 
3779   DMPlexComputeCellTypes() should be called after all calls to DMPlexSymmetrize() and DMPlexStratify()
3780 
3781   Level: developer
3782 
3783   Note: This function is normally called automatically by Plex when a cell type is requested. It creates an
3784   internal DMLabel named "celltype" which can be directly accessed using DMGetLabel(). A user may disable
3785   automatic creation by creating the label manually, using DMCreateLabel(dm, "celltype").
3786 
3787 .seealso: DMPlexCreate(), DMPlexSymmetrize(), DMPlexStratify(), DMGetLabel(), DMCreateLabel()
3788 @*/
3789 PetscErrorCode DMPlexComputeCellTypes(DM dm)
3790 {
3791   DM_Plex       *mesh;
3792   DMLabel        ctLabel;
3793   PetscInt       pStart, pEnd, p;
3794   PetscErrorCode ierr;
3795 
3796   PetscFunctionBegin;
3797   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3798   mesh = (DM_Plex *) dm->data;
3799   ierr = DMCreateLabel(dm, "celltype");CHKERRQ(ierr);
3800   ierr = DMPlexGetCellTypeLabel(dm, &ctLabel);CHKERRQ(ierr);
3801   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
3802   for (p = pStart; p < pEnd; ++p) {
3803     DMPolytopeType ct = DM_POLYTOPE_UNKNOWN;
3804     PetscInt       pdepth;
3805 
3806     ierr = DMPlexGetPointDepth(dm, p, &pdepth);CHKERRQ(ierr);
3807     ierr = DMPlexComputeCellType_Internal(dm, p, pdepth, &ct);CHKERRQ(ierr);
3808     if (ct == DM_POLYTOPE_UNKNOWN) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_SUP, "Point %D is screwed up", p);
3809     ierr = DMLabelSetValue(ctLabel, p, ct);CHKERRQ(ierr);
3810   }
3811   ierr = PetscObjectStateGet((PetscObject) ctLabel, &mesh->celltypeState);CHKERRQ(ierr);
3812   ierr = PetscObjectViewFromOptions((PetscObject) ctLabel, NULL, "-dm_plex_celltypes_view");CHKERRQ(ierr);
3813   PetscFunctionReturn(0);
3814 }
3815 
3816 /*@C
3817   DMPlexGetJoin - Get an array for the join of the set of points
3818 
3819   Not Collective
3820 
3821   Input Parameters:
3822 + dm - The DMPlex object
3823 . numPoints - The number of input points for the join
3824 - points - The input points
3825 
3826   Output Parameters:
3827 + numCoveredPoints - The number of points in the join
3828 - coveredPoints - The points in the join
3829 
3830   Level: intermediate
3831 
3832   Note: Currently, this is restricted to a single level join
3833 
3834   Fortran Notes:
3835   Since it returns an array, this routine is only available in Fortran 90, and you must
3836   include petsc.h90 in your code.
3837 
3838   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3839 
3840 .seealso: DMPlexRestoreJoin(), DMPlexGetMeet()
3841 @*/
3842 PetscErrorCode DMPlexGetJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
3843 {
3844   DM_Plex       *mesh = (DM_Plex*) dm->data;
3845   PetscInt      *join[2];
3846   PetscInt       joinSize, i = 0;
3847   PetscInt       dof, off, p, c, m;
3848   PetscErrorCode ierr;
3849 
3850   PetscFunctionBegin;
3851   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3852   PetscValidIntPointer(points, 3);
3853   PetscValidIntPointer(numCoveredPoints, 4);
3854   PetscValidPointer(coveredPoints, 5);
3855   ierr = DMGetWorkArray(dm, mesh->maxSupportSize, MPIU_INT, &join[0]);CHKERRQ(ierr);
3856   ierr = DMGetWorkArray(dm, mesh->maxSupportSize, MPIU_INT, &join[1]);CHKERRQ(ierr);
3857   /* Copy in support of first point */
3858   ierr = PetscSectionGetDof(mesh->supportSection, points[0], &dof);CHKERRQ(ierr);
3859   ierr = PetscSectionGetOffset(mesh->supportSection, points[0], &off);CHKERRQ(ierr);
3860   for (joinSize = 0; joinSize < dof; ++joinSize) {
3861     join[i][joinSize] = mesh->supports[off+joinSize];
3862   }
3863   /* Check each successive support */
3864   for (p = 1; p < numPoints; ++p) {
3865     PetscInt newJoinSize = 0;
3866 
3867     ierr = PetscSectionGetDof(mesh->supportSection, points[p], &dof);CHKERRQ(ierr);
3868     ierr = PetscSectionGetOffset(mesh->supportSection, points[p], &off);CHKERRQ(ierr);
3869     for (c = 0; c < dof; ++c) {
3870       const PetscInt point = mesh->supports[off+c];
3871 
3872       for (m = 0; m < joinSize; ++m) {
3873         if (point == join[i][m]) {
3874           join[1-i][newJoinSize++] = point;
3875           break;
3876         }
3877       }
3878     }
3879     joinSize = newJoinSize;
3880     i        = 1-i;
3881   }
3882   *numCoveredPoints = joinSize;
3883   *coveredPoints    = join[i];
3884   ierr              = DMRestoreWorkArray(dm, mesh->maxSupportSize, MPIU_INT, &join[1-i]);CHKERRQ(ierr);
3885   PetscFunctionReturn(0);
3886 }
3887 
3888 /*@C
3889   DMPlexRestoreJoin - Restore an array for the join of the set of points
3890 
3891   Not Collective
3892 
3893   Input Parameters:
3894 + dm - The DMPlex object
3895 . numPoints - The number of input points for the join
3896 - points - The input points
3897 
3898   Output Parameters:
3899 + numCoveredPoints - The number of points in the join
3900 - coveredPoints - The points in the join
3901 
3902   Fortran Notes:
3903   Since it returns an array, this routine is only available in Fortran 90, and you must
3904   include petsc.h90 in your code.
3905 
3906   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3907 
3908   Level: intermediate
3909 
3910 .seealso: DMPlexGetJoin(), DMPlexGetFullJoin(), DMPlexGetMeet()
3911 @*/
3912 PetscErrorCode DMPlexRestoreJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
3913 {
3914   PetscErrorCode ierr;
3915 
3916   PetscFunctionBegin;
3917   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3918   if (points) PetscValidIntPointer(points,3);
3919   if (numCoveredPoints) PetscValidIntPointer(numCoveredPoints,4);
3920   PetscValidPointer(coveredPoints, 5);
3921   ierr = DMRestoreWorkArray(dm, 0, MPIU_INT, (void*) coveredPoints);CHKERRQ(ierr);
3922   if (numCoveredPoints) *numCoveredPoints = 0;
3923   PetscFunctionReturn(0);
3924 }
3925 
3926 /*@C
3927   DMPlexGetFullJoin - Get an array for the join of the set of points
3928 
3929   Not Collective
3930 
3931   Input Parameters:
3932 + dm - The DMPlex object
3933 . numPoints - The number of input points for the join
3934 - points - The input points
3935 
3936   Output Parameters:
3937 + numCoveredPoints - The number of points in the join
3938 - coveredPoints - The points in the join
3939 
3940   Fortran Notes:
3941   Since it returns an array, this routine is only available in Fortran 90, and you must
3942   include petsc.h90 in your code.
3943 
3944   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3945 
3946   Level: intermediate
3947 
3948 .seealso: DMPlexGetJoin(), DMPlexRestoreJoin(), DMPlexGetMeet()
3949 @*/
3950 PetscErrorCode DMPlexGetFullJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
3951 {
3952   DM_Plex       *mesh = (DM_Plex*) dm->data;
3953   PetscInt      *offsets, **closures;
3954   PetscInt      *join[2];
3955   PetscInt       depth = 0, maxSize, joinSize = 0, i = 0;
3956   PetscInt       p, d, c, m, ms;
3957   PetscErrorCode ierr;
3958 
3959   PetscFunctionBegin;
3960   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3961   PetscValidIntPointer(points, 3);
3962   PetscValidIntPointer(numCoveredPoints, 4);
3963   PetscValidPointer(coveredPoints, 5);
3964 
3965   ierr    = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3966   ierr    = PetscCalloc1(numPoints, &closures);CHKERRQ(ierr);
3967   ierr    = DMGetWorkArray(dm, numPoints*(depth+2), MPIU_INT, &offsets);CHKERRQ(ierr);
3968   ms      = mesh->maxSupportSize;
3969   maxSize = (ms > 1) ? ((PetscPowInt(ms,depth+1)-1)/(ms-1)) : depth + 1;
3970   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &join[0]);CHKERRQ(ierr);
3971   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &join[1]);CHKERRQ(ierr);
3972 
3973   for (p = 0; p < numPoints; ++p) {
3974     PetscInt closureSize;
3975 
3976     ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_FALSE, &closureSize, &closures[p]);CHKERRQ(ierr);
3977 
3978     offsets[p*(depth+2)+0] = 0;
3979     for (d = 0; d < depth+1; ++d) {
3980       PetscInt pStart, pEnd, i;
3981 
3982       ierr = DMPlexGetDepthStratum(dm, d, &pStart, &pEnd);CHKERRQ(ierr);
3983       for (i = offsets[p*(depth+2)+d]; i < closureSize; ++i) {
3984         if ((pStart > closures[p][i*2]) || (pEnd <= closures[p][i*2])) {
3985           offsets[p*(depth+2)+d+1] = i;
3986           break;
3987         }
3988       }
3989       if (i == closureSize) offsets[p*(depth+2)+d+1] = i;
3990     }
3991     if (offsets[p*(depth+2)+depth+1] != closureSize) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Total size of closure %D should be %D", offsets[p*(depth+2)+depth+1], closureSize);
3992   }
3993   for (d = 0; d < depth+1; ++d) {
3994     PetscInt dof;
3995 
3996     /* Copy in support of first point */
3997     dof = offsets[d+1] - offsets[d];
3998     for (joinSize = 0; joinSize < dof; ++joinSize) {
3999       join[i][joinSize] = closures[0][(offsets[d]+joinSize)*2];
4000     }
4001     /* Check each successive cone */
4002     for (p = 1; p < numPoints && joinSize; ++p) {
4003       PetscInt newJoinSize = 0;
4004 
4005       dof = offsets[p*(depth+2)+d+1] - offsets[p*(depth+2)+d];
4006       for (c = 0; c < dof; ++c) {
4007         const PetscInt point = closures[p][(offsets[p*(depth+2)+d]+c)*2];
4008 
4009         for (m = 0; m < joinSize; ++m) {
4010           if (point == join[i][m]) {
4011             join[1-i][newJoinSize++] = point;
4012             break;
4013           }
4014         }
4015       }
4016       joinSize = newJoinSize;
4017       i        = 1-i;
4018     }
4019     if (joinSize) break;
4020   }
4021   *numCoveredPoints = joinSize;
4022   *coveredPoints    = join[i];
4023   for (p = 0; p < numPoints; ++p) {
4024     ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_FALSE, NULL, &closures[p]);CHKERRQ(ierr);
4025   }
4026   ierr = PetscFree(closures);CHKERRQ(ierr);
4027   ierr = DMRestoreWorkArray(dm, numPoints*(depth+2), MPIU_INT, &offsets);CHKERRQ(ierr);
4028   ierr = DMRestoreWorkArray(dm, mesh->maxSupportSize, MPIU_INT, &join[1-i]);CHKERRQ(ierr);
4029   PetscFunctionReturn(0);
4030 }
4031 
4032 /*@C
4033   DMPlexGetMeet - Get an array for the meet of the set of points
4034 
4035   Not Collective
4036 
4037   Input Parameters:
4038 + dm - The DMPlex object
4039 . numPoints - The number of input points for the meet
4040 - points - The input points
4041 
4042   Output Parameters:
4043 + numCoveredPoints - The number of points in the meet
4044 - coveredPoints - The points in the meet
4045 
4046   Level: intermediate
4047 
4048   Note: Currently, this is restricted to a single level meet
4049 
4050   Fortran Notes:
4051   Since it returns an array, this routine is only available in Fortran 90, and you must
4052   include petsc.h90 in your code.
4053 
4054   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
4055 
4056 .seealso: DMPlexRestoreMeet(), DMPlexGetJoin()
4057 @*/
4058 PetscErrorCode DMPlexGetMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveringPoints, const PetscInt **coveringPoints)
4059 {
4060   DM_Plex       *mesh = (DM_Plex*) dm->data;
4061   PetscInt      *meet[2];
4062   PetscInt       meetSize, i = 0;
4063   PetscInt       dof, off, p, c, m;
4064   PetscErrorCode ierr;
4065 
4066   PetscFunctionBegin;
4067   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4068   PetscValidPointer(points, 3);
4069   PetscValidPointer(numCoveringPoints, 4);
4070   PetscValidPointer(coveringPoints, 5);
4071   ierr = DMGetWorkArray(dm, mesh->maxConeSize, MPIU_INT, &meet[0]);CHKERRQ(ierr);
4072   ierr = DMGetWorkArray(dm, mesh->maxConeSize, MPIU_INT, &meet[1]);CHKERRQ(ierr);
4073   /* Copy in cone of first point */
4074   ierr = PetscSectionGetDof(mesh->coneSection, points[0], &dof);CHKERRQ(ierr);
4075   ierr = PetscSectionGetOffset(mesh->coneSection, points[0], &off);CHKERRQ(ierr);
4076   for (meetSize = 0; meetSize < dof; ++meetSize) {
4077     meet[i][meetSize] = mesh->cones[off+meetSize];
4078   }
4079   /* Check each successive cone */
4080   for (p = 1; p < numPoints; ++p) {
4081     PetscInt newMeetSize = 0;
4082 
4083     ierr = PetscSectionGetDof(mesh->coneSection, points[p], &dof);CHKERRQ(ierr);
4084     ierr = PetscSectionGetOffset(mesh->coneSection, points[p], &off);CHKERRQ(ierr);
4085     for (c = 0; c < dof; ++c) {
4086       const PetscInt point = mesh->cones[off+c];
4087 
4088       for (m = 0; m < meetSize; ++m) {
4089         if (point == meet[i][m]) {
4090           meet[1-i][newMeetSize++] = point;
4091           break;
4092         }
4093       }
4094     }
4095     meetSize = newMeetSize;
4096     i        = 1-i;
4097   }
4098   *numCoveringPoints = meetSize;
4099   *coveringPoints    = meet[i];
4100   ierr               = DMRestoreWorkArray(dm, mesh->maxConeSize, MPIU_INT, &meet[1-i]);CHKERRQ(ierr);
4101   PetscFunctionReturn(0);
4102 }
4103 
4104 /*@C
4105   DMPlexRestoreMeet - Restore an array for the meet of the set of points
4106 
4107   Not Collective
4108 
4109   Input Parameters:
4110 + dm - The DMPlex object
4111 . numPoints - The number of input points for the meet
4112 - points - The input points
4113 
4114   Output Parameters:
4115 + numCoveredPoints - The number of points in the meet
4116 - coveredPoints - The points in the meet
4117 
4118   Level: intermediate
4119 
4120   Fortran Notes:
4121   Since it returns an array, this routine is only available in Fortran 90, and you must
4122   include petsc.h90 in your code.
4123 
4124   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
4125 
4126 .seealso: DMPlexGetMeet(), DMPlexGetFullMeet(), DMPlexGetJoin()
4127 @*/
4128 PetscErrorCode DMPlexRestoreMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
4129 {
4130   PetscErrorCode ierr;
4131 
4132   PetscFunctionBegin;
4133   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4134   if (points) PetscValidIntPointer(points,3);
4135   if (numCoveredPoints) PetscValidIntPointer(numCoveredPoints,4);
4136   PetscValidPointer(coveredPoints,5);
4137   ierr = DMRestoreWorkArray(dm, 0, MPIU_INT, (void*) coveredPoints);CHKERRQ(ierr);
4138   if (numCoveredPoints) *numCoveredPoints = 0;
4139   PetscFunctionReturn(0);
4140 }
4141 
4142 /*@C
4143   DMPlexGetFullMeet - Get an array for the meet of the set of points
4144 
4145   Not Collective
4146 
4147   Input Parameters:
4148 + dm - The DMPlex object
4149 . numPoints - The number of input points for the meet
4150 - points - The input points
4151 
4152   Output Parameters:
4153 + numCoveredPoints - The number of points in the meet
4154 - coveredPoints - The points in the meet
4155 
4156   Level: intermediate
4157 
4158   Fortran Notes:
4159   Since it returns an array, this routine is only available in Fortran 90, and you must
4160   include petsc.h90 in your code.
4161 
4162   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
4163 
4164 .seealso: DMPlexGetMeet(), DMPlexRestoreMeet(), DMPlexGetJoin()
4165 @*/
4166 PetscErrorCode DMPlexGetFullMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
4167 {
4168   DM_Plex       *mesh = (DM_Plex*) dm->data;
4169   PetscInt      *offsets, **closures;
4170   PetscInt      *meet[2];
4171   PetscInt       height = 0, maxSize, meetSize = 0, i = 0;
4172   PetscInt       p, h, c, m, mc;
4173   PetscErrorCode ierr;
4174 
4175   PetscFunctionBegin;
4176   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4177   PetscValidPointer(points, 3);
4178   PetscValidPointer(numCoveredPoints, 4);
4179   PetscValidPointer(coveredPoints, 5);
4180 
4181   ierr    = DMPlexGetDepth(dm, &height);CHKERRQ(ierr);
4182   ierr    = PetscMalloc1(numPoints, &closures);CHKERRQ(ierr);
4183   ierr    = DMGetWorkArray(dm, numPoints*(height+2), MPIU_INT, &offsets);CHKERRQ(ierr);
4184   mc      = mesh->maxConeSize;
4185   maxSize = (mc > 1) ? ((PetscPowInt(mc,height+1)-1)/(mc-1)) : height + 1;
4186   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &meet[0]);CHKERRQ(ierr);
4187   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &meet[1]);CHKERRQ(ierr);
4188 
4189   for (p = 0; p < numPoints; ++p) {
4190     PetscInt closureSize;
4191 
4192     ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_TRUE, &closureSize, &closures[p]);CHKERRQ(ierr);
4193 
4194     offsets[p*(height+2)+0] = 0;
4195     for (h = 0; h < height+1; ++h) {
4196       PetscInt pStart, pEnd, i;
4197 
4198       ierr = DMPlexGetHeightStratum(dm, h, &pStart, &pEnd);CHKERRQ(ierr);
4199       for (i = offsets[p*(height+2)+h]; i < closureSize; ++i) {
4200         if ((pStart > closures[p][i*2]) || (pEnd <= closures[p][i*2])) {
4201           offsets[p*(height+2)+h+1] = i;
4202           break;
4203         }
4204       }
4205       if (i == closureSize) offsets[p*(height+2)+h+1] = i;
4206     }
4207     if (offsets[p*(height+2)+height+1] != closureSize) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Total size of closure %D should be %D", offsets[p*(height+2)+height+1], closureSize);
4208   }
4209   for (h = 0; h < height+1; ++h) {
4210     PetscInt dof;
4211 
4212     /* Copy in cone of first point */
4213     dof = offsets[h+1] - offsets[h];
4214     for (meetSize = 0; meetSize < dof; ++meetSize) {
4215       meet[i][meetSize] = closures[0][(offsets[h]+meetSize)*2];
4216     }
4217     /* Check each successive cone */
4218     for (p = 1; p < numPoints && meetSize; ++p) {
4219       PetscInt newMeetSize = 0;
4220 
4221       dof = offsets[p*(height+2)+h+1] - offsets[p*(height+2)+h];
4222       for (c = 0; c < dof; ++c) {
4223         const PetscInt point = closures[p][(offsets[p*(height+2)+h]+c)*2];
4224 
4225         for (m = 0; m < meetSize; ++m) {
4226           if (point == meet[i][m]) {
4227             meet[1-i][newMeetSize++] = point;
4228             break;
4229           }
4230         }
4231       }
4232       meetSize = newMeetSize;
4233       i        = 1-i;
4234     }
4235     if (meetSize) break;
4236   }
4237   *numCoveredPoints = meetSize;
4238   *coveredPoints    = meet[i];
4239   for (p = 0; p < numPoints; ++p) {
4240     ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_TRUE, NULL, &closures[p]);CHKERRQ(ierr);
4241   }
4242   ierr = PetscFree(closures);CHKERRQ(ierr);
4243   ierr = DMRestoreWorkArray(dm, numPoints*(height+2), MPIU_INT, &offsets);CHKERRQ(ierr);
4244   ierr = DMRestoreWorkArray(dm, mesh->maxConeSize, MPIU_INT, &meet[1-i]);CHKERRQ(ierr);
4245   PetscFunctionReturn(0);
4246 }
4247 
4248 /*@C
4249   DMPlexEqual - Determine if two DMs have the same topology
4250 
4251   Not Collective
4252 
4253   Input Parameters:
4254 + dmA - A DMPlex object
4255 - dmB - A DMPlex object
4256 
4257   Output Parameters:
4258 . equal - PETSC_TRUE if the topologies are identical
4259 
4260   Level: intermediate
4261 
4262   Notes:
4263   We are not solving graph isomorphism, so we do not permutation.
4264 
4265 .seealso: DMPlexGetCone()
4266 @*/
4267 PetscErrorCode DMPlexEqual(DM dmA, DM dmB, PetscBool *equal)
4268 {
4269   PetscInt       depth, depthB, pStart, pEnd, pStartB, pEndB, p;
4270   PetscErrorCode ierr;
4271 
4272   PetscFunctionBegin;
4273   PetscValidHeaderSpecific(dmA, DM_CLASSID, 1);
4274   PetscValidHeaderSpecific(dmB, DM_CLASSID, 2);
4275   PetscValidPointer(equal, 3);
4276 
4277   *equal = PETSC_FALSE;
4278   ierr = DMPlexGetDepth(dmA, &depth);CHKERRQ(ierr);
4279   ierr = DMPlexGetDepth(dmB, &depthB);CHKERRQ(ierr);
4280   if (depth != depthB) PetscFunctionReturn(0);
4281   ierr = DMPlexGetChart(dmA, &pStart,  &pEnd);CHKERRQ(ierr);
4282   ierr = DMPlexGetChart(dmB, &pStartB, &pEndB);CHKERRQ(ierr);
4283   if ((pStart != pStartB) || (pEnd != pEndB)) PetscFunctionReturn(0);
4284   for (p = pStart; p < pEnd; ++p) {
4285     const PetscInt *cone, *coneB, *ornt, *orntB, *support, *supportB;
4286     PetscInt        coneSize, coneSizeB, c, supportSize, supportSizeB, s;
4287 
4288     ierr = DMPlexGetConeSize(dmA, p, &coneSize);CHKERRQ(ierr);
4289     ierr = DMPlexGetCone(dmA, p, &cone);CHKERRQ(ierr);
4290     ierr = DMPlexGetConeOrientation(dmA, p, &ornt);CHKERRQ(ierr);
4291     ierr = DMPlexGetConeSize(dmB, p, &coneSizeB);CHKERRQ(ierr);
4292     ierr = DMPlexGetCone(dmB, p, &coneB);CHKERRQ(ierr);
4293     ierr = DMPlexGetConeOrientation(dmB, p, &orntB);CHKERRQ(ierr);
4294     if (coneSize != coneSizeB) PetscFunctionReturn(0);
4295     for (c = 0; c < coneSize; ++c) {
4296       if (cone[c] != coneB[c]) PetscFunctionReturn(0);
4297       if (ornt[c] != orntB[c]) PetscFunctionReturn(0);
4298     }
4299     ierr = DMPlexGetSupportSize(dmA, p, &supportSize);CHKERRQ(ierr);
4300     ierr = DMPlexGetSupport(dmA, p, &support);CHKERRQ(ierr);
4301     ierr = DMPlexGetSupportSize(dmB, p, &supportSizeB);CHKERRQ(ierr);
4302     ierr = DMPlexGetSupport(dmB, p, &supportB);CHKERRQ(ierr);
4303     if (supportSize != supportSizeB) PetscFunctionReturn(0);
4304     for (s = 0; s < supportSize; ++s) {
4305       if (support[s] != supportB[s]) PetscFunctionReturn(0);
4306     }
4307   }
4308   *equal = PETSC_TRUE;
4309   PetscFunctionReturn(0);
4310 }
4311 
4312 /*@C
4313   DMPlexGetNumFaceVertices - Returns the number of vertices on a face
4314 
4315   Not Collective
4316 
4317   Input Parameters:
4318 + dm         - The DMPlex
4319 . cellDim    - The cell dimension
4320 - numCorners - The number of vertices on a cell
4321 
4322   Output Parameters:
4323 . numFaceVertices - The number of vertices on a face
4324 
4325   Level: developer
4326 
4327   Notes:
4328   Of course this can only work for a restricted set of symmetric shapes
4329 
4330 .seealso: DMPlexGetCone()
4331 @*/
4332 PetscErrorCode DMPlexGetNumFaceVertices(DM dm, PetscInt cellDim, PetscInt numCorners, PetscInt *numFaceVertices)
4333 {
4334   MPI_Comm       comm;
4335   PetscErrorCode ierr;
4336 
4337   PetscFunctionBegin;
4338   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
4339   PetscValidPointer(numFaceVertices,4);
4340   switch (cellDim) {
4341   case 0:
4342     *numFaceVertices = 0;
4343     break;
4344   case 1:
4345     *numFaceVertices = 1;
4346     break;
4347   case 2:
4348     switch (numCorners) {
4349     case 3: /* triangle */
4350       *numFaceVertices = 2; /* Edge has 2 vertices */
4351       break;
4352     case 4: /* quadrilateral */
4353       *numFaceVertices = 2; /* Edge has 2 vertices */
4354       break;
4355     case 6: /* quadratic triangle, tri and quad cohesive Lagrange cells */
4356       *numFaceVertices = 3; /* Edge has 3 vertices */
4357       break;
4358     case 9: /* quadratic quadrilateral, quadratic quad cohesive Lagrange cells */
4359       *numFaceVertices = 3; /* Edge has 3 vertices */
4360       break;
4361     default:
4362       SETERRQ2(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid number of face corners %D for dimension %D", numCorners, cellDim);
4363     }
4364     break;
4365   case 3:
4366     switch (numCorners) {
4367     case 4: /* tetradehdron */
4368       *numFaceVertices = 3; /* Face has 3 vertices */
4369       break;
4370     case 6: /* tet cohesive cells */
4371       *numFaceVertices = 4; /* Face has 4 vertices */
4372       break;
4373     case 8: /* hexahedron */
4374       *numFaceVertices = 4; /* Face has 4 vertices */
4375       break;
4376     case 9: /* tet cohesive Lagrange cells */
4377       *numFaceVertices = 6; /* Face has 6 vertices */
4378       break;
4379     case 10: /* quadratic tetrahedron */
4380       *numFaceVertices = 6; /* Face has 6 vertices */
4381       break;
4382     case 12: /* hex cohesive Lagrange cells */
4383       *numFaceVertices = 6; /* Face has 6 vertices */
4384       break;
4385     case 18: /* quadratic tet cohesive Lagrange cells */
4386       *numFaceVertices = 6; /* Face has 6 vertices */
4387       break;
4388     case 27: /* quadratic hexahedron, quadratic hex cohesive Lagrange cells */
4389       *numFaceVertices = 9; /* Face has 9 vertices */
4390       break;
4391     default:
4392       SETERRQ2(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid number of face corners %D for dimension %D", numCorners, cellDim);
4393     }
4394     break;
4395   default:
4396     SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid cell dimension %D", cellDim);
4397   }
4398   PetscFunctionReturn(0);
4399 }
4400 
4401 /*@
4402   DMPlexGetDepthLabel - Get the DMLabel recording the depth of each point
4403 
4404   Not Collective
4405 
4406   Input Parameter:
4407 . dm    - The DMPlex object
4408 
4409   Output Parameter:
4410 . depthLabel - The DMLabel recording point depth
4411 
4412   Level: developer
4413 
4414 .seealso: DMPlexGetDepth(), DMPlexGetHeightStratum(), DMPlexGetDepthStratum(), DMPlexGetPointDepth(),
4415 @*/
4416 PetscErrorCode DMPlexGetDepthLabel(DM dm, DMLabel *depthLabel)
4417 {
4418   PetscFunctionBegin;
4419   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4420   PetscValidPointer(depthLabel, 2);
4421   *depthLabel = dm->depthLabel;
4422   PetscFunctionReturn(0);
4423 }
4424 
4425 /*@
4426   DMPlexGetDepth - Get the depth of the DAG representing this mesh
4427 
4428   Not Collective
4429 
4430   Input Parameter:
4431 . dm    - The DMPlex object
4432 
4433   Output Parameter:
4434 . depth - The number of strata (breadth first levels) in the DAG
4435 
4436   Level: developer
4437 
4438   Notes:
4439   This returns maximum of point depths over all points, i.e. maximum value of the label returned by DMPlexGetDepthLabel().
4440   The point depth is described more in detail in DMPlexGetDepthStratum().
4441   An empty mesh gives -1.
4442 
4443 .seealso: DMPlexGetDepthLabel(), DMPlexGetDepthStratum(), DMPlexGetPointDepth(), DMPlexSymmetrize()
4444 @*/
4445 PetscErrorCode DMPlexGetDepth(DM dm, PetscInt *depth)
4446 {
4447   DMLabel        label;
4448   PetscInt       d = 0;
4449   PetscErrorCode ierr;
4450 
4451   PetscFunctionBegin;
4452   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4453   PetscValidPointer(depth, 2);
4454   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
4455   if (label) {ierr = DMLabelGetNumValues(label, &d);CHKERRQ(ierr);}
4456   *depth = d-1;
4457   PetscFunctionReturn(0);
4458 }
4459 
4460 /*@
4461   DMPlexGetDepthStratum - Get the bounds [start, end) for all points at a certain depth.
4462 
4463   Not Collective
4464 
4465   Input Parameters:
4466 + dm           - The DMPlex object
4467 - stratumValue - The requested depth
4468 
4469   Output Parameters:
4470 + start - The first point at this depth
4471 - end   - One beyond the last point at this depth
4472 
4473   Notes:
4474   Depth indexing is related to topological dimension.  Depth stratum 0 contains the lowest topological dimension points,
4475   often "vertices".  If the mesh is "interpolated" (see DMPlexInterpolate()), then depth stratum 1 contains the next
4476   higher dimension, e.g., "edges".
4477 
4478   Level: developer
4479 
4480 .seealso: DMPlexGetHeightStratum(), DMPlexGetDepth(), DMPlexGetDepthLabel(), DMPlexGetPointDepth(), DMPlexSymmetrize(), DMPlexInterpolate()
4481 @*/
4482 PetscErrorCode DMPlexGetDepthStratum(DM dm, PetscInt stratumValue, PetscInt *start, PetscInt *end)
4483 {
4484   DMLabel        label;
4485   PetscInt       pStart, pEnd;
4486   PetscErrorCode ierr;
4487 
4488   PetscFunctionBegin;
4489   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4490   if (start) {PetscValidPointer(start, 3); *start = 0;}
4491   if (end)   {PetscValidPointer(end,   4); *end   = 0;}
4492   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
4493   if (pStart == pEnd) PetscFunctionReturn(0);
4494   if (stratumValue < 0) {
4495     if (start) *start = pStart;
4496     if (end)   *end   = pEnd;
4497     PetscFunctionReturn(0);
4498   }
4499   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
4500   if (!label) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "No label named depth was found");
4501   ierr = DMLabelGetStratumBounds(label, stratumValue, start, end);CHKERRQ(ierr);
4502   PetscFunctionReturn(0);
4503 }
4504 
4505 /*@
4506   DMPlexGetHeightStratum - Get the bounds [start, end) for all points at a certain height.
4507 
4508   Not Collective
4509 
4510   Input Parameters:
4511 + dm           - The DMPlex object
4512 - stratumValue - The requested height
4513 
4514   Output Parameters:
4515 + start - The first point at this height
4516 - end   - One beyond the last point at this height
4517 
4518   Notes:
4519   Height indexing is related to topological codimension.  Height stratum 0 contains the highest topological dimension
4520   points, often called "cells" or "elements".  If the mesh is "interpolated" (see DMPlexInterpolate()), then height
4521   stratum 1 contains the boundary of these "cells", often called "faces" or "facets".
4522 
4523   Level: developer
4524 
4525 .seealso: DMPlexGetDepthStratum(), DMPlexGetDepth(), DMPlexGetPointHeight()
4526 @*/
4527 PetscErrorCode DMPlexGetHeightStratum(DM dm, PetscInt stratumValue, PetscInt *start, PetscInt *end)
4528 {
4529   DMLabel        label;
4530   PetscInt       depth, pStart, pEnd;
4531   PetscErrorCode ierr;
4532 
4533   PetscFunctionBegin;
4534   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4535   if (start) {PetscValidPointer(start, 3); *start = 0;}
4536   if (end)   {PetscValidPointer(end,   4); *end   = 0;}
4537   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
4538   if (pStart == pEnd) PetscFunctionReturn(0);
4539   if (stratumValue < 0) {
4540     if (start) *start = pStart;
4541     if (end)   *end   = pEnd;
4542     PetscFunctionReturn(0);
4543   }
4544   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
4545   if (!label) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "No label named depth was found");
4546   ierr = DMLabelGetNumValues(label, &depth);CHKERRQ(ierr);
4547   ierr = DMLabelGetStratumBounds(label, depth-1-stratumValue, start, end);CHKERRQ(ierr);
4548   PetscFunctionReturn(0);
4549 }
4550 
4551 /*@
4552   DMPlexGetPointDepth - Get the depth of a given point
4553 
4554   Not Collective
4555 
4556   Input Parameter:
4557 + dm    - The DMPlex object
4558 - point - The point
4559 
4560   Output Parameter:
4561 . depth - The depth of the point
4562 
4563   Level: intermediate
4564 
4565 .seealso: DMPlexGetCellType(), DMPlexGetDepthLabel(), DMPlexGetDepth(), DMPlexGetPointHeight()
4566 @*/
4567 PetscErrorCode DMPlexGetPointDepth(DM dm, PetscInt point, PetscInt *depth)
4568 {
4569   PetscErrorCode ierr;
4570 
4571   PetscFunctionBegin;
4572   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4573   PetscValidIntPointer(depth, 3);
4574   ierr = DMLabelGetValue(dm->depthLabel, point, depth);CHKERRQ(ierr);
4575   PetscFunctionReturn(0);
4576 }
4577 
4578 /*@
4579   DMPlexGetPointHeight - Get the height of a given point
4580 
4581   Not Collective
4582 
4583   Input Parameter:
4584 + dm    - The DMPlex object
4585 - point - The point
4586 
4587   Output Parameter:
4588 . height - The height of the point
4589 
4590   Level: intermediate
4591 
4592 .seealso: DMPlexGetCellType(), DMPlexGetDepthLabel(), DMPlexGetDepth(), DMPlexGetPointDepth()
4593 @*/
4594 PetscErrorCode DMPlexGetPointHeight(DM dm, PetscInt point, PetscInt *height)
4595 {
4596   PetscInt       n, pDepth;
4597   PetscErrorCode ierr;
4598 
4599   PetscFunctionBegin;
4600   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4601   PetscValidIntPointer(height, 3);
4602   ierr = DMLabelGetNumValues(dm->depthLabel, &n);CHKERRQ(ierr);
4603   ierr = DMLabelGetValue(dm->depthLabel, point, &pDepth);CHKERRQ(ierr);
4604   *height = n - 1 - pDepth;  /* DAG depth is n-1 */
4605   PetscFunctionReturn(0);
4606 }
4607 
4608 /*@
4609   DMPlexGetCellTypeLabel - Get the DMLabel recording the polytope type of each cell
4610 
4611   Not Collective
4612 
4613   Input Parameter:
4614 . dm - The DMPlex object
4615 
4616   Output Parameter:
4617 . celltypeLabel - The DMLabel recording cell polytope type
4618 
4619   Note: This function will trigger automatica computation of cell types. This can be disabled by calling
4620   DMCreateLabel(dm, "celltype") beforehand.
4621 
4622   Level: developer
4623 
4624 .seealso: DMPlexGetCellType(), DMPlexGetDepthLabel(), DMCreateLabel()
4625 @*/
4626 PetscErrorCode DMPlexGetCellTypeLabel(DM dm, DMLabel *celltypeLabel)
4627 {
4628   PetscErrorCode ierr;
4629 
4630   PetscFunctionBegin;
4631   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4632   PetscValidPointer(celltypeLabel, 2);
4633   if (!dm->celltypeLabel) {ierr = DMPlexComputeCellTypes(dm);CHKERRQ(ierr);}
4634   *celltypeLabel = dm->celltypeLabel;
4635   PetscFunctionReturn(0);
4636 }
4637 
4638 /*@
4639   DMPlexGetCellType - Get the polytope type of a given cell
4640 
4641   Not Collective
4642 
4643   Input Parameter:
4644 + dm   - The DMPlex object
4645 - cell - The cell
4646 
4647   Output Parameter:
4648 . celltype - The polytope type of the cell
4649 
4650   Level: intermediate
4651 
4652 .seealso: DMPlexGetCellTypeLabel(), DMPlexGetDepthLabel(), DMPlexGetDepth()
4653 @*/
4654 PetscErrorCode DMPlexGetCellType(DM dm, PetscInt cell, DMPolytopeType *celltype)
4655 {
4656   DMLabel        label;
4657   PetscInt       ct;
4658   PetscErrorCode ierr;
4659 
4660   PetscFunctionBegin;
4661   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4662   PetscValidPointer(celltype, 3);
4663   ierr = DMPlexGetCellTypeLabel(dm, &label);CHKERRQ(ierr);
4664   ierr = DMLabelGetValue(label, cell, &ct);CHKERRQ(ierr);
4665   if (ct < 0) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Cell %D has not been assigned a cell type", cell);
4666   *celltype = (DMPolytopeType) ct;
4667   PetscFunctionReturn(0);
4668 }
4669 
4670 /*@
4671   DMPlexSetCellType - Set the polytope type of a given cell
4672 
4673   Not Collective
4674 
4675   Input Parameters:
4676 + dm   - The DMPlex object
4677 . cell - The cell
4678 - celltype - The polytope type of the cell
4679 
4680   Note: By default, cell types will be automatically computed using DMPlexComputeCellTypes() before this function
4681   is executed. This function will override the computed type. However, if automatic classification will not succeed
4682   and a user wants to manually specify all types, the classification must be disabled by calling
4683   DMCreaateLabel(dm, "celltype") before getting or setting any cell types.
4684 
4685   Level: advanced
4686 
4687 .seealso: DMPlexGetCellTypeLabel(), DMPlexGetDepthLabel(), DMPlexGetDepth(), DMPlexComputeCellTypes(), DMCreateLabel()
4688 @*/
4689 PetscErrorCode DMPlexSetCellType(DM dm, PetscInt cell, DMPolytopeType celltype)
4690 {
4691   DMLabel        label;
4692   PetscErrorCode ierr;
4693 
4694   PetscFunctionBegin;
4695   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4696   ierr = DMPlexGetCellTypeLabel(dm, &label);CHKERRQ(ierr);
4697   ierr = DMLabelSetValue(label, cell, celltype);CHKERRQ(ierr);
4698   PetscFunctionReturn(0);
4699 }
4700 
4701 PetscErrorCode DMCreateCoordinateDM_Plex(DM dm, DM *cdm)
4702 {
4703   PetscSection   section, s;
4704   Mat            m;
4705   PetscInt       maxHeight;
4706   PetscErrorCode ierr;
4707 
4708   PetscFunctionBegin;
4709   ierr = DMClone(dm, cdm);CHKERRQ(ierr);
4710   ierr = DMPlexGetMaxProjectionHeight(dm, &maxHeight);CHKERRQ(ierr);
4711   ierr = DMPlexSetMaxProjectionHeight(*cdm, maxHeight);CHKERRQ(ierr);
4712   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &section);CHKERRQ(ierr);
4713   ierr = DMSetLocalSection(*cdm, section);CHKERRQ(ierr);
4714   ierr = PetscSectionDestroy(&section);CHKERRQ(ierr);
4715   ierr = PetscSectionCreate(PETSC_COMM_SELF, &s);CHKERRQ(ierr);
4716   ierr = MatCreate(PETSC_COMM_SELF, &m);CHKERRQ(ierr);
4717   ierr = DMSetDefaultConstraints(*cdm, s, m);CHKERRQ(ierr);
4718   ierr = PetscSectionDestroy(&s);CHKERRQ(ierr);
4719   ierr = MatDestroy(&m);CHKERRQ(ierr);
4720 
4721   ierr = DMSetNumFields(*cdm, 1);CHKERRQ(ierr);
4722   ierr = DMCreateDS(*cdm);CHKERRQ(ierr);
4723   PetscFunctionReturn(0);
4724 }
4725 
4726 PetscErrorCode DMCreateCoordinateField_Plex(DM dm, DMField *field)
4727 {
4728   Vec            coordsLocal;
4729   DM             coordsDM;
4730   PetscErrorCode ierr;
4731 
4732   PetscFunctionBegin;
4733   *field = NULL;
4734   ierr = DMGetCoordinatesLocal(dm,&coordsLocal);CHKERRQ(ierr);
4735   ierr = DMGetCoordinateDM(dm,&coordsDM);CHKERRQ(ierr);
4736   if (coordsLocal && coordsDM) {
4737     ierr = DMFieldCreateDS(coordsDM, 0, coordsLocal, field);CHKERRQ(ierr);
4738   }
4739   PetscFunctionReturn(0);
4740 }
4741 
4742 /*@C
4743   DMPlexGetConeSection - Return a section which describes the layout of cone data
4744 
4745   Not Collective
4746 
4747   Input Parameters:
4748 . dm        - The DMPlex object
4749 
4750   Output Parameter:
4751 . section - The PetscSection object
4752 
4753   Level: developer
4754 
4755 .seealso: DMPlexGetSupportSection(), DMPlexGetCones(), DMPlexGetConeOrientations()
4756 @*/
4757 PetscErrorCode DMPlexGetConeSection(DM dm, PetscSection *section)
4758 {
4759   DM_Plex *mesh = (DM_Plex*) dm->data;
4760 
4761   PetscFunctionBegin;
4762   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4763   if (section) *section = mesh->coneSection;
4764   PetscFunctionReturn(0);
4765 }
4766 
4767 /*@C
4768   DMPlexGetSupportSection - Return a section which describes the layout of support data
4769 
4770   Not Collective
4771 
4772   Input Parameters:
4773 . dm        - The DMPlex object
4774 
4775   Output Parameter:
4776 . section - The PetscSection object
4777 
4778   Level: developer
4779 
4780 .seealso: DMPlexGetConeSection()
4781 @*/
4782 PetscErrorCode DMPlexGetSupportSection(DM dm, PetscSection *section)
4783 {
4784   DM_Plex *mesh = (DM_Plex*) dm->data;
4785 
4786   PetscFunctionBegin;
4787   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4788   if (section) *section = mesh->supportSection;
4789   PetscFunctionReturn(0);
4790 }
4791 
4792 /*@C
4793   DMPlexGetCones - Return cone data
4794 
4795   Not Collective
4796 
4797   Input Parameters:
4798 . dm        - The DMPlex object
4799 
4800   Output Parameter:
4801 . cones - The cone for each point
4802 
4803   Level: developer
4804 
4805 .seealso: DMPlexGetConeSection()
4806 @*/
4807 PetscErrorCode DMPlexGetCones(DM dm, PetscInt *cones[])
4808 {
4809   DM_Plex *mesh = (DM_Plex*) dm->data;
4810 
4811   PetscFunctionBegin;
4812   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4813   if (cones) *cones = mesh->cones;
4814   PetscFunctionReturn(0);
4815 }
4816 
4817 /*@C
4818   DMPlexGetConeOrientations - Return cone orientation data
4819 
4820   Not Collective
4821 
4822   Input Parameters:
4823 . dm        - The DMPlex object
4824 
4825   Output Parameter:
4826 . coneOrientations - The cone orientation for each point
4827 
4828   Level: developer
4829 
4830 .seealso: DMPlexGetConeSection()
4831 @*/
4832 PetscErrorCode DMPlexGetConeOrientations(DM dm, PetscInt *coneOrientations[])
4833 {
4834   DM_Plex *mesh = (DM_Plex*) dm->data;
4835 
4836   PetscFunctionBegin;
4837   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4838   if (coneOrientations) *coneOrientations = mesh->coneOrientations;
4839   PetscFunctionReturn(0);
4840 }
4841 
4842 /******************************** FEM Support **********************************/
4843 
4844 /*
4845  Returns number of components and tensor degree for the field.  For interpolated meshes, line should be a point
4846  representing a line in the section.
4847 */
4848 static PetscErrorCode PetscSectionFieldGetTensorDegree_Private(PetscSection section,PetscInt field,PetscInt line,PetscBool vertexchart,PetscInt *Nc,PetscInt *k)
4849 {
4850   PetscErrorCode ierr;
4851 
4852   PetscFunctionBeginHot;
4853   ierr = PetscSectionGetFieldComponents(section, field, Nc);CHKERRQ(ierr);
4854   if (line < 0) {
4855     *k = 0;
4856     *Nc = 0;
4857   } else if (vertexchart) {            /* If we only have a vertex chart, we must have degree k=1 */
4858     *k = 1;
4859   } else {                      /* Assume the full interpolated mesh is in the chart; lines in particular */
4860     /* An order k SEM disc has k-1 dofs on an edge */
4861     ierr = PetscSectionGetFieldDof(section, line, field, k);CHKERRQ(ierr);
4862     *k = *k / *Nc + 1;
4863   }
4864   PetscFunctionReturn(0);
4865 }
4866 
4867 /*@
4868 
4869   DMPlexSetClosurePermutationTensor - Create a permutation from the default (BFS) point ordering in the closure, to a
4870   lexicographic ordering over the tensor product cell (i.e., line, quad, hex, etc.), and set this permutation in the
4871   section provided (or the section of the DM).
4872 
4873   Input Parameters:
4874 + dm      - The DM
4875 . point   - Either a cell (highest dim point) or an edge (dim 1 point), or PETSC_DETERMINE
4876 - section - The PetscSection to reorder, or NULL for the default section
4877 
4878   Note: The point is used to determine the number of dofs/field on an edge. For SEM, this is related to the polynomial
4879   degree of the basis.
4880 
4881   Example:
4882   A typical interpolated single-quad mesh might order points as
4883 .vb
4884   [c0, v1, v2, v3, v4, e5, e6, e7, e8]
4885 
4886   v4 -- e6 -- v3
4887   |           |
4888   e7    c0    e8
4889   |           |
4890   v1 -- e5 -- v2
4891 .ve
4892 
4893   (There is no significance to the ordering described here.)  The default section for a Q3 quad might typically assign
4894   dofs in the order of points, e.g.,
4895 .vb
4896     c0 -> [0,1,2,3]
4897     v1 -> [4]
4898     ...
4899     e5 -> [8, 9]
4900 .ve
4901 
4902   which corresponds to the dofs
4903 .vb
4904     6   10  11  7
4905     13  2   3   15
4906     12  0   1   14
4907     4   8   9   5
4908 .ve
4909 
4910   The closure in BFS ordering works through height strata (cells, edges, vertices) to produce the ordering
4911 .vb
4912   0 1 2 3 8 9 14 15 11 10 13 12 4 5 7 6
4913 .ve
4914 
4915   After calling DMPlexSetClosurePermutationTensor(), the closure will be ordered lexicographically,
4916 .vb
4917    4 8 9 5 12 0 1 14 13 2 3 15 6 10 11 7
4918 .ve
4919 
4920   Level: developer
4921 
4922 .seealso: DMGetLocalSection(), PetscSectionSetClosurePermutation(), DMSetGlobalSection()
4923 @*/
4924 PetscErrorCode DMPlexSetClosurePermutationTensor(DM dm, PetscInt point, PetscSection section)
4925 {
4926   DMLabel        label;
4927   PetscInt       dim, depth = -1, eStart = -1, Nf;
4928   PetscBool      vertexchart;
4929   PetscErrorCode ierr;
4930 
4931   PetscFunctionBegin;
4932   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
4933   if (dim < 1) PetscFunctionReturn(0);
4934   if (point < 0) {
4935     PetscInt sStart,sEnd;
4936 
4937     ierr = DMPlexGetDepthStratum(dm, 1, &sStart, &sEnd);CHKERRQ(ierr);
4938     point = sEnd-sStart ? sStart : point;
4939   }
4940   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
4941   if (point >= 0) { ierr = DMLabelGetValue(label, point, &depth);CHKERRQ(ierr); }
4942   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
4943   if (depth == 1) {eStart = point;}
4944   else if  (depth == dim) {
4945     const PetscInt *cone;
4946 
4947     ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
4948     if (dim == 2) eStart = cone[0];
4949     else if (dim == 3) {
4950       const PetscInt *cone2;
4951       ierr = DMPlexGetCone(dm, cone[0], &cone2);CHKERRQ(ierr);
4952       eStart = cone2[0];
4953     } else SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Point %D of depth %D cannot be used to bootstrap spectral ordering for dim %D", point, depth, dim);
4954   } else if (depth >= 0) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Point %D of depth %D cannot be used to bootstrap spectral ordering for dim %D", point, depth, dim);
4955   {                             /* Determine whether the chart covers all points or just vertices. */
4956     PetscInt pStart,pEnd,cStart,cEnd;
4957     ierr = DMPlexGetDepthStratum(dm,0,&pStart,&pEnd);CHKERRQ(ierr);
4958     ierr = PetscSectionGetChart(section,&cStart,&cEnd);CHKERRQ(ierr);
4959     if (pStart == cStart && pEnd == cEnd) vertexchart = PETSC_TRUE; /* Just vertices */
4960     else vertexchart = PETSC_FALSE;                                 /* Assume all interpolated points are in chart */
4961   }
4962   ierr = PetscSectionGetNumFields(section, &Nf);CHKERRQ(ierr);
4963   for (PetscInt d=1; d<=dim; d++) {
4964     PetscInt k, f, Nc, c, i, j, size = 0, offset = 0, foffset = 0;
4965     PetscInt *perm;
4966 
4967     for (f = 0; f < Nf; ++f) {
4968       ierr = PetscSectionFieldGetTensorDegree_Private(section,f,eStart,vertexchart,&Nc,&k);CHKERRQ(ierr);
4969       size += PetscPowInt(k+1, d)*Nc;
4970     }
4971     ierr = PetscMalloc1(size, &perm);CHKERRQ(ierr);
4972     for (f = 0; f < Nf; ++f) {
4973       switch (d) {
4974       case 1:
4975         ierr = PetscSectionFieldGetTensorDegree_Private(section,f,eStart,vertexchart,&Nc,&k);CHKERRQ(ierr);
4976         /*
4977          Original ordering is [ edge of length k-1; vtx0; vtx1 ]
4978          We want              [ vtx0; edge of length k-1; vtx1 ]
4979          */
4980         for (c=0; c<Nc; c++,offset++) perm[offset] = (k-1)*Nc + c + foffset;
4981         for (i=0; i<k-1; i++) for (c=0; c<Nc; c++,offset++) perm[offset] = i*Nc + c + foffset;
4982         for (c=0; c<Nc; c++,offset++) perm[offset] = k*Nc + c + foffset;
4983         foffset = offset;
4984         break;
4985       case 2:
4986         /* The original quad closure is oriented clockwise, {f, e_b, e_r, e_t, e_l, v_lb, v_rb, v_tr, v_tl} */
4987         ierr = PetscSectionFieldGetTensorDegree_Private(section,f,eStart,vertexchart,&Nc,&k);CHKERRQ(ierr);
4988         /* The SEM order is
4989 
4990          v_lb, {e_b}, v_rb,
4991          e^{(k-1)-i}_l, {f^{i*(k-1)}}, e^i_r,
4992          v_lt, reverse {e_t}, v_rt
4993          */
4994         {
4995           const PetscInt of   = 0;
4996           const PetscInt oeb  = of   + PetscSqr(k-1);
4997           const PetscInt oer  = oeb  + (k-1);
4998           const PetscInt oet  = oer  + (k-1);
4999           const PetscInt oel  = oet  + (k-1);
5000           const PetscInt ovlb = oel  + (k-1);
5001           const PetscInt ovrb = ovlb + 1;
5002           const PetscInt ovrt = ovrb + 1;
5003           const PetscInt ovlt = ovrt + 1;
5004           PetscInt       o;
5005 
5006           /* bottom */
5007           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovlb*Nc + c + foffset;
5008           for (o = oeb; o < oer; ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5009           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovrb*Nc + c + foffset;
5010           /* middle */
5011           for (i = 0; i < k-1; ++i) {
5012             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oel+(k-2)-i)*Nc + c + foffset;
5013             for (o = of+(k-1)*i; o < of+(k-1)*(i+1); ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5014             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oer+i)*Nc + c + foffset;
5015           }
5016           /* top */
5017           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovlt*Nc + c + foffset;
5018           for (o = oel-1; o >= oet; --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5019           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovrt*Nc + c + foffset;
5020           foffset = offset;
5021         }
5022         break;
5023       case 3:
5024         /* The original hex closure is
5025 
5026          {c,
5027          f_b, f_t, f_f, f_b, f_r, f_l,
5028          e_bl, e_bb, e_br, e_bf,  e_tf, e_tr, e_tb, e_tl,  e_rf, e_lf, e_lb, e_rb,
5029          v_blf, v_blb, v_brb, v_brf, v_tlf, v_trf, v_trb, v_tlb}
5030          */
5031         ierr = PetscSectionFieldGetTensorDegree_Private(section,f,eStart,vertexchart,&Nc,&k);CHKERRQ(ierr);
5032         /* The SEM order is
5033          Bottom Slice
5034          v_blf, {e^{(k-1)-n}_bf}, v_brf,
5035          e^{i}_bl, f^{n*(k-1)+(k-1)-i}_b, e^{(k-1)-i}_br,
5036          v_blb, {e_bb}, v_brb,
5037 
5038          Middle Slice (j)
5039          {e^{(k-1)-j}_lf}, {f^{j*(k-1)+n}_f}, e^j_rf,
5040          f^{i*(k-1)+j}_l, {c^{(j*(k-1) + i)*(k-1)+n}_t}, f^{j*(k-1)+i}_r,
5041          e^j_lb, {f^{j*(k-1)+(k-1)-n}_b}, e^{(k-1)-j}_rb,
5042 
5043          Top Slice
5044          v_tlf, {e_tf}, v_trf,
5045          e^{(k-1)-i}_tl, {f^{i*(k-1)}_t}, e^{i}_tr,
5046          v_tlb, {e^{(k-1)-n}_tb}, v_trb,
5047          */
5048         {
5049           const PetscInt oc    = 0;
5050           const PetscInt ofb   = oc    + PetscSqr(k-1)*(k-1);
5051           const PetscInt oft   = ofb   + PetscSqr(k-1);
5052           const PetscInt off   = oft   + PetscSqr(k-1);
5053           const PetscInt ofk   = off   + PetscSqr(k-1);
5054           const PetscInt ofr   = ofk   + PetscSqr(k-1);
5055           const PetscInt ofl   = ofr   + PetscSqr(k-1);
5056           const PetscInt oebl  = ofl   + PetscSqr(k-1);
5057           const PetscInt oebb  = oebl  + (k-1);
5058           const PetscInt oebr  = oebb  + (k-1);
5059           const PetscInt oebf  = oebr  + (k-1);
5060           const PetscInt oetf  = oebf  + (k-1);
5061           const PetscInt oetr  = oetf  + (k-1);
5062           const PetscInt oetb  = oetr  + (k-1);
5063           const PetscInt oetl  = oetb  + (k-1);
5064           const PetscInt oerf  = oetl  + (k-1);
5065           const PetscInt oelf  = oerf  + (k-1);
5066           const PetscInt oelb  = oelf  + (k-1);
5067           const PetscInt oerb  = oelb  + (k-1);
5068           const PetscInt ovblf = oerb  + (k-1);
5069           const PetscInt ovblb = ovblf + 1;
5070           const PetscInt ovbrb = ovblb + 1;
5071           const PetscInt ovbrf = ovbrb + 1;
5072           const PetscInt ovtlf = ovbrf + 1;
5073           const PetscInt ovtrf = ovtlf + 1;
5074           const PetscInt ovtrb = ovtrf + 1;
5075           const PetscInt ovtlb = ovtrb + 1;
5076           PetscInt       o, n;
5077 
5078           /* Bottom Slice */
5079           /*   bottom */
5080           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovblf*Nc + c + foffset;
5081           for (o = oetf-1; o >= oebf; --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5082           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovbrf*Nc + c + foffset;
5083           /*   middle */
5084           for (i = 0; i < k-1; ++i) {
5085             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oebl+i)*Nc + c + foffset;
5086             for (n = 0; n < k-1; ++n) {o = ofb+n*(k-1)+i; for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;}
5087             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oebr+(k-2)-i)*Nc + c + foffset;
5088           }
5089           /*   top */
5090           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovblb*Nc + c + foffset;
5091           for (o = oebb; o < oebr; ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5092           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovbrb*Nc + c + foffset;
5093 
5094           /* Middle Slice */
5095           for (j = 0; j < k-1; ++j) {
5096             /*   bottom */
5097             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oelf+(k-2)-j)*Nc + c + foffset;
5098             for (o = off+j*(k-1); o < off+(j+1)*(k-1); ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5099             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oerf+j)*Nc + c + foffset;
5100             /*   middle */
5101             for (i = 0; i < k-1; ++i) {
5102               for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (ofl+i*(k-1)+j)*Nc + c + foffset;
5103               for (n = 0; n < k-1; ++n) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oc+(j*(k-1)+i)*(k-1)+n)*Nc + c + foffset;
5104               for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (ofr+j*(k-1)+i)*Nc + c + foffset;
5105             }
5106             /*   top */
5107             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oelb+j)*Nc + c + foffset;
5108             for (o = ofk+j*(k-1)+(k-2); o >= ofk+j*(k-1); --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5109             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oerb+(k-2)-j)*Nc + c + foffset;
5110           }
5111 
5112           /* Top Slice */
5113           /*   bottom */
5114           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtlf*Nc + c + foffset;
5115           for (o = oetf; o < oetr; ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5116           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtrf*Nc + c + foffset;
5117           /*   middle */
5118           for (i = 0; i < k-1; ++i) {
5119             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oetl+(k-2)-i)*Nc + c + foffset;
5120             for (n = 0; n < k-1; ++n) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oft+i*(k-1)+n)*Nc + c + foffset;
5121             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oetr+i)*Nc + c + foffset;
5122           }
5123           /*   top */
5124           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtlb*Nc + c + foffset;
5125           for (o = oetl-1; o >= oetb; --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5126           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtrb*Nc + c + foffset;
5127 
5128           foffset = offset;
5129         }
5130         break;
5131       default: SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_OUTOFRANGE, "No spectral ordering for dimension %D", d);
5132       }
5133     }
5134     if (offset != size) SETERRQ2(PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Number of permutation entries %D != %D", offset, size);
5135     /* Check permutation */
5136     {
5137       PetscInt *check;
5138 
5139       ierr = PetscMalloc1(size, &check);CHKERRQ(ierr);
5140       for (i = 0; i < size; ++i) {check[i] = -1; if (perm[i] < 0 || perm[i] >= size) SETERRQ2(PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Invalid permutation index p[%D] = %D", i, perm[i]);}
5141       for (i = 0; i < size; ++i) check[perm[i]] = i;
5142       for (i = 0; i < size; ++i) {if (check[i] < 0) SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Missing permutation index %D", i);}
5143       ierr = PetscFree(check);CHKERRQ(ierr);
5144     }
5145     ierr = PetscSectionSetClosurePermutation_Internal(section, (PetscObject) dm, d, size, PETSC_OWN_POINTER, perm);CHKERRQ(ierr);
5146   }
5147   PetscFunctionReturn(0);
5148 }
5149 
5150 PetscErrorCode DMPlexGetPointDualSpaceFEM(DM dm, PetscInt point, PetscInt field, PetscDualSpace *dspace)
5151 {
5152   PetscDS        prob;
5153   PetscInt       depth, Nf, h;
5154   DMLabel        label;
5155   PetscErrorCode ierr;
5156 
5157   PetscFunctionBeginHot;
5158   ierr = DMGetDS(dm, &prob);CHKERRQ(ierr);
5159   Nf      = prob->Nf;
5160   label   = dm->depthLabel;
5161   *dspace = NULL;
5162   if (field < Nf) {
5163     PetscObject disc = prob->disc[field];
5164 
5165     if (disc->classid == PETSCFE_CLASSID) {
5166       PetscDualSpace dsp;
5167 
5168       ierr = PetscFEGetDualSpace((PetscFE)disc,&dsp);CHKERRQ(ierr);
5169       ierr = DMLabelGetNumValues(label,&depth);CHKERRQ(ierr);
5170       ierr = DMLabelGetValue(label,point,&h);CHKERRQ(ierr);
5171       h    = depth - 1 - h;
5172       if (h) {
5173         ierr = PetscDualSpaceGetHeightSubspace(dsp,h,dspace);CHKERRQ(ierr);
5174       } else {
5175         *dspace = dsp;
5176       }
5177     }
5178   }
5179   PetscFunctionReturn(0);
5180 }
5181 
5182 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecGetClosure_Depth1_Static(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
5183 {
5184   PetscScalar    *array, *vArray;
5185   const PetscInt *cone, *coneO;
5186   PetscInt        pStart, pEnd, p, numPoints, size = 0, offset = 0;
5187   PetscErrorCode  ierr;
5188 
5189   PetscFunctionBeginHot;
5190   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
5191   ierr = DMPlexGetConeSize(dm, point, &numPoints);CHKERRQ(ierr);
5192   ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
5193   ierr = DMPlexGetConeOrientation(dm, point, &coneO);CHKERRQ(ierr);
5194   if (!values || !*values) {
5195     if ((point >= pStart) && (point < pEnd)) {
5196       PetscInt dof;
5197 
5198       ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
5199       size += dof;
5200     }
5201     for (p = 0; p < numPoints; ++p) {
5202       const PetscInt cp = cone[p];
5203       PetscInt       dof;
5204 
5205       if ((cp < pStart) || (cp >= pEnd)) continue;
5206       ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
5207       size += dof;
5208     }
5209     if (!values) {
5210       if (csize) *csize = size;
5211       PetscFunctionReturn(0);
5212     }
5213     ierr = DMGetWorkArray(dm, size, MPIU_SCALAR, &array);CHKERRQ(ierr);
5214   } else {
5215     array = *values;
5216   }
5217   size = 0;
5218   ierr = VecGetArray(v, &vArray);CHKERRQ(ierr);
5219   if ((point >= pStart) && (point < pEnd)) {
5220     PetscInt     dof, off, d;
5221     PetscScalar *varr;
5222 
5223     ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
5224     ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
5225     varr = &vArray[off];
5226     for (d = 0; d < dof; ++d, ++offset) {
5227       array[offset] = varr[d];
5228     }
5229     size += dof;
5230   }
5231   for (p = 0; p < numPoints; ++p) {
5232     const PetscInt cp = cone[p];
5233     PetscInt       o  = coneO[p];
5234     PetscInt       dof, off, d;
5235     PetscScalar   *varr;
5236 
5237     if ((cp < pStart) || (cp >= pEnd)) continue;
5238     ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
5239     ierr = PetscSectionGetOffset(section, cp, &off);CHKERRQ(ierr);
5240     varr = &vArray[off];
5241     if (o >= 0) {
5242       for (d = 0; d < dof; ++d, ++offset) {
5243         array[offset] = varr[d];
5244       }
5245     } else {
5246       for (d = dof-1; d >= 0; --d, ++offset) {
5247         array[offset] = varr[d];
5248       }
5249     }
5250     size += dof;
5251   }
5252   ierr = VecRestoreArray(v, &vArray);CHKERRQ(ierr);
5253   if (!*values) {
5254     if (csize) *csize = size;
5255     *values = array;
5256   } else {
5257     if (size > *csize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Size of input array %D < actual size %D", *csize, size);
5258     *csize = size;
5259   }
5260   PetscFunctionReturn(0);
5261 }
5262 
5263 /* Compress out points not in the section */
5264 PETSC_STATIC_INLINE PetscErrorCode CompressPoints_Private(PetscSection section, PetscInt *numPoints, PetscInt points[])
5265 {
5266   const PetscInt np = *numPoints;
5267   PetscInt       pStart, pEnd, p, q;
5268   PetscErrorCode ierr;
5269 
5270   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
5271   for (p = 0, q = 0; p < np; ++p) {
5272     const PetscInt r = points[p*2];
5273     if ((r >= pStart) && (r < pEnd)) {
5274       points[q*2]   = r;
5275       points[q*2+1] = points[p*2+1];
5276       ++q;
5277     }
5278   }
5279   *numPoints = q;
5280   return 0;
5281 }
5282 
5283 static PetscErrorCode DMPlexTransitiveClosure_Hybrid_Internal(DM dm, PetscInt point, PetscInt np, PetscInt *numPoints, PetscInt **points)
5284 {
5285   const PetscInt *cone, *ornt;
5286   PetscInt       *pts,  *closure = NULL;
5287   PetscInt        dim, coneSize, c, d, clSize, cl;
5288   PetscErrorCode  ierr;
5289 
5290   PetscFunctionBeginHot;
5291   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
5292   ierr = DMPlexGetConeSize(dm, point, &coneSize);CHKERRQ(ierr);
5293   ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
5294   ierr = DMPlexGetConeOrientation(dm, point, &ornt);CHKERRQ(ierr);
5295   ierr = DMPlexGetTransitiveClosure(dm, cone[0], PETSC_TRUE, &clSize, &closure);CHKERRQ(ierr);
5296   ierr = DMGetWorkArray(dm, np*2, MPIU_INT, &pts);CHKERRQ(ierr);
5297   c    = 0;
5298   pts[c*2+0] = point;
5299   pts[c*2+1] = 0;
5300   ++c;
5301   for (cl = 0; cl < clSize*2; cl += 2, ++c) {pts[c*2+0] = closure[cl]; pts[c*2+1] = closure[cl+1];}
5302   ierr = DMPlexGetTransitiveClosure(dm, cone[1], PETSC_TRUE, &clSize, &closure);CHKERRQ(ierr);
5303   for (cl = 0; cl < clSize*2; cl += 2, ++c) {pts[c*2+0] = closure[cl]; pts[c*2+1] = closure[cl+1];}
5304   ierr = DMPlexRestoreTransitiveClosure(dm, cone[0], PETSC_TRUE, &clSize, &closure);CHKERRQ(ierr);
5305   if (dim >= 2) {
5306     for (d = 2; d < coneSize; ++d, ++c) {pts[c*2+0] = cone[d]; pts[c*2+1] = ornt[d];}
5307   }
5308   if (dim >= 3) {
5309     for (d = 2; d < coneSize; ++d) {
5310       const PetscInt  fpoint = cone[d];
5311       const PetscInt *fcone;
5312       PetscInt        fconeSize, fc, i;
5313 
5314       ierr = DMPlexGetConeSize(dm, fpoint, &fconeSize);CHKERRQ(ierr);
5315       ierr = DMPlexGetCone(dm, fpoint, &fcone);CHKERRQ(ierr);
5316       for (fc = 0; fc < fconeSize; ++fc) {
5317         for (i = 0; i < c; ++i) if (pts[i*2] == fcone[fc]) break;
5318         if (i == c) {pts[c*2+0] = fcone[fc]; pts[c*2+1] = 0; ++c;}
5319       }
5320     }
5321   }
5322   if (c != np) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid closure for hybrid point %D, size %D != %D", point, c, np);
5323   *numPoints = np;
5324   *points    = pts;
5325   PetscFunctionReturn(0);
5326 }
5327 
5328 /* Compressed closure does not apply closure permutation */
5329 PetscErrorCode DMPlexGetCompressedClosure(DM dm, PetscSection section, PetscInt point, PetscInt *numPoints, PetscInt **points, PetscSection *clSec, IS *clPoints, const PetscInt **clp)
5330 {
5331   const PetscInt *cla = NULL;
5332   PetscInt       np, *pts = NULL;
5333   PetscErrorCode ierr;
5334 
5335   PetscFunctionBeginHot;
5336   ierr = PetscSectionGetClosureIndex(section, (PetscObject) dm, clSec, clPoints);CHKERRQ(ierr);
5337   if (*clPoints) {
5338     PetscInt dof, off;
5339 
5340     ierr = PetscSectionGetDof(*clSec, point, &dof);CHKERRQ(ierr);
5341     ierr = PetscSectionGetOffset(*clSec, point, &off);CHKERRQ(ierr);
5342     ierr = ISGetIndices(*clPoints, &cla);CHKERRQ(ierr);
5343     np   = dof/2;
5344     pts  = (PetscInt *) &cla[off];
5345   } else {
5346     DMPolytopeType ct;
5347 
5348     /* Do not make the label if it does not exist */
5349     if (!dm->celltypeLabel) {ct = DM_POLYTOPE_POINT;}
5350     else                    {ierr = DMPlexGetCellType(dm, point, &ct);CHKERRQ(ierr);}
5351     switch (ct) {
5352       case DM_POLYTOPE_SEG_PRISM_TENSOR:
5353         ierr = DMPlexTransitiveClosure_Hybrid_Internal(dm, point, 9, &np, &pts);CHKERRQ(ierr);
5354         break;
5355       case DM_POLYTOPE_TRI_PRISM_TENSOR:
5356         ierr = DMPlexTransitiveClosure_Hybrid_Internal(dm, point, 21, &np, &pts);CHKERRQ(ierr);
5357         break;
5358       case DM_POLYTOPE_QUAD_PRISM_TENSOR:
5359         ierr = DMPlexTransitiveClosure_Hybrid_Internal(dm, point, 27, &np, &pts);CHKERRQ(ierr);
5360         break;
5361       default:
5362         ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &np, &pts);CHKERRQ(ierr);
5363     }
5364     ierr = CompressPoints_Private(section, &np, pts);CHKERRQ(ierr);
5365   }
5366   *numPoints = np;
5367   *points    = pts;
5368   *clp       = cla;
5369   PetscFunctionReturn(0);
5370 }
5371 
5372 PetscErrorCode DMPlexRestoreCompressedClosure(DM dm, PetscSection section, PetscInt point, PetscInt *numPoints, PetscInt **points, PetscSection *clSec, IS *clPoints, const PetscInt **clp)
5373 {
5374   PetscErrorCode ierr;
5375 
5376   PetscFunctionBeginHot;
5377   if (!*clPoints) {
5378     ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, numPoints, points);CHKERRQ(ierr);
5379   } else {
5380     ierr = ISRestoreIndices(*clPoints, clp);CHKERRQ(ierr);
5381   }
5382   *numPoints = 0;
5383   *points    = NULL;
5384   *clSec     = NULL;
5385   *clPoints  = NULL;
5386   *clp       = NULL;
5387   PetscFunctionReturn(0);
5388 }
5389 
5390 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecGetClosure_Static(DM dm, PetscSection section, PetscInt numPoints, const PetscInt points[], const PetscInt clperm[], const PetscScalar vArray[], PetscInt *size, PetscScalar array[])
5391 {
5392   PetscInt          offset = 0, p;
5393   const PetscInt    **perms = NULL;
5394   const PetscScalar **flips = NULL;
5395   PetscErrorCode    ierr;
5396 
5397   PetscFunctionBeginHot;
5398   *size = 0;
5399   ierr = PetscSectionGetPointSyms(section,numPoints,points,&perms,&flips);CHKERRQ(ierr);
5400   for (p = 0; p < numPoints; p++) {
5401     const PetscInt    point = points[2*p];
5402     const PetscInt    *perm = perms ? perms[p] : NULL;
5403     const PetscScalar *flip = flips ? flips[p] : NULL;
5404     PetscInt          dof, off, d;
5405     const PetscScalar *varr;
5406 
5407     ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
5408     ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
5409     varr = &vArray[off];
5410     if (clperm) {
5411       if (perm) {
5412         for (d = 0; d < dof; d++) array[clperm[offset + perm[d]]]  = varr[d];
5413       } else {
5414         for (d = 0; d < dof; d++) array[clperm[offset +      d ]]  = varr[d];
5415       }
5416       if (flip) {
5417         for (d = 0; d < dof; d++) array[clperm[offset +      d ]] *= flip[d];
5418       }
5419     } else {
5420       if (perm) {
5421         for (d = 0; d < dof; d++) array[offset + perm[d]]  = varr[d];
5422       } else {
5423         for (d = 0; d < dof; d++) array[offset +      d ]  = varr[d];
5424       }
5425       if (flip) {
5426         for (d = 0; d < dof; d++) array[offset +      d ] *= flip[d];
5427       }
5428     }
5429     offset += dof;
5430   }
5431   ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms,&flips);CHKERRQ(ierr);
5432   *size = offset;
5433   PetscFunctionReturn(0);
5434 }
5435 
5436 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecGetClosure_Fields_Static(DM dm, PetscSection section, PetscInt numPoints, const PetscInt points[], PetscInt numFields, const PetscInt clperm[], const PetscScalar vArray[], PetscInt *size, PetscScalar array[])
5437 {
5438   PetscInt          offset = 0, f;
5439   PetscErrorCode    ierr;
5440 
5441   PetscFunctionBeginHot;
5442   *size = 0;
5443   for (f = 0; f < numFields; ++f) {
5444     PetscInt          p;
5445     const PetscInt    **perms = NULL;
5446     const PetscScalar **flips = NULL;
5447 
5448     ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
5449     for (p = 0; p < numPoints; p++) {
5450       const PetscInt    point = points[2*p];
5451       PetscInt          fdof, foff, b;
5452       const PetscScalar *varr;
5453       const PetscInt    *perm = perms ? perms[p] : NULL;
5454       const PetscScalar *flip = flips ? flips[p] : NULL;
5455 
5456       ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
5457       ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
5458       varr = &vArray[foff];
5459       if (clperm) {
5460         if (perm) {for (b = 0; b < fdof; b++) {array[clperm[offset + perm[b]]]  = varr[b];}}
5461         else      {for (b = 0; b < fdof; b++) {array[clperm[offset +      b ]]  = varr[b];}}
5462         if (flip) {for (b = 0; b < fdof; b++) {array[clperm[offset +      b ]] *= flip[b];}}
5463       } else {
5464         if (perm) {for (b = 0; b < fdof; b++) {array[offset + perm[b]]  = varr[b];}}
5465         else      {for (b = 0; b < fdof; b++) {array[offset +      b ]  = varr[b];}}
5466         if (flip) {for (b = 0; b < fdof; b++) {array[offset +      b ] *= flip[b];}}
5467       }
5468       offset += fdof;
5469     }
5470     ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
5471   }
5472   *size = offset;
5473   PetscFunctionReturn(0);
5474 }
5475 
5476 /*@C
5477   DMPlexVecGetClosure - Get an array of the values on the closure of 'point'
5478 
5479   Not collective
5480 
5481   Input Parameters:
5482 + dm - The DM
5483 . section - The section describing the layout in v, or NULL to use the default section
5484 . v - The local vector
5485 . point - The point in the DM
5486 . csize - The size of the input values array, or NULL
5487 - values - An array to use for the values, or NULL to have it allocated automatically
5488 
5489   Output Parameters:
5490 + csize - The number of values in the closure
5491 - values - The array of values. If the user provided NULL, it is a borrowed array and should not be freed
5492 
5493 $ Note that DMPlexVecGetClosure/DMPlexVecRestoreClosure only allocates the values array if it set to NULL in the
5494 $ calling function. This is because DMPlexVecGetClosure() is typically called in the inner loop of a Vec or Mat
5495 $ assembly function, and a user may already have allocated storage for this operation.
5496 $
5497 $ A typical use could be
5498 $
5499 $  values = NULL;
5500 $  ierr = DMPlexVecGetClosure(dm, NULL, v, p, &clSize, &values);CHKERRQ(ierr);
5501 $  for (cl = 0; cl < clSize; ++cl) {
5502 $    <Compute on closure>
5503 $  }
5504 $  ierr = DMPlexVecRestoreClosure(dm, NULL, v, p, &clSize, &values);CHKERRQ(ierr);
5505 $
5506 $ or
5507 $
5508 $  PetscMalloc1(clMaxSize, &values);
5509 $  for (p = pStart; p < pEnd; ++p) {
5510 $    clSize = clMaxSize;
5511 $    ierr = DMPlexVecGetClosure(dm, NULL, v, p, &clSize, &values);CHKERRQ(ierr);
5512 $    for (cl = 0; cl < clSize; ++cl) {
5513 $      <Compute on closure>
5514 $    }
5515 $  }
5516 $  PetscFree(values);
5517 
5518   Fortran Notes:
5519   Since it returns an array, this routine is only available in Fortran 90, and you must
5520   include petsc.h90 in your code.
5521 
5522   The csize argument is not present in the Fortran 90 binding since it is internal to the array.
5523 
5524   Level: intermediate
5525 
5526 .seealso DMPlexVecRestoreClosure(), DMPlexVecSetClosure(), DMPlexMatSetClosure()
5527 @*/
5528 PetscErrorCode DMPlexVecGetClosure(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
5529 {
5530   PetscSection       clSection;
5531   IS                 clPoints;
5532   PetscInt          *points = NULL;
5533   const PetscInt    *clp, *perm;
5534   PetscInt           depth, numFields, numPoints, asize;
5535   PetscErrorCode     ierr;
5536 
5537   PetscFunctionBeginHot;
5538   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5539   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
5540   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
5541   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
5542   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
5543   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5544   if (depth == 1 && numFields < 2) {
5545     ierr = DMPlexVecGetClosure_Depth1_Static(dm, section, v, point, csize, values);CHKERRQ(ierr);
5546     PetscFunctionReturn(0);
5547   }
5548   /* Get points */
5549   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5550   /* Get sizes */
5551   asize = 0;
5552   for (PetscInt p = 0; p < numPoints*2; p += 2) {
5553     PetscInt dof;
5554     ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5555     asize += dof;
5556   }
5557   if (values) {
5558     const PetscScalar *vArray;
5559     PetscInt          size;
5560 
5561     if (*values) {
5562       if (PetscUnlikely(*csize < asize)) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Provided array size %D not sufficient to hold closure size %D", *csize, asize);
5563     } else {ierr = DMGetWorkArray(dm, asize, MPIU_SCALAR, values);CHKERRQ(ierr);}
5564     ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, depth, asize, &perm);CHKERRQ(ierr);
5565     ierr = VecGetArrayRead(v, &vArray);CHKERRQ(ierr);
5566     /* Get values */
5567     if (numFields > 0) {ierr = DMPlexVecGetClosure_Fields_Static(dm, section, numPoints, points, numFields, perm, vArray, &size, *values);CHKERRQ(ierr);}
5568     else               {ierr = DMPlexVecGetClosure_Static(dm, section, numPoints, points, perm, vArray, &size, *values);CHKERRQ(ierr);}
5569     if (PetscUnlikely(asize != size)) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Section size %D does not match Vec closure size %D", asize, size);
5570     /* Cleanup array */
5571     ierr = VecRestoreArrayRead(v, &vArray);CHKERRQ(ierr);
5572   }
5573   if (csize) *csize = asize;
5574   /* Cleanup points */
5575   ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5576   PetscFunctionReturn(0);
5577 }
5578 
5579 PetscErrorCode DMPlexVecGetClosureAtDepth_Internal(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt depth, PetscInt *csize, PetscScalar *values[])
5580 {
5581   DMLabel            depthLabel;
5582   PetscSection       clSection;
5583   IS                 clPoints;
5584   PetscScalar       *array;
5585   const PetscScalar *vArray;
5586   PetscInt          *points = NULL;
5587   const PetscInt    *clp, *perm = NULL;
5588   PetscInt           mdepth, numFields, numPoints, Np = 0, p, clsize, size;
5589   PetscErrorCode     ierr;
5590 
5591   PetscFunctionBeginHot;
5592   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5593   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
5594   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
5595   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
5596   ierr = DMPlexGetDepth(dm, &mdepth);CHKERRQ(ierr);
5597   ierr = DMPlexGetDepthLabel(dm, &depthLabel);CHKERRQ(ierr);
5598   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5599   if (mdepth == 1 && numFields < 2) {
5600     ierr = DMPlexVecGetClosure_Depth1_Static(dm, section, v, point, csize, values);CHKERRQ(ierr);
5601     PetscFunctionReturn(0);
5602   }
5603   /* Get points */
5604   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5605   for (clsize=0,p=0; p<Np; p++) {
5606     PetscInt dof;
5607     ierr = PetscSectionGetDof(section, points[2*p], &dof);CHKERRQ(ierr);
5608     clsize += dof;
5609   }
5610   ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, depth, clsize, &perm);CHKERRQ(ierr);
5611   /* Filter points */
5612   for (p = 0; p < numPoints*2; p += 2) {
5613     PetscInt dep;
5614 
5615     ierr = DMLabelGetValue(depthLabel, points[p], &dep);CHKERRQ(ierr);
5616     if (dep != depth) continue;
5617     points[Np*2+0] = points[p];
5618     points[Np*2+1] = points[p+1];
5619     ++Np;
5620   }
5621   /* Get array */
5622   if (!values || !*values) {
5623     PetscInt asize = 0, dof;
5624 
5625     for (p = 0; p < Np*2; p += 2) {
5626       ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5627       asize += dof;
5628     }
5629     if (!values) {
5630       ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5631       if (csize) *csize = asize;
5632       PetscFunctionReturn(0);
5633     }
5634     ierr = DMGetWorkArray(dm, asize, MPIU_SCALAR, &array);CHKERRQ(ierr);
5635   } else {
5636     array = *values;
5637   }
5638   ierr = VecGetArrayRead(v, &vArray);CHKERRQ(ierr);
5639   /* Get values */
5640   if (numFields > 0) {ierr = DMPlexVecGetClosure_Fields_Static(dm, section, Np, points, numFields, perm, vArray, &size, array);CHKERRQ(ierr);}
5641   else               {ierr = DMPlexVecGetClosure_Static(dm, section, Np, points, perm, vArray, &size, array);CHKERRQ(ierr);}
5642   /* Cleanup points */
5643   ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5644   /* Cleanup array */
5645   ierr = VecRestoreArrayRead(v, &vArray);CHKERRQ(ierr);
5646   if (!*values) {
5647     if (csize) *csize = size;
5648     *values = array;
5649   } else {
5650     if (size > *csize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Size of input array %D < actual size %D", *csize, size);
5651     *csize = size;
5652   }
5653   PetscFunctionReturn(0);
5654 }
5655 
5656 /*@C
5657   DMPlexVecRestoreClosure - Restore the array of the values on the closure of 'point'
5658 
5659   Not collective
5660 
5661   Input Parameters:
5662 + dm - The DM
5663 . section - The section describing the layout in v, or NULL to use the default section
5664 . v - The local vector
5665 . point - The point in the DM
5666 . csize - The number of values in the closure, or NULL
5667 - values - The array of values, which is a borrowed array and should not be freed
5668 
5669   Note that the array values are discarded and not copied back into v. In order to copy values back to v, use DMPlexVecSetClosure()
5670 
5671   Fortran Notes:
5672   Since it returns an array, this routine is only available in Fortran 90, and you must
5673   include petsc.h90 in your code.
5674 
5675   The csize argument is not present in the Fortran 90 binding since it is internal to the array.
5676 
5677   Level: intermediate
5678 
5679 .seealso DMPlexVecGetClosure(), DMPlexVecSetClosure(), DMPlexMatSetClosure()
5680 @*/
5681 PetscErrorCode DMPlexVecRestoreClosure(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
5682 {
5683   PetscInt       size = 0;
5684   PetscErrorCode ierr;
5685 
5686   PetscFunctionBegin;
5687   /* Should work without recalculating size */
5688   ierr = DMRestoreWorkArray(dm, size, MPIU_SCALAR, (void*) values);CHKERRQ(ierr);
5689   *values = NULL;
5690   PetscFunctionReturn(0);
5691 }
5692 
5693 PETSC_STATIC_INLINE void add   (PetscScalar *x, PetscScalar y) {*x += y;}
5694 PETSC_STATIC_INLINE void insert(PetscScalar *x, PetscScalar y) {*x  = y;}
5695 
5696 PETSC_STATIC_INLINE PetscErrorCode updatePoint_private(PetscSection section, PetscInt point, PetscInt dof, void (*fuse)(PetscScalar*, PetscScalar), PetscBool setBC, const PetscInt perm[], const PetscScalar flip[], const PetscInt clperm[], const PetscScalar values[], PetscInt offset, PetscScalar array[])
5697 {
5698   PetscInt        cdof;   /* The number of constraints on this point */
5699   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
5700   PetscScalar    *a;
5701   PetscInt        off, cind = 0, k;
5702   PetscErrorCode  ierr;
5703 
5704   PetscFunctionBegin;
5705   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
5706   ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
5707   a    = &array[off];
5708   if (!cdof || setBC) {
5709     if (clperm) {
5710       if (perm) {for (k = 0; k < dof; ++k) {fuse(&a[k], values[clperm[offset+perm[k]]] * (flip ? flip[perm[k]] : 1.));}}
5711       else      {for (k = 0; k < dof; ++k) {fuse(&a[k], values[clperm[offset+     k ]] * (flip ? flip[     k ] : 1.));}}
5712     } else {
5713       if (perm) {for (k = 0; k < dof; ++k) {fuse(&a[k], values[offset+perm[k]] * (flip ? flip[perm[k]] : 1.));}}
5714       else      {for (k = 0; k < dof; ++k) {fuse(&a[k], values[offset+     k ] * (flip ? flip[     k ] : 1.));}}
5715     }
5716   } else {
5717     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
5718     if (clperm) {
5719       if (perm) {for (k = 0; k < dof; ++k) {
5720           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
5721           fuse(&a[k], values[clperm[offset+perm[k]]] * (flip ? flip[perm[k]] : 1.));
5722         }
5723       } else {
5724         for (k = 0; k < dof; ++k) {
5725           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
5726           fuse(&a[k], values[clperm[offset+     k ]] * (flip ? flip[     k ] : 1.));
5727         }
5728       }
5729     } else {
5730       if (perm) {
5731         for (k = 0; k < dof; ++k) {
5732           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
5733           fuse(&a[k], values[offset+perm[k]] * (flip ? flip[perm[k]] : 1.));
5734         }
5735       } else {
5736         for (k = 0; k < dof; ++k) {
5737           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
5738           fuse(&a[k], values[offset+     k ] * (flip ? flip[     k ] : 1.));
5739         }
5740       }
5741     }
5742   }
5743   PetscFunctionReturn(0);
5744 }
5745 
5746 PETSC_STATIC_INLINE PetscErrorCode updatePointBC_private(PetscSection section, PetscInt point, PetscInt dof, void (*fuse)(PetscScalar*, PetscScalar), const PetscInt perm[], const PetscScalar flip[], const PetscInt clperm[], const PetscScalar values[], PetscInt offset, PetscScalar array[])
5747 {
5748   PetscInt        cdof;   /* The number of constraints on this point */
5749   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
5750   PetscScalar    *a;
5751   PetscInt        off, cind = 0, k;
5752   PetscErrorCode  ierr;
5753 
5754   PetscFunctionBegin;
5755   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
5756   ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
5757   a    = &array[off];
5758   if (cdof) {
5759     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
5760     if (clperm) {
5761       if (perm) {
5762         for (k = 0; k < dof; ++k) {
5763           if ((cind < cdof) && (k == cdofs[cind])) {
5764             fuse(&a[k], values[clperm[offset+perm[k]]] * (flip ? flip[perm[k]] : 1.));
5765             cind++;
5766           }
5767         }
5768       } else {
5769         for (k = 0; k < dof; ++k) {
5770           if ((cind < cdof) && (k == cdofs[cind])) {
5771             fuse(&a[k], values[clperm[offset+     k ]] * (flip ? flip[     k ] : 1.));
5772             cind++;
5773           }
5774         }
5775       }
5776     } else {
5777       if (perm) {
5778         for (k = 0; k < dof; ++k) {
5779           if ((cind < cdof) && (k == cdofs[cind])) {
5780             fuse(&a[k], values[offset+perm[k]] * (flip ? flip[perm[k]] : 1.));
5781             cind++;
5782           }
5783         }
5784       } else {
5785         for (k = 0; k < dof; ++k) {
5786           if ((cind < cdof) && (k == cdofs[cind])) {
5787             fuse(&a[k], values[offset+     k ] * (flip ? flip[     k ] : 1.));
5788             cind++;
5789           }
5790         }
5791       }
5792     }
5793   }
5794   PetscFunctionReturn(0);
5795 }
5796 
5797 PETSC_STATIC_INLINE PetscErrorCode updatePointFields_private(PetscSection section, PetscInt point, const PetscInt *perm, const PetscScalar *flip, PetscInt f, void (*fuse)(PetscScalar*, PetscScalar), PetscBool setBC, const PetscInt clperm[], const PetscScalar values[], PetscInt *offset, PetscScalar array[])
5798 {
5799   PetscScalar    *a;
5800   PetscInt        fdof, foff, fcdof, foffset = *offset;
5801   const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
5802   PetscInt        cind = 0, b;
5803   PetscErrorCode  ierr;
5804 
5805   PetscFunctionBegin;
5806   ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
5807   ierr = PetscSectionGetFieldConstraintDof(section, point, f, &fcdof);CHKERRQ(ierr);
5808   ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
5809   a    = &array[foff];
5810   if (!fcdof || setBC) {
5811     if (clperm) {
5812       if (perm) {for (b = 0; b < fdof; b++) {fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));}}
5813       else      {for (b = 0; b < fdof; b++) {fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));}}
5814     } else {
5815       if (perm) {for (b = 0; b < fdof; b++) {fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));}}
5816       else      {for (b = 0; b < fdof; b++) {fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));}}
5817     }
5818   } else {
5819     ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
5820     if (clperm) {
5821       if (perm) {
5822         for (b = 0; b < fdof; b++) {
5823           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
5824           fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));
5825         }
5826       } else {
5827         for (b = 0; b < fdof; b++) {
5828           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
5829           fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));
5830         }
5831       }
5832     } else {
5833       if (perm) {
5834         for (b = 0; b < fdof; b++) {
5835           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
5836           fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));
5837         }
5838       } else {
5839         for (b = 0; b < fdof; b++) {
5840           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
5841           fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));
5842         }
5843       }
5844     }
5845   }
5846   *offset += fdof;
5847   PetscFunctionReturn(0);
5848 }
5849 
5850 PETSC_STATIC_INLINE PetscErrorCode updatePointFieldsBC_private(PetscSection section, PetscInt point, const PetscInt perm[], const PetscScalar flip[], PetscInt f, PetscInt Ncc, const PetscInt comps[], void (*fuse)(PetscScalar*, PetscScalar), const PetscInt clperm[], const PetscScalar values[], PetscInt *offset, PetscScalar array[])
5851 {
5852   PetscScalar    *a;
5853   PetscInt        fdof, foff, fcdof, foffset = *offset;
5854   const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
5855   PetscInt        Nc, cind = 0, ncind = 0, b;
5856   PetscBool       ncSet, fcSet;
5857   PetscErrorCode  ierr;
5858 
5859   PetscFunctionBegin;
5860   ierr = PetscSectionGetFieldComponents(section, f, &Nc);CHKERRQ(ierr);
5861   ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
5862   ierr = PetscSectionGetFieldConstraintDof(section, point, f, &fcdof);CHKERRQ(ierr);
5863   ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
5864   a    = &array[foff];
5865   if (fcdof) {
5866     /* We just override fcdof and fcdofs with Ncc and comps */
5867     ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
5868     if (clperm) {
5869       if (perm) {
5870         if (comps) {
5871           for (b = 0; b < fdof; b++) {
5872             ncSet = fcSet = PETSC_FALSE;
5873             if (b%Nc == comps[ncind]) {ncind = (ncind+1)%Ncc; ncSet = PETSC_TRUE;}
5874             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
5875             if (ncSet && fcSet) {fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));}
5876           }
5877         } else {
5878           for (b = 0; b < fdof; b++) {
5879             if ((cind < fcdof) && (b == fcdofs[cind])) {
5880               fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));
5881               ++cind;
5882             }
5883           }
5884         }
5885       } else {
5886         if (comps) {
5887           for (b = 0; b < fdof; b++) {
5888             ncSet = fcSet = PETSC_FALSE;
5889             if (b%Nc == comps[ncind]) {ncind = (ncind+1)%Ncc; ncSet = PETSC_TRUE;}
5890             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
5891             if (ncSet && fcSet) {fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));}
5892           }
5893         } else {
5894           for (b = 0; b < fdof; b++) {
5895             if ((cind < fcdof) && (b == fcdofs[cind])) {
5896               fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));
5897               ++cind;
5898             }
5899           }
5900         }
5901       }
5902     } else {
5903       if (perm) {
5904         if (comps) {
5905           for (b = 0; b < fdof; b++) {
5906             ncSet = fcSet = PETSC_FALSE;
5907             if (b%Nc == comps[ncind]) {ncind = (ncind+1)%Ncc; ncSet = PETSC_TRUE;}
5908             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
5909             if (ncSet && fcSet) {fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));}
5910           }
5911         } else {
5912           for (b = 0; b < fdof; b++) {
5913             if ((cind < fcdof) && (b == fcdofs[cind])) {
5914               fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));
5915               ++cind;
5916             }
5917           }
5918         }
5919       } else {
5920         if (comps) {
5921           for (b = 0; b < fdof; b++) {
5922             ncSet = fcSet = PETSC_FALSE;
5923             if (b%Nc == comps[ncind]) {ncind = (ncind+1)%Ncc; ncSet = PETSC_TRUE;}
5924             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
5925             if (ncSet && fcSet) {fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));}
5926           }
5927         } else {
5928           for (b = 0; b < fdof; b++) {
5929             if ((cind < fcdof) && (b == fcdofs[cind])) {
5930               fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));
5931               ++cind;
5932             }
5933           }
5934         }
5935       }
5936     }
5937   }
5938   *offset += fdof;
5939   PetscFunctionReturn(0);
5940 }
5941 
5942 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecSetClosure_Depth1_Static(DM dm, PetscSection section, Vec v, PetscInt point, const PetscScalar values[], InsertMode mode)
5943 {
5944   PetscScalar    *array;
5945   const PetscInt *cone, *coneO;
5946   PetscInt        pStart, pEnd, p, numPoints, off, dof;
5947   PetscErrorCode  ierr;
5948 
5949   PetscFunctionBeginHot;
5950   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
5951   ierr = DMPlexGetConeSize(dm, point, &numPoints);CHKERRQ(ierr);
5952   ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
5953   ierr = DMPlexGetConeOrientation(dm, point, &coneO);CHKERRQ(ierr);
5954   ierr = VecGetArray(v, &array);CHKERRQ(ierr);
5955   for (p = 0, off = 0; p <= numPoints; ++p, off += dof) {
5956     const PetscInt cp = !p ? point : cone[p-1];
5957     const PetscInt o  = !p ? 0     : coneO[p-1];
5958 
5959     if ((cp < pStart) || (cp >= pEnd)) {dof = 0; continue;}
5960     ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
5961     /* ADD_VALUES */
5962     {
5963       const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
5964       PetscScalar    *a;
5965       PetscInt        cdof, coff, cind = 0, k;
5966 
5967       ierr = PetscSectionGetConstraintDof(section, cp, &cdof);CHKERRQ(ierr);
5968       ierr = PetscSectionGetOffset(section, cp, &coff);CHKERRQ(ierr);
5969       a    = &array[coff];
5970       if (!cdof) {
5971         if (o >= 0) {
5972           for (k = 0; k < dof; ++k) {
5973             a[k] += values[off+k];
5974           }
5975         } else {
5976           for (k = 0; k < dof; ++k) {
5977             a[k] += values[off+dof-k-1];
5978           }
5979         }
5980       } else {
5981         ierr = PetscSectionGetConstraintIndices(section, cp, &cdofs);CHKERRQ(ierr);
5982         if (o >= 0) {
5983           for (k = 0; k < dof; ++k) {
5984             if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
5985             a[k] += values[off+k];
5986           }
5987         } else {
5988           for (k = 0; k < dof; ++k) {
5989             if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
5990             a[k] += values[off+dof-k-1];
5991           }
5992         }
5993       }
5994     }
5995   }
5996   ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
5997   PetscFunctionReturn(0);
5998 }
5999 
6000 /*@C
6001   DMPlexVecSetClosure - Set an array of the values on the closure of 'point'
6002 
6003   Not collective
6004 
6005   Input Parameters:
6006 + dm - The DM
6007 . section - The section describing the layout in v, or NULL to use the default section
6008 . v - The local vector
6009 . point - The point in the DM
6010 . values - The array of values
6011 - mode - The insert mode. One of INSERT_ALL_VALUES, ADD_ALL_VALUES, INSERT_VALUES, ADD_VALUES, INSERT_BC_VALUES, and ADD_BC_VALUES,
6012          where INSERT_ALL_VALUES and ADD_ALL_VALUES also overwrite boundary conditions.
6013 
6014   Fortran Notes:
6015   This routine is only available in Fortran 90, and you must include petsc.h90 in your code.
6016 
6017   Level: intermediate
6018 
6019 .seealso DMPlexVecGetClosure(), DMPlexMatSetClosure()
6020 @*/
6021 PetscErrorCode DMPlexVecSetClosure(DM dm, PetscSection section, Vec v, PetscInt point, const PetscScalar values[], InsertMode mode)
6022 {
6023   PetscSection    clSection;
6024   IS              clPoints;
6025   PetscScalar    *array;
6026   PetscInt       *points = NULL;
6027   const PetscInt *clp, *clperm = NULL;
6028   PetscInt        depth, numFields, numPoints, p, clsize;
6029   PetscErrorCode  ierr;
6030 
6031   PetscFunctionBeginHot;
6032   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6033   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
6034   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
6035   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
6036   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
6037   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
6038   if (depth == 1 && numFields < 2 && mode == ADD_VALUES) {
6039     ierr = DMPlexVecSetClosure_Depth1_Static(dm, section, v, point, values, mode);CHKERRQ(ierr);
6040     PetscFunctionReturn(0);
6041   }
6042   /* Get points */
6043   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
6044   for (clsize=0,p=0; p<numPoints; p++) {
6045     PetscInt dof;
6046     ierr = PetscSectionGetDof(section, points[2*p], &dof);CHKERRQ(ierr);
6047     clsize += dof;
6048   }
6049   ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, depth, clsize, &clperm);CHKERRQ(ierr);
6050   /* Get array */
6051   ierr = VecGetArray(v, &array);CHKERRQ(ierr);
6052   /* Get values */
6053   if (numFields > 0) {
6054     PetscInt offset = 0, f;
6055     for (f = 0; f < numFields; ++f) {
6056       const PetscInt    **perms = NULL;
6057       const PetscScalar **flips = NULL;
6058 
6059       ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
6060       switch (mode) {
6061       case INSERT_VALUES:
6062         for (p = 0; p < numPoints; p++) {
6063           const PetscInt    point = points[2*p];
6064           const PetscInt    *perm = perms ? perms[p] : NULL;
6065           const PetscScalar *flip = flips ? flips[p] : NULL;
6066           updatePointFields_private(section, point, perm, flip, f, insert, PETSC_FALSE, clperm, values, &offset, array);
6067         } break;
6068       case INSERT_ALL_VALUES:
6069         for (p = 0; p < numPoints; p++) {
6070           const PetscInt    point = points[2*p];
6071           const PetscInt    *perm = perms ? perms[p] : NULL;
6072           const PetscScalar *flip = flips ? flips[p] : NULL;
6073           updatePointFields_private(section, point, perm, flip, f, insert, PETSC_TRUE, clperm, values, &offset, array);
6074         } break;
6075       case INSERT_BC_VALUES:
6076         for (p = 0; p < numPoints; p++) {
6077           const PetscInt    point = points[2*p];
6078           const PetscInt    *perm = perms ? perms[p] : NULL;
6079           const PetscScalar *flip = flips ? flips[p] : NULL;
6080           updatePointFieldsBC_private(section, point, perm, flip, f, -1, NULL, insert, clperm, values, &offset, array);
6081         } break;
6082       case ADD_VALUES:
6083         for (p = 0; p < numPoints; p++) {
6084           const PetscInt    point = points[2*p];
6085           const PetscInt    *perm = perms ? perms[p] : NULL;
6086           const PetscScalar *flip = flips ? flips[p] : NULL;
6087           updatePointFields_private(section, point, perm, flip, f, add, PETSC_FALSE, clperm, values, &offset, array);
6088         } break;
6089       case ADD_ALL_VALUES:
6090         for (p = 0; p < numPoints; p++) {
6091           const PetscInt    point = points[2*p];
6092           const PetscInt    *perm = perms ? perms[p] : NULL;
6093           const PetscScalar *flip = flips ? flips[p] : NULL;
6094           updatePointFields_private(section, point, perm, flip, f, add, PETSC_TRUE, clperm, values, &offset, array);
6095         } break;
6096       case ADD_BC_VALUES:
6097         for (p = 0; p < numPoints; p++) {
6098           const PetscInt    point = points[2*p];
6099           const PetscInt    *perm = perms ? perms[p] : NULL;
6100           const PetscScalar *flip = flips ? flips[p] : NULL;
6101           updatePointFieldsBC_private(section, point, perm, flip, f, -1, NULL, add, clperm, values, &offset, array);
6102         } break;
6103       default:
6104         SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %d", mode);
6105       }
6106       ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
6107     }
6108   } else {
6109     PetscInt dof, off;
6110     const PetscInt    **perms = NULL;
6111     const PetscScalar **flips = NULL;
6112 
6113     ierr = PetscSectionGetPointSyms(section,numPoints,points,&perms,&flips);CHKERRQ(ierr);
6114     switch (mode) {
6115     case INSERT_VALUES:
6116       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
6117         const PetscInt    point = points[2*p];
6118         const PetscInt    *perm = perms ? perms[p] : NULL;
6119         const PetscScalar *flip = flips ? flips[p] : NULL;
6120         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
6121         updatePoint_private(section, point, dof, insert, PETSC_FALSE, perm, flip, clperm, values, off, array);
6122       } break;
6123     case INSERT_ALL_VALUES:
6124       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
6125         const PetscInt    point = points[2*p];
6126         const PetscInt    *perm = perms ? perms[p] : NULL;
6127         const PetscScalar *flip = flips ? flips[p] : NULL;
6128         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
6129         updatePoint_private(section, point, dof, insert, PETSC_TRUE,  perm, flip, clperm, values, off, array);
6130       } break;
6131     case INSERT_BC_VALUES:
6132       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
6133         const PetscInt    point = points[2*p];
6134         const PetscInt    *perm = perms ? perms[p] : NULL;
6135         const PetscScalar *flip = flips ? flips[p] : NULL;
6136         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
6137         updatePointBC_private(section, point, dof, insert,  perm, flip, clperm, values, off, array);
6138       } break;
6139     case ADD_VALUES:
6140       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
6141         const PetscInt    point = points[2*p];
6142         const PetscInt    *perm = perms ? perms[p] : NULL;
6143         const PetscScalar *flip = flips ? flips[p] : NULL;
6144         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
6145         updatePoint_private(section, point, dof, add,    PETSC_FALSE, perm, flip, clperm, values, off, array);
6146       } break;
6147     case ADD_ALL_VALUES:
6148       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
6149         const PetscInt    point = points[2*p];
6150         const PetscInt    *perm = perms ? perms[p] : NULL;
6151         const PetscScalar *flip = flips ? flips[p] : NULL;
6152         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
6153         updatePoint_private(section, point, dof, add,    PETSC_TRUE,  perm, flip, clperm, values, off, array);
6154       } break;
6155     case ADD_BC_VALUES:
6156       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
6157         const PetscInt    point = points[2*p];
6158         const PetscInt    *perm = perms ? perms[p] : NULL;
6159         const PetscScalar *flip = flips ? flips[p] : NULL;
6160         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
6161         updatePointBC_private(section, point, dof, add,  perm, flip, clperm, values, off, array);
6162       } break;
6163     default:
6164       SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %d", mode);
6165     }
6166     ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms,&flips);CHKERRQ(ierr);
6167   }
6168   /* Cleanup points */
6169   ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
6170   /* Cleanup array */
6171   ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
6172   PetscFunctionReturn(0);
6173 }
6174 
6175 /* Check whether the given point is in the label. If not, update the offset to skip this point */
6176 PETSC_STATIC_INLINE PetscErrorCode CheckPoint_Private(DMLabel label, PetscInt labelId, PetscSection section, PetscInt point, PetscInt f, PetscInt *offset)
6177 {
6178   PetscFunctionBegin;
6179   if (label) {
6180     PetscInt       val, fdof;
6181     PetscErrorCode ierr;
6182 
6183     /* There is a problem with this:
6184          Suppose we have two label values, defining surfaces, interecting along a line in 3D. When we add cells to the label, the cells that
6185        touch both surfaces must pick a label value. Thus we miss setting values for the surface with that other value intersecting that cell.
6186        Thus I am only going to check val != -1, not val != labelId
6187     */
6188     ierr = DMLabelGetValue(label, point, &val);CHKERRQ(ierr);
6189     if (val < 0) {
6190       ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
6191       *offset += fdof;
6192       PetscFunctionReturn(1);
6193     }
6194   }
6195   PetscFunctionReturn(0);
6196 }
6197 
6198 /* Unlike DMPlexVecSetClosure(), this uses plex-native closure permutation, not a user-specified permutation such as DMPlexSetClosurePermutationTensor(). */
6199 PetscErrorCode DMPlexVecSetFieldClosure_Internal(DM dm, PetscSection section, Vec v, PetscBool fieldActive[], PetscInt point, PetscInt Ncc, const PetscInt comps[], DMLabel label, PetscInt labelId, const PetscScalar values[], InsertMode mode)
6200 {
6201   PetscSection      clSection;
6202   IS                clPoints;
6203   PetscScalar       *array;
6204   PetscInt          *points = NULL;
6205   const PetscInt    *clp;
6206   PetscInt          numFields, numPoints, p;
6207   PetscInt          offset = 0, f;
6208   PetscErrorCode    ierr;
6209 
6210   PetscFunctionBeginHot;
6211   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6212   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
6213   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
6214   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
6215   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
6216   /* Get points */
6217   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
6218   /* Get array */
6219   ierr = VecGetArray(v, &array);CHKERRQ(ierr);
6220   /* Get values */
6221   for (f = 0; f < numFields; ++f) {
6222     const PetscInt    **perms = NULL;
6223     const PetscScalar **flips = NULL;
6224 
6225     if (!fieldActive[f]) {
6226       for (p = 0; p < numPoints*2; p += 2) {
6227         PetscInt fdof;
6228         ierr = PetscSectionGetFieldDof(section, points[p], f, &fdof);CHKERRQ(ierr);
6229         offset += fdof;
6230       }
6231       continue;
6232     }
6233     ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
6234     switch (mode) {
6235     case INSERT_VALUES:
6236       for (p = 0; p < numPoints; p++) {
6237         const PetscInt    point = points[2*p];
6238         const PetscInt    *perm = perms ? perms[p] : NULL;
6239         const PetscScalar *flip = flips ? flips[p] : NULL;
6240         ierr = CheckPoint_Private(label, labelId, section, point, f, &offset); if (ierr) continue;
6241         updatePointFields_private(section, point, perm, flip, f, insert, PETSC_FALSE, NULL, values, &offset, array);
6242       } break;
6243     case INSERT_ALL_VALUES:
6244       for (p = 0; p < numPoints; p++) {
6245         const PetscInt    point = points[2*p];
6246         const PetscInt    *perm = perms ? perms[p] : NULL;
6247         const PetscScalar *flip = flips ? flips[p] : NULL;
6248         ierr = CheckPoint_Private(label, labelId, section, point, f, &offset); if (ierr) continue;
6249         updatePointFields_private(section, point, perm, flip, f, insert, PETSC_TRUE, NULL, values, &offset, array);
6250       } break;
6251     case INSERT_BC_VALUES:
6252       for (p = 0; p < numPoints; p++) {
6253         const PetscInt    point = points[2*p];
6254         const PetscInt    *perm = perms ? perms[p] : NULL;
6255         const PetscScalar *flip = flips ? flips[p] : NULL;
6256         ierr = CheckPoint_Private(label, labelId, section, point, f, &offset); if (ierr) continue;
6257         updatePointFieldsBC_private(section, point, perm, flip, f, Ncc, comps, insert, NULL, values, &offset, array);
6258       } break;
6259     case ADD_VALUES:
6260       for (p = 0; p < numPoints; p++) {
6261         const PetscInt    point = points[2*p];
6262         const PetscInt    *perm = perms ? perms[p] : NULL;
6263         const PetscScalar *flip = flips ? flips[p] : NULL;
6264         ierr = CheckPoint_Private(label, labelId, section, point, f, &offset); if (ierr) continue;
6265         updatePointFields_private(section, point, perm, flip, f, add, PETSC_FALSE, NULL, values, &offset, array);
6266       } break;
6267     case ADD_ALL_VALUES:
6268       for (p = 0; p < numPoints; p++) {
6269         const PetscInt    point = points[2*p];
6270         const PetscInt    *perm = perms ? perms[p] : NULL;
6271         const PetscScalar *flip = flips ? flips[p] : NULL;
6272         ierr = CheckPoint_Private(label, labelId, section, point, f, &offset); if (ierr) continue;
6273         updatePointFields_private(section, point, perm, flip, f, add, PETSC_TRUE, NULL, values, &offset, array);
6274       } break;
6275     default:
6276       SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %d", mode);
6277     }
6278     ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
6279   }
6280   /* Cleanup points */
6281   ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
6282   /* Cleanup array */
6283   ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
6284   PetscFunctionReturn(0);
6285 }
6286 
6287 static PetscErrorCode DMPlexPrintMatSetValues(PetscViewer viewer, Mat A, PetscInt point, PetscInt numRIndices, const PetscInt rindices[], PetscInt numCIndices, const PetscInt cindices[], const PetscScalar values[])
6288 {
6289   PetscMPIInt    rank;
6290   PetscInt       i, j;
6291   PetscErrorCode ierr;
6292 
6293   PetscFunctionBegin;
6294   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRMPI(ierr);
6295   ierr = PetscViewerASCIIPrintf(viewer, "[%d]mat for point %D\n", rank, point);CHKERRQ(ierr);
6296   for (i = 0; i < numRIndices; i++) {ierr = PetscViewerASCIIPrintf(viewer, "[%d]mat row indices[%D] = %D\n", rank, i, rindices[i]);CHKERRQ(ierr);}
6297   for (i = 0; i < numCIndices; i++) {ierr = PetscViewerASCIIPrintf(viewer, "[%d]mat col indices[%D] = %D\n", rank, i, cindices[i]);CHKERRQ(ierr);}
6298   numCIndices = numCIndices ? numCIndices : numRIndices;
6299   if (!values) PetscFunctionReturn(0);
6300   for (i = 0; i < numRIndices; i++) {
6301     ierr = PetscViewerASCIIPrintf(viewer, "[%d]", rank);CHKERRQ(ierr);
6302     for (j = 0; j < numCIndices; j++) {
6303 #if defined(PETSC_USE_COMPLEX)
6304       ierr = PetscViewerASCIIPrintf(viewer, " (%g,%g)", (double)PetscRealPart(values[i*numCIndices+j]), (double)PetscImaginaryPart(values[i*numCIndices+j]));CHKERRQ(ierr);
6305 #else
6306       ierr = PetscViewerASCIIPrintf(viewer, " %g", (double)values[i*numCIndices+j]);CHKERRQ(ierr);
6307 #endif
6308     }
6309     ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
6310   }
6311   PetscFunctionReturn(0);
6312 }
6313 
6314 /*
6315   DMPlexGetIndicesPoint_Internal - Add the indices for dofs on a point to an index array
6316 
6317   Input Parameters:
6318 + section - The section for this data layout
6319 . islocal - Is the section (and thus indices being requested) local or global?
6320 . point   - The point contributing dofs with these indices
6321 . off     - The global offset of this point
6322 . loff    - The local offset of each field
6323 . setBC   - The flag determining whether to include indices of bounsary values
6324 . perm    - A permutation of the dofs on this point, or NULL
6325 - indperm - A permutation of the entire indices array, or NULL
6326 
6327   Output Parameter:
6328 . indices - Indices for dofs on this point
6329 
6330   Level: developer
6331 
6332   Note: The indices could be local or global, depending on the value of 'off'.
6333 */
6334 PetscErrorCode DMPlexGetIndicesPoint_Internal(PetscSection section, PetscBool islocal,PetscInt point, PetscInt off, PetscInt *loff, PetscBool setBC, const PetscInt perm[], const PetscInt indperm[], PetscInt indices[])
6335 {
6336   PetscInt        dof;   /* The number of unknowns on this point */
6337   PetscInt        cdof;  /* The number of constraints on this point */
6338   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
6339   PetscInt        cind = 0, k;
6340   PetscErrorCode  ierr;
6341 
6342   PetscFunctionBegin;
6343   if (!islocal && setBC) SETERRQ(PetscObjectComm((PetscObject)section),PETSC_ERR_ARG_INCOMP,"setBC incompatible with global indices; use a local section or disable setBC");
6344   ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
6345   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
6346   if (!cdof || setBC) {
6347     for (k = 0; k < dof; ++k) {
6348       const PetscInt preind = perm ? *loff+perm[k] : *loff+k;
6349       const PetscInt ind    = indperm ? indperm[preind] : preind;
6350 
6351       indices[ind] = off + k;
6352     }
6353   } else {
6354     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
6355     for (k = 0; k < dof; ++k) {
6356       const PetscInt preind = perm ? *loff+perm[k] : *loff+k;
6357       const PetscInt ind    = indperm ? indperm[preind] : preind;
6358 
6359       if ((cind < cdof) && (k == cdofs[cind])) {
6360         /* Insert check for returning constrained indices */
6361         indices[ind] = -(off+k+1);
6362         ++cind;
6363       } else {
6364         indices[ind] = off + k - (islocal ? 0 : cind);
6365       }
6366     }
6367   }
6368   *loff += dof;
6369   PetscFunctionReturn(0);
6370 }
6371 
6372 /*
6373  DMPlexGetIndicesPointFields_Internal - gets section indices for a point in its canonical ordering.
6374 
6375  Input Parameters:
6376 + section - a section (global or local)
6377 - islocal - PETSC_TRUE if requesting local indices (i.e., section is local); PETSC_FALSE for global
6378 . point - point within section
6379 . off - The offset of this point in the (local or global) indexed space - should match islocal and (usually) the section
6380 . foffs - array of length numFields containing the offset in canonical point ordering (the location in indices) of each field
6381 . setBC - identify constrained (boundary condition) points via involution.
6382 . perms - perms[f][permsoff][:] is a permutation of dofs within each field
6383 . permsoff - offset
6384 - indperm - index permutation
6385 
6386  Output Parameter:
6387 . foffs - each entry is incremented by the number of (unconstrained if setBC=FALSE) dofs in that field
6388 . indices - array to hold indices (as defined by section) of each dof associated with point
6389 
6390  Notes:
6391  If section is local and setBC=true, there is no distinction between constrained and unconstrained dofs.
6392  If section is local and setBC=false, the indices for constrained points are the involution -(i+1) of their position
6393  in the local vector.
6394 
6395  If section is global and setBC=false, the indices for constrained points are negative (and their value is not
6396  significant).  It is invalid to call with a global section and setBC=true.
6397 
6398  Developer Note:
6399  The section is only used for field layout, so islocal is technically a statement about the offset (off).  At some point
6400  in the future, global sections may have fields set, in which case we could pass the global section and obtain the
6401  offset could be obtained from the section instead of passing it explicitly as we do now.
6402 
6403  Example:
6404  Suppose a point contains one field with three components, and for which the unconstrained indices are {10, 11, 12}.
6405  When the middle component is constrained, we get the array {10, -12, 12} for (islocal=TRUE, setBC=FALSE).
6406  Note that -12 is the involution of 11, so the user can involute negative indices to recover local indices.
6407  The global vector does not store constrained dofs, so when this function returns global indices, say {110, -112, 111}, the value of -112 is an arbitrary flag that should not be interpreted beyond its sign.
6408 
6409  Level: developer
6410 */
6411 PetscErrorCode DMPlexGetIndicesPointFields_Internal(PetscSection section, PetscBool islocal, PetscInt point, PetscInt off, PetscInt foffs[], PetscBool setBC, const PetscInt ***perms, PetscInt permsoff, const PetscInt indperm[], PetscInt indices[])
6412 {
6413   PetscInt       numFields, foff, f;
6414   PetscErrorCode ierr;
6415 
6416   PetscFunctionBegin;
6417   if (!islocal && setBC) SETERRQ(PetscObjectComm((PetscObject)section),PETSC_ERR_ARG_INCOMP,"setBC incompatible with global indices; use a local section or disable setBC");
6418   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
6419   for (f = 0, foff = 0; f < numFields; ++f) {
6420     PetscInt        fdof, cfdof;
6421     const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
6422     PetscInt        cind = 0, b;
6423     const PetscInt  *perm = (perms && perms[f]) ? perms[f][permsoff] : NULL;
6424 
6425     ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
6426     ierr = PetscSectionGetFieldConstraintDof(section, point, f, &cfdof);CHKERRQ(ierr);
6427     if (!cfdof || setBC) {
6428       for (b = 0; b < fdof; ++b) {
6429         const PetscInt preind = perm ? foffs[f]+perm[b] : foffs[f]+b;
6430         const PetscInt ind    = indperm ? indperm[preind] : preind;
6431 
6432         indices[ind] = off+foff+b;
6433       }
6434     } else {
6435       ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
6436       for (b = 0; b < fdof; ++b) {
6437         const PetscInt preind = perm ? foffs[f]+perm[b] : foffs[f]+b;
6438         const PetscInt ind    = indperm ? indperm[preind] : preind;
6439 
6440         if ((cind < cfdof) && (b == fcdofs[cind])) {
6441           indices[ind] = -(off+foff+b+1);
6442           ++cind;
6443         } else {
6444           indices[ind] = off + foff + b - (islocal ? 0 : cind);
6445         }
6446       }
6447     }
6448     foff     += (setBC || islocal ? fdof : (fdof - cfdof));
6449     foffs[f] += fdof;
6450   }
6451   PetscFunctionReturn(0);
6452 }
6453 
6454 /*
6455   This version believes the globalSection offsets for each field, rather than just the point offset
6456 
6457  . foffs - The offset into 'indices' for each field, since it is segregated by field
6458 
6459  Notes:
6460  The semantics of this function relate to that of setBC=FALSE in DMPlexGetIndicesPointFields_Internal.
6461  Since this function uses global indices, setBC=TRUE would be invalid, so no such argument exists.
6462 */
6463 static PetscErrorCode DMPlexGetIndicesPointFieldsSplit_Internal(PetscSection section, PetscSection globalSection, PetscInt point, PetscInt foffs[], const PetscInt ***perms, PetscInt permsoff, const PetscInt indperm[], PetscInt indices[])
6464 {
6465   PetscInt       numFields, foff, f;
6466   PetscErrorCode ierr;
6467 
6468   PetscFunctionBegin;
6469   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
6470   for (f = 0; f < numFields; ++f) {
6471     PetscInt        fdof, cfdof;
6472     const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
6473     PetscInt        cind = 0, b;
6474     const PetscInt  *perm = (perms && perms[f]) ? perms[f][permsoff] : NULL;
6475 
6476     ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
6477     ierr = PetscSectionGetFieldConstraintDof(section, point, f, &cfdof);CHKERRQ(ierr);
6478     ierr = PetscSectionGetFieldOffset(globalSection, point, f, &foff);CHKERRQ(ierr);
6479     if (!cfdof) {
6480       for (b = 0; b < fdof; ++b) {
6481         const PetscInt preind = perm ? foffs[f]+perm[b] : foffs[f]+b;
6482         const PetscInt ind    = indperm ? indperm[preind] : preind;
6483 
6484         indices[ind] = foff+b;
6485       }
6486     } else {
6487       ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
6488       for (b = 0; b < fdof; ++b) {
6489         const PetscInt preind = perm ? foffs[f]+perm[b] : foffs[f]+b;
6490         const PetscInt ind    = indperm ? indperm[preind] : preind;
6491 
6492         if ((cind < cfdof) && (b == fcdofs[cind])) {
6493           indices[ind] = -(foff+b+1);
6494           ++cind;
6495         } else {
6496           indices[ind] = foff+b-cind;
6497         }
6498       }
6499     }
6500     foffs[f] += fdof;
6501   }
6502   PetscFunctionReturn(0);
6503 }
6504 
6505 PetscErrorCode DMPlexAnchorsModifyMat(DM dm, PetscSection section, PetscInt numPoints, PetscInt numIndices, const PetscInt points[], const PetscInt ***perms, const PetscScalar values[], PetscInt *outNumPoints, PetscInt *outNumIndices, PetscInt *outPoints[], PetscScalar *outValues[], PetscInt offsets[], PetscBool multiplyLeft)
6506 {
6507   Mat             cMat;
6508   PetscSection    aSec, cSec;
6509   IS              aIS;
6510   PetscInt        aStart = -1, aEnd = -1;
6511   const PetscInt  *anchors;
6512   PetscInt        numFields, f, p, q, newP = 0;
6513   PetscInt        newNumPoints = 0, newNumIndices = 0;
6514   PetscInt        *newPoints, *indices, *newIndices;
6515   PetscInt        maxAnchor, maxDof;
6516   PetscInt        newOffsets[32];
6517   PetscInt        *pointMatOffsets[32];
6518   PetscInt        *newPointOffsets[32];
6519   PetscScalar     *pointMat[32];
6520   PetscScalar     *newValues=NULL,*tmpValues;
6521   PetscBool       anyConstrained = PETSC_FALSE;
6522   PetscErrorCode  ierr;
6523 
6524   PetscFunctionBegin;
6525   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6526   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
6527   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
6528 
6529   ierr = DMPlexGetAnchors(dm,&aSec,&aIS);CHKERRQ(ierr);
6530   /* if there are point-to-point constraints */
6531   if (aSec) {
6532     ierr = PetscArrayzero(newOffsets, 32);CHKERRQ(ierr);
6533     ierr = ISGetIndices(aIS,&anchors);CHKERRQ(ierr);
6534     ierr = PetscSectionGetChart(aSec,&aStart,&aEnd);CHKERRQ(ierr);
6535     /* figure out how many points are going to be in the new element matrix
6536      * (we allow double counting, because it's all just going to be summed
6537      * into the global matrix anyway) */
6538     for (p = 0; p < 2*numPoints; p+=2) {
6539       PetscInt b    = points[p];
6540       PetscInt bDof = 0, bSecDof;
6541 
6542       ierr = PetscSectionGetDof(section,b,&bSecDof);CHKERRQ(ierr);
6543       if (!bSecDof) {
6544         continue;
6545       }
6546       if (b >= aStart && b < aEnd) {
6547         ierr = PetscSectionGetDof(aSec,b,&bDof);CHKERRQ(ierr);
6548       }
6549       if (bDof) {
6550         /* this point is constrained */
6551         /* it is going to be replaced by its anchors */
6552         PetscInt bOff, q;
6553 
6554         anyConstrained = PETSC_TRUE;
6555         newNumPoints  += bDof;
6556         ierr = PetscSectionGetOffset(aSec,b,&bOff);CHKERRQ(ierr);
6557         for (q = 0; q < bDof; q++) {
6558           PetscInt a = anchors[bOff + q];
6559           PetscInt aDof;
6560 
6561           ierr           = PetscSectionGetDof(section,a,&aDof);CHKERRQ(ierr);
6562           newNumIndices += aDof;
6563           for (f = 0; f < numFields; ++f) {
6564             PetscInt fDof;
6565 
6566             ierr             = PetscSectionGetFieldDof(section, a, f, &fDof);CHKERRQ(ierr);
6567             newOffsets[f+1] += fDof;
6568           }
6569         }
6570       }
6571       else {
6572         /* this point is not constrained */
6573         newNumPoints++;
6574         newNumIndices += bSecDof;
6575         for (f = 0; f < numFields; ++f) {
6576           PetscInt fDof;
6577 
6578           ierr = PetscSectionGetFieldDof(section, b, f, &fDof);CHKERRQ(ierr);
6579           newOffsets[f+1] += fDof;
6580         }
6581       }
6582     }
6583   }
6584   if (!anyConstrained) {
6585     if (outNumPoints)  *outNumPoints  = 0;
6586     if (outNumIndices) *outNumIndices = 0;
6587     if (outPoints)     *outPoints     = NULL;
6588     if (outValues)     *outValues     = NULL;
6589     if (aSec) {ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);}
6590     PetscFunctionReturn(0);
6591   }
6592 
6593   if (outNumPoints)  *outNumPoints  = newNumPoints;
6594   if (outNumIndices) *outNumIndices = newNumIndices;
6595 
6596   for (f = 0; f < numFields; ++f) newOffsets[f+1] += newOffsets[f];
6597 
6598   if (!outPoints && !outValues) {
6599     if (offsets) {
6600       for (f = 0; f <= numFields; f++) {
6601         offsets[f] = newOffsets[f];
6602       }
6603     }
6604     if (aSec) {ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);}
6605     PetscFunctionReturn(0);
6606   }
6607 
6608   if (numFields && newOffsets[numFields] != newNumIndices) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", newOffsets[numFields], newNumIndices);
6609 
6610   ierr = DMGetDefaultConstraints(dm, &cSec, &cMat);CHKERRQ(ierr);
6611 
6612   /* workspaces */
6613   if (numFields) {
6614     for (f = 0; f < numFields; f++) {
6615       ierr = DMGetWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[f]);CHKERRQ(ierr);
6616       ierr = DMGetWorkArray(dm,numPoints+1,MPIU_INT,&newPointOffsets[f]);CHKERRQ(ierr);
6617     }
6618   }
6619   else {
6620     ierr = DMGetWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[0]);CHKERRQ(ierr);
6621     ierr = DMGetWorkArray(dm,numPoints,MPIU_INT,&newPointOffsets[0]);CHKERRQ(ierr);
6622   }
6623 
6624   /* get workspaces for the point-to-point matrices */
6625   if (numFields) {
6626     PetscInt totalOffset, totalMatOffset;
6627 
6628     for (p = 0; p < numPoints; p++) {
6629       PetscInt b    = points[2*p];
6630       PetscInt bDof = 0, bSecDof;
6631 
6632       ierr = PetscSectionGetDof(section,b,&bSecDof);CHKERRQ(ierr);
6633       if (!bSecDof) {
6634         for (f = 0; f < numFields; f++) {
6635           newPointOffsets[f][p + 1] = 0;
6636           pointMatOffsets[f][p + 1] = 0;
6637         }
6638         continue;
6639       }
6640       if (b >= aStart && b < aEnd) {
6641         ierr = PetscSectionGetDof(aSec, b, &bDof);CHKERRQ(ierr);
6642       }
6643       if (bDof) {
6644         for (f = 0; f < numFields; f++) {
6645           PetscInt fDof, q, bOff, allFDof = 0;
6646 
6647           ierr = PetscSectionGetFieldDof(section, b, f, &fDof);CHKERRQ(ierr);
6648           ierr = PetscSectionGetOffset(aSec, b, &bOff);CHKERRQ(ierr);
6649           for (q = 0; q < bDof; q++) {
6650             PetscInt a = anchors[bOff + q];
6651             PetscInt aFDof;
6652 
6653             ierr     = PetscSectionGetFieldDof(section, a, f, &aFDof);CHKERRQ(ierr);
6654             allFDof += aFDof;
6655           }
6656           newPointOffsets[f][p+1] = allFDof;
6657           pointMatOffsets[f][p+1] = fDof * allFDof;
6658         }
6659       }
6660       else {
6661         for (f = 0; f < numFields; f++) {
6662           PetscInt fDof;
6663 
6664           ierr = PetscSectionGetFieldDof(section, b, f, &fDof);CHKERRQ(ierr);
6665           newPointOffsets[f][p+1] = fDof;
6666           pointMatOffsets[f][p+1] = 0;
6667         }
6668       }
6669     }
6670     for (f = 0, totalOffset = 0, totalMatOffset = 0; f < numFields; f++) {
6671       newPointOffsets[f][0] = totalOffset;
6672       pointMatOffsets[f][0] = totalMatOffset;
6673       for (p = 0; p < numPoints; p++) {
6674         newPointOffsets[f][p+1] += newPointOffsets[f][p];
6675         pointMatOffsets[f][p+1] += pointMatOffsets[f][p];
6676       }
6677       totalOffset    = newPointOffsets[f][numPoints];
6678       totalMatOffset = pointMatOffsets[f][numPoints];
6679       ierr = DMGetWorkArray(dm,pointMatOffsets[f][numPoints],MPIU_SCALAR,&pointMat[f]);CHKERRQ(ierr);
6680     }
6681   }
6682   else {
6683     for (p = 0; p < numPoints; p++) {
6684       PetscInt b    = points[2*p];
6685       PetscInt bDof = 0, bSecDof;
6686 
6687       ierr = PetscSectionGetDof(section,b,&bSecDof);CHKERRQ(ierr);
6688       if (!bSecDof) {
6689         newPointOffsets[0][p + 1] = 0;
6690         pointMatOffsets[0][p + 1] = 0;
6691         continue;
6692       }
6693       if (b >= aStart && b < aEnd) {
6694         ierr = PetscSectionGetDof(aSec, b, &bDof);CHKERRQ(ierr);
6695       }
6696       if (bDof) {
6697         PetscInt bOff, q, allDof = 0;
6698 
6699         ierr = PetscSectionGetOffset(aSec, b, &bOff);CHKERRQ(ierr);
6700         for (q = 0; q < bDof; q++) {
6701           PetscInt a = anchors[bOff + q], aDof;
6702 
6703           ierr    = PetscSectionGetDof(section, a, &aDof);CHKERRQ(ierr);
6704           allDof += aDof;
6705         }
6706         newPointOffsets[0][p+1] = allDof;
6707         pointMatOffsets[0][p+1] = bSecDof * allDof;
6708       }
6709       else {
6710         newPointOffsets[0][p+1] = bSecDof;
6711         pointMatOffsets[0][p+1] = 0;
6712       }
6713     }
6714     newPointOffsets[0][0] = 0;
6715     pointMatOffsets[0][0] = 0;
6716     for (p = 0; p < numPoints; p++) {
6717       newPointOffsets[0][p+1] += newPointOffsets[0][p];
6718       pointMatOffsets[0][p+1] += pointMatOffsets[0][p];
6719     }
6720     ierr = DMGetWorkArray(dm,pointMatOffsets[0][numPoints],MPIU_SCALAR,&pointMat[0]);CHKERRQ(ierr);
6721   }
6722 
6723   /* output arrays */
6724   ierr = DMGetWorkArray(dm,2*newNumPoints,MPIU_INT,&newPoints);CHKERRQ(ierr);
6725 
6726   /* get the point-to-point matrices; construct newPoints */
6727   ierr = PetscSectionGetMaxDof(aSec, &maxAnchor);CHKERRQ(ierr);
6728   ierr = PetscSectionGetMaxDof(section, &maxDof);CHKERRQ(ierr);
6729   ierr = DMGetWorkArray(dm,maxDof,MPIU_INT,&indices);CHKERRQ(ierr);
6730   ierr = DMGetWorkArray(dm,maxAnchor*maxDof,MPIU_INT,&newIndices);CHKERRQ(ierr);
6731   if (numFields) {
6732     for (p = 0, newP = 0; p < numPoints; p++) {
6733       PetscInt b    = points[2*p];
6734       PetscInt o    = points[2*p+1];
6735       PetscInt bDof = 0, bSecDof;
6736 
6737       ierr = PetscSectionGetDof(section, b, &bSecDof);CHKERRQ(ierr);
6738       if (!bSecDof) {
6739         continue;
6740       }
6741       if (b >= aStart && b < aEnd) {
6742         ierr = PetscSectionGetDof(aSec, b, &bDof);CHKERRQ(ierr);
6743       }
6744       if (bDof) {
6745         PetscInt fStart[32], fEnd[32], fAnchorStart[32], fAnchorEnd[32], bOff, q;
6746 
6747         fStart[0] = 0;
6748         fEnd[0]   = 0;
6749         for (f = 0; f < numFields; f++) {
6750           PetscInt fDof;
6751 
6752           ierr        = PetscSectionGetFieldDof(cSec, b, f, &fDof);CHKERRQ(ierr);
6753           fStart[f+1] = fStart[f] + fDof;
6754           fEnd[f+1]   = fStart[f+1];
6755         }
6756         ierr = PetscSectionGetOffset(cSec, b, &bOff);CHKERRQ(ierr);
6757         ierr = DMPlexGetIndicesPointFields_Internal(cSec, PETSC_TRUE, b, bOff, fEnd, PETSC_TRUE, perms, p, NULL, indices);CHKERRQ(ierr);
6758 
6759         fAnchorStart[0] = 0;
6760         fAnchorEnd[0]   = 0;
6761         for (f = 0; f < numFields; f++) {
6762           PetscInt fDof = newPointOffsets[f][p + 1] - newPointOffsets[f][p];
6763 
6764           fAnchorStart[f+1] = fAnchorStart[f] + fDof;
6765           fAnchorEnd[f+1]   = fAnchorStart[f + 1];
6766         }
6767         ierr = PetscSectionGetOffset(aSec, b, &bOff);CHKERRQ(ierr);
6768         for (q = 0; q < bDof; q++) {
6769           PetscInt a = anchors[bOff + q], aOff;
6770 
6771           /* we take the orientation of ap into account in the order that we constructed the indices above: the newly added points have no orientation */
6772           newPoints[2*(newP + q)]     = a;
6773           newPoints[2*(newP + q) + 1] = 0;
6774           ierr = PetscSectionGetOffset(section, a, &aOff);CHKERRQ(ierr);
6775           ierr = DMPlexGetIndicesPointFields_Internal(section, PETSC_TRUE, a, aOff, fAnchorEnd, PETSC_TRUE, NULL, -1, NULL, newIndices);CHKERRQ(ierr);
6776         }
6777         newP += bDof;
6778 
6779         if (outValues) {
6780           /* get the point-to-point submatrix */
6781           for (f = 0; f < numFields; f++) {
6782             ierr = MatGetValues(cMat,fEnd[f]-fStart[f],indices + fStart[f],fAnchorEnd[f] - fAnchorStart[f],newIndices + fAnchorStart[f],pointMat[f] + pointMatOffsets[f][p]);CHKERRQ(ierr);
6783           }
6784         }
6785       }
6786       else {
6787         newPoints[2 * newP]     = b;
6788         newPoints[2 * newP + 1] = o;
6789         newP++;
6790       }
6791     }
6792   } else {
6793     for (p = 0; p < numPoints; p++) {
6794       PetscInt b    = points[2*p];
6795       PetscInt o    = points[2*p+1];
6796       PetscInt bDof = 0, bSecDof;
6797 
6798       ierr = PetscSectionGetDof(section, b, &bSecDof);CHKERRQ(ierr);
6799       if (!bSecDof) {
6800         continue;
6801       }
6802       if (b >= aStart && b < aEnd) {
6803         ierr = PetscSectionGetDof(aSec, b, &bDof);CHKERRQ(ierr);
6804       }
6805       if (bDof) {
6806         PetscInt bEnd = 0, bAnchorEnd = 0, bOff;
6807 
6808         ierr = PetscSectionGetOffset(cSec, b, &bOff);CHKERRQ(ierr);
6809         ierr = DMPlexGetIndicesPoint_Internal(cSec, PETSC_TRUE, b, bOff, &bEnd, PETSC_TRUE, (perms && perms[0]) ? perms[0][p] : NULL, NULL, indices);CHKERRQ(ierr);
6810 
6811         ierr = PetscSectionGetOffset (aSec, b, &bOff);CHKERRQ(ierr);
6812         for (q = 0; q < bDof; q++) {
6813           PetscInt a = anchors[bOff + q], aOff;
6814 
6815           /* we take the orientation of ap into account in the order that we constructed the indices above: the newly added points have no orientation */
6816 
6817           newPoints[2*(newP + q)]     = a;
6818           newPoints[2*(newP + q) + 1] = 0;
6819           ierr = PetscSectionGetOffset(section, a, &aOff);CHKERRQ(ierr);
6820           ierr = DMPlexGetIndicesPoint_Internal(section, PETSC_TRUE, a, aOff, &bAnchorEnd, PETSC_TRUE, NULL, NULL, newIndices);CHKERRQ(ierr);
6821         }
6822         newP += bDof;
6823 
6824         /* get the point-to-point submatrix */
6825         if (outValues) {
6826           ierr = MatGetValues(cMat,bEnd,indices,bAnchorEnd,newIndices,pointMat[0] + pointMatOffsets[0][p]);CHKERRQ(ierr);
6827         }
6828       }
6829       else {
6830         newPoints[2 * newP]     = b;
6831         newPoints[2 * newP + 1] = o;
6832         newP++;
6833       }
6834     }
6835   }
6836 
6837   if (outValues) {
6838     ierr = DMGetWorkArray(dm,newNumIndices*numIndices,MPIU_SCALAR,&tmpValues);CHKERRQ(ierr);
6839     ierr = PetscArrayzero(tmpValues,newNumIndices*numIndices);CHKERRQ(ierr);
6840     /* multiply constraints on the right */
6841     if (numFields) {
6842       for (f = 0; f < numFields; f++) {
6843         PetscInt oldOff = offsets[f];
6844 
6845         for (p = 0; p < numPoints; p++) {
6846           PetscInt cStart = newPointOffsets[f][p];
6847           PetscInt b      = points[2 * p];
6848           PetscInt c, r, k;
6849           PetscInt dof;
6850 
6851           ierr = PetscSectionGetFieldDof(section,b,f,&dof);CHKERRQ(ierr);
6852           if (!dof) {
6853             continue;
6854           }
6855           if (pointMatOffsets[f][p] < pointMatOffsets[f][p + 1]) {
6856             PetscInt nCols         = newPointOffsets[f][p+1]-cStart;
6857             const PetscScalar *mat = pointMat[f] + pointMatOffsets[f][p];
6858 
6859             for (r = 0; r < numIndices; r++) {
6860               for (c = 0; c < nCols; c++) {
6861                 for (k = 0; k < dof; k++) {
6862                   tmpValues[r * newNumIndices + cStart + c] += values[r * numIndices + oldOff + k] * mat[k * nCols + c];
6863                 }
6864               }
6865             }
6866           }
6867           else {
6868             /* copy this column as is */
6869             for (r = 0; r < numIndices; r++) {
6870               for (c = 0; c < dof; c++) {
6871                 tmpValues[r * newNumIndices + cStart + c] = values[r * numIndices + oldOff + c];
6872               }
6873             }
6874           }
6875           oldOff += dof;
6876         }
6877       }
6878     }
6879     else {
6880       PetscInt oldOff = 0;
6881       for (p = 0; p < numPoints; p++) {
6882         PetscInt cStart = newPointOffsets[0][p];
6883         PetscInt b      = points[2 * p];
6884         PetscInt c, r, k;
6885         PetscInt dof;
6886 
6887         ierr = PetscSectionGetDof(section,b,&dof);CHKERRQ(ierr);
6888         if (!dof) {
6889           continue;
6890         }
6891         if (pointMatOffsets[0][p] < pointMatOffsets[0][p + 1]) {
6892           PetscInt nCols         = newPointOffsets[0][p+1]-cStart;
6893           const PetscScalar *mat = pointMat[0] + pointMatOffsets[0][p];
6894 
6895           for (r = 0; r < numIndices; r++) {
6896             for (c = 0; c < nCols; c++) {
6897               for (k = 0; k < dof; k++) {
6898                 tmpValues[r * newNumIndices + cStart + c] += mat[k * nCols + c] * values[r * numIndices + oldOff + k];
6899               }
6900             }
6901           }
6902         }
6903         else {
6904           /* copy this column as is */
6905           for (r = 0; r < numIndices; r++) {
6906             for (c = 0; c < dof; c++) {
6907               tmpValues[r * newNumIndices + cStart + c] = values[r * numIndices + oldOff + c];
6908             }
6909           }
6910         }
6911         oldOff += dof;
6912       }
6913     }
6914 
6915     if (multiplyLeft) {
6916       ierr = DMGetWorkArray(dm,newNumIndices*newNumIndices,MPIU_SCALAR,&newValues);CHKERRQ(ierr);
6917       ierr = PetscArrayzero(newValues,newNumIndices*newNumIndices);CHKERRQ(ierr);
6918       /* multiply constraints transpose on the left */
6919       if (numFields) {
6920         for (f = 0; f < numFields; f++) {
6921           PetscInt oldOff = offsets[f];
6922 
6923           for (p = 0; p < numPoints; p++) {
6924             PetscInt rStart = newPointOffsets[f][p];
6925             PetscInt b      = points[2 * p];
6926             PetscInt c, r, k;
6927             PetscInt dof;
6928 
6929             ierr = PetscSectionGetFieldDof(section,b,f,&dof);CHKERRQ(ierr);
6930             if (pointMatOffsets[f][p] < pointMatOffsets[f][p + 1]) {
6931               PetscInt nRows                        = newPointOffsets[f][p+1]-rStart;
6932               const PetscScalar *PETSC_RESTRICT mat = pointMat[f] + pointMatOffsets[f][p];
6933 
6934               for (r = 0; r < nRows; r++) {
6935                 for (c = 0; c < newNumIndices; c++) {
6936                   for (k = 0; k < dof; k++) {
6937                     newValues[(rStart + r) * newNumIndices + c] += mat[k * nRows + r] * tmpValues[(oldOff + k) * newNumIndices + c];
6938                   }
6939                 }
6940               }
6941             }
6942             else {
6943               /* copy this row as is */
6944               for (r = 0; r < dof; r++) {
6945                 for (c = 0; c < newNumIndices; c++) {
6946                   newValues[(rStart + r) * newNumIndices + c] = tmpValues[(oldOff + r) * newNumIndices + c];
6947                 }
6948               }
6949             }
6950             oldOff += dof;
6951           }
6952         }
6953       }
6954       else {
6955         PetscInt oldOff = 0;
6956 
6957         for (p = 0; p < numPoints; p++) {
6958           PetscInt rStart = newPointOffsets[0][p];
6959           PetscInt b      = points[2 * p];
6960           PetscInt c, r, k;
6961           PetscInt dof;
6962 
6963           ierr = PetscSectionGetDof(section,b,&dof);CHKERRQ(ierr);
6964           if (pointMatOffsets[0][p] < pointMatOffsets[0][p + 1]) {
6965             PetscInt nRows                        = newPointOffsets[0][p+1]-rStart;
6966             const PetscScalar *PETSC_RESTRICT mat = pointMat[0] + pointMatOffsets[0][p];
6967 
6968             for (r = 0; r < nRows; r++) {
6969               for (c = 0; c < newNumIndices; c++) {
6970                 for (k = 0; k < dof; k++) {
6971                   newValues[(rStart + r) * newNumIndices + c] += mat[k * nRows + r] * tmpValues[(oldOff + k) * newNumIndices + c];
6972                 }
6973               }
6974             }
6975           }
6976           else {
6977             /* copy this row as is */
6978             for (r = 0; r < dof; r++) {
6979               for (c = 0; c < newNumIndices; c++) {
6980                 newValues[(rStart + r) * newNumIndices + c] = tmpValues[(oldOff + r) * newNumIndices + c];
6981               }
6982             }
6983           }
6984           oldOff += dof;
6985         }
6986       }
6987 
6988       ierr = DMRestoreWorkArray(dm,newNumIndices*numIndices,MPIU_SCALAR,&tmpValues);CHKERRQ(ierr);
6989     }
6990     else {
6991       newValues = tmpValues;
6992     }
6993   }
6994 
6995   /* clean up */
6996   ierr = DMRestoreWorkArray(dm,maxDof,MPIU_INT,&indices);CHKERRQ(ierr);
6997   ierr = DMRestoreWorkArray(dm,maxAnchor*maxDof,MPIU_INT,&newIndices);CHKERRQ(ierr);
6998 
6999   if (numFields) {
7000     for (f = 0; f < numFields; f++) {
7001       ierr = DMRestoreWorkArray(dm,pointMatOffsets[f][numPoints],MPIU_SCALAR,&pointMat[f]);CHKERRQ(ierr);
7002       ierr = DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[f]);CHKERRQ(ierr);
7003       ierr = DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&newPointOffsets[f]);CHKERRQ(ierr);
7004     }
7005   }
7006   else {
7007     ierr = DMRestoreWorkArray(dm,pointMatOffsets[0][numPoints],MPIU_SCALAR,&pointMat[0]);CHKERRQ(ierr);
7008     ierr = DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[0]);CHKERRQ(ierr);
7009     ierr = DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&newPointOffsets[0]);CHKERRQ(ierr);
7010   }
7011   ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);
7012 
7013   /* output */
7014   if (outPoints) {
7015     *outPoints = newPoints;
7016   }
7017   else {
7018     ierr = DMRestoreWorkArray(dm,2*newNumPoints,MPIU_INT,&newPoints);CHKERRQ(ierr);
7019   }
7020   if (outValues) {
7021     *outValues = newValues;
7022   }
7023   for (f = 0; f <= numFields; f++) {
7024     offsets[f] = newOffsets[f];
7025   }
7026   PetscFunctionReturn(0);
7027 }
7028 
7029 /*@C
7030   DMPlexGetClosureIndices - Gets the global dof indices associated with the closure of the given point within the provided sections.
7031 
7032   Not collective
7033 
7034   Input Parameters:
7035 + dm         - The DM
7036 . section    - The PetscSection describing the points (a local section)
7037 . idxSection - The PetscSection from which to obtain indices (may be local or global)
7038 . point      - The point defining the closure
7039 - useClPerm  - Use the closure point permutation if available
7040 
7041   Output Parameters:
7042 + numIndices - The number of dof indices in the closure of point with the input sections
7043 . indices    - The dof indices
7044 . outOffsets - Array to write the field offsets into, or NULL
7045 - values     - The input values, which may be modified if sign flips are induced by the point symmetries, or NULL
7046 
7047   Notes:
7048   Must call DMPlexRestoreClosureIndices() to free allocated memory
7049 
7050   If idxSection is global, any constrained dofs (see DMAddBoundary(), for example) will get negative indices.  The value
7051   of those indices is not significant.  If idxSection is local, the constrained dofs will yield the involution -(idx+1)
7052   of their index in a local vector.  A caller who does not wish to distinguish those points may recover the nonnegative
7053   indices via involution, -(-(idx+1)+1)==idx.  Local indices are provided when idxSection == section, otherwise global
7054   indices (with the above semantics) are implied.
7055 
7056   Level: advanced
7057 
7058 .seealso DMPlexRestoreClosureIndices(), DMPlexVecGetClosure(), DMPlexMatSetClosure(), DMGetLocalSection(), DMGetGlobalSection()
7059 @*/
7060 PetscErrorCode DMPlexGetClosureIndices(DM dm, PetscSection section, PetscSection idxSection, PetscInt point, PetscBool useClPerm,
7061                                        PetscInt *numIndices, PetscInt *indices[], PetscInt outOffsets[], PetscScalar *values[])
7062 {
7063   /* Closure ordering */
7064   PetscSection        clSection;
7065   IS                  clPoints;
7066   const PetscInt     *clp;
7067   PetscInt           *points;
7068   const PetscInt     *clperm = NULL;
7069   /* Dof permutation and sign flips */
7070   const PetscInt    **perms[32] = {NULL};
7071   const PetscScalar **flips[32] = {NULL};
7072   PetscScalar        *valCopy   = NULL;
7073   /* Hanging node constraints */
7074   PetscInt           *pointsC = NULL;
7075   PetscScalar        *valuesC = NULL;
7076   PetscInt            NclC, NiC;
7077 
7078   PetscInt           *idx;
7079   PetscInt            Nf, Ncl, Ni = 0, offsets[32], p, f;
7080   PetscBool           isLocal = (section == idxSection) ? PETSC_TRUE : PETSC_FALSE;
7081   PetscErrorCode      ierr;
7082 
7083   PetscFunctionBeginHot;
7084   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7085   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
7086   PetscValidHeaderSpecific(idxSection, PETSC_SECTION_CLASSID, 3);
7087   if (numIndices) PetscValidPointer(numIndices, 6);
7088   if (indices)    PetscValidPointer(indices, 7);
7089   if (outOffsets) PetscValidPointer(outOffsets, 8);
7090   if (values)     PetscValidPointer(values, 9);
7091   ierr = PetscSectionGetNumFields(section, &Nf);CHKERRQ(ierr);
7092   if (Nf > 31) SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", Nf);
7093   ierr = PetscArrayzero(offsets, 32);CHKERRQ(ierr);
7094   /* 1) Get points in closure */
7095   ierr = DMPlexGetCompressedClosure(dm, section, point, &Ncl, &points, &clSection, &clPoints, &clp);CHKERRQ(ierr);
7096   if (useClPerm) {
7097     PetscInt depth, clsize;
7098     ierr = DMPlexGetPointDepth(dm, point, &depth);CHKERRQ(ierr);
7099     for (clsize=0,p=0; p<Ncl; p++) {
7100       PetscInt dof;
7101       ierr = PetscSectionGetDof(section, points[2*p], &dof);CHKERRQ(ierr);
7102       clsize += dof;
7103     }
7104     ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, depth, clsize, &clperm);CHKERRQ(ierr);
7105   }
7106   /* 2) Get number of indices on these points and field offsets from section */
7107   for (p = 0; p < Ncl*2; p += 2) {
7108     PetscInt dof, fdof;
7109 
7110     ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
7111     for (f = 0; f < Nf; ++f) {
7112       ierr = PetscSectionGetFieldDof(section, points[p], f, &fdof);CHKERRQ(ierr);
7113       offsets[f+1] += fdof;
7114     }
7115     Ni += dof;
7116   }
7117   for (f = 1; f < Nf; ++f) offsets[f+1] += offsets[f];
7118   if (Nf && offsets[Nf] != Ni) SETERRQ2(PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", offsets[Nf], Ni);
7119   /* 3) Get symmetries and sign flips. Apply sign flips to values if passed in (only works for square values matrix) */
7120   for (f = 0; f < PetscMax(1, Nf); ++f) {
7121     if (Nf) {ierr = PetscSectionGetFieldPointSyms(section, f, Ncl, points, &perms[f], &flips[f]);CHKERRQ(ierr);}
7122     else    {ierr = PetscSectionGetPointSyms(section, Ncl, points, &perms[f], &flips[f]);CHKERRQ(ierr);}
7123     /* may need to apply sign changes to the element matrix */
7124     if (values && flips[f]) {
7125       PetscInt foffset = offsets[f];
7126 
7127       for (p = 0; p < Ncl; ++p) {
7128         PetscInt           pnt  = points[2*p], fdof;
7129         const PetscScalar *flip = flips[f] ? flips[f][p] : NULL;
7130 
7131         if (!Nf) {ierr = PetscSectionGetDof(section, pnt, &fdof);CHKERRQ(ierr);}
7132         else     {ierr = PetscSectionGetFieldDof(section, pnt, f, &fdof);CHKERRQ(ierr);}
7133         if (flip) {
7134           PetscInt i, j, k;
7135 
7136           if (!valCopy) {
7137             ierr = DMGetWorkArray(dm, Ni*Ni, MPIU_SCALAR, &valCopy);CHKERRQ(ierr);
7138             for (j = 0; j < Ni * Ni; ++j) valCopy[j] = (*values)[j];
7139             *values = valCopy;
7140           }
7141           for (i = 0; i < fdof; ++i) {
7142             PetscScalar fval = flip[i];
7143 
7144             for (k = 0; k < Ni; ++k) {
7145               valCopy[Ni * (foffset + i) + k] *= fval;
7146               valCopy[Ni * k + (foffset + i)] *= fval;
7147             }
7148           }
7149         }
7150         foffset += fdof;
7151       }
7152     }
7153   }
7154   /* 4) Apply hanging node constraints. Get new symmetries and replace all storage with constrained storage */
7155   ierr = DMPlexAnchorsModifyMat(dm, section, Ncl, Ni, points, perms, values ? *values : NULL, &NclC, &NiC, &pointsC, values ? &valuesC : NULL, offsets, PETSC_TRUE);CHKERRQ(ierr);
7156   if (NclC) {
7157     if (valCopy) {ierr = DMRestoreWorkArray(dm, Ni*Ni, MPIU_SCALAR, &valCopy);CHKERRQ(ierr);}
7158     for (f = 0; f < PetscMax(1, Nf); ++f) {
7159       if (Nf) {ierr = PetscSectionRestoreFieldPointSyms(section, f, Ncl, points, &perms[f], &flips[f]);CHKERRQ(ierr);}
7160       else    {ierr = PetscSectionRestorePointSyms(section, Ncl, points, &perms[f], &flips[f]);CHKERRQ(ierr);}
7161     }
7162     for (f = 0; f < PetscMax(1, Nf); ++f) {
7163       if (Nf) {ierr = PetscSectionGetFieldPointSyms(section, f, NclC, pointsC, &perms[f], &flips[f]);CHKERRQ(ierr);}
7164       else    {ierr = PetscSectionGetPointSyms(section, NclC, pointsC, &perms[f], &flips[f]);CHKERRQ(ierr);}
7165     }
7166     ierr = DMPlexRestoreCompressedClosure(dm, section, point, &Ncl, &points, &clSection, &clPoints, &clp);CHKERRQ(ierr);
7167     Ncl     = NclC;
7168     Ni      = NiC;
7169     points  = pointsC;
7170     if (values) *values = valuesC;
7171   }
7172   /* 5) Calculate indices */
7173   ierr = DMGetWorkArray(dm, Ni, MPIU_INT, &idx);CHKERRQ(ierr);
7174   if (Nf) {
7175     PetscInt  idxOff;
7176     PetscBool useFieldOffsets;
7177 
7178     if (outOffsets) {for (f = 0; f <= Nf; f++) outOffsets[f] = offsets[f];}
7179     ierr = PetscSectionGetUseFieldOffsets(idxSection, &useFieldOffsets);CHKERRQ(ierr);
7180     if (useFieldOffsets) {
7181       for (p = 0; p < Ncl; ++p) {
7182         const PetscInt pnt = points[p*2];
7183 
7184         ierr = DMPlexGetIndicesPointFieldsSplit_Internal(section, idxSection, pnt, offsets, perms, p, clperm, idx);CHKERRQ(ierr);
7185       }
7186     } else {
7187       for (p = 0; p < Ncl; ++p) {
7188         const PetscInt pnt = points[p*2];
7189 
7190         ierr = PetscSectionGetOffset(idxSection, pnt, &idxOff);CHKERRQ(ierr);
7191         /* Note that we pass a local section even though we're using global offsets.  This is because global sections do
7192          * not (at the time of this writing) have fields set. They probably should, in which case we would pass the
7193          * global section. */
7194         ierr = DMPlexGetIndicesPointFields_Internal(section, isLocal, pnt, idxOff < 0 ? -(idxOff+1) : idxOff, offsets, PETSC_FALSE, perms, p, clperm, idx);CHKERRQ(ierr);
7195       }
7196     }
7197   } else {
7198     PetscInt off = 0, idxOff;
7199 
7200     for (p = 0; p < Ncl; ++p) {
7201       const PetscInt  pnt  = points[p*2];
7202       const PetscInt *perm = perms[0] ? perms[0][p] : NULL;
7203 
7204       ierr = PetscSectionGetOffset(idxSection, pnt, &idxOff);CHKERRQ(ierr);
7205       /* Note that we pass a local section even though we're using global offsets.  This is because global sections do
7206        * not (at the time of this writing) have fields set. They probably should, in which case we would pass the global section. */
7207       ierr = DMPlexGetIndicesPoint_Internal(section, isLocal, pnt, idxOff < 0 ? -(idxOff+1) : idxOff, &off, PETSC_FALSE, perm, clperm, idx);CHKERRQ(ierr);
7208     }
7209   }
7210   /* 6) Cleanup */
7211   for (f = 0; f < PetscMax(1, Nf); ++f) {
7212     if (Nf) {ierr = PetscSectionRestoreFieldPointSyms(section, f, Ncl, points, &perms[f], &flips[f]);CHKERRQ(ierr);}
7213     else    {ierr = PetscSectionRestorePointSyms(section, Ncl, points, &perms[f], &flips[f]);CHKERRQ(ierr);}
7214   }
7215   if (NclC) {
7216     ierr = DMRestoreWorkArray(dm, NclC*2, MPIU_INT, &pointsC);CHKERRQ(ierr);
7217   } else {
7218     ierr = DMPlexRestoreCompressedClosure(dm, section, point, &Ncl, &points, &clSection, &clPoints, &clp);CHKERRQ(ierr);
7219   }
7220 
7221   if (numIndices) *numIndices = Ni;
7222   if (indices)    *indices    = idx;
7223   PetscFunctionReturn(0);
7224 }
7225 
7226 /*@C
7227   DMPlexRestoreClosureIndices - Restores the global dof indices associated with the closure of the given point within the provided sections.
7228 
7229   Not collective
7230 
7231   Input Parameters:
7232 + dm         - The DM
7233 . section    - The PetscSection describing the points (a local section)
7234 . idxSection - The PetscSection from which to obtain indices (may be local or global)
7235 . point      - The point defining the closure
7236 - useClPerm  - Use the closure point permutation if available
7237 
7238   Output Parameters:
7239 + numIndices - The number of dof indices in the closure of point with the input sections
7240 . indices    - The dof indices
7241 . outOffsets - Array to write the field offsets into, or NULL
7242 - values     - The input values, which may be modified if sign flips are induced by the point symmetries, or NULL
7243 
7244   Notes:
7245   If values were modified, the user is responsible for calling DMRestoreWorkArray(dm, 0, MPIU_SCALAR, &values).
7246 
7247   If idxSection is global, any constrained dofs (see DMAddBoundary(), for example) will get negative indices.  The value
7248   of those indices is not significant.  If idxSection is local, the constrained dofs will yield the involution -(idx+1)
7249   of their index in a local vector.  A caller who does not wish to distinguish those points may recover the nonnegative
7250   indices via involution, -(-(idx+1)+1)==idx.  Local indices are provided when idxSection == section, otherwise global
7251   indices (with the above semantics) are implied.
7252 
7253   Level: advanced
7254 
7255 .seealso DMPlexGetClosureIndices(), DMPlexVecGetClosure(), DMPlexMatSetClosure(), DMGetLocalSection(), DMGetGlobalSection()
7256 @*/
7257 PetscErrorCode DMPlexRestoreClosureIndices(DM dm, PetscSection section, PetscSection idxSection, PetscInt point, PetscBool useClPerm,
7258                                            PetscInt *numIndices, PetscInt *indices[], PetscInt outOffsets[], PetscScalar *values[])
7259 {
7260   PetscErrorCode ierr;
7261 
7262   PetscFunctionBegin;
7263   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7264   PetscValidPointer(indices, 7);
7265   ierr = DMRestoreWorkArray(dm, 0, MPIU_INT, indices);CHKERRQ(ierr);
7266   PetscFunctionReturn(0);
7267 }
7268 
7269 /*@C
7270   DMPlexMatSetClosure - Set an array of the values on the closure of 'point'
7271 
7272   Not collective
7273 
7274   Input Parameters:
7275 + dm - The DM
7276 . section - The section describing the layout in v, or NULL to use the default section
7277 . globalSection - The section describing the layout in v, or NULL to use the default global section
7278 . A - The matrix
7279 . point - The point in the DM
7280 . values - The array of values
7281 - mode - The insert mode, where INSERT_ALL_VALUES and ADD_ALL_VALUES also overwrite boundary conditions
7282 
7283   Fortran Notes:
7284   This routine is only available in Fortran 90, and you must include petsc.h90 in your code.
7285 
7286   Level: intermediate
7287 
7288 .seealso DMPlexMatSetClosureGeneral(), DMPlexVecGetClosure(), DMPlexVecSetClosure()
7289 @*/
7290 PetscErrorCode DMPlexMatSetClosure(DM dm, PetscSection section, PetscSection globalSection, Mat A, PetscInt point, const PetscScalar values[], InsertMode mode)
7291 {
7292   DM_Plex           *mesh = (DM_Plex*) dm->data;
7293   PetscInt          *indices;
7294   PetscInt           numIndices;
7295   const PetscScalar *valuesOrig = values;
7296   PetscErrorCode     ierr;
7297 
7298   PetscFunctionBegin;
7299   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7300   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
7301   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
7302   if (!globalSection) {ierr = DMGetGlobalSection(dm, &globalSection);CHKERRQ(ierr);}
7303   PetscValidHeaderSpecific(globalSection, PETSC_SECTION_CLASSID, 3);
7304   PetscValidHeaderSpecific(A, MAT_CLASSID, 4);
7305 
7306   ierr = DMPlexGetClosureIndices(dm, section, globalSection, point, PETSC_TRUE, &numIndices, &indices, NULL, (PetscScalar **) &values);CHKERRQ(ierr);
7307 
7308   if (mesh->printSetValues) {ierr = DMPlexPrintMatSetValues(PETSC_VIEWER_STDOUT_SELF, A, point, numIndices, indices, 0, NULL, values);CHKERRQ(ierr);}
7309   ierr = MatSetValues(A, numIndices, indices, numIndices, indices, values, mode);
7310   if (ierr) {
7311     PetscMPIInt    rank;
7312     PetscErrorCode ierr2;
7313 
7314     ierr2 = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRMPI(ierr2);
7315     ierr2 = (*PetscErrorPrintf)("[%d]ERROR in DMPlexMatSetClosure\n", rank);CHKERRQ(ierr2);
7316     ierr2 = DMPlexPrintMatSetValues(PETSC_VIEWER_STDERR_SELF, A, point, numIndices, indices, 0, NULL, values);CHKERRQ(ierr2);
7317     ierr2 = DMPlexRestoreClosureIndices(dm, section, globalSection, point, PETSC_TRUE, &numIndices, &indices, NULL, (PetscScalar **) &values);CHKERRQ(ierr2);
7318     if (values != valuesOrig) {ierr2 = DMRestoreWorkArray(dm, 0, MPIU_SCALAR, &values);CHKERRQ(ierr2);}
7319     CHKERRQ(ierr);
7320   }
7321   if (mesh->printFEM > 1) {
7322     PetscInt i;
7323     ierr = PetscPrintf(PETSC_COMM_SELF, "  Indices:");CHKERRQ(ierr);
7324     for (i = 0; i < numIndices; ++i) {ierr = PetscPrintf(PETSC_COMM_SELF, " %D", indices[i]);CHKERRQ(ierr);}
7325     ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
7326   }
7327 
7328   ierr = DMPlexRestoreClosureIndices(dm, section, globalSection, point, PETSC_TRUE, &numIndices, &indices, NULL, (PetscScalar **) &values);CHKERRQ(ierr);
7329   if (values != valuesOrig) {ierr = DMRestoreWorkArray(dm, 0, MPIU_SCALAR, &values);CHKERRQ(ierr);}
7330   PetscFunctionReturn(0);
7331 }
7332 
7333 /*@C
7334   DMPlexMatSetClosure - Set an array of the values on the closure of 'point' using a different row and column section
7335 
7336   Not collective
7337 
7338   Input Parameters:
7339 + dmRow - The DM for the row fields
7340 . sectionRow - The section describing the layout, or NULL to use the default section in dmRow
7341 . globalSectionRow - The section describing the layout, or NULL to use the default global section in dmRow
7342 . dmCol - The DM for the column fields
7343 . sectionCol - The section describing the layout, or NULL to use the default section in dmCol
7344 . globalSectionCol - The section describing the layout, or NULL to use the default global section in dmCol
7345 . A - The matrix
7346 . point - The point in the DMs
7347 . values - The array of values
7348 - mode - The insert mode, where INSERT_ALL_VALUES and ADD_ALL_VALUES also overwrite boundary conditions
7349 
7350   Level: intermediate
7351 
7352 .seealso DMPlexMatSetClosure(), DMPlexVecGetClosure(), DMPlexVecSetClosure()
7353 @*/
7354 PetscErrorCode DMPlexMatSetClosureGeneral(DM dmRow, PetscSection sectionRow, PetscSection globalSectionRow, DM dmCol, PetscSection sectionCol, PetscSection globalSectionCol, Mat A, PetscInt point, const PetscScalar values[], InsertMode mode)
7355 {
7356   DM_Plex           *mesh = (DM_Plex*) dmRow->data;
7357   PetscInt          *indicesRow, *indicesCol;
7358   PetscInt           numIndicesRow, numIndicesCol;
7359   const PetscScalar *valuesOrig = values;
7360   PetscErrorCode     ierr;
7361 
7362   PetscFunctionBegin;
7363   PetscValidHeaderSpecific(dmRow, DM_CLASSID, 1);
7364   if (!sectionRow) {ierr = DMGetLocalSection(dmRow, &sectionRow);CHKERRQ(ierr);}
7365   PetscValidHeaderSpecific(sectionRow, PETSC_SECTION_CLASSID, 2);
7366   if (!globalSectionRow) {ierr = DMGetGlobalSection(dmRow, &globalSectionRow);CHKERRQ(ierr);}
7367   PetscValidHeaderSpecific(globalSectionRow, PETSC_SECTION_CLASSID, 3);
7368   PetscValidHeaderSpecific(dmCol, DM_CLASSID, 4);
7369   if (!sectionCol) {ierr = DMGetLocalSection(dmCol, &sectionCol);CHKERRQ(ierr);}
7370   PetscValidHeaderSpecific(sectionCol, PETSC_SECTION_CLASSID, 5);
7371   if (!globalSectionCol) {ierr = DMGetGlobalSection(dmCol, &globalSectionCol);CHKERRQ(ierr);}
7372   PetscValidHeaderSpecific(globalSectionCol, PETSC_SECTION_CLASSID, 6);
7373   PetscValidHeaderSpecific(A, MAT_CLASSID, 7);
7374 
7375   ierr = DMPlexGetClosureIndices(dmRow, sectionRow, globalSectionRow, point, PETSC_TRUE, &numIndicesRow, &indicesRow, NULL, (PetscScalar **) &values);CHKERRQ(ierr);
7376   ierr = DMPlexGetClosureIndices(dmCol, sectionCol, globalSectionCol, point, PETSC_TRUE, &numIndicesCol, &indicesCol, NULL, (PetscScalar **) &values);CHKERRQ(ierr);
7377 
7378   if (mesh->printSetValues) {ierr = DMPlexPrintMatSetValues(PETSC_VIEWER_STDOUT_SELF, A, point, numIndicesRow, indicesRow, numIndicesCol, indicesCol, values);CHKERRQ(ierr);}
7379   ierr = MatSetValues(A, numIndicesRow, indicesRow, numIndicesCol, indicesCol, values, mode);
7380   if (ierr) {
7381     PetscMPIInt    rank;
7382     PetscErrorCode ierr2;
7383 
7384     ierr2 = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRMPI(ierr2);
7385     ierr2 = (*PetscErrorPrintf)("[%d]ERROR in DMPlexMatSetClosure\n", rank);CHKERRQ(ierr2);
7386     ierr2 = DMPlexPrintMatSetValues(PETSC_VIEWER_STDERR_SELF, A, point, numIndicesRow, indicesRow, numIndicesCol, indicesCol, values);CHKERRQ(ierr2);
7387     ierr2 = DMPlexRestoreClosureIndices(dmRow, sectionRow, globalSectionRow, point, PETSC_TRUE, &numIndicesRow, &indicesRow, NULL, (PetscScalar **) &values);CHKERRQ(ierr2);
7388     ierr2 = DMPlexRestoreClosureIndices(dmCol, sectionCol, globalSectionCol, point, PETSC_TRUE, &numIndicesCol, &indicesRow, NULL, (PetscScalar **) &values);CHKERRQ(ierr2);
7389     if (values != valuesOrig) {ierr2 = DMRestoreWorkArray(dmRow, 0, MPIU_SCALAR, &values);CHKERRQ(ierr2);}
7390     CHKERRQ(ierr);
7391   }
7392 
7393   ierr = DMPlexRestoreClosureIndices(dmRow, sectionRow, globalSectionRow, point, PETSC_TRUE, &numIndicesRow, &indicesRow, NULL, (PetscScalar **) &values);CHKERRQ(ierr);
7394   ierr = DMPlexRestoreClosureIndices(dmCol, sectionCol, globalSectionCol, point, PETSC_TRUE, &numIndicesCol, &indicesCol, NULL, (PetscScalar **) &values);CHKERRQ(ierr);
7395   if (values != valuesOrig) {ierr = DMRestoreWorkArray(dmRow, 0, MPIU_SCALAR, &values);CHKERRQ(ierr);}
7396   PetscFunctionReturn(0);
7397 }
7398 
7399 PetscErrorCode DMPlexMatSetClosureRefined(DM dmf, PetscSection fsection, PetscSection globalFSection, DM dmc, PetscSection csection, PetscSection globalCSection, Mat A, PetscInt point, const PetscScalar values[], InsertMode mode)
7400 {
7401   DM_Plex        *mesh   = (DM_Plex*) dmf->data;
7402   PetscInt       *fpoints = NULL, *ftotpoints = NULL;
7403   PetscInt       *cpoints = NULL;
7404   PetscInt       *findices, *cindices;
7405   const PetscInt *fclperm = NULL, *cclperm = NULL; /* Closure permutations cannot work here */
7406   PetscInt        foffsets[32], coffsets[32];
7407   DMPolytopeType  ct;
7408   PetscInt        numFields, numSubcells, maxFPoints, numFPoints, numCPoints, numFIndices, numCIndices, dof, off, globalOff, pStart, pEnd, p, q, r, s, f;
7409   PetscErrorCode  ierr;
7410 
7411   PetscFunctionBegin;
7412   PetscValidHeaderSpecific(dmf, DM_CLASSID, 1);
7413   PetscValidHeaderSpecific(dmc, DM_CLASSID, 4);
7414   if (!fsection) {ierr = DMGetLocalSection(dmf, &fsection);CHKERRQ(ierr);}
7415   PetscValidHeaderSpecific(fsection, PETSC_SECTION_CLASSID, 2);
7416   if (!csection) {ierr = DMGetLocalSection(dmc, &csection);CHKERRQ(ierr);}
7417   PetscValidHeaderSpecific(csection, PETSC_SECTION_CLASSID, 5);
7418   if (!globalFSection) {ierr = DMGetGlobalSection(dmf, &globalFSection);CHKERRQ(ierr);}
7419   PetscValidHeaderSpecific(globalFSection, PETSC_SECTION_CLASSID, 3);
7420   if (!globalCSection) {ierr = DMGetGlobalSection(dmc, &globalCSection);CHKERRQ(ierr);}
7421   PetscValidHeaderSpecific(globalCSection, PETSC_SECTION_CLASSID, 6);
7422   PetscValidHeaderSpecific(A, MAT_CLASSID, 7);
7423   ierr = PetscSectionGetNumFields(fsection, &numFields);CHKERRQ(ierr);
7424   if (numFields > 31) SETERRQ1(PetscObjectComm((PetscObject)dmf), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", numFields);
7425   ierr = PetscArrayzero(foffsets, 32);CHKERRQ(ierr);
7426   ierr = PetscArrayzero(coffsets, 32);CHKERRQ(ierr);
7427   /* Column indices */
7428   ierr = DMPlexGetTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
7429   maxFPoints = numCPoints;
7430   /* Compress out points not in the section */
7431   /*   TODO: Squeeze out points with 0 dof as well */
7432   ierr = PetscSectionGetChart(csection, &pStart, &pEnd);CHKERRQ(ierr);
7433   for (p = 0, q = 0; p < numCPoints*2; p += 2) {
7434     if ((cpoints[p] >= pStart) && (cpoints[p] < pEnd)) {
7435       cpoints[q*2]   = cpoints[p];
7436       cpoints[q*2+1] = cpoints[p+1];
7437       ++q;
7438     }
7439   }
7440   numCPoints = q;
7441   for (p = 0, numCIndices = 0; p < numCPoints*2; p += 2) {
7442     PetscInt fdof;
7443 
7444     ierr = PetscSectionGetDof(csection, cpoints[p], &dof);CHKERRQ(ierr);
7445     if (!dof) continue;
7446     for (f = 0; f < numFields; ++f) {
7447       ierr           = PetscSectionGetFieldDof(csection, cpoints[p], f, &fdof);CHKERRQ(ierr);
7448       coffsets[f+1] += fdof;
7449     }
7450     numCIndices += dof;
7451   }
7452   for (f = 1; f < numFields; ++f) coffsets[f+1] += coffsets[f];
7453   /* Row indices */
7454   ierr = DMPlexGetCellType(dmc, point, &ct);CHKERRQ(ierr);
7455   {
7456     DMPlexCellRefiner cr;
7457     ierr = DMPlexCellRefinerCreate(dmc, &cr);CHKERRQ(ierr);
7458     ierr = DMPlexCellRefinerGetAffineTransforms(cr, ct, &numSubcells, NULL, NULL, NULL);CHKERRQ(ierr);
7459     ierr = DMPlexCellRefinerDestroy(&cr);CHKERRQ(ierr);
7460   }
7461   ierr = DMGetWorkArray(dmf, maxFPoints*2*numSubcells, MPIU_INT, &ftotpoints);CHKERRQ(ierr);
7462   for (r = 0, q = 0; r < numSubcells; ++r) {
7463     /* TODO Map from coarse to fine cells */
7464     ierr = DMPlexGetTransitiveClosure(dmf, point*numSubcells + r, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
7465     /* Compress out points not in the section */
7466     ierr = PetscSectionGetChart(fsection, &pStart, &pEnd);CHKERRQ(ierr);
7467     for (p = 0; p < numFPoints*2; p += 2) {
7468       if ((fpoints[p] >= pStart) && (fpoints[p] < pEnd)) {
7469         ierr = PetscSectionGetDof(fsection, fpoints[p], &dof);CHKERRQ(ierr);
7470         if (!dof) continue;
7471         for (s = 0; s < q; ++s) if (fpoints[p] == ftotpoints[s*2]) break;
7472         if (s < q) continue;
7473         ftotpoints[q*2]   = fpoints[p];
7474         ftotpoints[q*2+1] = fpoints[p+1];
7475         ++q;
7476       }
7477     }
7478     ierr = DMPlexRestoreTransitiveClosure(dmf, point, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
7479   }
7480   numFPoints = q;
7481   for (p = 0, numFIndices = 0; p < numFPoints*2; p += 2) {
7482     PetscInt fdof;
7483 
7484     ierr = PetscSectionGetDof(fsection, ftotpoints[p], &dof);CHKERRQ(ierr);
7485     if (!dof) continue;
7486     for (f = 0; f < numFields; ++f) {
7487       ierr           = PetscSectionGetFieldDof(fsection, ftotpoints[p], f, &fdof);CHKERRQ(ierr);
7488       foffsets[f+1] += fdof;
7489     }
7490     numFIndices += dof;
7491   }
7492   for (f = 1; f < numFields; ++f) foffsets[f+1] += foffsets[f];
7493 
7494   if (numFields && foffsets[numFields] != numFIndices) SETERRQ2(PetscObjectComm((PetscObject)dmf), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", foffsets[numFields], numFIndices);
7495   if (numFields && coffsets[numFields] != numCIndices) SETERRQ2(PetscObjectComm((PetscObject)dmc), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", coffsets[numFields], numCIndices);
7496   ierr = DMGetWorkArray(dmf, numFIndices, MPIU_INT, &findices);CHKERRQ(ierr);
7497   ierr = DMGetWorkArray(dmc, numCIndices, MPIU_INT, &cindices);CHKERRQ(ierr);
7498   if (numFields) {
7499     const PetscInt **permsF[32] = {NULL};
7500     const PetscInt **permsC[32] = {NULL};
7501 
7502     for (f = 0; f < numFields; f++) {
7503       ierr = PetscSectionGetFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL);CHKERRQ(ierr);
7504       ierr = PetscSectionGetFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL);CHKERRQ(ierr);
7505     }
7506     for (p = 0; p < numFPoints; p++) {
7507       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff);CHKERRQ(ierr);
7508       ierr = DMPlexGetIndicesPointFields_Internal(fsection, PETSC_FALSE, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, foffsets, PETSC_FALSE, permsF, p, fclperm, findices);CHKERRQ(ierr);
7509     }
7510     for (p = 0; p < numCPoints; p++) {
7511       ierr = PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff);CHKERRQ(ierr);
7512       ierr = DMPlexGetIndicesPointFields_Internal(csection, PETSC_FALSE, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, coffsets, PETSC_FALSE, permsC, p, cclperm, cindices);CHKERRQ(ierr);
7513     }
7514     for (f = 0; f < numFields; f++) {
7515       ierr = PetscSectionRestoreFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL);CHKERRQ(ierr);
7516       ierr = PetscSectionRestoreFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL);CHKERRQ(ierr);
7517     }
7518   } else {
7519     const PetscInt **permsF = NULL;
7520     const PetscInt **permsC = NULL;
7521 
7522     ierr = PetscSectionGetPointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL);CHKERRQ(ierr);
7523     ierr = PetscSectionGetPointSyms(csection,numCPoints,cpoints,&permsC,NULL);CHKERRQ(ierr);
7524     for (p = 0, off = 0; p < numFPoints; p++) {
7525       const PetscInt *perm = permsF ? permsF[p] : NULL;
7526 
7527       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff);CHKERRQ(ierr);
7528       ierr = DMPlexGetIndicesPoint_Internal(fsection, PETSC_FALSE, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, fclperm, findices);CHKERRQ(ierr);
7529     }
7530     for (p = 0, off = 0; p < numCPoints; p++) {
7531       const PetscInt *perm = permsC ? permsC[p] : NULL;
7532 
7533       ierr = PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff);CHKERRQ(ierr);
7534       ierr = DMPlexGetIndicesPoint_Internal(csection, PETSC_FALSE, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, cclperm, cindices);CHKERRQ(ierr);
7535     }
7536     ierr = PetscSectionRestorePointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL);CHKERRQ(ierr);
7537     ierr = PetscSectionRestorePointSyms(csection,numCPoints,cpoints,&permsC,NULL);CHKERRQ(ierr);
7538   }
7539   if (mesh->printSetValues) {ierr = DMPlexPrintMatSetValues(PETSC_VIEWER_STDOUT_SELF, A, point, numFIndices, findices, numCIndices, cindices, values);CHKERRQ(ierr);}
7540   /* TODO: flips */
7541   ierr = MatSetValues(A, numFIndices, findices, numCIndices, cindices, values, mode);
7542   if (ierr) {
7543     PetscMPIInt    rank;
7544     PetscErrorCode ierr2;
7545 
7546     ierr2 = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRMPI(ierr2);
7547     ierr2 = (*PetscErrorPrintf)("[%d]ERROR in DMPlexMatSetClosure\n", rank);CHKERRQ(ierr2);
7548     ierr2 = DMPlexPrintMatSetValues(PETSC_VIEWER_STDERR_SELF, A, point, numFIndices, findices, numCIndices, cindices, values);CHKERRQ(ierr2);
7549     ierr2 = DMRestoreWorkArray(dmf, numFIndices, MPIU_INT, &findices);CHKERRQ(ierr2);
7550     ierr2 = DMRestoreWorkArray(dmc, numCIndices, MPIU_INT, &cindices);CHKERRQ(ierr2);
7551     CHKERRQ(ierr);
7552   }
7553   ierr = DMRestoreWorkArray(dmf, numCPoints*2*4, MPIU_INT, &ftotpoints);CHKERRQ(ierr);
7554   ierr = DMPlexRestoreTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
7555   ierr = DMRestoreWorkArray(dmf, numFIndices, MPIU_INT, &findices);CHKERRQ(ierr);
7556   ierr = DMRestoreWorkArray(dmc, numCIndices, MPIU_INT, &cindices);CHKERRQ(ierr);
7557   PetscFunctionReturn(0);
7558 }
7559 
7560 PetscErrorCode DMPlexMatGetClosureIndicesRefined(DM dmf, PetscSection fsection, PetscSection globalFSection, DM dmc, PetscSection csection, PetscSection globalCSection, PetscInt point, PetscInt cindices[], PetscInt findices[])
7561 {
7562   PetscInt      *fpoints = NULL, *ftotpoints = NULL;
7563   PetscInt      *cpoints = NULL;
7564   PetscInt       foffsets[32], coffsets[32];
7565   const PetscInt *fclperm = NULL, *cclperm = NULL; /* Closure permutations cannot work here */
7566   DMPolytopeType ct;
7567   PetscInt       numFields, numSubcells, maxFPoints, numFPoints, numCPoints, numFIndices, numCIndices, dof, off, globalOff, pStart, pEnd, p, q, r, s, f;
7568   PetscErrorCode ierr;
7569 
7570   PetscFunctionBegin;
7571   PetscValidHeaderSpecific(dmf, DM_CLASSID, 1);
7572   PetscValidHeaderSpecific(dmc, DM_CLASSID, 4);
7573   if (!fsection) {ierr = DMGetLocalSection(dmf, &fsection);CHKERRQ(ierr);}
7574   PetscValidHeaderSpecific(fsection, PETSC_SECTION_CLASSID, 2);
7575   if (!csection) {ierr = DMGetLocalSection(dmc, &csection);CHKERRQ(ierr);}
7576   PetscValidHeaderSpecific(csection, PETSC_SECTION_CLASSID, 5);
7577   if (!globalFSection) {ierr = DMGetGlobalSection(dmf, &globalFSection);CHKERRQ(ierr);}
7578   PetscValidHeaderSpecific(globalFSection, PETSC_SECTION_CLASSID, 3);
7579   if (!globalCSection) {ierr = DMGetGlobalSection(dmc, &globalCSection);CHKERRQ(ierr);}
7580   PetscValidHeaderSpecific(globalCSection, PETSC_SECTION_CLASSID, 6);
7581   ierr = PetscSectionGetNumFields(fsection, &numFields);CHKERRQ(ierr);
7582   if (numFields > 31) SETERRQ1(PetscObjectComm((PetscObject)dmf), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", numFields);
7583   ierr = PetscArrayzero(foffsets, 32);CHKERRQ(ierr);
7584   ierr = PetscArrayzero(coffsets, 32);CHKERRQ(ierr);
7585   /* Column indices */
7586   ierr = DMPlexGetTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
7587   maxFPoints = numCPoints;
7588   /* Compress out points not in the section */
7589   /*   TODO: Squeeze out points with 0 dof as well */
7590   ierr = PetscSectionGetChart(csection, &pStart, &pEnd);CHKERRQ(ierr);
7591   for (p = 0, q = 0; p < numCPoints*2; p += 2) {
7592     if ((cpoints[p] >= pStart) && (cpoints[p] < pEnd)) {
7593       cpoints[q*2]   = cpoints[p];
7594       cpoints[q*2+1] = cpoints[p+1];
7595       ++q;
7596     }
7597   }
7598   numCPoints = q;
7599   for (p = 0, numCIndices = 0; p < numCPoints*2; p += 2) {
7600     PetscInt fdof;
7601 
7602     ierr = PetscSectionGetDof(csection, cpoints[p], &dof);CHKERRQ(ierr);
7603     if (!dof) continue;
7604     for (f = 0; f < numFields; ++f) {
7605       ierr           = PetscSectionGetFieldDof(csection, cpoints[p], f, &fdof);CHKERRQ(ierr);
7606       coffsets[f+1] += fdof;
7607     }
7608     numCIndices += dof;
7609   }
7610   for (f = 1; f < numFields; ++f) coffsets[f+1] += coffsets[f];
7611   /* Row indices */
7612   ierr = DMPlexGetCellType(dmc, point, &ct);CHKERRQ(ierr);
7613   {
7614     DMPlexCellRefiner cr;
7615     ierr = DMPlexCellRefinerCreate(dmc, &cr);CHKERRQ(ierr);
7616     ierr = DMPlexCellRefinerGetAffineTransforms(cr, ct, &numSubcells, NULL, NULL, NULL);CHKERRQ(ierr);
7617     ierr = DMPlexCellRefinerDestroy(&cr);CHKERRQ(ierr);
7618   }
7619   ierr = DMGetWorkArray(dmf, maxFPoints*2*numSubcells, MPIU_INT, &ftotpoints);CHKERRQ(ierr);
7620   for (r = 0, q = 0; r < numSubcells; ++r) {
7621     /* TODO Map from coarse to fine cells */
7622     ierr = DMPlexGetTransitiveClosure(dmf, point*numSubcells + r, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
7623     /* Compress out points not in the section */
7624     ierr = PetscSectionGetChart(fsection, &pStart, &pEnd);CHKERRQ(ierr);
7625     for (p = 0; p < numFPoints*2; p += 2) {
7626       if ((fpoints[p] >= pStart) && (fpoints[p] < pEnd)) {
7627         ierr = PetscSectionGetDof(fsection, fpoints[p], &dof);CHKERRQ(ierr);
7628         if (!dof) continue;
7629         for (s = 0; s < q; ++s) if (fpoints[p] == ftotpoints[s*2]) break;
7630         if (s < q) continue;
7631         ftotpoints[q*2]   = fpoints[p];
7632         ftotpoints[q*2+1] = fpoints[p+1];
7633         ++q;
7634       }
7635     }
7636     ierr = DMPlexRestoreTransitiveClosure(dmf, point, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
7637   }
7638   numFPoints = q;
7639   for (p = 0, numFIndices = 0; p < numFPoints*2; p += 2) {
7640     PetscInt fdof;
7641 
7642     ierr = PetscSectionGetDof(fsection, ftotpoints[p], &dof);CHKERRQ(ierr);
7643     if (!dof) continue;
7644     for (f = 0; f < numFields; ++f) {
7645       ierr           = PetscSectionGetFieldDof(fsection, ftotpoints[p], f, &fdof);CHKERRQ(ierr);
7646       foffsets[f+1] += fdof;
7647     }
7648     numFIndices += dof;
7649   }
7650   for (f = 1; f < numFields; ++f) foffsets[f+1] += foffsets[f];
7651 
7652   if (numFields && foffsets[numFields] != numFIndices) SETERRQ2(PetscObjectComm((PetscObject)dmf), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", foffsets[numFields], numFIndices);
7653   if (numFields && coffsets[numFields] != numCIndices) SETERRQ2(PetscObjectComm((PetscObject)dmc), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", coffsets[numFields], numCIndices);
7654   if (numFields) {
7655     const PetscInt **permsF[32] = {NULL};
7656     const PetscInt **permsC[32] = {NULL};
7657 
7658     for (f = 0; f < numFields; f++) {
7659       ierr = PetscSectionGetFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL);CHKERRQ(ierr);
7660       ierr = PetscSectionGetFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL);CHKERRQ(ierr);
7661     }
7662     for (p = 0; p < numFPoints; p++) {
7663       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff);CHKERRQ(ierr);
7664       ierr = DMPlexGetIndicesPointFields_Internal(fsection, PETSC_FALSE, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, foffsets, PETSC_FALSE, permsF, p, fclperm, findices);CHKERRQ(ierr);
7665     }
7666     for (p = 0; p < numCPoints; p++) {
7667       ierr = PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff);CHKERRQ(ierr);
7668       ierr = DMPlexGetIndicesPointFields_Internal(csection, PETSC_FALSE, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, coffsets, PETSC_FALSE, permsC, p, cclperm, cindices);CHKERRQ(ierr);
7669     }
7670     for (f = 0; f < numFields; f++) {
7671       ierr = PetscSectionRestoreFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL);CHKERRQ(ierr);
7672       ierr = PetscSectionRestoreFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL);CHKERRQ(ierr);
7673     }
7674   } else {
7675     const PetscInt **permsF = NULL;
7676     const PetscInt **permsC = NULL;
7677 
7678     ierr = PetscSectionGetPointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL);CHKERRQ(ierr);
7679     ierr = PetscSectionGetPointSyms(csection,numCPoints,cpoints,&permsC,NULL);CHKERRQ(ierr);
7680     for (p = 0, off = 0; p < numFPoints; p++) {
7681       const PetscInt *perm = permsF ? permsF[p] : NULL;
7682 
7683       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff);CHKERRQ(ierr);
7684       ierr = DMPlexGetIndicesPoint_Internal(fsection, PETSC_FALSE, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, fclperm, findices);CHKERRQ(ierr);
7685     }
7686     for (p = 0, off = 0; p < numCPoints; p++) {
7687       const PetscInt *perm = permsC ? permsC[p] : NULL;
7688 
7689       ierr = PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff);CHKERRQ(ierr);
7690       ierr = DMPlexGetIndicesPoint_Internal(csection, PETSC_FALSE, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, cclperm, cindices);CHKERRQ(ierr);
7691     }
7692     ierr = PetscSectionRestorePointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL);CHKERRQ(ierr);
7693     ierr = PetscSectionRestorePointSyms(csection,numCPoints,cpoints,&permsC,NULL);CHKERRQ(ierr);
7694   }
7695   ierr = DMRestoreWorkArray(dmf, numCPoints*2*4, MPIU_INT, &ftotpoints);CHKERRQ(ierr);
7696   ierr = DMPlexRestoreTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
7697   PetscFunctionReturn(0);
7698 }
7699 
7700 /*@C
7701   DMPlexGetVTKCellHeight - Returns the height in the DAG used to determine which points are cells (normally 0)
7702 
7703   Input Parameter:
7704 . dm   - The DMPlex object
7705 
7706   Output Parameter:
7707 . cellHeight - The height of a cell
7708 
7709   Level: developer
7710 
7711 .seealso DMPlexSetVTKCellHeight()
7712 @*/
7713 PetscErrorCode DMPlexGetVTKCellHeight(DM dm, PetscInt *cellHeight)
7714 {
7715   DM_Plex *mesh = (DM_Plex*) dm->data;
7716 
7717   PetscFunctionBegin;
7718   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7719   PetscValidPointer(cellHeight, 2);
7720   *cellHeight = mesh->vtkCellHeight;
7721   PetscFunctionReturn(0);
7722 }
7723 
7724 /*@C
7725   DMPlexSetVTKCellHeight - Sets the height in the DAG used to determine which points are cells (normally 0)
7726 
7727   Input Parameters:
7728 + dm   - The DMPlex object
7729 - cellHeight - The height of a cell
7730 
7731   Level: developer
7732 
7733 .seealso DMPlexGetVTKCellHeight()
7734 @*/
7735 PetscErrorCode DMPlexSetVTKCellHeight(DM dm, PetscInt cellHeight)
7736 {
7737   DM_Plex *mesh = (DM_Plex*) dm->data;
7738 
7739   PetscFunctionBegin;
7740   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7741   mesh->vtkCellHeight = cellHeight;
7742   PetscFunctionReturn(0);
7743 }
7744 
7745 /*@
7746   DMPlexGetGhostCellStratum - Get the range of cells which are used to enforce FV boundary conditions
7747 
7748   Input Parameter:
7749 . dm - The DMPlex object
7750 
7751   Output Parameters:
7752 + gcStart - The first ghost cell, or NULL
7753 - gcEnd   - The upper bound on ghost cells, or NULL
7754 
7755   Level: advanced
7756 
7757 .seealso DMPlexConstructGhostCells(), DMPlexGetGhostCellStratum()
7758 @*/
7759 PetscErrorCode DMPlexGetGhostCellStratum(DM dm, PetscInt *gcStart, PetscInt *gcEnd)
7760 {
7761   DMLabel        ctLabel;
7762   PetscErrorCode ierr;
7763 
7764   PetscFunctionBegin;
7765   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7766   ierr = DMPlexGetCellTypeLabel(dm, &ctLabel);CHKERRQ(ierr);
7767   ierr = DMLabelGetStratumBounds(ctLabel, DM_POLYTOPE_FV_GHOST, gcStart, gcEnd);CHKERRQ(ierr);
7768   PetscFunctionReturn(0);
7769 }
7770 
7771 /* We can easily have a form that takes an IS instead */
7772 PetscErrorCode DMPlexCreateNumbering_Plex(DM dm, PetscInt pStart, PetscInt pEnd, PetscInt shift, PetscInt *globalSize, PetscSF sf, IS *numbering)
7773 {
7774   PetscSection   section, globalSection;
7775   PetscInt      *numbers, p;
7776   PetscErrorCode ierr;
7777 
7778   PetscFunctionBegin;
7779   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &section);CHKERRQ(ierr);
7780   ierr = PetscSectionSetChart(section, pStart, pEnd);CHKERRQ(ierr);
7781   for (p = pStart; p < pEnd; ++p) {
7782     ierr = PetscSectionSetDof(section, p, 1);CHKERRQ(ierr);
7783   }
7784   ierr = PetscSectionSetUp(section);CHKERRQ(ierr);
7785   ierr = PetscSectionCreateGlobalSection(section, sf, PETSC_FALSE, PETSC_FALSE, &globalSection);CHKERRQ(ierr);
7786   ierr = PetscMalloc1(pEnd - pStart, &numbers);CHKERRQ(ierr);
7787   for (p = pStart; p < pEnd; ++p) {
7788     ierr = PetscSectionGetOffset(globalSection, p, &numbers[p-pStart]);CHKERRQ(ierr);
7789     if (numbers[p-pStart] < 0) numbers[p-pStart] -= shift;
7790     else                       numbers[p-pStart] += shift;
7791   }
7792   ierr = ISCreateGeneral(PetscObjectComm((PetscObject) dm), pEnd - pStart, numbers, PETSC_OWN_POINTER, numbering);CHKERRQ(ierr);
7793   if (globalSize) {
7794     PetscLayout layout;
7795     ierr = PetscSectionGetPointLayout(PetscObjectComm((PetscObject) dm), globalSection, &layout);CHKERRQ(ierr);
7796     ierr = PetscLayoutGetSize(layout, globalSize);CHKERRQ(ierr);
7797     ierr = PetscLayoutDestroy(&layout);CHKERRQ(ierr);
7798   }
7799   ierr = PetscSectionDestroy(&section);CHKERRQ(ierr);
7800   ierr = PetscSectionDestroy(&globalSection);CHKERRQ(ierr);
7801   PetscFunctionReturn(0);
7802 }
7803 
7804 PetscErrorCode DMPlexCreateCellNumbering_Internal(DM dm, PetscBool includeHybrid, IS *globalCellNumbers)
7805 {
7806   PetscInt       cellHeight, cStart, cEnd;
7807   PetscErrorCode ierr;
7808 
7809   PetscFunctionBegin;
7810   ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
7811   if (includeHybrid) {ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);}
7812   else               {ierr = DMPlexGetSimplexOrBoxCells(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);}
7813   ierr = DMPlexCreateNumbering_Plex(dm, cStart, cEnd, 0, NULL, dm->sf, globalCellNumbers);CHKERRQ(ierr);
7814   PetscFunctionReturn(0);
7815 }
7816 
7817 /*@
7818   DMPlexGetCellNumbering - Get a global cell numbering for all cells on this process
7819 
7820   Input Parameter:
7821 . dm   - The DMPlex object
7822 
7823   Output Parameter:
7824 . globalCellNumbers - Global cell numbers for all cells on this process
7825 
7826   Level: developer
7827 
7828 .seealso DMPlexGetVertexNumbering()
7829 @*/
7830 PetscErrorCode DMPlexGetCellNumbering(DM dm, IS *globalCellNumbers)
7831 {
7832   DM_Plex       *mesh = (DM_Plex*) dm->data;
7833   PetscErrorCode ierr;
7834 
7835   PetscFunctionBegin;
7836   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7837   if (!mesh->globalCellNumbers) {ierr = DMPlexCreateCellNumbering_Internal(dm, PETSC_FALSE, &mesh->globalCellNumbers);CHKERRQ(ierr);}
7838   *globalCellNumbers = mesh->globalCellNumbers;
7839   PetscFunctionReturn(0);
7840 }
7841 
7842 PetscErrorCode DMPlexCreateVertexNumbering_Internal(DM dm, PetscBool includeHybrid, IS *globalVertexNumbers)
7843 {
7844   PetscInt       vStart, vEnd;
7845   PetscErrorCode ierr;
7846 
7847   PetscFunctionBegin;
7848   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7849   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
7850   ierr = DMPlexCreateNumbering_Plex(dm, vStart, vEnd, 0, NULL, dm->sf, globalVertexNumbers);CHKERRQ(ierr);
7851   PetscFunctionReturn(0);
7852 }
7853 
7854 /*@
7855   DMPlexGetVertexNumbering - Get a global vertex numbering for all vertices on this process
7856 
7857   Input Parameter:
7858 . dm   - The DMPlex object
7859 
7860   Output Parameter:
7861 . globalVertexNumbers - Global vertex numbers for all vertices on this process
7862 
7863   Level: developer
7864 
7865 .seealso DMPlexGetCellNumbering()
7866 @*/
7867 PetscErrorCode DMPlexGetVertexNumbering(DM dm, IS *globalVertexNumbers)
7868 {
7869   DM_Plex       *mesh = (DM_Plex*) dm->data;
7870   PetscErrorCode ierr;
7871 
7872   PetscFunctionBegin;
7873   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7874   if (!mesh->globalVertexNumbers) {ierr = DMPlexCreateVertexNumbering_Internal(dm, PETSC_FALSE, &mesh->globalVertexNumbers);CHKERRQ(ierr);}
7875   *globalVertexNumbers = mesh->globalVertexNumbers;
7876   PetscFunctionReturn(0);
7877 }
7878 
7879 /*@
7880   DMPlexCreatePointNumbering - Create a global numbering for all points on this process
7881 
7882   Input Parameter:
7883 . dm   - The DMPlex object
7884 
7885   Output Parameter:
7886 . globalPointNumbers - Global numbers for all points on this process
7887 
7888   Level: developer
7889 
7890 .seealso DMPlexGetCellNumbering()
7891 @*/
7892 PetscErrorCode DMPlexCreatePointNumbering(DM dm, IS *globalPointNumbers)
7893 {
7894   IS             nums[4];
7895   PetscInt       depths[4], gdepths[4], starts[4];
7896   PetscInt       depth, d, shift = 0;
7897   PetscErrorCode ierr;
7898 
7899   PetscFunctionBegin;
7900   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7901   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
7902   /* For unstratified meshes use dim instead of depth */
7903   if (depth < 0) {ierr = DMGetDimension(dm, &depth);CHKERRQ(ierr);}
7904   for (d = 0; d <= depth; ++d) {
7905     PetscInt end;
7906 
7907     depths[d] = depth-d;
7908     ierr = DMPlexGetDepthStratum(dm, depths[d], &starts[d], &end);CHKERRQ(ierr);
7909     if (!(starts[d]-end)) { starts[d] = depths[d] = -1; }
7910   }
7911   ierr = PetscSortIntWithArray(depth+1, starts, depths);CHKERRQ(ierr);
7912   ierr = MPIU_Allreduce(depths, gdepths, depth+1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject) dm));CHKERRMPI(ierr);
7913   for (d = 0; d <= depth; ++d) {
7914     if (starts[d] >= 0 && depths[d] != gdepths[d]) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Expected depth %D, found %D",depths[d],gdepths[d]);
7915   }
7916   for (d = 0; d <= depth; ++d) {
7917     PetscInt pStart, pEnd, gsize;
7918 
7919     ierr = DMPlexGetDepthStratum(dm, gdepths[d], &pStart, &pEnd);CHKERRQ(ierr);
7920     ierr = DMPlexCreateNumbering_Plex(dm, pStart, pEnd, shift, &gsize, dm->sf, &nums[d]);CHKERRQ(ierr);
7921     shift += gsize;
7922   }
7923   ierr = ISConcatenate(PetscObjectComm((PetscObject) dm), depth+1, nums, globalPointNumbers);CHKERRQ(ierr);
7924   for (d = 0; d <= depth; ++d) {ierr = ISDestroy(&nums[d]);CHKERRQ(ierr);}
7925   PetscFunctionReturn(0);
7926 }
7927 
7928 /*@
7929   DMPlexCreateRankField - Create a cell field whose value is the rank of the owner
7930 
7931   Input Parameter:
7932 . dm - The DMPlex object
7933 
7934   Output Parameter:
7935 . ranks - The rank field
7936 
7937   Options Database Keys:
7938 . -dm_partition_view - Adds the rank field into the DM output from -dm_view using the same viewer
7939 
7940   Level: intermediate
7941 
7942 .seealso: DMView()
7943 @*/
7944 PetscErrorCode DMPlexCreateRankField(DM dm, Vec *ranks)
7945 {
7946   DM             rdm;
7947   PetscFE        fe;
7948   PetscScalar   *r;
7949   PetscMPIInt    rank;
7950   DMPolytopeType ct;
7951   PetscInt       dim, cStart, cEnd, c;
7952   PetscBool      simplex;
7953   PetscErrorCode ierr;
7954 
7955   PetscFunctionBeginUser;
7956   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7957   PetscValidPointer(ranks, 2);
7958   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRMPI(ierr);
7959   ierr = DMClone(dm, &rdm);CHKERRQ(ierr);
7960   ierr = DMGetDimension(rdm, &dim);CHKERRQ(ierr);
7961   ierr = DMPlexGetHeightStratum(rdm, 0, &cStart, &cEnd);CHKERRQ(ierr);
7962   ierr = DMPlexGetCellType(dm, cStart, &ct);CHKERRQ(ierr);
7963   simplex = DMPolytopeTypeGetNumVertices(ct) == DMPolytopeTypeGetDim(ct)+1 ? PETSC_TRUE : PETSC_FALSE;
7964   ierr = PetscFECreateDefault(PETSC_COMM_SELF, dim, 1, simplex, "PETSc___rank_", -1, &fe);CHKERRQ(ierr);
7965   ierr = PetscObjectSetName((PetscObject) fe, "rank");CHKERRQ(ierr);
7966   ierr = DMSetField(rdm, 0, NULL, (PetscObject) fe);CHKERRQ(ierr);
7967   ierr = PetscFEDestroy(&fe);CHKERRQ(ierr);
7968   ierr = DMCreateDS(rdm);CHKERRQ(ierr);
7969   ierr = DMCreateGlobalVector(rdm, ranks);CHKERRQ(ierr);
7970   ierr = PetscObjectSetName((PetscObject) *ranks, "partition");CHKERRQ(ierr);
7971   ierr = VecGetArray(*ranks, &r);CHKERRQ(ierr);
7972   for (c = cStart; c < cEnd; ++c) {
7973     PetscScalar *lr;
7974 
7975     ierr = DMPlexPointGlobalRef(rdm, c, r, &lr);CHKERRQ(ierr);
7976     if (lr) *lr = rank;
7977   }
7978   ierr = VecRestoreArray(*ranks, &r);CHKERRQ(ierr);
7979   ierr = DMDestroy(&rdm);CHKERRQ(ierr);
7980   PetscFunctionReturn(0);
7981 }
7982 
7983 /*@
7984   DMPlexCreateLabelField - Create a cell field whose value is the label value for that cell
7985 
7986   Input Parameters:
7987 + dm    - The DMPlex
7988 - label - The DMLabel
7989 
7990   Output Parameter:
7991 . val - The label value field
7992 
7993   Options Database Keys:
7994 . -dm_label_view - Adds the label value field into the DM output from -dm_view using the same viewer
7995 
7996   Level: intermediate
7997 
7998 .seealso: DMView()
7999 @*/
8000 PetscErrorCode DMPlexCreateLabelField(DM dm, DMLabel label, Vec *val)
8001 {
8002   DM             rdm;
8003   PetscFE        fe;
8004   PetscScalar   *v;
8005   PetscInt       dim, cStart, cEnd, c;
8006   PetscErrorCode ierr;
8007 
8008   PetscFunctionBeginUser;
8009   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8010   PetscValidPointer(label, 2);
8011   PetscValidPointer(val, 3);
8012   ierr = DMClone(dm, &rdm);CHKERRQ(ierr);
8013   ierr = DMGetDimension(rdm, &dim);CHKERRQ(ierr);
8014   ierr = PetscFECreateDefault(PetscObjectComm((PetscObject) rdm), dim, 1, PETSC_TRUE, "PETSc___label_value_", -1, &fe);CHKERRQ(ierr);
8015   ierr = PetscObjectSetName((PetscObject) fe, "label_value");CHKERRQ(ierr);
8016   ierr = DMSetField(rdm, 0, NULL, (PetscObject) fe);CHKERRQ(ierr);
8017   ierr = PetscFEDestroy(&fe);CHKERRQ(ierr);
8018   ierr = DMCreateDS(rdm);CHKERRQ(ierr);
8019   ierr = DMPlexGetHeightStratum(rdm, 0, &cStart, &cEnd);CHKERRQ(ierr);
8020   ierr = DMCreateGlobalVector(rdm, val);CHKERRQ(ierr);
8021   ierr = PetscObjectSetName((PetscObject) *val, "label_value");CHKERRQ(ierr);
8022   ierr = VecGetArray(*val, &v);CHKERRQ(ierr);
8023   for (c = cStart; c < cEnd; ++c) {
8024     PetscScalar *lv;
8025     PetscInt     cval;
8026 
8027     ierr = DMPlexPointGlobalRef(rdm, c, v, &lv);CHKERRQ(ierr);
8028     ierr = DMLabelGetValue(label, c, &cval);CHKERRQ(ierr);
8029     *lv = cval;
8030   }
8031   ierr = VecRestoreArray(*val, &v);CHKERRQ(ierr);
8032   ierr = DMDestroy(&rdm);CHKERRQ(ierr);
8033   PetscFunctionReturn(0);
8034 }
8035 
8036 /*@
8037   DMPlexCheckSymmetry - Check that the adjacency information in the mesh is symmetric.
8038 
8039   Input Parameter:
8040 . dm - The DMPlex object
8041 
8042   Notes:
8043   This is a useful diagnostic when creating meshes programmatically.
8044 
8045   For the complete list of DMPlexCheck* functions, see DMSetFromOptions().
8046 
8047   Level: developer
8048 
8049 .seealso: DMCreate(), DMSetFromOptions()
8050 @*/
8051 PetscErrorCode DMPlexCheckSymmetry(DM dm)
8052 {
8053   PetscSection    coneSection, supportSection;
8054   const PetscInt *cone, *support;
8055   PetscInt        coneSize, c, supportSize, s;
8056   PetscInt        pStart, pEnd, p, pp, csize, ssize;
8057   PetscBool       storagecheck = PETSC_TRUE;
8058   PetscErrorCode  ierr;
8059 
8060   PetscFunctionBegin;
8061   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8062   ierr = DMViewFromOptions(dm, NULL, "-sym_dm_view");CHKERRQ(ierr);
8063   ierr = DMPlexGetConeSection(dm, &coneSection);CHKERRQ(ierr);
8064   ierr = DMPlexGetSupportSection(dm, &supportSection);CHKERRQ(ierr);
8065   /* Check that point p is found in the support of its cone points, and vice versa */
8066   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
8067   for (p = pStart; p < pEnd; ++p) {
8068     ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
8069     ierr = DMPlexGetCone(dm, p, &cone);CHKERRQ(ierr);
8070     for (c = 0; c < coneSize; ++c) {
8071       PetscBool dup = PETSC_FALSE;
8072       PetscInt  d;
8073       for (d = c-1; d >= 0; --d) {
8074         if (cone[c] == cone[d]) {dup = PETSC_TRUE; break;}
8075       }
8076       ierr = DMPlexGetSupportSize(dm, cone[c], &supportSize);CHKERRQ(ierr);
8077       ierr = DMPlexGetSupport(dm, cone[c], &support);CHKERRQ(ierr);
8078       for (s = 0; s < supportSize; ++s) {
8079         if (support[s] == p) break;
8080       }
8081       if ((s >= supportSize) || (dup && (support[s+1] != p))) {
8082         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %D cone: ", p);CHKERRQ(ierr);
8083         for (s = 0; s < coneSize; ++s) {
8084           ierr = PetscPrintf(PETSC_COMM_SELF, "%D, ", cone[s]);CHKERRQ(ierr);
8085         }
8086         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
8087         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %D support: ", cone[c]);CHKERRQ(ierr);
8088         for (s = 0; s < supportSize; ++s) {
8089           ierr = PetscPrintf(PETSC_COMM_SELF, "%D, ", support[s]);CHKERRQ(ierr);
8090         }
8091         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
8092         if (dup) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %D not repeatedly found in support of repeated cone point %D", p, cone[c]);
8093         else SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %D not found in support of cone point %D", p, cone[c]);
8094       }
8095     }
8096     ierr = DMPlexGetTreeParent(dm, p, &pp, NULL);CHKERRQ(ierr);
8097     if (p != pp) { storagecheck = PETSC_FALSE; continue; }
8098     ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
8099     ierr = DMPlexGetSupport(dm, p, &support);CHKERRQ(ierr);
8100     for (s = 0; s < supportSize; ++s) {
8101       ierr = DMPlexGetConeSize(dm, support[s], &coneSize);CHKERRQ(ierr);
8102       ierr = DMPlexGetCone(dm, support[s], &cone);CHKERRQ(ierr);
8103       for (c = 0; c < coneSize; ++c) {
8104         ierr = DMPlexGetTreeParent(dm, cone[c], &pp, NULL);CHKERRQ(ierr);
8105         if (cone[c] != pp) { c = 0; break; }
8106         if (cone[c] == p) break;
8107       }
8108       if (c >= coneSize) {
8109         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %D support: ", p);CHKERRQ(ierr);
8110         for (c = 0; c < supportSize; ++c) {
8111           ierr = PetscPrintf(PETSC_COMM_SELF, "%D, ", support[c]);CHKERRQ(ierr);
8112         }
8113         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
8114         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %D cone: ", support[s]);CHKERRQ(ierr);
8115         for (c = 0; c < coneSize; ++c) {
8116           ierr = PetscPrintf(PETSC_COMM_SELF, "%D, ", cone[c]);CHKERRQ(ierr);
8117         }
8118         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
8119         SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %D not found in cone of support point %D", p, support[s]);
8120       }
8121     }
8122   }
8123   if (storagecheck) {
8124     ierr = PetscSectionGetStorageSize(coneSection, &csize);CHKERRQ(ierr);
8125     ierr = PetscSectionGetStorageSize(supportSection, &ssize);CHKERRQ(ierr);
8126     if (csize != ssize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "Total cone size %D != Total support size %D", csize, ssize);
8127   }
8128   PetscFunctionReturn(0);
8129 }
8130 
8131 /*
8132   For submeshes with cohesive cells (see DMPlexConstructCohesiveCells()), we allow a special case where some of the boundary of a face (edges and vertices) are not duplicated. We call these special boundary points "unsplit", since the same edge or vertex appears in both copies of the face. These unsplit points throw off our counting, so we have to explicitly account for them here.
8133 */
8134 static PetscErrorCode DMPlexCellUnsplitVertices_Private(DM dm, PetscInt c, DMPolytopeType ct, PetscInt *unsplit)
8135 {
8136   DMPolytopeType  cct;
8137   PetscInt        ptpoints[4];
8138   const PetscInt *cone, *ccone, *ptcone;
8139   PetscInt        coneSize, cp, cconeSize, ccp, npt = 0, pt;
8140   PetscErrorCode  ierr;
8141 
8142   PetscFunctionBegin;
8143   *unsplit = 0;
8144   switch (ct) {
8145     case DM_POLYTOPE_SEG_PRISM_TENSOR:
8146       ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
8147       ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
8148       for (cp = 0; cp < coneSize; ++cp) {
8149         ierr = DMPlexGetCellType(dm, cone[cp], &cct);CHKERRQ(ierr);
8150         if (cct == DM_POLYTOPE_POINT_PRISM_TENSOR) ptpoints[npt++] = cone[cp];
8151       }
8152       break;
8153     case DM_POLYTOPE_TRI_PRISM_TENSOR:
8154     case DM_POLYTOPE_QUAD_PRISM_TENSOR:
8155       ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
8156       ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
8157       for (cp = 0; cp < coneSize; ++cp) {
8158         ierr = DMPlexGetCone(dm, cone[cp], &ccone);CHKERRQ(ierr);
8159         ierr = DMPlexGetConeSize(dm, cone[cp], &cconeSize);CHKERRQ(ierr);
8160         for (ccp = 0; ccp < cconeSize; ++ccp) {
8161           ierr = DMPlexGetCellType(dm, ccone[ccp], &cct);CHKERRQ(ierr);
8162           if (cct == DM_POLYTOPE_POINT_PRISM_TENSOR) {
8163             PetscInt p;
8164             for (p = 0; p < npt; ++p) if (ptpoints[p] == ccone[ccp]) break;
8165             if (p == npt) ptpoints[npt++] = ccone[ccp];
8166           }
8167         }
8168       }
8169       break;
8170     default: break;
8171   }
8172   for (pt = 0; pt < npt; ++pt) {
8173     ierr = DMPlexGetCone(dm, ptpoints[pt], &ptcone);CHKERRQ(ierr);
8174     if (ptcone[0] == ptcone[1]) ++(*unsplit);
8175   }
8176   PetscFunctionReturn(0);
8177 }
8178 
8179 /*@
8180   DMPlexCheckSkeleton - Check that each cell has the correct number of vertices
8181 
8182   Input Parameters:
8183 + dm - The DMPlex object
8184 - cellHeight - Normally 0
8185 
8186   Notes:
8187   This is a useful diagnostic when creating meshes programmatically.
8188   Currently applicable only to homogeneous simplex or tensor meshes.
8189 
8190   For the complete list of DMPlexCheck* functions, see DMSetFromOptions().
8191 
8192   Level: developer
8193 
8194 .seealso: DMCreate(), DMSetFromOptions()
8195 @*/
8196 PetscErrorCode DMPlexCheckSkeleton(DM dm, PetscInt cellHeight)
8197 {
8198   DMPlexInterpolatedFlag interp;
8199   DMPolytopeType         ct;
8200   PetscInt               vStart, vEnd, cStart, cEnd, c;
8201   PetscErrorCode         ierr;
8202 
8203   PetscFunctionBegin;
8204   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8205   ierr = DMPlexIsInterpolated(dm, &interp);CHKERRQ(ierr);
8206   ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
8207   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
8208   for (c = cStart; c < cEnd; ++c) {
8209     PetscInt *closure = NULL;
8210     PetscInt  coneSize, closureSize, cl, Nv = 0;
8211 
8212     ierr = DMPlexGetCellType(dm, c, &ct);CHKERRQ(ierr);
8213     if ((PetscInt) ct < 0) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %D has no cell type", c);
8214     if (ct == DM_POLYTOPE_UNKNOWN) continue;
8215     if (interp == DMPLEX_INTERPOLATED_FULL) {
8216       ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
8217       if (coneSize != DMPolytopeTypeGetConeSize(ct)) SETERRQ4(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %D of type %s has cone size %D != %D", c, DMPolytopeTypes[ct], coneSize, DMPolytopeTypeGetConeSize(ct));
8218     }
8219     ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
8220     for (cl = 0; cl < closureSize*2; cl += 2) {
8221       const PetscInt p = closure[cl];
8222       if ((p >= vStart) && (p < vEnd)) ++Nv;
8223     }
8224     ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
8225     /* Special Case: Tensor faces with identified vertices */
8226     if (Nv < DMPolytopeTypeGetNumVertices(ct)) {
8227       PetscInt unsplit;
8228 
8229       ierr = DMPlexCellUnsplitVertices_Private(dm, c, ct, &unsplit);CHKERRQ(ierr);
8230       if (Nv + unsplit == DMPolytopeTypeGetNumVertices(ct)) continue;
8231     }
8232     if (Nv != DMPolytopeTypeGetNumVertices(ct)) SETERRQ4(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %D of type %s has %D vertices != %D", c, DMPolytopeTypes[ct], Nv, DMPolytopeTypeGetNumVertices(ct));
8233   }
8234   PetscFunctionReturn(0);
8235 }
8236 
8237 /*@
8238   DMPlexCheckFaces - Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type
8239 
8240   Not Collective
8241 
8242   Input Parameters:
8243 + dm - The DMPlex object
8244 - cellHeight - Normally 0
8245 
8246   Notes:
8247   This is a useful diagnostic when creating meshes programmatically.
8248   This routine is only relevant for meshes that are fully interpolated across all ranks.
8249   It will error out if a partially interpolated mesh is given on some rank.
8250   It will do nothing for locally uninterpolated mesh (as there is nothing to check).
8251 
8252   For the complete list of DMPlexCheck* functions, see DMSetFromOptions().
8253 
8254   Level: developer
8255 
8256 .seealso: DMCreate(), DMPlexGetVTKCellHeight(), DMSetFromOptions()
8257 @*/
8258 PetscErrorCode DMPlexCheckFaces(DM dm, PetscInt cellHeight)
8259 {
8260   PetscInt       dim, depth, vStart, vEnd, cStart, cEnd, c, h;
8261   PetscErrorCode ierr;
8262   DMPlexInterpolatedFlag interpEnum;
8263 
8264   PetscFunctionBegin;
8265   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8266   ierr = DMPlexIsInterpolated(dm, &interpEnum);CHKERRQ(ierr);
8267   if (interpEnum == DMPLEX_INTERPOLATED_NONE) PetscFunctionReturn(0);
8268   if (interpEnum == DMPLEX_INTERPOLATED_PARTIAL) {
8269     PetscMPIInt rank;
8270     MPI_Comm    comm;
8271 
8272     ierr = PetscObjectGetComm((PetscObject) dm, &comm);CHKERRQ(ierr);
8273     ierr = MPI_Comm_rank(comm, &rank);CHKERRMPI(ierr);
8274     SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_SUP, "Mesh is only partially interpolated on rank %d, this is currently not supported", rank);
8275   }
8276 
8277   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
8278   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
8279   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
8280   for (h = cellHeight; h < PetscMin(depth, dim); ++h) {
8281     ierr = DMPlexGetHeightStratum(dm, h, &cStart, &cEnd);CHKERRQ(ierr);
8282     for (c = cStart; c < cEnd; ++c) {
8283       const PetscInt      *cone, *ornt, *faceSizes, *faces;
8284       const DMPolytopeType *faceTypes;
8285       DMPolytopeType        ct;
8286       PetscInt              numFaces, coneSize, f;
8287       PetscInt             *closure = NULL, closureSize, cl, numCorners = 0, fOff = 0, unsplit;
8288 
8289       ierr = DMPlexGetCellType(dm, c, &ct);CHKERRQ(ierr);
8290       ierr = DMPlexCellUnsplitVertices_Private(dm, c, ct, &unsplit);CHKERRQ(ierr);
8291       if (unsplit) continue;
8292       ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
8293       ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
8294       ierr = DMPlexGetConeOrientation(dm, c, &ornt);CHKERRQ(ierr);
8295       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
8296       for (cl = 0; cl < closureSize*2; cl += 2) {
8297         const PetscInt p = closure[cl];
8298         if ((p >= vStart) && (p < vEnd)) closure[numCorners++] = p;
8299       }
8300       ierr = DMPlexGetRawFaces_Internal(dm, ct, closure, &numFaces, &faceTypes, &faceSizes, &faces);CHKERRQ(ierr);
8301       if (coneSize != numFaces) SETERRQ4(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %D of type %s has %D faces but should have %D", c, DMPolytopeTypes[ct], coneSize, numFaces);
8302       for (f = 0; f < numFaces; ++f) {
8303         DMPolytopeType fct;
8304         PetscInt       *fclosure = NULL, fclosureSize, cl, fnumCorners = 0, v;
8305 
8306         ierr = DMPlexGetCellType(dm, cone[f], &fct);CHKERRQ(ierr);
8307         ierr = DMPlexGetTransitiveClosure_Internal(dm, cone[f], ornt[f], PETSC_TRUE, &fclosureSize, &fclosure);CHKERRQ(ierr);
8308         for (cl = 0; cl < fclosureSize*2; cl += 2) {
8309           const PetscInt p = fclosure[cl];
8310           if ((p >= vStart) && (p < vEnd)) fclosure[fnumCorners++] = p;
8311         }
8312         if (fnumCorners != faceSizes[f]) SETERRQ7(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %D of type %s (cone idx %D) of cell %D of type %s has %D vertices but should have %D", cone[f], DMPolytopeTypes[fct], f, c, DMPolytopeTypes[ct], fnumCorners, faceSizes[f]);
8313         for (v = 0; v < fnumCorners; ++v) {
8314           if (fclosure[v] != faces[fOff+v]) SETERRQ8(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %D of type %s (cone idx %d) of cell %D of type %s vertex %D, %D != %D", cone[f], DMPolytopeTypes[fct], f, c, DMPolytopeTypes[ct], v, fclosure[v], faces[fOff+v]);
8315         }
8316         ierr = DMPlexRestoreTransitiveClosure(dm, cone[f], PETSC_TRUE, &fclosureSize, &fclosure);CHKERRQ(ierr);
8317         fOff += faceSizes[f];
8318       }
8319       ierr = DMPlexRestoreRawFaces_Internal(dm, ct, closure, &numFaces, &faceTypes, &faceSizes, &faces);CHKERRQ(ierr);
8320       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
8321     }
8322   }
8323   PetscFunctionReturn(0);
8324 }
8325 
8326 /*@
8327   DMPlexCheckGeometry - Check the geometry of mesh cells
8328 
8329   Input Parameter:
8330 . dm - The DMPlex object
8331 
8332   Notes:
8333   This is a useful diagnostic when creating meshes programmatically.
8334 
8335   For the complete list of DMPlexCheck* functions, see DMSetFromOptions().
8336 
8337   Level: developer
8338 
8339 .seealso: DMCreate(), DMSetFromOptions()
8340 @*/
8341 PetscErrorCode DMPlexCheckGeometry(DM dm)
8342 {
8343   Vec            coordinates;
8344   PetscReal      detJ, J[9], refVol = 1.0;
8345   PetscReal      vol;
8346   PetscBool      periodic;
8347   PetscInt       dim, depth, dE, d, cStart, cEnd, c;
8348   PetscErrorCode ierr;
8349 
8350   PetscFunctionBegin;
8351   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
8352   ierr = DMGetCoordinateDim(dm, &dE);CHKERRQ(ierr);
8353   if (dim != dE) PetscFunctionReturn(0);
8354   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
8355   ierr = DMGetPeriodicity(dm, &periodic, NULL, NULL, NULL);CHKERRQ(ierr);
8356   for (d = 0; d < dim; ++d) refVol *= 2.0;
8357   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
8358   /* Make sure local coordinates are created, because that step is collective */
8359   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
8360   for (c = cStart; c < cEnd; ++c) {
8361     DMPolytopeType ct;
8362     PetscInt       unsplit;
8363     PetscBool      ignoreZeroVol = PETSC_FALSE;
8364 
8365     ierr = DMPlexGetCellType(dm, c, &ct);CHKERRQ(ierr);
8366     switch (ct) {
8367       case DM_POLYTOPE_SEG_PRISM_TENSOR:
8368       case DM_POLYTOPE_TRI_PRISM_TENSOR:
8369       case DM_POLYTOPE_QUAD_PRISM_TENSOR:
8370         ignoreZeroVol = PETSC_TRUE; break;
8371       default: break;
8372     }
8373     switch (ct) {
8374       case DM_POLYTOPE_TRI_PRISM:
8375       case DM_POLYTOPE_TRI_PRISM_TENSOR:
8376       case DM_POLYTOPE_QUAD_PRISM_TENSOR:
8377       case DM_POLYTOPE_PYRAMID:
8378         continue;
8379       default: break;
8380     }
8381     ierr = DMPlexCellUnsplitVertices_Private(dm, c, ct, &unsplit);CHKERRQ(ierr);
8382     if (unsplit) continue;
8383     ierr = DMPlexComputeCellGeometryFEM(dm, c, NULL, NULL, J, NULL, &detJ);CHKERRQ(ierr);
8384     if (detJ < -PETSC_SMALL || (detJ <= 0.0 && !ignoreZeroVol)) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Mesh cell %D of type %s is inverted, |J| = %g", c, DMPolytopeTypes[ct], (double) detJ);
8385     ierr = PetscInfo2(dm, "Cell %D FEM Volume %g\n", c, (double) detJ*refVol);CHKERRQ(ierr);
8386     if (depth > 1 && !periodic) {
8387       ierr = DMPlexComputeCellGeometryFVM(dm, c, &vol, NULL, NULL);CHKERRQ(ierr);
8388       if (vol < -PETSC_SMALL || (vol <= 0.0 && !ignoreZeroVol)) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Mesh cell %D of type %s is inverted, vol = %g", c, DMPolytopeTypes[ct], (double) vol);
8389       ierr = PetscInfo2(dm, "Cell %D FVM Volume %g\n", c, (double) vol);CHKERRQ(ierr);
8390     }
8391   }
8392   PetscFunctionReturn(0);
8393 }
8394 
8395 /*@
8396   DMPlexCheckPointSF - Check that several necessary conditions are met for the point SF of this plex.
8397 
8398   Input Parameters:
8399 . dm - The DMPlex object
8400 
8401   Notes:
8402   This is mainly intended for debugging/testing purposes.
8403   It currently checks only meshes with no partition overlapping.
8404 
8405   For the complete list of DMPlexCheck* functions, see DMSetFromOptions().
8406 
8407   Level: developer
8408 
8409 .seealso: DMGetPointSF(), DMSetFromOptions()
8410 @*/
8411 PetscErrorCode DMPlexCheckPointSF(DM dm)
8412 {
8413   PetscSF         pointSF;
8414   PetscInt        cellHeight, cStart, cEnd, l, nleaves, nroots, overlap;
8415   const PetscInt *locals, *rootdegree;
8416   PetscBool       distributed;
8417   PetscErrorCode  ierr;
8418 
8419   PetscFunctionBegin;
8420   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8421   ierr = DMGetPointSF(dm, &pointSF);CHKERRQ(ierr);
8422   ierr = DMPlexIsDistributed(dm, &distributed);CHKERRQ(ierr);
8423   if (!distributed) PetscFunctionReturn(0);
8424   ierr = DMPlexGetOverlap(dm, &overlap);CHKERRQ(ierr);
8425   if (overlap) {
8426     ierr = PetscPrintf(PetscObjectComm((PetscObject)dm), "Warning: DMPlexCheckPointSF() is currently not implemented for meshes with partition overlapping");CHKERRQ(ierr);
8427     PetscFunctionReturn(0);
8428   }
8429   if (!pointSF) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONGSTATE, "This DMPlex is distributed but does not have PointSF attached");
8430   ierr = PetscSFGetGraph(pointSF, &nroots, &nleaves, &locals, NULL);CHKERRQ(ierr);
8431   if (nroots < 0) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONGSTATE, "This DMPlex is distributed but its PointSF has no graph set");
8432   ierr = PetscSFComputeDegreeBegin(pointSF, &rootdegree);CHKERRQ(ierr);
8433   ierr = PetscSFComputeDegreeEnd(pointSF, &rootdegree);CHKERRQ(ierr);
8434 
8435   /* 1) check there are no faces in 2D, cells in 3D, in interface */
8436   ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
8437   ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
8438   for (l = 0; l < nleaves; ++l) {
8439     const PetscInt point = locals[l];
8440 
8441     if (point >= cStart && point < cEnd) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point SF contains %D which is a cell", point);
8442   }
8443 
8444   /* 2) if some point is in interface, then all its cone points must be also in interface (either as leaves or roots) */
8445   for (l = 0; l < nleaves; ++l) {
8446     const PetscInt  point = locals[l];
8447     const PetscInt *cone;
8448     PetscInt        coneSize, c, idx;
8449 
8450     ierr = DMPlexGetConeSize(dm, point, &coneSize);CHKERRQ(ierr);
8451     ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
8452     for (c = 0; c < coneSize; ++c) {
8453       if (!rootdegree[cone[c]]) {
8454         ierr = PetscFindInt(cone[c], nleaves, locals, &idx);CHKERRQ(ierr);
8455         if (idx < 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point SF contains %D but not %D from its cone", point, cone[c]);
8456       }
8457     }
8458   }
8459   PetscFunctionReturn(0);
8460 }
8461 
8462 typedef struct cell_stats
8463 {
8464   PetscReal min, max, sum, squaresum;
8465   PetscInt  count;
8466 } cell_stats_t;
8467 
8468 static void MPIAPI cell_stats_reduce(void *a, void *b, int * len, MPI_Datatype *datatype)
8469 {
8470   PetscInt i, N = *len;
8471 
8472   for (i = 0; i < N; i++) {
8473     cell_stats_t *A = (cell_stats_t *) a;
8474     cell_stats_t *B = (cell_stats_t *) b;
8475 
8476     B->min = PetscMin(A->min,B->min);
8477     B->max = PetscMax(A->max,B->max);
8478     B->sum += A->sum;
8479     B->squaresum += A->squaresum;
8480     B->count += A->count;
8481   }
8482 }
8483 
8484 /*@
8485   DMPlexCheckCellShape - Checks the Jacobian of the mapping from reference to real cells and computes some minimal statistics.
8486 
8487   Collective on dm
8488 
8489   Input Parameters:
8490 + dm        - The DMPlex object
8491 . output    - If true, statistics will be displayed on stdout
8492 - condLimit - Display all cells above this condition number, or PETSC_DETERMINE for no cell output
8493 
8494   Notes:
8495   This is mainly intended for debugging/testing purposes.
8496 
8497   For the complete list of DMPlexCheck* functions, see DMSetFromOptions().
8498 
8499   Level: developer
8500 
8501 .seealso: DMSetFromOptions(), DMPlexComputeOrthogonalQuality()
8502 @*/
8503 PetscErrorCode DMPlexCheckCellShape(DM dm, PetscBool output, PetscReal condLimit)
8504 {
8505   DM             dmCoarse;
8506   cell_stats_t   stats, globalStats;
8507   MPI_Comm       comm = PetscObjectComm((PetscObject)dm);
8508   PetscReal      *J, *invJ, min = 0, max = 0, mean = 0, stdev = 0;
8509   PetscReal      limit = condLimit > 0 ? condLimit : PETSC_MAX_REAL;
8510   PetscInt       cdim, cStart, cEnd, c, eStart, eEnd, count = 0;
8511   PetscMPIInt    rank,size;
8512   PetscErrorCode ierr;
8513 
8514   PetscFunctionBegin;
8515   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8516   stats.min   = PETSC_MAX_REAL;
8517   stats.max   = PETSC_MIN_REAL;
8518   stats.sum   = stats.squaresum = 0.;
8519   stats.count = 0;
8520 
8521   ierr = MPI_Comm_size(comm, &size);CHKERRMPI(ierr);
8522   ierr = MPI_Comm_rank(comm, &rank);CHKERRMPI(ierr);
8523   ierr = DMGetCoordinateDim(dm,&cdim);CHKERRQ(ierr);
8524   ierr = PetscMalloc2(PetscSqr(cdim), &J, PetscSqr(cdim), &invJ);CHKERRQ(ierr);
8525   ierr = DMPlexGetSimplexOrBoxCells(dm,0,&cStart,&cEnd);CHKERRQ(ierr);
8526   ierr = DMPlexGetDepthStratum(dm,1,&eStart,&eEnd);CHKERRQ(ierr);
8527   for (c = cStart; c < cEnd; c++) {
8528     PetscInt  i;
8529     PetscReal frobJ = 0., frobInvJ = 0., cond2, cond, detJ;
8530 
8531     ierr = DMPlexComputeCellGeometryAffineFEM(dm,c,NULL,J,invJ,&detJ);CHKERRQ(ierr);
8532     if (detJ < 0.0) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Mesh cell %D is inverted", c);
8533     for (i = 0; i < PetscSqr(cdim); ++i) {
8534       frobJ    += J[i] * J[i];
8535       frobInvJ += invJ[i] * invJ[i];
8536     }
8537     cond2 = frobJ * frobInvJ;
8538     cond  = PetscSqrtReal(cond2);
8539 
8540     stats.min        = PetscMin(stats.min,cond);
8541     stats.max        = PetscMax(stats.max,cond);
8542     stats.sum       += cond;
8543     stats.squaresum += cond2;
8544     stats.count++;
8545     if (output && cond > limit) {
8546       PetscSection coordSection;
8547       Vec          coordsLocal;
8548       PetscScalar *coords = NULL;
8549       PetscInt     Nv, d, clSize, cl, *closure = NULL;
8550 
8551       ierr = DMGetCoordinatesLocal(dm, &coordsLocal);CHKERRQ(ierr);
8552       ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
8553       ierr = DMPlexVecGetClosure(dm, coordSection, coordsLocal, c, &Nv, &coords);CHKERRQ(ierr);
8554       ierr = PetscSynchronizedPrintf(comm, "[%d] Cell %D cond %g\n", rank, c, (double) cond);CHKERRQ(ierr);
8555       for (i = 0; i < Nv/cdim; ++i) {
8556         ierr = PetscSynchronizedPrintf(comm, "  Vertex %D: (", i);CHKERRQ(ierr);
8557         for (d = 0; d < cdim; ++d) {
8558           if (d > 0) {ierr = PetscSynchronizedPrintf(comm, ", ");CHKERRQ(ierr);}
8559           ierr = PetscSynchronizedPrintf(comm, "%g", (double) PetscRealPart(coords[i*cdim+d]));CHKERRQ(ierr);
8560         }
8561         ierr = PetscSynchronizedPrintf(comm, ")\n");CHKERRQ(ierr);
8562       }
8563       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &clSize, &closure);CHKERRQ(ierr);
8564       for (cl = 0; cl < clSize*2; cl += 2) {
8565         const PetscInt edge = closure[cl];
8566 
8567         if ((edge >= eStart) && (edge < eEnd)) {
8568           PetscReal len;
8569 
8570           ierr = DMPlexComputeCellGeometryFVM(dm, edge, &len, NULL, NULL);CHKERRQ(ierr);
8571           ierr = PetscSynchronizedPrintf(comm, "  Edge %D: length %g\n", edge, (double) len);CHKERRQ(ierr);
8572         }
8573       }
8574       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &clSize, &closure);CHKERRQ(ierr);
8575       ierr = DMPlexVecRestoreClosure(dm, coordSection, coordsLocal, c, &Nv, &coords);CHKERRQ(ierr);
8576     }
8577   }
8578   if (output) {ierr = PetscSynchronizedFlush(comm, NULL);CHKERRQ(ierr);}
8579 
8580   if (size > 1) {
8581     PetscMPIInt   blockLengths[2] = {4,1};
8582     MPI_Aint      blockOffsets[2] = {offsetof(cell_stats_t,min),offsetof(cell_stats_t,count)};
8583     MPI_Datatype  blockTypes[2]   = {MPIU_REAL,MPIU_INT}, statType;
8584     MPI_Op        statReduce;
8585 
8586     ierr = MPI_Type_create_struct(2,blockLengths,blockOffsets,blockTypes,&statType);CHKERRMPI(ierr);
8587     ierr = MPI_Type_commit(&statType);CHKERRMPI(ierr);
8588     ierr = MPI_Op_create(cell_stats_reduce, PETSC_TRUE, &statReduce);CHKERRMPI(ierr);
8589     ierr = MPI_Reduce(&stats,&globalStats,1,statType,statReduce,0,comm);CHKERRMPI(ierr);
8590     ierr = MPI_Op_free(&statReduce);CHKERRMPI(ierr);
8591     ierr = MPI_Type_free(&statType);CHKERRMPI(ierr);
8592   } else {
8593     ierr = PetscArraycpy(&globalStats,&stats,1);CHKERRQ(ierr);
8594   }
8595   if (!rank) {
8596     count = globalStats.count;
8597     min   = globalStats.min;
8598     max   = globalStats.max;
8599     mean  = globalStats.sum / globalStats.count;
8600     stdev = globalStats.count > 1 ? PetscSqrtReal(PetscMax((globalStats.squaresum - globalStats.count * mean * mean) / (globalStats.count - 1),0)) : 0.0;
8601   }
8602 
8603   if (output) {
8604     ierr = PetscPrintf(comm,"Mesh with %D cells, shape condition numbers: min = %g, max = %g, mean = %g, stddev = %g\n", count, (double) min, (double) max, (double) mean, (double) stdev);CHKERRQ(ierr);
8605   }
8606   ierr = PetscFree2(J,invJ);CHKERRQ(ierr);
8607 
8608   ierr = DMGetCoarseDM(dm,&dmCoarse);CHKERRQ(ierr);
8609   if (dmCoarse) {
8610     PetscBool isplex;
8611 
8612     ierr = PetscObjectTypeCompare((PetscObject)dmCoarse,DMPLEX,&isplex);CHKERRQ(ierr);
8613     if (isplex) {
8614       ierr = DMPlexCheckCellShape(dmCoarse,output,condLimit);CHKERRQ(ierr);
8615     }
8616   }
8617   PetscFunctionReturn(0);
8618 }
8619 
8620 /*@
8621   DMPlexComputeOrthogonalQuality - Compute cell-wise orthogonal quality mesh statistic. Optionally tags all cells with
8622   orthogonal quality below given tolerance.
8623 
8624   Collective on dm
8625 
8626   Input Parameters:
8627 + dm   - The DMPlex object
8628 . fv   - Optional PetscFV object for pre-computed cell/face centroid information
8629 - atol - [0, 1] Absolute tolerance for tagging cells.
8630 
8631   Output Parameters:
8632 + OrthQual      - Vec containing orthogonal quality per cell
8633 - OrthQualLabel - DMLabel tagging cells below atol with DM_ADAPT_REFINE
8634 
8635   Options Database Keys:
8636 + -dm_plex_orthogonal_quality_label_view - view OrthQualLabel if label is requested. Currently only PETSCVIEWERASCII is
8637 supported.
8638 - -dm_plex_orthogonal_quality_vec_view - view OrthQual vector.
8639 
8640   Notes:
8641   Orthogonal quality is given by the following formula:
8642 
8643   \min \left[ \frac{A_i \cdot f_i}{\|A_i\| \|f_i\|} , \frac{A_i \cdot c_i}{\|A_i\| \|c_i\|} \right]
8644 
8645   Where A_i is the i'th face-normal vector, f_i is the vector from the cell centroid to the i'th face centroid, and c_i
8646   is the vector from the current cells centroid to the centroid of its i'th neighbor (which shares a face with the
8647   current cell). This computes the vector similarity between each cell face and its corresponding neighbor centroid by
8648   calculating the cosine of the angle between these vectors.
8649 
8650   Orthogonal quality ranges from 1 (best) to 0 (worst).
8651 
8652   This routine is mainly useful for FVM, however is not restricted to only FVM. The PetscFV object is optionally used to check for
8653   pre-computed FVM cell data, but if it is not passed in then this data will be computed.
8654 
8655   Cells are tagged if they have an orthogonal quality less than or equal to the absolute tolerance.
8656 
8657   Level: intermediate
8658 
8659 .seealso: DMPlexCheckCellShape(), DMCreateLabel()
8660 @*/
8661 PetscErrorCode DMPlexComputeOrthogonalQuality(DM dm, PetscFV fv, PetscReal atol, Vec *OrthQual, DMLabel *OrthQualLabel)
8662 {
8663   PetscInt                nc, cellHeight, cStart, cEnd, cell, cellIter = 0;
8664   PetscInt                *idx;
8665   PetscScalar             *oqVals;
8666   const PetscScalar       *cellGeomArr, *faceGeomArr;
8667   PetscReal               *ci, *fi, *Ai;
8668   MPI_Comm                comm;
8669   Vec                     cellgeom, facegeom;
8670   DM                      dmFace, dmCell;
8671   IS                      glob;
8672   ISLocalToGlobalMapping  ltog;
8673   PetscViewer             vwr;
8674   PetscErrorCode          ierr;
8675 
8676   PetscFunctionBegin;
8677   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8678   if (fv) {PetscValidHeaderSpecific(fv, PETSCFV_CLASSID, 2);}
8679   PetscValidPointer(OrthQual, 4);
8680   if (PetscUnlikelyDebug(atol < 0.0 || atol > 1.0)) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"Absolute tolerance %g not in [0,1]",(double)atol);
8681   ierr = PetscObjectGetComm((PetscObject) dm, &comm);CHKERRQ(ierr);
8682   ierr = DMGetDimension(dm, &nc);CHKERRQ(ierr);
8683   if (nc < 2) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONGSTATE, "DM must have dimension >= 2 (current %D)", nc);
8684   {
8685     DMPlexInterpolatedFlag interpFlag;
8686 
8687     ierr = DMPlexIsInterpolated(dm, &interpFlag);CHKERRQ(ierr);
8688     if (interpFlag != DMPLEX_INTERPOLATED_FULL) {
8689       PetscMPIInt rank;
8690 
8691       ierr = MPI_Comm_rank(comm, &rank);CHKERRMPI(ierr);
8692       SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONGSTATE, "DM must be fully interpolated, DM on rank %d is not fully interpolated", rank);
8693     }
8694   }
8695   if (OrthQualLabel) {
8696     PetscValidPointer(OrthQualLabel, 5);
8697     ierr = DMCreateLabel(dm, "Orthogonal_Quality");CHKERRQ(ierr);
8698     ierr = DMGetLabel(dm, "Orthogonal_Quality", OrthQualLabel);CHKERRQ(ierr);
8699   } else {*OrthQualLabel = NULL;}
8700   ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
8701   ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
8702   ierr = DMPlexCreateCellNumbering_Internal(dm, PETSC_TRUE, &glob);CHKERRQ(ierr);
8703   ierr = ISLocalToGlobalMappingCreateIS(glob, &ltog);CHKERRQ(ierr);
8704   ierr = ISLocalToGlobalMappingSetType(ltog, ISLOCALTOGLOBALMAPPINGHASH);CHKERRQ(ierr);
8705   ierr = VecCreate(comm, OrthQual);CHKERRQ(ierr);
8706   ierr = VecSetType(*OrthQual, VECSTANDARD);CHKERRQ(ierr);
8707   ierr = VecSetSizes(*OrthQual, cEnd-cStart, PETSC_DETERMINE);CHKERRQ(ierr);
8708   ierr = VecSetLocalToGlobalMapping(*OrthQual, ltog);CHKERRQ(ierr);
8709   ierr = VecSetUp(*OrthQual);CHKERRQ(ierr);
8710   ierr = ISDestroy(&glob);CHKERRQ(ierr);
8711   ierr = ISLocalToGlobalMappingDestroy(&ltog);CHKERRQ(ierr);
8712   ierr = DMPlexGetDataFVM(dm, fv, &cellgeom, &facegeom, NULL);CHKERRQ(ierr);
8713   ierr = VecGetArrayRead(cellgeom, &cellGeomArr);CHKERRQ(ierr);
8714   ierr = VecGetArrayRead(facegeom, &faceGeomArr);CHKERRQ(ierr);
8715   ierr = VecGetDM(cellgeom, &dmCell);CHKERRQ(ierr);
8716   ierr = VecGetDM(facegeom, &dmFace);CHKERRQ(ierr);
8717   ierr = PetscMalloc5(cEnd-cStart, &idx, cEnd-cStart, &oqVals, nc, &ci, nc, &fi, nc, &Ai);CHKERRQ(ierr);
8718   for (cell = cStart; cell < cEnd; cellIter++,cell++) {
8719     PetscInt           cellneigh, cellneighiter = 0, adjSize = PETSC_DETERMINE;
8720     PetscInt           cellarr[2], *adj = NULL;
8721     PetscScalar        *cArr, *fArr;
8722     PetscReal          minvalc = 1.0, minvalf = 1.0;
8723     PetscFVCellGeom    *cg;
8724 
8725     idx[cellIter] = cell-cStart;
8726     cellarr[0] = cell;
8727     /* Make indexing into cellGeom easier */
8728     ierr = DMPlexPointLocalRead(dmCell, cell, cellGeomArr, &cg);CHKERRQ(ierr);
8729     ierr = DMPlexGetAdjacency_Internal(dm, cell, PETSC_TRUE, PETSC_FALSE, PETSC_FALSE, &adjSize, &adj);CHKERRQ(ierr);
8730     /* Technically 1 too big, but easier than fiddling with empty adjacency array */
8731     ierr = PetscCalloc2(adjSize, &cArr, adjSize, &fArr);CHKERRQ(ierr);
8732     for (cellneigh = 0; cellneigh < adjSize; cellneighiter++,cellneigh++) {
8733       PetscInt         i;
8734       const PetscInt   neigh = adj[cellneigh];
8735       PetscReal        normci = 0, normfi = 0, normai = 0;
8736       PetscFVCellGeom  *cgneigh;
8737       PetscFVFaceGeom  *fg;
8738 
8739       /* Don't count ourselves in the neighbor list */
8740       if (neigh == cell) continue;
8741       ierr = DMPlexPointLocalRead(dmCell, neigh, cellGeomArr, &cgneigh);CHKERRQ(ierr);
8742       cellarr[1] = neigh;
8743       {
8744         PetscInt       numcovpts;
8745         const PetscInt *covpts;
8746 
8747         ierr = DMPlexGetMeet(dm, 2, cellarr, &numcovpts, &covpts);CHKERRQ(ierr);
8748         ierr = DMPlexPointLocalRead(dmFace, covpts[0], faceGeomArr, &fg);CHKERRQ(ierr);
8749         ierr = DMPlexRestoreMeet(dm, 2, cellarr, &numcovpts, &covpts);CHKERRQ(ierr);
8750       }
8751 
8752       /* Compute c_i, f_i and their norms */
8753       for (i = 0; i < nc; i++) {
8754         ci[i] = cgneigh->centroid[i] - cg->centroid[i];
8755         fi[i] = fg->centroid[i] - cg->centroid[i];
8756         Ai[i] = fg->normal[i];
8757         normci += PetscPowReal(ci[i], 2);
8758         normfi += PetscPowReal(fi[i], 2);
8759         normai += PetscPowReal(Ai[i], 2);
8760       }
8761       normci = PetscSqrtReal(normci);
8762       normfi = PetscSqrtReal(normfi);
8763       normai = PetscSqrtReal(normai);
8764 
8765       /* Normalize and compute for each face-cell-normal pair */
8766       for (i = 0; i < nc; i++) {
8767         ci[i] = ci[i]/normci;
8768         fi[i] = fi[i]/normfi;
8769         Ai[i] = Ai[i]/normai;
8770         /* PetscAbs because I don't know if normals are guaranteed to point out */
8771         cArr[cellneighiter] += PetscAbs(Ai[i]*ci[i]);
8772         fArr[cellneighiter] += PetscAbs(Ai[i]*fi[i]);
8773       }
8774       if (PetscRealPart(cArr[cellneighiter]) < minvalc) {
8775         minvalc = PetscRealPart(cArr[cellneighiter]);
8776       }
8777       if (PetscRealPart(fArr[cellneighiter]) < minvalf) {
8778         minvalf = PetscRealPart(fArr[cellneighiter]);
8779       }
8780     }
8781     ierr = PetscFree(adj);CHKERRQ(ierr);
8782     ierr = PetscFree2(cArr, fArr);CHKERRQ(ierr);
8783     /* Defer to cell if they're equal */
8784     oqVals[cellIter] = PetscMin(minvalf, minvalc);
8785     if (OrthQualLabel) {
8786       if (PetscRealPart(oqVals[cellIter]) <= atol) {ierr = DMLabelSetValue(*OrthQualLabel, cell, DM_ADAPT_REFINE);CHKERRQ(ierr);}
8787     }
8788   }
8789   ierr = VecSetValuesLocal(*OrthQual, cEnd-cStart, idx, oqVals, INSERT_VALUES);CHKERRQ(ierr);
8790   ierr = VecAssemblyBegin(*OrthQual);CHKERRQ(ierr);
8791   ierr = VecAssemblyEnd(*OrthQual);CHKERRQ(ierr);
8792   ierr = VecRestoreArrayRead(cellgeom, &cellGeomArr);CHKERRQ(ierr);
8793   ierr = VecRestoreArrayRead(facegeom, &faceGeomArr);CHKERRQ(ierr);
8794   ierr = PetscOptionsGetViewer(comm, NULL, NULL, "-dm_plex_orthogonal_quality_label_view", &vwr, NULL, NULL);CHKERRQ(ierr);
8795   if (OrthQualLabel) {
8796     if (vwr) {ierr = DMLabelView(*OrthQualLabel, vwr);CHKERRQ(ierr);}
8797   }
8798   ierr = PetscFree5(idx, oqVals, ci, fi, Ai);CHKERRQ(ierr);
8799   ierr = PetscViewerDestroy(&vwr);CHKERRQ(ierr);
8800   ierr = VecViewFromOptions(*OrthQual, NULL, "-dm_plex_orthogonal_quality_vec_view");CHKERRQ(ierr);
8801   PetscFunctionReturn(0);
8802 }
8803 
8804 /* this is here insead of DMGetOutputDM because output DM still has constraints in the local indices that affect
8805  * interpolator construction */
8806 static PetscErrorCode DMGetFullDM(DM dm, DM *odm)
8807 {
8808   PetscSection   section, newSection, gsection;
8809   PetscSF        sf;
8810   PetscBool      hasConstraints, ghasConstraints;
8811   PetscErrorCode ierr;
8812 
8813   PetscFunctionBegin;
8814   PetscValidHeaderSpecific(dm,DM_CLASSID,1);
8815   PetscValidPointer(odm,2);
8816   ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);
8817   ierr = PetscSectionHasConstraints(section, &hasConstraints);CHKERRQ(ierr);
8818   ierr = MPI_Allreduce(&hasConstraints, &ghasConstraints, 1, MPIU_BOOL, MPI_LOR, PetscObjectComm((PetscObject) dm));CHKERRMPI(ierr);
8819   if (!ghasConstraints) {
8820     ierr = PetscObjectReference((PetscObject)dm);CHKERRQ(ierr);
8821     *odm = dm;
8822     PetscFunctionReturn(0);
8823   }
8824   ierr = DMClone(dm, odm);CHKERRQ(ierr);
8825   ierr = DMCopyFields(dm, *odm);CHKERRQ(ierr);
8826   ierr = DMGetLocalSection(*odm, &newSection);CHKERRQ(ierr);
8827   ierr = DMGetPointSF(*odm, &sf);CHKERRQ(ierr);
8828   ierr = PetscSectionCreateGlobalSection(newSection, sf, PETSC_TRUE, PETSC_FALSE, &gsection);CHKERRQ(ierr);
8829   ierr = DMSetGlobalSection(*odm, gsection);CHKERRQ(ierr);
8830   ierr = PetscSectionDestroy(&gsection);CHKERRQ(ierr);
8831   PetscFunctionReturn(0);
8832 }
8833 
8834 static PetscErrorCode DMCreateAffineInterpolationCorrection_Plex(DM dmc, DM dmf, Vec *shift)
8835 {
8836   DM             dmco, dmfo;
8837   Mat            interpo;
8838   Vec            rscale;
8839   Vec            cglobalo, clocal;
8840   Vec            fglobal, fglobalo, flocal;
8841   PetscBool      regular;
8842   PetscErrorCode ierr;
8843 
8844   PetscFunctionBegin;
8845   ierr = DMGetFullDM(dmc, &dmco);CHKERRQ(ierr);
8846   ierr = DMGetFullDM(dmf, &dmfo);CHKERRQ(ierr);
8847   ierr = DMSetCoarseDM(dmfo, dmco);CHKERRQ(ierr);
8848   ierr = DMPlexGetRegularRefinement(dmf, &regular);CHKERRQ(ierr);
8849   ierr = DMPlexSetRegularRefinement(dmfo, regular);CHKERRQ(ierr);
8850   ierr = DMCreateInterpolation(dmco, dmfo, &interpo, &rscale);CHKERRQ(ierr);
8851   ierr = DMCreateGlobalVector(dmco, &cglobalo);CHKERRQ(ierr);
8852   ierr = DMCreateLocalVector(dmc, &clocal);CHKERRQ(ierr);
8853   ierr = VecSet(cglobalo, 0.);CHKERRQ(ierr);
8854   ierr = VecSet(clocal, 0.);CHKERRQ(ierr);
8855   ierr = DMCreateGlobalVector(dmf, &fglobal);CHKERRQ(ierr);
8856   ierr = DMCreateGlobalVector(dmfo, &fglobalo);CHKERRQ(ierr);
8857   ierr = DMCreateLocalVector(dmf, &flocal);CHKERRQ(ierr);
8858   ierr = VecSet(fglobal, 0.);CHKERRQ(ierr);
8859   ierr = VecSet(fglobalo, 0.);CHKERRQ(ierr);
8860   ierr = VecSet(flocal, 0.);CHKERRQ(ierr);
8861   ierr = DMPlexInsertBoundaryValues(dmc, PETSC_TRUE, clocal, 0., NULL, NULL, NULL);CHKERRQ(ierr);
8862   ierr = DMLocalToGlobalBegin(dmco, clocal, INSERT_VALUES, cglobalo);CHKERRQ(ierr);
8863   ierr = DMLocalToGlobalEnd(dmco, clocal, INSERT_VALUES, cglobalo);CHKERRQ(ierr);
8864   ierr = MatMult(interpo, cglobalo, fglobalo);CHKERRQ(ierr);
8865   ierr = DMGlobalToLocalBegin(dmfo, fglobalo, INSERT_VALUES, flocal);CHKERRQ(ierr);
8866   ierr = DMGlobalToLocalEnd(dmfo, fglobalo, INSERT_VALUES, flocal);CHKERRQ(ierr);
8867   ierr = DMLocalToGlobalBegin(dmf, flocal, INSERT_VALUES, fglobal);CHKERRQ(ierr);
8868   ierr = DMLocalToGlobalEnd(dmf, flocal, INSERT_VALUES, fglobal);CHKERRQ(ierr);
8869   *shift = fglobal;
8870   ierr = VecDestroy(&flocal);CHKERRQ(ierr);
8871   ierr = VecDestroy(&fglobalo);CHKERRQ(ierr);
8872   ierr = VecDestroy(&clocal);CHKERRQ(ierr);
8873   ierr = VecDestroy(&cglobalo);CHKERRQ(ierr);
8874   ierr = VecDestroy(&rscale);CHKERRQ(ierr);
8875   ierr = MatDestroy(&interpo);CHKERRQ(ierr);
8876   ierr = DMDestroy(&dmfo);CHKERRQ(ierr);
8877   ierr = DMDestroy(&dmco);CHKERRQ(ierr);
8878   PetscFunctionReturn(0);
8879 }
8880 
8881 PETSC_INTERN PetscErrorCode DMInterpolateSolution_Plex(DM coarse, DM fine, Mat interp, Vec coarseSol, Vec fineSol)
8882 {
8883   PetscObject    shifto;
8884   Vec            shift;
8885 
8886   PetscErrorCode ierr;
8887 
8888   PetscFunctionBegin;
8889   if (!interp) {
8890     Vec rscale;
8891 
8892     ierr = DMCreateInterpolation(coarse, fine, &interp, &rscale);CHKERRQ(ierr);
8893     ierr = VecDestroy(&rscale);CHKERRQ(ierr);
8894   } else {
8895     ierr = PetscObjectReference((PetscObject)interp);CHKERRQ(ierr);
8896   }
8897   ierr = PetscObjectQuery((PetscObject)interp, "_DMInterpolateSolution_Plex_Vec", &shifto);CHKERRQ(ierr);
8898   if (!shifto) {
8899     ierr = DMCreateAffineInterpolationCorrection_Plex(coarse, fine, &shift);CHKERRQ(ierr);
8900     ierr = PetscObjectCompose((PetscObject)interp, "_DMInterpolateSolution_Plex_Vec", (PetscObject) shift);CHKERRQ(ierr);
8901     shifto = (PetscObject) shift;
8902     ierr = VecDestroy(&shift);CHKERRQ(ierr);
8903   }
8904   shift = (Vec) shifto;
8905   ierr = MatInterpolate(interp, coarseSol, fineSol);CHKERRQ(ierr);
8906   ierr = VecAXPY(fineSol, 1.0, shift);CHKERRQ(ierr);
8907   ierr = MatDestroy(&interp);CHKERRQ(ierr);
8908   PetscFunctionReturn(0);
8909 }
8910 
8911 /* Pointwise interpolation
8912      Just code FEM for now
8913      u^f = I u^c
8914      sum_k u^f_k phi^f_k = I sum_j u^c_j phi^c_j
8915      u^f_i = sum_j psi^f_i I phi^c_j u^c_j
8916      I_{ij} = psi^f_i phi^c_j
8917 */
8918 PetscErrorCode DMCreateInterpolation_Plex(DM dmCoarse, DM dmFine, Mat *interpolation, Vec *scaling)
8919 {
8920   PetscSection   gsc, gsf;
8921   PetscInt       m, n;
8922   void          *ctx;
8923   DM             cdm;
8924   PetscBool      regular, ismatis, isRefined = dmCoarse->data == dmFine->data ? PETSC_FALSE : PETSC_TRUE;
8925   PetscErrorCode ierr;
8926 
8927   PetscFunctionBegin;
8928   ierr = DMGetGlobalSection(dmFine, &gsf);CHKERRQ(ierr);
8929   ierr = PetscSectionGetConstrainedStorageSize(gsf, &m);CHKERRQ(ierr);
8930   ierr = DMGetGlobalSection(dmCoarse, &gsc);CHKERRQ(ierr);
8931   ierr = PetscSectionGetConstrainedStorageSize(gsc, &n);CHKERRQ(ierr);
8932 
8933   ierr = PetscStrcmp(dmCoarse->mattype, MATIS, &ismatis);CHKERRQ(ierr);
8934   ierr = MatCreate(PetscObjectComm((PetscObject) dmCoarse), interpolation);CHKERRQ(ierr);
8935   ierr = MatSetSizes(*interpolation, m, n, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
8936   ierr = MatSetType(*interpolation, ismatis ? MATAIJ : dmCoarse->mattype);CHKERRQ(ierr);
8937   ierr = DMGetApplicationContext(dmFine, &ctx);CHKERRQ(ierr);
8938 
8939   ierr = DMGetCoarseDM(dmFine, &cdm);CHKERRQ(ierr);
8940   ierr = DMPlexGetRegularRefinement(dmFine, &regular);CHKERRQ(ierr);
8941   if (!isRefined || (regular && cdm == dmCoarse)) {ierr = DMPlexComputeInterpolatorNested(dmCoarse, dmFine, isRefined, *interpolation, ctx);CHKERRQ(ierr);}
8942   else                                            {ierr = DMPlexComputeInterpolatorGeneral(dmCoarse, dmFine, *interpolation, ctx);CHKERRQ(ierr);}
8943   ierr = MatViewFromOptions(*interpolation, NULL, "-interp_mat_view");CHKERRQ(ierr);
8944   if (scaling) {
8945     /* Use naive scaling */
8946     ierr = DMCreateInterpolationScale(dmCoarse, dmFine, *interpolation, scaling);CHKERRQ(ierr);
8947   }
8948   PetscFunctionReturn(0);
8949 }
8950 
8951 PetscErrorCode DMCreateInjection_Plex(DM dmCoarse, DM dmFine, Mat *mat)
8952 {
8953   PetscErrorCode ierr;
8954   VecScatter     ctx;
8955 
8956   PetscFunctionBegin;
8957   ierr = DMPlexComputeInjectorFEM(dmCoarse, dmFine, &ctx, NULL);CHKERRQ(ierr);
8958   ierr = MatCreateScatter(PetscObjectComm((PetscObject)ctx), ctx, mat);CHKERRQ(ierr);
8959   ierr = VecScatterDestroy(&ctx);CHKERRQ(ierr);
8960   PetscFunctionReturn(0);
8961 }
8962 
8963 static void g0_identity_private(PetscInt dim, PetscInt Nf, PetscInt NfAux,
8964                                 const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[],
8965                                 const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[],
8966                                 PetscReal t, PetscReal u_tShift, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar g0[])
8967 {
8968   g0[0] = 1.0;
8969 }
8970 
8971 PetscErrorCode DMCreateMassMatrix_Plex(DM dmCoarse, DM dmFine, Mat *mass)
8972 {
8973   PetscSection   gsc, gsf;
8974   PetscInt       m, n;
8975   void          *ctx;
8976   DM             cdm;
8977   PetscBool      regular;
8978   PetscErrorCode ierr;
8979 
8980   PetscFunctionBegin;
8981   if (dmFine == dmCoarse) {
8982     DM       dmc;
8983     PetscDS  ds;
8984     Vec      u;
8985     IS       cellIS;
8986     PetscFormKey key;
8987     PetscInt depth;
8988 
8989     ierr = DMClone(dmFine, &dmc);CHKERRQ(ierr);
8990     ierr = DMCopyDisc(dmFine, dmc);CHKERRQ(ierr);
8991     ierr = DMGetDS(dmc, &ds);CHKERRQ(ierr);
8992     ierr = PetscDSSetJacobian(ds, 0, 0, g0_identity_private, NULL, NULL, NULL);CHKERRQ(ierr);
8993     ierr = DMCreateMatrix(dmc, mass);CHKERRQ(ierr);
8994     ierr = DMGetGlobalVector(dmc, &u);CHKERRQ(ierr);
8995     ierr = DMPlexGetDepth(dmc, &depth);CHKERRQ(ierr);
8996     ierr = DMGetStratumIS(dmc, "depth", depth, &cellIS);CHKERRQ(ierr);
8997     ierr = MatZeroEntries(*mass);CHKERRQ(ierr);
8998     key.label = NULL;
8999     key.value = 0;
9000     key.field = 0;
9001     key.part  = 0;
9002     ierr = DMPlexComputeJacobian_Internal(dmc, key, cellIS, 0.0, 0.0, u, NULL, *mass, *mass, NULL);CHKERRQ(ierr);
9003     ierr = ISDestroy(&cellIS);CHKERRQ(ierr);
9004     ierr = DMRestoreGlobalVector(dmc, &u);CHKERRQ(ierr);
9005     ierr = DMDestroy(&dmc);CHKERRQ(ierr);
9006   } else {
9007     ierr = DMGetGlobalSection(dmFine, &gsf);CHKERRQ(ierr);
9008     ierr = PetscSectionGetConstrainedStorageSize(gsf, &m);CHKERRQ(ierr);
9009     ierr = DMGetGlobalSection(dmCoarse, &gsc);CHKERRQ(ierr);
9010     ierr = PetscSectionGetConstrainedStorageSize(gsc, &n);CHKERRQ(ierr);
9011 
9012     ierr = MatCreate(PetscObjectComm((PetscObject) dmCoarse), mass);CHKERRQ(ierr);
9013     ierr = MatSetSizes(*mass, m, n, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
9014     ierr = MatSetType(*mass, dmCoarse->mattype);CHKERRQ(ierr);
9015     ierr = DMGetApplicationContext(dmFine, &ctx);CHKERRQ(ierr);
9016 
9017     ierr = DMGetCoarseDM(dmFine, &cdm);CHKERRQ(ierr);
9018     ierr = DMPlexGetRegularRefinement(dmFine, &regular);CHKERRQ(ierr);
9019     if (regular && cdm == dmCoarse) {ierr = DMPlexComputeMassMatrixNested(dmCoarse, dmFine, *mass, ctx);CHKERRQ(ierr);}
9020     else                            {ierr = DMPlexComputeMassMatrixGeneral(dmCoarse, dmFine, *mass, ctx);CHKERRQ(ierr);}
9021   }
9022   ierr = MatViewFromOptions(*mass, NULL, "-mass_mat_view");CHKERRQ(ierr);
9023   PetscFunctionReturn(0);
9024 }
9025 
9026 /*@
9027   DMPlexGetRegularRefinement - Get the flag indicating that this mesh was obtained by regular refinement from its coarse mesh
9028 
9029   Input Parameter:
9030 . dm - The DMPlex object
9031 
9032   Output Parameter:
9033 . regular - The flag
9034 
9035   Level: intermediate
9036 
9037 .seealso: DMPlexSetRegularRefinement()
9038 @*/
9039 PetscErrorCode DMPlexGetRegularRefinement(DM dm, PetscBool *regular)
9040 {
9041   PetscFunctionBegin;
9042   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9043   PetscValidPointer(regular, 2);
9044   *regular = ((DM_Plex *) dm->data)->regularRefinement;
9045   PetscFunctionReturn(0);
9046 }
9047 
9048 /*@
9049   DMPlexSetRegularRefinement - Set the flag indicating that this mesh was obtained by regular refinement from its coarse mesh
9050 
9051   Input Parameters:
9052 + dm - The DMPlex object
9053 - regular - The flag
9054 
9055   Level: intermediate
9056 
9057 .seealso: DMPlexGetRegularRefinement()
9058 @*/
9059 PetscErrorCode DMPlexSetRegularRefinement(DM dm, PetscBool regular)
9060 {
9061   PetscFunctionBegin;
9062   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9063   ((DM_Plex *) dm->data)->regularRefinement = regular;
9064   PetscFunctionReturn(0);
9065 }
9066 
9067 /*@
9068   DMPlexGetCellRefinerType - Get the strategy for refining a cell
9069 
9070   Input Parameter:
9071 . dm - The DMPlex object
9072 
9073   Output Parameter:
9074 . cr - The strategy number
9075 
9076   Level: intermediate
9077 
9078 .seealso: DMPlexSetCellRefinerType(), DMPlexSetRegularRefinement()
9079 @*/
9080 PetscErrorCode DMPlexGetCellRefinerType(DM dm, DMPlexCellRefinerType *cr)
9081 {
9082   PetscFunctionBegin;
9083   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9084   PetscValidPointer(cr, 2);
9085   *cr = ((DM_Plex *) dm->data)->cellRefiner;
9086   PetscFunctionReturn(0);
9087 }
9088 
9089 /*@
9090   DMPlexSetCellRefinerType - Set the strategy for refining a cell
9091 
9092   Input Parameters:
9093 + dm - The DMPlex object
9094 - cr - The strategy number
9095 
9096   Level: intermediate
9097 
9098 .seealso: DMPlexGetCellRefinerType(), DMPlexGetRegularRefinement()
9099 @*/
9100 PetscErrorCode DMPlexSetCellRefinerType(DM dm, DMPlexCellRefinerType cr)
9101 {
9102   PetscFunctionBegin;
9103   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9104   ((DM_Plex *) dm->data)->cellRefiner = cr;
9105   PetscFunctionReturn(0);
9106 }
9107 
9108 /* anchors */
9109 /*@
9110   DMPlexGetAnchors - Get the layout of the anchor (point-to-point) constraints.  Typically, the user will not have to
9111   call DMPlexGetAnchors() directly: if there are anchors, then DMPlexGetAnchors() is called during DMGetConstraints().
9112 
9113   not collective
9114 
9115   Input Parameters:
9116 . dm - The DMPlex object
9117 
9118   Output Parameters:
9119 + anchorSection - If not NULL, set to the section describing which points anchor the constrained points.
9120 - anchorIS - If not NULL, set to the list of anchors indexed by anchorSection
9121 
9122   Level: intermediate
9123 
9124 .seealso: DMPlexSetAnchors(), DMGetConstraints(), DMSetConstraints()
9125 @*/
9126 PetscErrorCode DMPlexGetAnchors(DM dm, PetscSection *anchorSection, IS *anchorIS)
9127 {
9128   DM_Plex *plex = (DM_Plex *)dm->data;
9129   PetscErrorCode ierr;
9130 
9131   PetscFunctionBegin;
9132   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9133   if (!plex->anchorSection && !plex->anchorIS && plex->createanchors) {ierr = (*plex->createanchors)(dm);CHKERRQ(ierr);}
9134   if (anchorSection) *anchorSection = plex->anchorSection;
9135   if (anchorIS) *anchorIS = plex->anchorIS;
9136   PetscFunctionReturn(0);
9137 }
9138 
9139 /*@
9140   DMPlexSetAnchors - Set the layout of the local anchor (point-to-point) constraints.  Unlike boundary conditions,
9141   when a point's degrees of freedom in a section are constrained to an outside value, the anchor constraints set a
9142   point's degrees of freedom to be a linear combination of other points' degrees of freedom.
9143 
9144   After specifying the layout of constraints with DMPlexSetAnchors(), one specifies the constraints by calling
9145   DMGetConstraints() and filling in the entries in the constraint matrix.
9146 
9147   collective on dm
9148 
9149   Input Parameters:
9150 + dm - The DMPlex object
9151 . anchorSection - The section that describes the mapping from constrained points to the anchor points listed in anchorIS.  Must have a local communicator (PETSC_COMM_SELF or derivative).
9152 - anchorIS - The list of all anchor points.  Must have a local communicator (PETSC_COMM_SELF or derivative).
9153 
9154   The reference counts of anchorSection and anchorIS are incremented.
9155 
9156   Level: intermediate
9157 
9158 .seealso: DMPlexGetAnchors(), DMGetConstraints(), DMSetConstraints()
9159 @*/
9160 PetscErrorCode DMPlexSetAnchors(DM dm, PetscSection anchorSection, IS anchorIS)
9161 {
9162   DM_Plex        *plex = (DM_Plex *)dm->data;
9163   PetscMPIInt    result;
9164   PetscErrorCode ierr;
9165 
9166   PetscFunctionBegin;
9167   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9168   if (anchorSection) {
9169     PetscValidHeaderSpecific(anchorSection,PETSC_SECTION_CLASSID,2);
9170     ierr = MPI_Comm_compare(PETSC_COMM_SELF,PetscObjectComm((PetscObject)anchorSection),&result);CHKERRMPI(ierr);
9171     if (result != MPI_CONGRUENT && result != MPI_IDENT) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_NOTSAMECOMM,"anchor section must have local communicator");
9172   }
9173   if (anchorIS) {
9174     PetscValidHeaderSpecific(anchorIS,IS_CLASSID,3);
9175     ierr = MPI_Comm_compare(PETSC_COMM_SELF,PetscObjectComm((PetscObject)anchorIS),&result);CHKERRMPI(ierr);
9176     if (result != MPI_CONGRUENT && result != MPI_IDENT) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_NOTSAMECOMM,"anchor IS must have local communicator");
9177   }
9178 
9179   ierr = PetscObjectReference((PetscObject)anchorSection);CHKERRQ(ierr);
9180   ierr = PetscSectionDestroy(&plex->anchorSection);CHKERRQ(ierr);
9181   plex->anchorSection = anchorSection;
9182 
9183   ierr = PetscObjectReference((PetscObject)anchorIS);CHKERRQ(ierr);
9184   ierr = ISDestroy(&plex->anchorIS);CHKERRQ(ierr);
9185   plex->anchorIS = anchorIS;
9186 
9187   if (PetscUnlikelyDebug(anchorIS && anchorSection)) {
9188     PetscInt size, a, pStart, pEnd;
9189     const PetscInt *anchors;
9190 
9191     ierr = PetscSectionGetChart(anchorSection,&pStart,&pEnd);CHKERRQ(ierr);
9192     ierr = ISGetLocalSize(anchorIS,&size);CHKERRQ(ierr);
9193     ierr = ISGetIndices(anchorIS,&anchors);CHKERRQ(ierr);
9194     for (a = 0; a < size; a++) {
9195       PetscInt p;
9196 
9197       p = anchors[a];
9198       if (p >= pStart && p < pEnd) {
9199         PetscInt dof;
9200 
9201         ierr = PetscSectionGetDof(anchorSection,p,&dof);CHKERRQ(ierr);
9202         if (dof) {
9203           PetscErrorCode ierr2;
9204 
9205           ierr2 = ISRestoreIndices(anchorIS,&anchors);CHKERRQ(ierr2);
9206           SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_INCOMP,"Point %D cannot be constrained and an anchor",p);
9207         }
9208       }
9209     }
9210     ierr = ISRestoreIndices(anchorIS,&anchors);CHKERRQ(ierr);
9211   }
9212   /* reset the generic constraints */
9213   ierr = DMSetDefaultConstraints(dm,NULL,NULL);CHKERRQ(ierr);
9214   PetscFunctionReturn(0);
9215 }
9216 
9217 static PetscErrorCode DMPlexCreateConstraintSection_Anchors(DM dm, PetscSection section, PetscSection *cSec)
9218 {
9219   PetscSection anchorSection;
9220   PetscInt pStart, pEnd, sStart, sEnd, p, dof, numFields, f;
9221   PetscErrorCode ierr;
9222 
9223   PetscFunctionBegin;
9224   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9225   ierr = DMPlexGetAnchors(dm,&anchorSection,NULL);CHKERRQ(ierr);
9226   ierr = PetscSectionCreate(PETSC_COMM_SELF,cSec);CHKERRQ(ierr);
9227   ierr = PetscSectionGetNumFields(section,&numFields);CHKERRQ(ierr);
9228   if (numFields) {
9229     PetscInt f;
9230     ierr = PetscSectionSetNumFields(*cSec,numFields);CHKERRQ(ierr);
9231 
9232     for (f = 0; f < numFields; f++) {
9233       PetscInt numComp;
9234 
9235       ierr = PetscSectionGetFieldComponents(section,f,&numComp);CHKERRQ(ierr);
9236       ierr = PetscSectionSetFieldComponents(*cSec,f,numComp);CHKERRQ(ierr);
9237     }
9238   }
9239   ierr = PetscSectionGetChart(anchorSection,&pStart,&pEnd);CHKERRQ(ierr);
9240   ierr = PetscSectionGetChart(section,&sStart,&sEnd);CHKERRQ(ierr);
9241   pStart = PetscMax(pStart,sStart);
9242   pEnd   = PetscMin(pEnd,sEnd);
9243   pEnd   = PetscMax(pStart,pEnd);
9244   ierr = PetscSectionSetChart(*cSec,pStart,pEnd);CHKERRQ(ierr);
9245   for (p = pStart; p < pEnd; p++) {
9246     ierr = PetscSectionGetDof(anchorSection,p,&dof);CHKERRQ(ierr);
9247     if (dof) {
9248       ierr = PetscSectionGetDof(section,p,&dof);CHKERRQ(ierr);
9249       ierr = PetscSectionSetDof(*cSec,p,dof);CHKERRQ(ierr);
9250       for (f = 0; f < numFields; f++) {
9251         ierr = PetscSectionGetFieldDof(section,p,f,&dof);CHKERRQ(ierr);
9252         ierr = PetscSectionSetFieldDof(*cSec,p,f,dof);CHKERRQ(ierr);
9253       }
9254     }
9255   }
9256   ierr = PetscSectionSetUp(*cSec);CHKERRQ(ierr);
9257   ierr = PetscObjectSetName((PetscObject) *cSec, "Constraint Section");CHKERRQ(ierr);
9258   PetscFunctionReturn(0);
9259 }
9260 
9261 static PetscErrorCode DMPlexCreateConstraintMatrix_Anchors(DM dm, PetscSection section, PetscSection cSec, Mat *cMat)
9262 {
9263   PetscSection   aSec;
9264   PetscInt       pStart, pEnd, p, sStart, sEnd, dof, aDof, aOff, off, nnz, annz, m, n, q, a, offset, *i, *j;
9265   const PetscInt *anchors;
9266   PetscInt       numFields, f;
9267   IS             aIS;
9268   PetscErrorCode ierr;
9269   MatType        mtype;
9270   PetscBool      iscuda,iskokkos;
9271 
9272   PetscFunctionBegin;
9273   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9274   ierr = PetscSectionGetStorageSize(cSec, &m);CHKERRQ(ierr);
9275   ierr = PetscSectionGetStorageSize(section, &n);CHKERRQ(ierr);
9276   ierr = MatCreate(PETSC_COMM_SELF,cMat);CHKERRQ(ierr);
9277   ierr = MatSetSizes(*cMat,m,n,m,n);CHKERRQ(ierr);
9278   ierr = PetscStrcmp(dm->mattype,MATSEQAIJCUSPARSE,&iscuda);CHKERRQ(ierr);
9279   if (!iscuda) { ierr = PetscStrcmp(dm->mattype,MATMPIAIJCUSPARSE,&iscuda);CHKERRQ(ierr); }
9280   ierr = PetscStrcmp(dm->mattype,MATSEQAIJKOKKOS,&iskokkos);CHKERRQ(ierr);
9281   if (!iskokkos) { ierr = PetscStrcmp(dm->mattype,MATMPIAIJKOKKOS,&iskokkos);CHKERRQ(ierr); }
9282   if (iscuda) mtype = MATSEQAIJCUSPARSE;
9283   else if (iskokkos) mtype = MATSEQAIJKOKKOS;
9284   else mtype = MATSEQAIJ;
9285   ierr = MatSetType(*cMat,mtype);CHKERRQ(ierr);
9286   ierr = DMPlexGetAnchors(dm,&aSec,&aIS);CHKERRQ(ierr);
9287   ierr = ISGetIndices(aIS,&anchors);CHKERRQ(ierr);
9288   /* cSec will be a subset of aSec and section */
9289   ierr = PetscSectionGetChart(cSec,&pStart,&pEnd);CHKERRQ(ierr);
9290   ierr = PetscSectionGetChart(section,&sStart,&sEnd);CHKERRQ(ierr);
9291   ierr = PetscMalloc1(m+1,&i);CHKERRQ(ierr);
9292   i[0] = 0;
9293   ierr = PetscSectionGetNumFields(section,&numFields);CHKERRQ(ierr);
9294   for (p = pStart; p < pEnd; p++) {
9295     PetscInt rDof, rOff, r;
9296 
9297     ierr = PetscSectionGetDof(aSec,p,&rDof);CHKERRQ(ierr);
9298     if (!rDof) continue;
9299     ierr = PetscSectionGetOffset(aSec,p,&rOff);CHKERRQ(ierr);
9300     if (numFields) {
9301       for (f = 0; f < numFields; f++) {
9302         annz = 0;
9303         for (r = 0; r < rDof; r++) {
9304           a = anchors[rOff + r];
9305           if (a < sStart || a >= sEnd) continue;
9306           ierr = PetscSectionGetFieldDof(section,a,f,&aDof);CHKERRQ(ierr);
9307           annz += aDof;
9308         }
9309         ierr = PetscSectionGetFieldDof(cSec,p,f,&dof);CHKERRQ(ierr);
9310         ierr = PetscSectionGetFieldOffset(cSec,p,f,&off);CHKERRQ(ierr);
9311         for (q = 0; q < dof; q++) {
9312           i[off + q + 1] = i[off + q] + annz;
9313         }
9314       }
9315     }
9316     else {
9317       annz = 0;
9318       ierr = PetscSectionGetDof(cSec,p,&dof);CHKERRQ(ierr);
9319       for (q = 0; q < dof; q++) {
9320         a = anchors[rOff + q];
9321         if (a < sStart || a >= sEnd) continue;
9322         ierr = PetscSectionGetDof(section,a,&aDof);CHKERRQ(ierr);
9323         annz += aDof;
9324       }
9325       ierr = PetscSectionGetDof(cSec,p,&dof);CHKERRQ(ierr);
9326       ierr = PetscSectionGetOffset(cSec,p,&off);CHKERRQ(ierr);
9327       for (q = 0; q < dof; q++) {
9328         i[off + q + 1] = i[off + q] + annz;
9329       }
9330     }
9331   }
9332   nnz = i[m];
9333   ierr = PetscMalloc1(nnz,&j);CHKERRQ(ierr);
9334   offset = 0;
9335   for (p = pStart; p < pEnd; p++) {
9336     if (numFields) {
9337       for (f = 0; f < numFields; f++) {
9338         ierr = PetscSectionGetFieldDof(cSec,p,f,&dof);CHKERRQ(ierr);
9339         for (q = 0; q < dof; q++) {
9340           PetscInt rDof, rOff, r;
9341           ierr = PetscSectionGetDof(aSec,p,&rDof);CHKERRQ(ierr);
9342           ierr = PetscSectionGetOffset(aSec,p,&rOff);CHKERRQ(ierr);
9343           for (r = 0; r < rDof; r++) {
9344             PetscInt s;
9345 
9346             a = anchors[rOff + r];
9347             if (a < sStart || a >= sEnd) continue;
9348             ierr = PetscSectionGetFieldDof(section,a,f,&aDof);CHKERRQ(ierr);
9349             ierr = PetscSectionGetFieldOffset(section,a,f,&aOff);CHKERRQ(ierr);
9350             for (s = 0; s < aDof; s++) {
9351               j[offset++] = aOff + s;
9352             }
9353           }
9354         }
9355       }
9356     }
9357     else {
9358       ierr = PetscSectionGetDof(cSec,p,&dof);CHKERRQ(ierr);
9359       for (q = 0; q < dof; q++) {
9360         PetscInt rDof, rOff, r;
9361         ierr = PetscSectionGetDof(aSec,p,&rDof);CHKERRQ(ierr);
9362         ierr = PetscSectionGetOffset(aSec,p,&rOff);CHKERRQ(ierr);
9363         for (r = 0; r < rDof; r++) {
9364           PetscInt s;
9365 
9366           a = anchors[rOff + r];
9367           if (a < sStart || a >= sEnd) continue;
9368           ierr = PetscSectionGetDof(section,a,&aDof);CHKERRQ(ierr);
9369           ierr = PetscSectionGetOffset(section,a,&aOff);CHKERRQ(ierr);
9370           for (s = 0; s < aDof; s++) {
9371             j[offset++] = aOff + s;
9372           }
9373         }
9374       }
9375     }
9376   }
9377   ierr = MatSeqAIJSetPreallocationCSR(*cMat,i,j,NULL);CHKERRQ(ierr);
9378   ierr = PetscFree(i);CHKERRQ(ierr);
9379   ierr = PetscFree(j);CHKERRQ(ierr);
9380   ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);
9381   PetscFunctionReturn(0);
9382 }
9383 
9384 PetscErrorCode DMCreateDefaultConstraints_Plex(DM dm)
9385 {
9386   DM_Plex        *plex = (DM_Plex *)dm->data;
9387   PetscSection   anchorSection, section, cSec;
9388   Mat            cMat;
9389   PetscErrorCode ierr;
9390 
9391   PetscFunctionBegin;
9392   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9393   ierr = DMPlexGetAnchors(dm,&anchorSection,NULL);CHKERRQ(ierr);
9394   if (anchorSection) {
9395     PetscInt Nf;
9396 
9397     ierr = DMGetLocalSection(dm,&section);CHKERRQ(ierr);
9398     ierr = DMPlexCreateConstraintSection_Anchors(dm,section,&cSec);CHKERRQ(ierr);
9399     ierr = DMPlexCreateConstraintMatrix_Anchors(dm,section,cSec,&cMat);CHKERRQ(ierr);
9400     ierr = DMGetNumFields(dm,&Nf);CHKERRQ(ierr);
9401     if (Nf && plex->computeanchormatrix) {ierr = (*plex->computeanchormatrix)(dm,section,cSec,cMat);CHKERRQ(ierr);}
9402     ierr = DMSetDefaultConstraints(dm,cSec,cMat);CHKERRQ(ierr);
9403     ierr = PetscSectionDestroy(&cSec);CHKERRQ(ierr);
9404     ierr = MatDestroy(&cMat);CHKERRQ(ierr);
9405   }
9406   PetscFunctionReturn(0);
9407 }
9408 
9409 PetscErrorCode DMCreateSubDomainDM_Plex(DM dm, DMLabel label, PetscInt value, IS *is, DM *subdm)
9410 {
9411   IS             subis;
9412   PetscSection   section, subsection;
9413   PetscErrorCode ierr;
9414 
9415   PetscFunctionBegin;
9416   ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);
9417   if (!section) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Must set default section for DM before splitting subdomain");
9418   if (!subdm)   SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Must set output subDM for splitting subdomain");
9419   /* Create subdomain */
9420   ierr = DMPlexFilter(dm, label, value, subdm);CHKERRQ(ierr);
9421   /* Create submodel */
9422   ierr = DMPlexGetSubpointIS(*subdm, &subis);CHKERRQ(ierr);
9423   ierr = PetscSectionCreateSubmeshSection(section, subis, &subsection);CHKERRQ(ierr);
9424   ierr = DMSetLocalSection(*subdm, subsection);CHKERRQ(ierr);
9425   ierr = PetscSectionDestroy(&subsection);CHKERRQ(ierr);
9426   ierr = DMCopyDisc(dm, *subdm);CHKERRQ(ierr);
9427   /* Create map from submodel to global model */
9428   if (is) {
9429     PetscSection    sectionGlobal, subsectionGlobal;
9430     IS              spIS;
9431     const PetscInt *spmap;
9432     PetscInt       *subIndices;
9433     PetscInt        subSize = 0, subOff = 0, pStart, pEnd, p;
9434     PetscInt        Nf, f, bs = -1, bsLocal[2], bsMinMax[2];
9435 
9436     ierr = DMPlexGetSubpointIS(*subdm, &spIS);CHKERRQ(ierr);
9437     ierr = ISGetIndices(spIS, &spmap);CHKERRQ(ierr);
9438     ierr = PetscSectionGetNumFields(section, &Nf);CHKERRQ(ierr);
9439     ierr = DMGetGlobalSection(dm, &sectionGlobal);CHKERRQ(ierr);
9440     ierr = DMGetGlobalSection(*subdm, &subsectionGlobal);CHKERRQ(ierr);
9441     ierr = PetscSectionGetChart(subsection, &pStart, &pEnd);CHKERRQ(ierr);
9442     for (p = pStart; p < pEnd; ++p) {
9443       PetscInt gdof, pSubSize  = 0;
9444 
9445       ierr = PetscSectionGetDof(sectionGlobal, p, &gdof);CHKERRQ(ierr);
9446       if (gdof > 0) {
9447         for (f = 0; f < Nf; ++f) {
9448           PetscInt fdof, fcdof;
9449 
9450           ierr     = PetscSectionGetFieldDof(subsection, p, f, &fdof);CHKERRQ(ierr);
9451           ierr     = PetscSectionGetFieldConstraintDof(subsection, p, f, &fcdof);CHKERRQ(ierr);
9452           pSubSize += fdof-fcdof;
9453         }
9454         subSize += pSubSize;
9455         if (pSubSize) {
9456           if (bs < 0) {
9457             bs = pSubSize;
9458           } else if (bs != pSubSize) {
9459             /* Layout does not admit a pointwise block size */
9460             bs = 1;
9461           }
9462         }
9463       }
9464     }
9465     /* Must have same blocksize on all procs (some might have no points) */
9466     bsLocal[0] = bs < 0 ? PETSC_MAX_INT : bs; bsLocal[1] = bs;
9467     ierr = PetscGlobalMinMaxInt(PetscObjectComm((PetscObject) dm), bsLocal, bsMinMax);CHKERRQ(ierr);
9468     if (bsMinMax[0] != bsMinMax[1]) {bs = 1;}
9469     else                            {bs = bsMinMax[0];}
9470     ierr = PetscMalloc1(subSize, &subIndices);CHKERRQ(ierr);
9471     for (p = pStart; p < pEnd; ++p) {
9472       PetscInt gdof, goff;
9473 
9474       ierr = PetscSectionGetDof(subsectionGlobal, p, &gdof);CHKERRQ(ierr);
9475       if (gdof > 0) {
9476         const PetscInt point = spmap[p];
9477 
9478         ierr = PetscSectionGetOffset(sectionGlobal, point, &goff);CHKERRQ(ierr);
9479         for (f = 0; f < Nf; ++f) {
9480           PetscInt fdof, fcdof, fc, f2, poff = 0;
9481 
9482           /* Can get rid of this loop by storing field information in the global section */
9483           for (f2 = 0; f2 < f; ++f2) {
9484             ierr  = PetscSectionGetFieldDof(section, p, f2, &fdof);CHKERRQ(ierr);
9485             ierr  = PetscSectionGetFieldConstraintDof(section, p, f2, &fcdof);CHKERRQ(ierr);
9486             poff += fdof-fcdof;
9487           }
9488           ierr = PetscSectionGetFieldDof(section, p, f, &fdof);CHKERRQ(ierr);
9489           ierr = PetscSectionGetFieldConstraintDof(section, p, f, &fcdof);CHKERRQ(ierr);
9490           for (fc = 0; fc < fdof-fcdof; ++fc, ++subOff) {
9491             subIndices[subOff] = goff+poff+fc;
9492           }
9493         }
9494       }
9495     }
9496     ierr = ISRestoreIndices(spIS, &spmap);CHKERRQ(ierr);
9497     ierr = ISCreateGeneral(PetscObjectComm((PetscObject)dm), subSize, subIndices, PETSC_OWN_POINTER, is);CHKERRQ(ierr);
9498     if (bs > 1) {
9499       /* We need to check that the block size does not come from non-contiguous fields */
9500       PetscInt i, j, set = 1;
9501       for (i = 0; i < subSize; i += bs) {
9502         for (j = 0; j < bs; ++j) {
9503           if (subIndices[i+j] != subIndices[i]+j) {set = 0; break;}
9504         }
9505       }
9506       if (set) {ierr = ISSetBlockSize(*is, bs);CHKERRQ(ierr);}
9507     }
9508     /* Attach nullspace */
9509     for (f = 0; f < Nf; ++f) {
9510       (*subdm)->nullspaceConstructors[f] = dm->nullspaceConstructors[f];
9511       if ((*subdm)->nullspaceConstructors[f]) break;
9512     }
9513     if (f < Nf) {
9514       MatNullSpace nullSpace;
9515       ierr = (*(*subdm)->nullspaceConstructors[f])(*subdm, f, f, &nullSpace);CHKERRQ(ierr);
9516 
9517       ierr = PetscObjectCompose((PetscObject) *is, "nullspace", (PetscObject) nullSpace);CHKERRQ(ierr);
9518       ierr = MatNullSpaceDestroy(&nullSpace);CHKERRQ(ierr);
9519     }
9520   }
9521   PetscFunctionReturn(0);
9522 }
9523 
9524 /*@
9525   DMPlexMonitorThroughput - Report the cell throughput of FE integration
9526 
9527   Input Parameter:
9528 - dm - The DM
9529 
9530   Level: developer
9531 
9532   Options Database Keys:
9533 . -dm_plex_monitor_throughput - Activate the monitor
9534 
9535 .seealso: DMSetFromOptions(), DMPlexCreate()
9536 @*/
9537 PetscErrorCode DMPlexMonitorThroughput(DM dm, void *dummy)
9538 {
9539 #if defined(PETSC_USE_LOG)
9540   PetscStageLog      stageLog;
9541   PetscLogEvent      event;
9542   PetscLogStage      stage;
9543   PetscEventPerfInfo eventInfo;
9544   PetscReal          cellRate, flopRate;
9545   PetscInt           cStart, cEnd, Nf, N;
9546   const char        *name;
9547   PetscErrorCode     ierr;
9548 #endif
9549 
9550   PetscFunctionBegin;
9551   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9552 #if defined(PETSC_USE_LOG)
9553   ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
9554   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
9555   ierr = DMGetNumFields(dm, &Nf);CHKERRQ(ierr);
9556   ierr = PetscLogGetStageLog(&stageLog);CHKERRQ(ierr);
9557   ierr = PetscStageLogGetCurrent(stageLog, &stage);CHKERRQ(ierr);
9558   ierr = PetscLogEventGetId("DMPlexResidualFE", &event);CHKERRQ(ierr);
9559   ierr = PetscLogEventGetPerfInfo(stage, event, &eventInfo);CHKERRQ(ierr);
9560   N        = (cEnd - cStart)*Nf*eventInfo.count;
9561   flopRate = eventInfo.flops/eventInfo.time;
9562   cellRate = N/eventInfo.time;
9563   ierr = PetscPrintf(PetscObjectComm((PetscObject) dm), "DM (%s) FE Residual Integration: %D integrals %D reps\n  Cell rate: %.2g/s flop rate: %.2g MF/s\n", name ? name : "unknown", N, eventInfo.count, (double) cellRate, (double) (flopRate/1.e6));CHKERRQ(ierr);
9564 #else
9565   SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Plex Throughput Monitor is not supported if logging is turned off. Reconfigure using --with-log.");
9566 #endif
9567   PetscFunctionReturn(0);
9568 }
9569