xref: /petsc/src/dm/impls/plex/plex.c (revision b5a892a1a70151a25c27b9422f362b5a6d017e0f) 
1 #include <petsc/private/dmpleximpl.h>   /*I      "petscdmplex.h"   I*/
2 #include <petsc/private/isimpl.h>
3 #include <petsc/private/vecimpl.h>
4 #include <petsc/private/glvisvecimpl.h>
5 #include <petscsf.h>
6 #include <petscds.h>
7 #include <petscdraw.h>
8 #include <petscdmfield.h>
9 
10 /* Logging support */
11 PetscLogEvent DMPLEX_Interpolate, DMPLEX_Partition, DMPLEX_Distribute, DMPLEX_DistributeCones, DMPLEX_DistributeLabels, DMPLEX_DistributeSF, DMPLEX_DistributeOverlap, DMPLEX_DistributeField, DMPLEX_DistributeData, DMPLEX_Migrate, DMPLEX_InterpolateSF, DMPLEX_GlobalToNaturalBegin, DMPLEX_GlobalToNaturalEnd, DMPLEX_NaturalToGlobalBegin, DMPLEX_NaturalToGlobalEnd, DMPLEX_Stratify, DMPLEX_Symmetrize, DMPLEX_Preallocate, DMPLEX_ResidualFEM, DMPLEX_JacobianFEM, DMPLEX_InterpolatorFEM, DMPLEX_InjectorFEM, DMPLEX_IntegralFEM, DMPLEX_CreateGmsh, DMPLEX_RebalanceSharedPoints, DMPLEX_PartSelf, DMPLEX_PartLabelInvert, DMPLEX_PartLabelCreateSF, DMPLEX_PartStratSF, DMPLEX_CreatePointSF,DMPLEX_LocatePoints;
12 
13 PETSC_EXTERN PetscErrorCode VecView_MPI(Vec, PetscViewer);
14 
15 /*@
16   DMPlexIsSimplex - Is the first cell in this mesh a simplex?
17 
18   Input Parameter:
19 . dm      - The DMPlex object
20 
21   Output Parameter:
22 . simplex - Flag checking for a simplex
23 
24   Note: This just gives the first range of cells found. If the mesh has several cell types, it will only give the first.
25   If the mesh has no cells, this returns PETSC_FALSE.
26 
27   Level: intermediate
28 
29 .seealso DMPlexGetSimplexOrBoxCells(), DMPlexGetCellType(), DMPlexGetHeightStratum(), DMPolytopeTypeGetNumVertices()
30 @*/
31 PetscErrorCode DMPlexIsSimplex(DM dm, PetscBool *simplex)
32 {
33   DMPolytopeType ct;
34   PetscInt       cStart, cEnd;
35   PetscErrorCode ierr;
36 
37   PetscFunctionBegin;
38   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
39   if (cEnd <= cStart) {*simplex = PETSC_FALSE; PetscFunctionReturn(0);}
40   ierr = DMPlexGetCellType(dm, cStart, &ct);CHKERRQ(ierr);
41   *simplex = DMPolytopeTypeGetNumVertices(ct) == DMPolytopeTypeGetDim(ct)+1 ? PETSC_TRUE : PETSC_FALSE;
42   PetscFunctionReturn(0);
43 }
44 
45 /*@
46   DMPlexGetSimplexOrBoxCells - Get the range of cells which are neither prisms nor ghost FV cells
47 
48   Input Parameter:
49 + dm     - The DMPlex object
50 - height - The cell height in the Plex, 0 is the default
51 
52   Output Parameters:
53 + cStart - The first "normal" cell
54 - cEnd   - The upper bound on "normal"" cells
55 
56   Note: This just gives the first range of cells found. If the mesh has several cell types, it will only give the first.
57 
58   Level: developer
59 
60 .seealso DMPlexConstructGhostCells(), DMPlexGetGhostCellStratum()
61 @*/
62 PetscErrorCode DMPlexGetSimplexOrBoxCells(DM dm, PetscInt height, PetscInt *cStart, PetscInt *cEnd)
63 {
64   DMPolytopeType ct = DM_POLYTOPE_UNKNOWN;
65   PetscInt       cS, cE, c;
66   PetscErrorCode ierr;
67 
68   PetscFunctionBegin;
69   ierr = DMPlexGetHeightStratum(dm, PetscMax(height, 0), &cS, &cE);CHKERRQ(ierr);
70   for (c = cS; c < cE; ++c) {
71     DMPolytopeType cct;
72 
73     ierr = DMPlexGetCellType(dm, c, &cct);CHKERRQ(ierr);
74     if ((PetscInt) cct < 0) break;
75     switch (cct) {
76       case DM_POLYTOPE_POINT:
77       case DM_POLYTOPE_SEGMENT:
78       case DM_POLYTOPE_TRIANGLE:
79       case DM_POLYTOPE_QUADRILATERAL:
80       case DM_POLYTOPE_TETRAHEDRON:
81       case DM_POLYTOPE_HEXAHEDRON:
82         ct = cct;
83         break;
84       default: break;
85     }
86     if (ct != DM_POLYTOPE_UNKNOWN) break;
87   }
88   if (ct != DM_POLYTOPE_UNKNOWN) {
89     DMLabel ctLabel;
90 
91     ierr = DMPlexGetCellTypeLabel(dm, &ctLabel);CHKERRQ(ierr);
92     ierr = DMLabelGetStratumBounds(ctLabel, ct, &cS, &cE);CHKERRQ(ierr);
93   }
94   if (cStart) *cStart = cS;
95   if (cEnd)   *cEnd   = cE;
96   PetscFunctionReturn(0);
97 }
98 
99 PetscErrorCode DMPlexGetFieldType_Internal(DM dm, PetscSection section, PetscInt field, PetscInt *sStart, PetscInt *sEnd, PetscViewerVTKFieldType *ft)
100 {
101   PetscInt       cdim, pStart, pEnd, vStart, vEnd, cStart, cEnd;
102   PetscInt       vcdof[2] = {0,0}, globalvcdof[2];
103   PetscErrorCode ierr;
104 
105   PetscFunctionBegin;
106   *ft  = PETSC_VTK_INVALID;
107   ierr = DMGetCoordinateDim(dm, &cdim);CHKERRQ(ierr);
108   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
109   ierr = DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
110   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
111   if (field >= 0) {
112     if ((vStart >= pStart) && (vStart < pEnd)) {ierr = PetscSectionGetFieldDof(section, vStart, field, &vcdof[0]);CHKERRQ(ierr);}
113     if ((cStart >= pStart) && (cStart < pEnd)) {ierr = PetscSectionGetFieldDof(section, cStart, field, &vcdof[1]);CHKERRQ(ierr);}
114   } else {
115     if ((vStart >= pStart) && (vStart < pEnd)) {ierr = PetscSectionGetDof(section, vStart, &vcdof[0]);CHKERRQ(ierr);}
116     if ((cStart >= pStart) && (cStart < pEnd)) {ierr = PetscSectionGetDof(section, cStart, &vcdof[1]);CHKERRQ(ierr);}
117   }
118   ierr = MPI_Allreduce(vcdof, globalvcdof, 2, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject)dm));CHKERRMPI(ierr);
119   if (globalvcdof[0]) {
120     *sStart = vStart;
121     *sEnd   = vEnd;
122     if (globalvcdof[0] == cdim) *ft = PETSC_VTK_POINT_VECTOR_FIELD;
123     else                        *ft = PETSC_VTK_POINT_FIELD;
124   } else if (globalvcdof[1]) {
125     *sStart = cStart;
126     *sEnd   = cEnd;
127     if (globalvcdof[1] == cdim) *ft = PETSC_VTK_CELL_VECTOR_FIELD;
128     else                        *ft = PETSC_VTK_CELL_FIELD;
129   } else {
130     if (field >= 0) {
131       const char *fieldname;
132 
133       ierr = PetscSectionGetFieldName(section, field, &fieldname);CHKERRQ(ierr);
134       ierr = PetscInfo2((PetscObject) dm, "Could not classify VTK output type of section field %D \"%s\"\n", field, fieldname);CHKERRQ(ierr);
135     } else {
136       ierr = PetscInfo((PetscObject) dm, "Could not classify VTK output type of section\"%s\"\n");CHKERRQ(ierr);
137     }
138   }
139   PetscFunctionReturn(0);
140 }
141 
142 static PetscErrorCode VecView_Plex_Local_Draw(Vec v, PetscViewer viewer)
143 {
144   DM                 dm;
145   PetscSection       s;
146   PetscDraw          draw, popup;
147   DM                 cdm;
148   PetscSection       coordSection;
149   Vec                coordinates;
150   const PetscScalar *coords, *array;
151   PetscReal          bound[4] = {PETSC_MAX_REAL, PETSC_MAX_REAL, PETSC_MIN_REAL, PETSC_MIN_REAL};
152   PetscReal          vbound[2], time;
153   PetscBool          isnull, flg;
154   PetscInt           dim, Nf, f, Nc, comp, vStart, vEnd, cStart, cEnd, c, N, level, step, w = 0;
155   const char        *name;
156   char               title[PETSC_MAX_PATH_LEN];
157   PetscErrorCode     ierr;
158 
159   PetscFunctionBegin;
160   ierr = PetscViewerDrawGetDraw(viewer, 0, &draw);CHKERRQ(ierr);
161   ierr = PetscDrawIsNull(draw, &isnull);CHKERRQ(ierr);
162   if (isnull) PetscFunctionReturn(0);
163 
164   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
165   ierr = DMGetCoordinateDim(dm, &dim);CHKERRQ(ierr);
166   if (dim != 2) SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Cannot draw meshes of dimension %D. Use PETSCVIEWERGLVIS", dim);
167   ierr = DMGetLocalSection(dm, &s);CHKERRQ(ierr);
168   ierr = PetscSectionGetNumFields(s, &Nf);CHKERRQ(ierr);
169   ierr = DMGetCoarsenLevel(dm, &level);CHKERRQ(ierr);
170   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
171   ierr = DMGetLocalSection(cdm, &coordSection);CHKERRQ(ierr);
172   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
173   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
174   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
175 
176   ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
177   ierr = DMGetOutputSequenceNumber(dm, &step, &time);CHKERRQ(ierr);
178 
179   ierr = VecGetLocalSize(coordinates, &N);CHKERRQ(ierr);
180   ierr = VecGetArrayRead(coordinates, &coords);CHKERRQ(ierr);
181   for (c = 0; c < N; c += dim) {
182     bound[0] = PetscMin(bound[0], PetscRealPart(coords[c]));   bound[2] = PetscMax(bound[2], PetscRealPart(coords[c]));
183     bound[1] = PetscMin(bound[1], PetscRealPart(coords[c+1])); bound[3] = PetscMax(bound[3], PetscRealPart(coords[c+1]));
184   }
185   ierr = VecRestoreArrayRead(coordinates, &coords);CHKERRQ(ierr);
186   ierr = PetscDrawClear(draw);CHKERRQ(ierr);
187 
188   /* Could implement something like DMDASelectFields() */
189   for (f = 0; f < Nf; ++f) {
190     DM   fdm = dm;
191     Vec  fv  = v;
192     IS   fis;
193     char prefix[PETSC_MAX_PATH_LEN];
194     const char *fname;
195 
196     ierr = PetscSectionGetFieldComponents(s, f, &Nc);CHKERRQ(ierr);
197     ierr = PetscSectionGetFieldName(s, f, &fname);CHKERRQ(ierr);
198 
199     if (v->hdr.prefix) {ierr = PetscStrncpy(prefix, v->hdr.prefix,sizeof(prefix));CHKERRQ(ierr);}
200     else               {prefix[0] = '\0';}
201     if (Nf > 1) {
202       ierr = DMCreateSubDM(dm, 1, &f, &fis, &fdm);CHKERRQ(ierr);
203       ierr = VecGetSubVector(v, fis, &fv);CHKERRQ(ierr);
204       ierr = PetscStrlcat(prefix, fname,sizeof(prefix));CHKERRQ(ierr);
205       ierr = PetscStrlcat(prefix, "_",sizeof(prefix));CHKERRQ(ierr);
206     }
207     for (comp = 0; comp < Nc; ++comp, ++w) {
208       PetscInt nmax = 2;
209 
210       ierr = PetscViewerDrawGetDraw(viewer, w, &draw);CHKERRQ(ierr);
211       if (Nc > 1) {ierr = PetscSNPrintf(title, sizeof(title), "%s:%s_%D Step: %D Time: %.4g", name, fname, comp, step, time);CHKERRQ(ierr);}
212       else        {ierr = PetscSNPrintf(title, sizeof(title), "%s:%s Step: %D Time: %.4g", name, fname, step, time);CHKERRQ(ierr);}
213       ierr = PetscDrawSetTitle(draw, title);CHKERRQ(ierr);
214 
215       /* TODO Get max and min only for this component */
216       ierr = PetscOptionsGetRealArray(NULL, prefix, "-vec_view_bounds", vbound, &nmax, &flg);CHKERRQ(ierr);
217       if (!flg) {
218         ierr = VecMin(fv, NULL, &vbound[0]);CHKERRQ(ierr);
219         ierr = VecMax(fv, NULL, &vbound[1]);CHKERRQ(ierr);
220         if (vbound[1] <= vbound[0]) vbound[1] = vbound[0] + 1.0;
221       }
222       ierr = PetscDrawGetPopup(draw, &popup);CHKERRQ(ierr);
223       ierr = PetscDrawScalePopup(popup, vbound[0], vbound[1]);CHKERRQ(ierr);
224       ierr = PetscDrawSetCoordinates(draw, bound[0], bound[1], bound[2], bound[3]);CHKERRQ(ierr);
225 
226       ierr = VecGetArrayRead(fv, &array);CHKERRQ(ierr);
227       for (c = cStart; c < cEnd; ++c) {
228         PetscScalar *coords = NULL, *a = NULL;
229         PetscInt     numCoords, color[4] = {-1,-1,-1,-1};
230 
231         ierr = DMPlexPointLocalRead(fdm, c, array, &a);CHKERRQ(ierr);
232         if (a) {
233           color[0] = PetscDrawRealToColor(PetscRealPart(a[comp]), vbound[0], vbound[1]);
234           color[1] = color[2] = color[3] = color[0];
235         } else {
236           PetscScalar *vals = NULL;
237           PetscInt     numVals, va;
238 
239           ierr = DMPlexVecGetClosure(fdm, NULL, fv, c, &numVals, &vals);CHKERRQ(ierr);
240           if (numVals % Nc) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "The number of components %D does not divide the number of values in the closure %D", Nc, numVals);
241           switch (numVals/Nc) {
242           case 3: /* P1 Triangle */
243           case 4: /* P1 Quadrangle */
244             for (va = 0; va < numVals/Nc; ++va) color[va] = PetscDrawRealToColor(PetscRealPart(vals[va*Nc+comp]), vbound[0], vbound[1]);
245             break;
246           case 6: /* P2 Triangle */
247           case 8: /* P2 Quadrangle */
248             for (va = 0; va < numVals/(Nc*2); ++va) color[va] = PetscDrawRealToColor(PetscRealPart(vals[va*Nc+comp + numVals/(Nc*2)]), vbound[0], vbound[1]);
249             break;
250           default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of values for cell closure %D cannot be handled", numVals/Nc);
251           }
252           ierr = DMPlexVecRestoreClosure(fdm, NULL, fv, c, &numVals, &vals);CHKERRQ(ierr);
253         }
254         ierr = DMPlexVecGetClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
255         switch (numCoords) {
256         case 6:
257         case 12: /* Localized triangle */
258           ierr = PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), color[0], color[1], color[2]);CHKERRQ(ierr);
259           break;
260         case 8:
261         case 16: /* Localized quadrilateral */
262           ierr = PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), color[0], color[1], color[2]);CHKERRQ(ierr);
263           ierr = PetscDrawTriangle(draw, PetscRealPart(coords[4]), PetscRealPart(coords[5]), PetscRealPart(coords[6]), PetscRealPart(coords[7]), PetscRealPart(coords[0]), PetscRealPart(coords[1]), color[2], color[3], color[0]);CHKERRQ(ierr);
264           break;
265         default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_SUP, "Cannot draw cells with %D coordinates", numCoords);
266         }
267         ierr = DMPlexVecRestoreClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
268       }
269       ierr = VecRestoreArrayRead(fv, &array);CHKERRQ(ierr);
270       ierr = PetscDrawFlush(draw);CHKERRQ(ierr);
271       ierr = PetscDrawPause(draw);CHKERRQ(ierr);
272       ierr = PetscDrawSave(draw);CHKERRQ(ierr);
273     }
274     if (Nf > 1) {
275       ierr = VecRestoreSubVector(v, fis, &fv);CHKERRQ(ierr);
276       ierr = ISDestroy(&fis);CHKERRQ(ierr);
277       ierr = DMDestroy(&fdm);CHKERRQ(ierr);
278     }
279   }
280   PetscFunctionReturn(0);
281 }
282 
283 static PetscErrorCode VecView_Plex_Local_VTK(Vec v, PetscViewer viewer)
284 {
285   DM                      dm;
286   Vec                     locv;
287   const char              *name;
288   PetscSection            section;
289   PetscInt                pStart, pEnd;
290   PetscInt                numFields;
291   PetscViewerVTKFieldType ft;
292   PetscErrorCode          ierr;
293 
294   PetscFunctionBegin;
295   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
296   ierr = DMCreateLocalVector(dm, &locv);CHKERRQ(ierr); /* VTK viewer requires exclusive ownership of the vector */
297   ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
298   ierr = PetscObjectSetName((PetscObject) locv, name);CHKERRQ(ierr);
299   ierr = VecCopy(v, locv);CHKERRQ(ierr);
300   ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);
301   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
302   if (!numFields) {
303     ierr = DMPlexGetFieldType_Internal(dm, section, PETSC_DETERMINE, &pStart, &pEnd, &ft);CHKERRQ(ierr);
304     ierr = PetscViewerVTKAddField(viewer, (PetscObject) dm, DMPlexVTKWriteAll, PETSC_DEFAULT, ft, PETSC_TRUE,(PetscObject) locv);CHKERRQ(ierr);
305   } else {
306     PetscInt f;
307 
308     for (f = 0; f < numFields; f++) {
309       ierr = DMPlexGetFieldType_Internal(dm, section, f, &pStart, &pEnd, &ft);CHKERRQ(ierr);
310       if (ft == PETSC_VTK_INVALID) continue;
311       ierr = PetscObjectReference((PetscObject)locv);CHKERRQ(ierr);
312       ierr = PetscViewerVTKAddField(viewer, (PetscObject) dm, DMPlexVTKWriteAll, f, ft, PETSC_TRUE,(PetscObject) locv);CHKERRQ(ierr);
313     }
314     ierr = VecDestroy(&locv);CHKERRQ(ierr);
315   }
316   PetscFunctionReturn(0);
317 }
318 
319 PetscErrorCode VecView_Plex_Local(Vec v, PetscViewer viewer)
320 {
321   DM             dm;
322   PetscBool      isvtk, ishdf5, isdraw, isglvis;
323   PetscErrorCode ierr;
324 
325   PetscFunctionBegin;
326   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
327   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
328   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,   &isvtk);CHKERRQ(ierr);
329   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,  &ishdf5);CHKERRQ(ierr);
330   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERDRAW,  &isdraw);CHKERRQ(ierr);
331   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERGLVIS, &isglvis);CHKERRQ(ierr);
332   if (isvtk || ishdf5 || isdraw || isglvis) {
333     PetscInt    i,numFields;
334     PetscObject fe;
335     PetscBool   fem = PETSC_FALSE;
336     Vec         locv = v;
337     const char  *name;
338     PetscInt    step;
339     PetscReal   time;
340 
341     ierr = DMGetNumFields(dm, &numFields);CHKERRQ(ierr);
342     for (i=0; i<numFields; i++) {
343       ierr = DMGetField(dm, i, NULL, &fe);CHKERRQ(ierr);
344       if (fe->classid == PETSCFE_CLASSID) { fem = PETSC_TRUE; break; }
345     }
346     if (fem) {
347       PetscObject isZero;
348 
349       ierr = DMGetLocalVector(dm, &locv);CHKERRQ(ierr);
350       ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
351       ierr = PetscObjectSetName((PetscObject) locv, name);CHKERRQ(ierr);
352       ierr = PetscObjectQuery((PetscObject) v, "__Vec_bc_zero__", &isZero);CHKERRQ(ierr);
353       ierr = PetscObjectCompose((PetscObject) locv, "__Vec_bc_zero__", isZero);CHKERRQ(ierr);
354       ierr = VecCopy(v, locv);CHKERRQ(ierr);
355       ierr = DMGetOutputSequenceNumber(dm, NULL, &time);CHKERRQ(ierr);
356       ierr = DMPlexInsertBoundaryValues(dm, PETSC_TRUE, locv, time, NULL, NULL, NULL);CHKERRQ(ierr);
357     }
358     if (isvtk) {
359       ierr = VecView_Plex_Local_VTK(locv, viewer);CHKERRQ(ierr);
360     } else if (ishdf5) {
361 #if defined(PETSC_HAVE_HDF5)
362       ierr = VecView_Plex_Local_HDF5_Internal(locv, viewer);CHKERRQ(ierr);
363 #else
364       SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
365 #endif
366     } else if (isdraw) {
367       ierr = VecView_Plex_Local_Draw(locv, viewer);CHKERRQ(ierr);
368     } else if (isglvis) {
369       ierr = DMGetOutputSequenceNumber(dm, &step, NULL);CHKERRQ(ierr);
370       ierr = PetscViewerGLVisSetSnapId(viewer, step);CHKERRQ(ierr);
371       ierr = VecView_GLVis(locv, viewer);CHKERRQ(ierr);
372     }
373     if (fem) {
374       ierr = PetscObjectCompose((PetscObject) locv, "__Vec_bc_zero__", NULL);CHKERRQ(ierr);
375       ierr = DMRestoreLocalVector(dm, &locv);CHKERRQ(ierr);
376     }
377   } else {
378     PetscBool isseq;
379 
380     ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
381     if (isseq) {ierr = VecView_Seq(v, viewer);CHKERRQ(ierr);}
382     else       {ierr = VecView_MPI(v, viewer);CHKERRQ(ierr);}
383   }
384   PetscFunctionReturn(0);
385 }
386 
387 PetscErrorCode VecView_Plex(Vec v, PetscViewer viewer)
388 {
389   DM             dm;
390   PetscBool      isvtk, ishdf5, isdraw, isglvis, isexodusii;
391   PetscErrorCode ierr;
392 
393   PetscFunctionBegin;
394   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
395   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
396   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,      &isvtk);CHKERRQ(ierr);
397   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,     &ishdf5);CHKERRQ(ierr);
398   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERDRAW,     &isdraw);CHKERRQ(ierr);
399   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERGLVIS,    &isglvis);CHKERRQ(ierr);
400   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWEREXODUSII, &isexodusii);CHKERRQ(ierr);
401   if (isvtk || isdraw || isglvis) {
402     Vec         locv;
403     PetscObject isZero;
404     const char *name;
405 
406     ierr = DMGetLocalVector(dm, &locv);CHKERRQ(ierr);
407     ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
408     ierr = PetscObjectSetName((PetscObject) locv, name);CHKERRQ(ierr);
409     ierr = DMGlobalToLocalBegin(dm, v, INSERT_VALUES, locv);CHKERRQ(ierr);
410     ierr = DMGlobalToLocalEnd(dm, v, INSERT_VALUES, locv);CHKERRQ(ierr);
411     ierr = PetscObjectQuery((PetscObject) v, "__Vec_bc_zero__", &isZero);CHKERRQ(ierr);
412     ierr = PetscObjectCompose((PetscObject) locv, "__Vec_bc_zero__", isZero);CHKERRQ(ierr);
413     ierr = VecView_Plex_Local(locv, viewer);CHKERRQ(ierr);
414     ierr = PetscObjectCompose((PetscObject) locv, "__Vec_bc_zero__", NULL);CHKERRQ(ierr);
415     ierr = DMRestoreLocalVector(dm, &locv);CHKERRQ(ierr);
416   } else if (ishdf5) {
417 #if defined(PETSC_HAVE_HDF5)
418     ierr = VecView_Plex_HDF5_Internal(v, viewer);CHKERRQ(ierr);
419 #else
420     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
421 #endif
422   } else if (isexodusii) {
423 #if defined(PETSC_HAVE_EXODUSII)
424     ierr = VecView_PlexExodusII_Internal(v, viewer);CHKERRQ(ierr);
425 #else
426     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "ExodusII not supported in this build.\nPlease reconfigure using --download-exodusii");
427 #endif
428   } else {
429     PetscBool isseq;
430 
431     ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
432     if (isseq) {ierr = VecView_Seq(v, viewer);CHKERRQ(ierr);}
433     else       {ierr = VecView_MPI(v, viewer);CHKERRQ(ierr);}
434   }
435   PetscFunctionReturn(0);
436 }
437 
438 PetscErrorCode VecView_Plex_Native(Vec originalv, PetscViewer viewer)
439 {
440   DM                dm;
441   MPI_Comm          comm;
442   PetscViewerFormat format;
443   Vec               v;
444   PetscBool         isvtk, ishdf5;
445   PetscErrorCode    ierr;
446 
447   PetscFunctionBegin;
448   ierr = VecGetDM(originalv, &dm);CHKERRQ(ierr);
449   ierr = PetscObjectGetComm((PetscObject) originalv, &comm);CHKERRQ(ierr);
450   if (!dm) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
451   ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
452   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
453   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,  &isvtk);CHKERRQ(ierr);
454   if (format == PETSC_VIEWER_NATIVE) {
455     /* Natural ordering is the common case for DMDA, NATIVE means plain vector, for PLEX is the opposite */
456     /* this need a better fix */
457     if (dm->useNatural) {
458       if (dm->sfNatural) {
459         const char *vecname;
460         PetscInt    n, nroots;
461 
462         ierr = VecGetLocalSize(originalv, &n);CHKERRQ(ierr);
463         ierr = PetscSFGetGraph(dm->sfNatural, &nroots, NULL, NULL, NULL);CHKERRQ(ierr);
464         if (n == nroots) {
465           ierr = DMGetGlobalVector(dm, &v);CHKERRQ(ierr);
466           ierr = DMPlexGlobalToNaturalBegin(dm, originalv, v);CHKERRQ(ierr);
467           ierr = DMPlexGlobalToNaturalEnd(dm, originalv, v);CHKERRQ(ierr);
468           ierr = PetscObjectGetName((PetscObject) originalv, &vecname);CHKERRQ(ierr);
469           ierr = PetscObjectSetName((PetscObject) v, vecname);CHKERRQ(ierr);
470         } else SETERRQ(comm, PETSC_ERR_ARG_WRONG, "DM global to natural SF only handles global vectors");
471       } else SETERRQ(comm, PETSC_ERR_ARG_WRONGSTATE, "DM global to natural SF was not created");
472     } else v = originalv;
473   } else v = originalv;
474 
475   if (ishdf5) {
476 #if defined(PETSC_HAVE_HDF5)
477     ierr = VecView_Plex_HDF5_Native_Internal(v, viewer);CHKERRQ(ierr);
478 #else
479     SETERRQ(comm, PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
480 #endif
481   } else if (isvtk) {
482     SETERRQ(comm, PETSC_ERR_SUP, "VTK format does not support viewing in natural order. Please switch to HDF5.");
483   } else {
484     PetscBool isseq;
485 
486     ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
487     if (isseq) {ierr = VecView_Seq(v, viewer);CHKERRQ(ierr);}
488     else       {ierr = VecView_MPI(v, viewer);CHKERRQ(ierr);}
489   }
490   if (v != originalv) {ierr = DMRestoreGlobalVector(dm, &v);CHKERRQ(ierr);}
491   PetscFunctionReturn(0);
492 }
493 
494 PetscErrorCode VecLoad_Plex_Local(Vec v, PetscViewer viewer)
495 {
496   DM             dm;
497   PetscBool      ishdf5;
498   PetscErrorCode ierr;
499 
500   PetscFunctionBegin;
501   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
502   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
503   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
504   if (ishdf5) {
505     DM          dmBC;
506     Vec         gv;
507     const char *name;
508 
509     ierr = DMGetOutputDM(dm, &dmBC);CHKERRQ(ierr);
510     ierr = DMGetGlobalVector(dmBC, &gv);CHKERRQ(ierr);
511     ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
512     ierr = PetscObjectSetName((PetscObject) gv, name);CHKERRQ(ierr);
513     ierr = VecLoad_Default(gv, viewer);CHKERRQ(ierr);
514     ierr = DMGlobalToLocalBegin(dmBC, gv, INSERT_VALUES, v);CHKERRQ(ierr);
515     ierr = DMGlobalToLocalEnd(dmBC, gv, INSERT_VALUES, v);CHKERRQ(ierr);
516     ierr = DMRestoreGlobalVector(dmBC, &gv);CHKERRQ(ierr);
517   } else {
518     ierr = VecLoad_Default(v, viewer);CHKERRQ(ierr);
519   }
520   PetscFunctionReturn(0);
521 }
522 
523 PetscErrorCode VecLoad_Plex(Vec v, PetscViewer viewer)
524 {
525   DM             dm;
526   PetscBool      ishdf5,isexodusii;
527   PetscErrorCode ierr;
528 
529   PetscFunctionBegin;
530   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
531   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
532   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,     &ishdf5);CHKERRQ(ierr);
533   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWEREXODUSII, &isexodusii);CHKERRQ(ierr);
534   if (ishdf5) {
535 #if defined(PETSC_HAVE_HDF5)
536     ierr = VecLoad_Plex_HDF5_Internal(v, viewer);CHKERRQ(ierr);
537 #else
538     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
539 #endif
540   } else if (isexodusii) {
541 #if defined(PETSC_HAVE_EXODUSII)
542     ierr = VecLoad_PlexExodusII_Internal(v, viewer);CHKERRQ(ierr);
543 #else
544     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "ExodusII not supported in this build.\nPlease reconfigure using --download-exodusii");
545 #endif
546   } else {
547     ierr = VecLoad_Default(v, viewer);CHKERRQ(ierr);
548   }
549   PetscFunctionReturn(0);
550 }
551 
552 PetscErrorCode VecLoad_Plex_Native(Vec originalv, PetscViewer viewer)
553 {
554   DM                dm;
555   PetscViewerFormat format;
556   PetscBool         ishdf5;
557   PetscErrorCode    ierr;
558 
559   PetscFunctionBegin;
560   ierr = VecGetDM(originalv, &dm);CHKERRQ(ierr);
561   if (!dm) SETERRQ(PetscObjectComm((PetscObject) originalv), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
562   ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
563   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
564   if (format == PETSC_VIEWER_NATIVE) {
565     if (dm->useNatural) {
566       if (dm->sfNatural) {
567         if (ishdf5) {
568 #if defined(PETSC_HAVE_HDF5)
569           Vec         v;
570           const char *vecname;
571 
572           ierr = DMGetGlobalVector(dm, &v);CHKERRQ(ierr);
573           ierr = PetscObjectGetName((PetscObject) originalv, &vecname);CHKERRQ(ierr);
574           ierr = PetscObjectSetName((PetscObject) v, vecname);CHKERRQ(ierr);
575           ierr = VecLoad_Plex_HDF5_Native_Internal(v, viewer);CHKERRQ(ierr);
576           ierr = DMPlexNaturalToGlobalBegin(dm, v, originalv);CHKERRQ(ierr);
577           ierr = DMPlexNaturalToGlobalEnd(dm, v, originalv);CHKERRQ(ierr);
578           ierr = DMRestoreGlobalVector(dm, &v);CHKERRQ(ierr);
579 #else
580           SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
581 #endif
582         } else SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Reading in natural order is not supported for anything but HDF5.");
583       }
584     } else {
585       ierr = VecLoad_Default(originalv, viewer);CHKERRQ(ierr);
586     }
587   }
588   PetscFunctionReturn(0);
589 }
590 
591 PETSC_UNUSED static PetscErrorCode DMPlexView_Ascii_Geometry(DM dm, PetscViewer viewer)
592 {
593   PetscSection       coordSection;
594   Vec                coordinates;
595   DMLabel            depthLabel, celltypeLabel;
596   const char        *name[4];
597   const PetscScalar *a;
598   PetscInt           dim, pStart, pEnd, cStart, cEnd, c;
599   PetscErrorCode     ierr;
600 
601   PetscFunctionBegin;
602   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
603   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
604   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
605   ierr = DMPlexGetDepthLabel(dm, &depthLabel);CHKERRQ(ierr);
606   ierr = DMPlexGetCellTypeLabel(dm, &celltypeLabel);CHKERRQ(ierr);
607   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
608   ierr = PetscSectionGetChart(coordSection, &pStart, &pEnd);CHKERRQ(ierr);
609   ierr = VecGetArrayRead(coordinates, &a);CHKERRQ(ierr);
610   name[0]     = "vertex";
611   name[1]     = "edge";
612   name[dim-1] = "face";
613   name[dim]   = "cell";
614   for (c = cStart; c < cEnd; ++c) {
615     PetscInt *closure = NULL;
616     PetscInt  closureSize, cl, ct;
617 
618     ierr = DMLabelGetValue(celltypeLabel, c, &ct);CHKERRQ(ierr);
619     ierr = PetscViewerASCIIPrintf(viewer, "Geometry for cell %D polytope type %s:\n", c, DMPolytopeTypes[ct]);CHKERRQ(ierr);
620     ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
621     ierr = PetscViewerASCIIPushTab(viewer);CHKERRQ(ierr);
622     for (cl = 0; cl < closureSize*2; cl += 2) {
623       PetscInt point = closure[cl], depth, dof, off, d, p;
624 
625       if ((point < pStart) || (point >= pEnd)) continue;
626       ierr = PetscSectionGetDof(coordSection, point, &dof);CHKERRQ(ierr);
627       if (!dof) continue;
628       ierr = DMLabelGetValue(depthLabel, point, &depth);CHKERRQ(ierr);
629       ierr = PetscSectionGetOffset(coordSection, point, &off);CHKERRQ(ierr);
630       ierr = PetscViewerASCIIPrintf(viewer, "%s %D coords:", name[depth], point);CHKERRQ(ierr);
631       for (p = 0; p < dof/dim; ++p) {
632         ierr = PetscViewerASCIIPrintf(viewer, " (");CHKERRQ(ierr);
633         for (d = 0; d < dim; ++d) {
634           if (d > 0) {ierr = PetscViewerASCIIPrintf(viewer, ", ");CHKERRQ(ierr);}
635           ierr = PetscViewerASCIIPrintf(viewer, "%g", (double) PetscRealPart(a[off+p*dim+d]));CHKERRQ(ierr);
636         }
637         ierr = PetscViewerASCIIPrintf(viewer, ")");CHKERRQ(ierr);
638       }
639       ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
640     }
641     ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
642     ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr);
643   }
644   ierr = VecRestoreArrayRead(coordinates, &a);CHKERRQ(ierr);
645   PetscFunctionReturn(0);
646 }
647 
648 static PetscErrorCode DMPlexView_Ascii(DM dm, PetscViewer viewer)
649 {
650   DM_Plex          *mesh = (DM_Plex*) dm->data;
651   DM                cdm;
652   PetscSection      coordSection;
653   Vec               coordinates;
654   PetscViewerFormat format;
655   PetscErrorCode    ierr;
656 
657   PetscFunctionBegin;
658   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
659   ierr = DMGetLocalSection(cdm, &coordSection);CHKERRQ(ierr);
660   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
661   ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
662   if (format == PETSC_VIEWER_ASCII_INFO_DETAIL) {
663     const char *name;
664     PetscInt    dim, cellHeight, maxConeSize, maxSupportSize;
665     PetscInt    pStart, pEnd, p, numLabels, l;
666     PetscMPIInt rank, size;
667 
668     ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRMPI(ierr);
669     ierr = MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size);CHKERRMPI(ierr);
670     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
671     ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
672     ierr = DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize);CHKERRQ(ierr);
673     ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
674     ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
675     if (name) {ierr = PetscViewerASCIIPrintf(viewer, "%s in %D dimension%s:\n", name, dim, dim == 1 ? "" : "s");CHKERRQ(ierr);}
676     else      {ierr = PetscViewerASCIIPrintf(viewer, "Mesh in %D dimension%s:\n", dim, dim == 1 ? "" : "s");CHKERRQ(ierr);}
677     if (cellHeight) {ierr = PetscViewerASCIIPrintf(viewer, "  Cells are at height %D\n", cellHeight);CHKERRQ(ierr);}
678     ierr = PetscViewerASCIIPrintf(viewer, "Supports:\n", name);CHKERRQ(ierr);
679     ierr = PetscViewerASCIIPushSynchronized(viewer);CHKERRQ(ierr);
680     ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%d] Max support size: %D\n", rank, maxSupportSize);CHKERRQ(ierr);
681     for (p = pStart; p < pEnd; ++p) {
682       PetscInt dof, off, s;
683 
684       ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
685       ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
686       for (s = off; s < off+dof; ++s) {
687         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%d]: %D ----> %D\n", rank, p, mesh->supports[s]);CHKERRQ(ierr);
688       }
689     }
690     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
691     ierr = PetscViewerASCIIPrintf(viewer, "Cones:\n", name);CHKERRQ(ierr);
692     ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%d] Max cone size: %D\n", rank, maxConeSize);CHKERRQ(ierr);
693     for (p = pStart; p < pEnd; ++p) {
694       PetscInt dof, off, c;
695 
696       ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
697       ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
698       for (c = off; c < off+dof; ++c) {
699         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%d]: %D <---- %D (%D)\n", rank, p, mesh->cones[c], mesh->coneOrientations[c]);CHKERRQ(ierr);
700       }
701     }
702     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
703     ierr = PetscViewerASCIIPopSynchronized(viewer);CHKERRQ(ierr);
704     if (coordSection && coordinates) {
705       ierr = PetscSectionVecView(coordSection, coordinates, viewer);CHKERRQ(ierr);
706     }
707     ierr = DMGetNumLabels(dm, &numLabels);CHKERRQ(ierr);
708     if (numLabels) {ierr = PetscViewerASCIIPrintf(viewer, "Labels:\n");CHKERRQ(ierr);}
709     for (l = 0; l < numLabels; ++l) {
710       DMLabel     label;
711       PetscBool   isdepth;
712       const char *name;
713 
714       ierr = DMGetLabelName(dm, l, &name);CHKERRQ(ierr);
715       ierr = PetscStrcmp(name, "depth", &isdepth);CHKERRQ(ierr);
716       if (isdepth) continue;
717       ierr = DMGetLabel(dm, name, &label);CHKERRQ(ierr);
718       ierr = DMLabelView(label, viewer);CHKERRQ(ierr);
719     }
720     if (size > 1) {
721       PetscSF sf;
722 
723       ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr);
724       ierr = PetscSFView(sf, viewer);CHKERRQ(ierr);
725     }
726     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
727   } else if (format == PETSC_VIEWER_ASCII_LATEX) {
728     const char  *name, *color;
729     const char  *defcolors[3]  = {"gray", "orange", "green"};
730     const char  *deflcolors[4] = {"blue", "cyan", "red", "magenta"};
731     char         lname[PETSC_MAX_PATH_LEN];
732     PetscReal    scale         = 2.0;
733     PetscReal    tikzscale     = 1.0;
734     PetscBool    useNumbers    = PETSC_TRUE, drawNumbers[4], drawColors[4], useLabels, useColors, plotEdges, drawHasse = PETSC_FALSE;
735     double       tcoords[3];
736     PetscScalar *coords;
737     PetscInt     numLabels, l, numColors, numLColors, dim, d, depth, cStart, cEnd, c, vStart, vEnd, v, eStart = 0, eEnd = 0, e, p, n;
738     PetscMPIInt  rank, size;
739     char         **names, **colors, **lcolors;
740     PetscBool    flg, lflg;
741     PetscBT      wp = NULL;
742     PetscInt     pEnd, pStart;
743 
744     ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
745     ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
746     ierr = DMGetNumLabels(dm, &numLabels);CHKERRQ(ierr);
747     numLabels  = PetscMax(numLabels, 10);
748     numColors  = 10;
749     numLColors = 10;
750     ierr = PetscCalloc3(numLabels, &names, numColors, &colors, numLColors, &lcolors);CHKERRQ(ierr);
751     ierr = PetscOptionsGetReal(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_scale", &scale, NULL);CHKERRQ(ierr);
752     ierr = PetscOptionsGetReal(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_tikzscale", &tikzscale, NULL);CHKERRQ(ierr);
753     ierr = PetscOptionsGetBool(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_numbers", &useNumbers, NULL);CHKERRQ(ierr);
754     for (d = 0; d < 4; ++d) drawNumbers[d] = useNumbers;
755     for (d = 0; d < 4; ++d) drawColors[d]  = PETSC_TRUE;
756     n = 4;
757     ierr = PetscOptionsGetBoolArray(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_numbers_depth", drawNumbers, &n, &flg);CHKERRQ(ierr);
758     if (flg && n != dim+1) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_SIZ, "Number of flags %D != %D dim+1", n, dim+1);
759     ierr = PetscOptionsGetBoolArray(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_colors_depth", drawColors, &n, &flg);CHKERRQ(ierr);
760     if (flg && n != dim+1) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_SIZ, "Number of flags %D != %D dim+1", n, dim+1);
761     ierr = PetscOptionsGetStringArray(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_labels", names, &numLabels, &useLabels);CHKERRQ(ierr);
762     if (!useLabels) numLabels = 0;
763     ierr = PetscOptionsGetStringArray(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_colors", colors, &numColors, &useColors);CHKERRQ(ierr);
764     if (!useColors) {
765       numColors = 3;
766       for (c = 0; c < numColors; ++c) {ierr = PetscStrallocpy(defcolors[c], &colors[c]);CHKERRQ(ierr);}
767     }
768     ierr = PetscOptionsGetStringArray(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_lcolors", lcolors, &numLColors, &useColors);CHKERRQ(ierr);
769     if (!useColors) {
770       numLColors = 4;
771       for (c = 0; c < numLColors; ++c) {ierr = PetscStrallocpy(deflcolors[c], &lcolors[c]);CHKERRQ(ierr);}
772     }
773     ierr = PetscOptionsGetString(((PetscObject) viewer)->options, ((PetscObject) viewer)->prefix, "-dm_plex_view_label_filter", lname, sizeof(lname), &lflg);CHKERRQ(ierr);
774     plotEdges = (PetscBool)(depth > 1 && drawNumbers[1] && dim < 3);
775     ierr = PetscOptionsGetBool(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_edges", &plotEdges, &flg);CHKERRQ(ierr);
776     if (flg && plotEdges && depth < dim) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Mesh must be interpolated");
777     if (depth < dim) plotEdges = PETSC_FALSE;
778     ierr = PetscOptionsGetBool(((PetscObject) viewer)->options, ((PetscObject) viewer)->prefix, "-dm_plex_view_hasse", &drawHasse, NULL);CHKERRQ(ierr);
779 
780     /* filter points with labelvalue != labeldefaultvalue */
781     ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
782     ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
783     ierr = DMPlexGetDepthStratum(dm, 1, &eStart, &eEnd);CHKERRQ(ierr);
784     ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
785     if (lflg) {
786       DMLabel lbl;
787 
788       ierr = DMGetLabel(dm, lname, &lbl);CHKERRQ(ierr);
789       if (lbl) {
790         PetscInt val, defval;
791 
792         ierr = DMLabelGetDefaultValue(lbl, &defval);CHKERRQ(ierr);
793         ierr = PetscBTCreate(pEnd-pStart, &wp);CHKERRQ(ierr);
794         for (c = pStart;  c < pEnd; c++) {
795           PetscInt *closure = NULL;
796           PetscInt  closureSize;
797 
798           ierr = DMLabelGetValue(lbl, c, &val);CHKERRQ(ierr);
799           if (val == defval) continue;
800 
801           ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
802           for (p = 0; p < closureSize*2; p += 2) {
803             ierr = PetscBTSet(wp, closure[p] - pStart);CHKERRQ(ierr);
804           }
805           ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
806         }
807       }
808     }
809 
810     ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRMPI(ierr);
811     ierr = MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size);CHKERRMPI(ierr);
812     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
813     ierr = PetscViewerASCIIPrintf(viewer, "\
814 \\documentclass[tikz]{standalone}\n\n\
815 \\usepackage{pgflibraryshapes}\n\
816 \\usetikzlibrary{backgrounds}\n\
817 \\usetikzlibrary{arrows}\n\
818 \\begin{document}\n");CHKERRQ(ierr);
819     if (size > 1) {
820       ierr = PetscViewerASCIIPrintf(viewer, "%s for process ", name);CHKERRQ(ierr);
821       for (p = 0; p < size; ++p) {
822         if (p > 0 && p == size-1) {
823           ierr = PetscViewerASCIIPrintf(viewer, ", and ", colors[p%numColors], p);CHKERRQ(ierr);
824         } else if (p > 0) {
825           ierr = PetscViewerASCIIPrintf(viewer, ", ", colors[p%numColors], p);CHKERRQ(ierr);
826         }
827         ierr = PetscViewerASCIIPrintf(viewer, "{\\textcolor{%s}%D}", colors[p%numColors], p);CHKERRQ(ierr);
828       }
829       ierr = PetscViewerASCIIPrintf(viewer, ".\n\n\n");CHKERRQ(ierr);
830     }
831     if (drawHasse) {
832       PetscInt maxStratum = PetscMax(vEnd-vStart, PetscMax(eEnd-eStart, cEnd-cStart));
833 
834       ierr = PetscViewerASCIIPrintf(viewer, "\\newcommand{\\vStart}{%D}\n", vStart);CHKERRQ(ierr);
835       ierr = PetscViewerASCIIPrintf(viewer, "\\newcommand{\\vEnd}{%D}\n", vEnd-1);CHKERRQ(ierr);
836       ierr = PetscViewerASCIIPrintf(viewer, "\\newcommand{\\numVertices}{%D}\n", vEnd-vStart);CHKERRQ(ierr);
837       ierr = PetscViewerASCIIPrintf(viewer, "\\newcommand{\\vShift}{%.2f}\n", 3 + (maxStratum-(vEnd-vStart))/2.);CHKERRQ(ierr);
838       ierr = PetscViewerASCIIPrintf(viewer, "\\newcommand{\\eStart}{%D}\n", eStart);CHKERRQ(ierr);
839       ierr = PetscViewerASCIIPrintf(viewer, "\\newcommand{\\eEnd}{%D}\n", eEnd-1);CHKERRQ(ierr);
840       ierr = PetscViewerASCIIPrintf(viewer, "\\newcommand{\\eShift}{%.2f}\n", 3 + (maxStratum-(eEnd-eStart))/2.);CHKERRQ(ierr);
841       ierr = PetscViewerASCIIPrintf(viewer, "\\newcommand{\\numEdges}{%D}\n", eEnd-eStart);CHKERRQ(ierr);
842       ierr = PetscViewerASCIIPrintf(viewer, "\\newcommand{\\cStart}{%D}\n", cStart);CHKERRQ(ierr);
843       ierr = PetscViewerASCIIPrintf(viewer, "\\newcommand{\\cEnd}{%D}\n", cEnd-1);CHKERRQ(ierr);
844       ierr = PetscViewerASCIIPrintf(viewer, "\\newcommand{\\numCells}{%D}\n", cEnd-cStart);CHKERRQ(ierr);
845       ierr = PetscViewerASCIIPrintf(viewer, "\\newcommand{\\cShift}{%.2f}\n", 3 + (maxStratum-(cEnd-cStart))/2.);CHKERRQ(ierr);
846     }
847     ierr = PetscViewerASCIIPrintf(viewer, "\\begin{tikzpicture}[scale = %g,font=\\fontsize{8}{8}\\selectfont]\n", (double) tikzscale);CHKERRQ(ierr);
848 
849     /* Plot vertices */
850     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
851     ierr = PetscViewerASCIIPushSynchronized(viewer);CHKERRQ(ierr);
852     for (v = vStart; v < vEnd; ++v) {
853       PetscInt  off, dof, d;
854       PetscBool isLabeled = PETSC_FALSE;
855 
856       if (wp && !PetscBTLookup(wp,v - pStart)) continue;
857       ierr = PetscSectionGetDof(coordSection, v, &dof);CHKERRQ(ierr);
858       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
859       ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\path (");CHKERRQ(ierr);
860       if (PetscUnlikely(dof > 3)) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"coordSection vertex %D has dof %D > 3",v,dof);
861       for (d = 0; d < dof; ++d) {
862         tcoords[d] = (double) (scale*PetscRealPart(coords[off+d]));
863         tcoords[d] = PetscAbs(tcoords[d]) < 1e-10 ? 0.0 : tcoords[d];
864       }
865       /* Rotate coordinates since PGF makes z point out of the page instead of up */
866       if (dim == 3) {PetscReal tmp = tcoords[1]; tcoords[1] = tcoords[2]; tcoords[2] = -tmp;}
867       for (d = 0; d < dof; ++d) {
868         if (d > 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, ",");CHKERRQ(ierr);}
869         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "%g", (double) tcoords[d]);CHKERRQ(ierr);
870       }
871       if (drawHasse) color = colors[0%numColors];
872       else           color = colors[rank%numColors];
873       for (l = 0; l < numLabels; ++l) {
874         PetscInt val;
875         ierr = DMGetLabelValue(dm, names[l], v, &val);CHKERRQ(ierr);
876         if (val >= 0) {color = lcolors[l%numLColors]; isLabeled = PETSC_TRUE; break;}
877       }
878       if (drawNumbers[0]) {
879         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [draw,shape=circle,color=%s] {%D};\n", v, rank, color, v);CHKERRQ(ierr);
880       } else if (drawColors[0]) {
881         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [fill,inner sep=%dpt,shape=circle,color=%s] {};\n", v, rank, !isLabeled ? 1 : 2, color);CHKERRQ(ierr);
882       } else {
883         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [] {};\n", v, rank);CHKERRQ(ierr);
884       }
885     }
886     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
887     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
888     /* Plot edges */
889     if (plotEdges) {
890       ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
891       ierr = PetscViewerASCIIPrintf(viewer, "\\path\n");CHKERRQ(ierr);
892       for (e = eStart; e < eEnd; ++e) {
893         const PetscInt *cone;
894         PetscInt        coneSize, offA, offB, dof, d;
895 
896         if (wp && !PetscBTLookup(wp,e - pStart)) continue;
897         ierr = DMPlexGetConeSize(dm, e, &coneSize);CHKERRQ(ierr);
898         if (coneSize != 2) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Edge %D cone should have two vertices, not %D", e, coneSize);
899         ierr = DMPlexGetCone(dm, e, &cone);CHKERRQ(ierr);
900         ierr = PetscSectionGetDof(coordSection, cone[0], &dof);CHKERRQ(ierr);
901         ierr = PetscSectionGetOffset(coordSection, cone[0], &offA);CHKERRQ(ierr);
902         ierr = PetscSectionGetOffset(coordSection, cone[1], &offB);CHKERRQ(ierr);
903         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "(");CHKERRQ(ierr);
904         for (d = 0; d < dof; ++d) {
905           tcoords[d] = (double) (0.5*scale*PetscRealPart(coords[offA+d]+coords[offB+d]));
906           tcoords[d] = PetscAbs(tcoords[d]) < 1e-10 ? 0.0 : tcoords[d];
907         }
908         /* Rotate coordinates since PGF makes z point out of the page instead of up */
909         if (dim == 3) {PetscReal tmp = tcoords[1]; tcoords[1] = tcoords[2]; tcoords[2] = -tmp;}
910         for (d = 0; d < dof; ++d) {
911           if (d > 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, ",");CHKERRQ(ierr);}
912           ierr = PetscViewerASCIISynchronizedPrintf(viewer, "%g", (double)tcoords[d]);CHKERRQ(ierr);
913         }
914         if (drawHasse) color = colors[1%numColors];
915         else           color = colors[rank%numColors];
916         for (l = 0; l < numLabels; ++l) {
917           PetscInt val;
918           ierr = DMGetLabelValue(dm, names[l], v, &val);CHKERRQ(ierr);
919           if (val >= 0) {color = lcolors[l%numLColors]; break;}
920         }
921         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [draw,shape=circle,color=%s] {%D} --\n", e, rank, color, e);CHKERRQ(ierr);
922       }
923       ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
924       ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
925       ierr = PetscViewerASCIIPrintf(viewer, "(0,0);\n");CHKERRQ(ierr);
926     }
927     /* Plot cells */
928     if (dim == 3 || !drawNumbers[1]) {
929       for (e = eStart; e < eEnd; ++e) {
930         const PetscInt *cone;
931 
932         if (wp && !PetscBTLookup(wp,e - pStart)) continue;
933         color = colors[rank%numColors];
934         for (l = 0; l < numLabels; ++l) {
935           PetscInt val;
936           ierr = DMGetLabelValue(dm, names[l], e, &val);CHKERRQ(ierr);
937           if (val >= 0) {color = lcolors[l%numLColors]; break;}
938         }
939         ierr = DMPlexGetCone(dm, e, &cone);CHKERRQ(ierr);
940         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\draw[color=%s] (%D_%d) -- (%D_%d);\n", color, cone[0], rank, cone[1], rank);CHKERRQ(ierr);
941       }
942     } else {
943        DMPolytopeType ct;
944 
945       /* Drawing a 2D polygon */
946       for (c = cStart; c < cEnd; ++c) {
947         if (wp && !PetscBTLookup(wp, c - pStart)) continue;
948         ierr = DMPlexGetCellType(dm, c, &ct);CHKERRQ(ierr);
949         if (ct == DM_POLYTOPE_SEG_PRISM_TENSOR ||
950             ct == DM_POLYTOPE_TRI_PRISM_TENSOR ||
951             ct == DM_POLYTOPE_QUAD_PRISM_TENSOR) {
952           const PetscInt *cone;
953           PetscInt        coneSize, e;
954 
955           ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
956           ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
957           for (e = 0; e < coneSize; ++e) {
958             const PetscInt *econe;
959 
960             ierr = DMPlexGetCone(dm, cone[e], &econe);CHKERRQ(ierr);
961             ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\draw[color=%s] (%D_%d) -- (%D_%d) -- (%D_%d);\n", colors[rank%numColors], econe[0], rank, cone[e], rank, econe[1], rank);CHKERRQ(ierr);
962           }
963         } else {
964           PetscInt *closure = NULL;
965           PetscInt  closureSize, Nv = 0, v;
966 
967           ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
968           for (p = 0; p < closureSize*2; p += 2) {
969             const PetscInt point = closure[p];
970 
971             if ((point >= vStart) && (point < vEnd)) closure[Nv++] = point;
972           }
973           ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\draw[color=%s] ", colors[rank%numColors]);CHKERRQ(ierr);
974           for (v = 0; v <= Nv; ++v) {
975             const PetscInt vertex = closure[v%Nv];
976 
977             if (v > 0) {
978               if (plotEdges) {
979                 const PetscInt *edge;
980                 PetscInt        endpoints[2], ne;
981 
982                 endpoints[0] = closure[v-1]; endpoints[1] = vertex;
983                 ierr = DMPlexGetJoin(dm, 2, endpoints, &ne, &edge);CHKERRQ(ierr);
984                 if (ne != 1) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Could not find edge for vertices %D, %D", endpoints[0], endpoints[1]);
985                 ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- (%D_%d) -- ", edge[0], rank);CHKERRQ(ierr);
986                 ierr = DMPlexRestoreJoin(dm, 2, endpoints, &ne, &edge);CHKERRQ(ierr);
987               } else {
988                 ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- ");CHKERRQ(ierr);
989               }
990             }
991             ierr = PetscViewerASCIISynchronizedPrintf(viewer, "(%D_%d)", vertex, rank);CHKERRQ(ierr);
992           }
993           ierr = PetscViewerASCIISynchronizedPrintf(viewer, ";\n");CHKERRQ(ierr);
994           ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
995         }
996       }
997     }
998     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
999     for (c = cStart; c < cEnd; ++c) {
1000       double    ccoords[3] = {0.0, 0.0, 0.0};
1001       PetscBool isLabeled  = PETSC_FALSE;
1002       PetscInt *closure    = NULL;
1003       PetscInt  closureSize, dof, d, n = 0;
1004 
1005       if (wp && !PetscBTLookup(wp,c - pStart)) continue;
1006       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1007       ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\path (");CHKERRQ(ierr);
1008       for (p = 0; p < closureSize*2; p += 2) {
1009         const PetscInt point = closure[p];
1010         PetscInt       off;
1011 
1012         if ((point < vStart) || (point >= vEnd)) continue;
1013         ierr = PetscSectionGetDof(coordSection, point, &dof);CHKERRQ(ierr);
1014         ierr = PetscSectionGetOffset(coordSection, point, &off);CHKERRQ(ierr);
1015         for (d = 0; d < dof; ++d) {
1016           tcoords[d] = (double) (scale*PetscRealPart(coords[off+d]));
1017           tcoords[d] = PetscAbs(tcoords[d]) < 1e-10 ? 0.0 : tcoords[d];
1018         }
1019         /* Rotate coordinates since PGF makes z point out of the page instead of up */
1020         if (dof == 3) {PetscReal tmp = tcoords[1]; tcoords[1] = tcoords[2]; tcoords[2] = -tmp;}
1021         for (d = 0; d < dof; ++d) {ccoords[d] += tcoords[d];}
1022         ++n;
1023       }
1024       for (d = 0; d < dof; ++d) {ccoords[d] /= n;}
1025       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1026       for (d = 0; d < dof; ++d) {
1027         if (d > 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, ",");CHKERRQ(ierr);}
1028         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "%g", (double) ccoords[d]);CHKERRQ(ierr);
1029       }
1030       if (drawHasse) color = colors[depth%numColors];
1031       else           color = colors[rank%numColors];
1032       for (l = 0; l < numLabels; ++l) {
1033         PetscInt val;
1034         ierr = DMGetLabelValue(dm, names[l], c, &val);CHKERRQ(ierr);
1035         if (val >= 0) {color = lcolors[l%numLColors]; isLabeled = PETSC_TRUE; break;}
1036       }
1037       if (drawNumbers[dim]) {
1038         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [draw,shape=circle,color=%s] {%D};\n", c, rank, color, c);CHKERRQ(ierr);
1039       } else if (drawColors[dim]) {
1040         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [fill,inner sep=%dpt,shape=circle,color=%s] {};\n", c, rank, !isLabeled ? 1 : 2, color);CHKERRQ(ierr);
1041       } else {
1042         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [] {};\n", c, rank);CHKERRQ(ierr);
1043       }
1044     }
1045     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
1046     if (drawHasse) {
1047       color = colors[depth%numColors];
1048       ierr = PetscViewerASCIIPrintf(viewer, "%% Cells\n");CHKERRQ(ierr);
1049       ierr = PetscViewerASCIIPrintf(viewer, "\\foreach \\c in {\\cStart,...,\\cEnd}\n");CHKERRQ(ierr);
1050       ierr = PetscViewerASCIIPrintf(viewer, "{\n");CHKERRQ(ierr);
1051       ierr = PetscViewerASCIIPrintf(viewer, "  \\node(\\c_%d) [draw,shape=circle,color=%s,minimum size = 6mm] at (\\cShift+\\c-\\cStart,0) {\\c};\n", rank, color);CHKERRQ(ierr);
1052       ierr = PetscViewerASCIIPrintf(viewer, "}\n");CHKERRQ(ierr);
1053 
1054       color = colors[1%numColors];
1055       ierr = PetscViewerASCIIPrintf(viewer, "%% Edges\n");CHKERRQ(ierr);
1056       ierr = PetscViewerASCIIPrintf(viewer, "\\foreach \\e in {\\eStart,...,\\eEnd}\n");CHKERRQ(ierr);
1057       ierr = PetscViewerASCIIPrintf(viewer, "{\n");CHKERRQ(ierr);
1058       ierr = PetscViewerASCIIPrintf(viewer, "  \\node(\\e_%d) [draw,shape=circle,color=%s,minimum size = 6mm] at (\\eShift+\\e-\\eStart,1) {\\e};\n", rank, color);CHKERRQ(ierr);
1059       ierr = PetscViewerASCIIPrintf(viewer, "}\n");CHKERRQ(ierr);
1060 
1061       color = colors[0%numColors];
1062       ierr = PetscViewerASCIIPrintf(viewer, "%% Vertices\n");CHKERRQ(ierr);
1063       ierr = PetscViewerASCIIPrintf(viewer, "\\foreach \\v in {\\vStart,...,\\vEnd}\n");CHKERRQ(ierr);
1064       ierr = PetscViewerASCIIPrintf(viewer, "{\n");CHKERRQ(ierr);
1065       ierr = PetscViewerASCIIPrintf(viewer, "  \\node(\\v_%d) [draw,shape=circle,color=%s,minimum size = 6mm] at (\\vShift+\\v-\\vStart,2) {\\v};\n", rank, color);CHKERRQ(ierr);
1066       ierr = PetscViewerASCIIPrintf(viewer, "}\n");CHKERRQ(ierr);
1067 
1068       for (p = pStart; p < pEnd; ++p) {
1069         const PetscInt *cone;
1070         PetscInt        coneSize, cp;
1071 
1072         ierr = DMPlexGetCone(dm, p, &cone);CHKERRQ(ierr);
1073         ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
1074         for (cp = 0; cp < coneSize; ++cp) {
1075           ierr = PetscViewerASCIIPrintf(viewer, "\\draw[->, shorten >=1pt] (%D_%d) -- (%D_%d);\n", cone[cp], rank, p, rank);CHKERRQ(ierr);
1076         }
1077       }
1078     }
1079     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
1080     ierr = PetscViewerASCIIPopSynchronized(viewer);CHKERRQ(ierr);
1081     ierr = PetscViewerASCIIPrintf(viewer, "\\end{tikzpicture}\n");CHKERRQ(ierr);
1082     ierr = PetscViewerASCIIPrintf(viewer, "\\end{document}\n", name);CHKERRQ(ierr);
1083     for (l = 0; l < numLabels;  ++l) {ierr = PetscFree(names[l]);CHKERRQ(ierr);}
1084     for (c = 0; c < numColors;  ++c) {ierr = PetscFree(colors[c]);CHKERRQ(ierr);}
1085     for (c = 0; c < numLColors; ++c) {ierr = PetscFree(lcolors[c]);CHKERRQ(ierr);}
1086     ierr = PetscFree3(names, colors, lcolors);CHKERRQ(ierr);
1087     ierr = PetscBTDestroy(&wp);CHKERRQ(ierr);
1088   } else if (format == PETSC_VIEWER_LOAD_BALANCE) {
1089     Vec                    cown,acown;
1090     VecScatter             sct;
1091     ISLocalToGlobalMapping g2l;
1092     IS                     gid,acis;
1093     MPI_Comm               comm,ncomm = MPI_COMM_NULL;
1094     MPI_Group              ggroup,ngroup;
1095     PetscScalar            *array,nid;
1096     const PetscInt         *idxs;
1097     PetscInt               *idxs2,*start,*adjacency,*work;
1098     PetscInt64             lm[3],gm[3];
1099     PetscInt               i,c,cStart,cEnd,cum,numVertices,ect,ectn,cellHeight;
1100     PetscMPIInt            d1,d2,rank;
1101 
1102     ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
1103     ierr = MPI_Comm_rank(comm,&rank);CHKERRMPI(ierr);
1104 #if defined(PETSC_HAVE_MPI_PROCESS_SHARED_MEMORY)
1105     ierr = MPI_Comm_split_type(comm,MPI_COMM_TYPE_SHARED,rank,MPI_INFO_NULL,&ncomm);CHKERRMPI(ierr);
1106 #endif
1107     if (ncomm != MPI_COMM_NULL) {
1108       ierr = MPI_Comm_group(comm,&ggroup);CHKERRMPI(ierr);
1109       ierr = MPI_Comm_group(ncomm,&ngroup);CHKERRMPI(ierr);
1110       d1   = 0;
1111       ierr = MPI_Group_translate_ranks(ngroup,1,&d1,ggroup,&d2);CHKERRMPI(ierr);
1112       nid  = d2;
1113       ierr = MPI_Group_free(&ggroup);CHKERRMPI(ierr);
1114       ierr = MPI_Group_free(&ngroup);CHKERRMPI(ierr);
1115       ierr = MPI_Comm_free(&ncomm);CHKERRMPI(ierr);
1116     } else nid = 0.0;
1117 
1118     /* Get connectivity */
1119     ierr = DMPlexGetVTKCellHeight(dm,&cellHeight);CHKERRQ(ierr);
1120     ierr = DMPlexCreatePartitionerGraph(dm,cellHeight,&numVertices,&start,&adjacency,&gid);CHKERRQ(ierr);
1121 
1122     /* filter overlapped local cells */
1123     ierr = DMPlexGetHeightStratum(dm,cellHeight,&cStart,&cEnd);CHKERRQ(ierr);
1124     ierr = ISGetIndices(gid,&idxs);CHKERRQ(ierr);
1125     ierr = ISGetLocalSize(gid,&cum);CHKERRQ(ierr);
1126     ierr = PetscMalloc1(cum,&idxs2);CHKERRQ(ierr);
1127     for (c = cStart, cum = 0; c < cEnd; c++) {
1128       if (idxs[c-cStart] < 0) continue;
1129       idxs2[cum++] = idxs[c-cStart];
1130     }
1131     ierr = ISRestoreIndices(gid,&idxs);CHKERRQ(ierr);
1132     if (numVertices != cum) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Unexpected %D != %D",numVertices,cum);
1133     ierr = ISDestroy(&gid);CHKERRQ(ierr);
1134     ierr = ISCreateGeneral(comm,numVertices,idxs2,PETSC_OWN_POINTER,&gid);CHKERRQ(ierr);
1135 
1136     /* support for node-aware cell locality */
1137     ierr = ISCreateGeneral(comm,start[numVertices],adjacency,PETSC_USE_POINTER,&acis);CHKERRQ(ierr);
1138     ierr = VecCreateSeq(PETSC_COMM_SELF,start[numVertices],&acown);CHKERRQ(ierr);
1139     ierr = VecCreateMPI(comm,numVertices,PETSC_DECIDE,&cown);CHKERRQ(ierr);
1140     ierr = VecGetArray(cown,&array);CHKERRQ(ierr);
1141     for (c = 0; c < numVertices; c++) array[c] = nid;
1142     ierr = VecRestoreArray(cown,&array);CHKERRQ(ierr);
1143     ierr = VecScatterCreate(cown,acis,acown,NULL,&sct);CHKERRQ(ierr);
1144     ierr = VecScatterBegin(sct,cown,acown,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
1145     ierr = VecScatterEnd(sct,cown,acown,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
1146     ierr = ISDestroy(&acis);CHKERRQ(ierr);
1147     ierr = VecScatterDestroy(&sct);CHKERRQ(ierr);
1148     ierr = VecDestroy(&cown);CHKERRQ(ierr);
1149 
1150     /* compute edgeCut */
1151     for (c = 0, cum = 0; c < numVertices; c++) cum = PetscMax(cum,start[c+1]-start[c]);
1152     ierr = PetscMalloc1(cum,&work);CHKERRQ(ierr);
1153     ierr = ISLocalToGlobalMappingCreateIS(gid,&g2l);CHKERRQ(ierr);
1154     ierr = ISLocalToGlobalMappingSetType(g2l,ISLOCALTOGLOBALMAPPINGHASH);CHKERRQ(ierr);
1155     ierr = ISDestroy(&gid);CHKERRQ(ierr);
1156     ierr = VecGetArray(acown,&array);CHKERRQ(ierr);
1157     for (c = 0, ect = 0, ectn = 0; c < numVertices; c++) {
1158       PetscInt totl;
1159 
1160       totl = start[c+1]-start[c];
1161       ierr = ISGlobalToLocalMappingApply(g2l,IS_GTOLM_MASK,totl,adjacency+start[c],NULL,work);CHKERRQ(ierr);
1162       for (i = 0; i < totl; i++) {
1163         if (work[i] < 0) {
1164           ect  += 1;
1165           ectn += (array[i + start[c]] != nid) ? 0 : 1;
1166         }
1167       }
1168     }
1169     ierr  = PetscFree(work);CHKERRQ(ierr);
1170     ierr  = VecRestoreArray(acown,&array);CHKERRQ(ierr);
1171     lm[0] = numVertices > 0 ?  numVertices : PETSC_MAX_INT;
1172     lm[1] = -numVertices;
1173     ierr  = MPIU_Allreduce(lm,gm,2,MPIU_INT64,MPI_MIN,comm);CHKERRMPI(ierr);
1174     ierr  = PetscViewerASCIIPrintf(viewer,"  Cell balance: %.2f (max %D, min %D",-((double)gm[1])/((double)gm[0]),-(PetscInt)gm[1],(PetscInt)gm[0]);CHKERRQ(ierr);
1175     lm[0] = ect; /* edgeCut */
1176     lm[1] = ectn; /* node-aware edgeCut */
1177     lm[2] = numVertices > 0 ? 0 : 1; /* empty processes */
1178     ierr  = MPIU_Allreduce(lm,gm,3,MPIU_INT64,MPI_SUM,comm);CHKERRMPI(ierr);
1179     ierr  = PetscViewerASCIIPrintf(viewer,", empty %D)\n",(PetscInt)gm[2]);CHKERRQ(ierr);
1180 #if defined(PETSC_HAVE_MPI_PROCESS_SHARED_MEMORY)
1181     ierr  = PetscViewerASCIIPrintf(viewer,"  Edge Cut: %D (on node %.3f)\n",(PetscInt)(gm[0]/2),gm[0] ? ((double)(gm[1]))/((double)gm[0]) : 1.);CHKERRQ(ierr);
1182 #else
1183     ierr  = PetscViewerASCIIPrintf(viewer,"  Edge Cut: %D (on node %.3f)\n",(PetscInt)(gm[0]/2),0.0);CHKERRQ(ierr);
1184 #endif
1185     ierr  = ISLocalToGlobalMappingDestroy(&g2l);CHKERRQ(ierr);
1186     ierr  = PetscFree(start);CHKERRQ(ierr);
1187     ierr  = PetscFree(adjacency);CHKERRQ(ierr);
1188     ierr  = VecDestroy(&acown);CHKERRQ(ierr);
1189   } else {
1190     const char    *name;
1191     PetscInt      *sizes, *hybsizes, *ghostsizes;
1192     PetscInt       locDepth, depth, cellHeight, dim, d;
1193     PetscInt       pStart, pEnd, p, gcStart, gcEnd, gcNum;
1194     PetscInt       numLabels, l;
1195     DMPolytopeType ct0 = DM_POLYTOPE_UNKNOWN;
1196     MPI_Comm       comm;
1197     PetscMPIInt    size, rank;
1198 
1199     ierr = PetscObjectGetComm((PetscObject) dm, &comm);CHKERRQ(ierr);
1200     ierr = MPI_Comm_size(comm, &size);CHKERRMPI(ierr);
1201     ierr = MPI_Comm_rank(comm, &rank);CHKERRMPI(ierr);
1202     ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
1203     ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
1204     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
1205     if (name) {ierr = PetscViewerASCIIPrintf(viewer, "%s in %D dimension%s:\n", name, dim, dim == 1 ? "" : "s");CHKERRQ(ierr);}
1206     else      {ierr = PetscViewerASCIIPrintf(viewer, "Mesh in %D dimension%s:\n", dim, dim == 1 ? "" : "s");CHKERRQ(ierr);}
1207     if (cellHeight) {ierr = PetscViewerASCIIPrintf(viewer, "  Cells are at height %D\n", cellHeight);CHKERRQ(ierr);}
1208     ierr = DMPlexGetDepth(dm, &locDepth);CHKERRQ(ierr);
1209     ierr = MPIU_Allreduce(&locDepth, &depth, 1, MPIU_INT, MPI_MAX, comm);CHKERRMPI(ierr);
1210     ierr = DMPlexGetGhostCellStratum(dm, &gcStart, &gcEnd);CHKERRQ(ierr);
1211     gcNum = gcEnd - gcStart;
1212     ierr = PetscCalloc3(size,&sizes,size,&hybsizes,size,&ghostsizes);CHKERRQ(ierr);
1213     for (d = 0; d <= depth; d++) {
1214       PetscInt Nc[2] = {0, 0}, ict;
1215 
1216       ierr = DMPlexGetDepthStratum(dm, d, &pStart, &pEnd);CHKERRQ(ierr);
1217       if (pStart < pEnd) {ierr = DMPlexGetCellType(dm, pStart, &ct0);CHKERRQ(ierr);}
1218       ict  = ct0;
1219       ierr = MPI_Bcast(&ict, 1, MPIU_INT, 0, comm);CHKERRMPI(ierr);
1220       ct0  = (DMPolytopeType) ict;
1221       for (p = pStart; p < pEnd; ++p) {
1222         DMPolytopeType ct;
1223 
1224         ierr = DMPlexGetCellType(dm, p, &ct);CHKERRQ(ierr);
1225         if (ct == ct0) ++Nc[0];
1226         else           ++Nc[1];
1227       }
1228       ierr = MPI_Gather(&Nc[0], 1, MPIU_INT, sizes,    1, MPIU_INT, 0, comm);CHKERRMPI(ierr);
1229       ierr = MPI_Gather(&Nc[1], 1, MPIU_INT, hybsizes, 1, MPIU_INT, 0, comm);CHKERRMPI(ierr);
1230       if (d == depth) {ierr = MPI_Gather(&gcNum, 1, MPIU_INT, ghostsizes, 1, MPIU_INT, 0, comm);CHKERRMPI(ierr);}
1231       ierr = PetscViewerASCIIPrintf(viewer, "  %D-cells:", (depth == 1) && d ? dim : d);CHKERRQ(ierr);
1232       for (p = 0; p < size; ++p) {
1233         if (!rank) {
1234           ierr = PetscViewerASCIIPrintf(viewer, " %D", sizes[p]+hybsizes[p]);CHKERRQ(ierr);
1235           if (hybsizes[p]   > 0) {ierr = PetscViewerASCIIPrintf(viewer, " (%D)", hybsizes[p]);CHKERRQ(ierr);}
1236           if (ghostsizes[p] > 0) {ierr = PetscViewerASCIIPrintf(viewer, " [%D]", ghostsizes[p]);CHKERRQ(ierr);}
1237         }
1238       }
1239       ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
1240     }
1241     ierr = PetscFree3(sizes,hybsizes,ghostsizes);CHKERRQ(ierr);
1242     {
1243       const PetscReal      *maxCell;
1244       const PetscReal      *L;
1245       const DMBoundaryType *bd;
1246       PetscBool             per, localized;
1247 
1248       ierr = DMGetPeriodicity(dm, &per, &maxCell, &L, &bd);CHKERRQ(ierr);
1249       ierr = DMGetCoordinatesLocalized(dm, &localized);CHKERRQ(ierr);
1250       if (per) {
1251         ierr = PetscViewerASCIIPrintf(viewer, "Periodic mesh (");CHKERRQ(ierr);
1252         ierr = PetscViewerASCIIUseTabs(viewer, PETSC_FALSE);CHKERRQ(ierr);
1253         for (d = 0; d < dim; ++d) {
1254           if (bd && d > 0) {ierr = PetscViewerASCIIPrintf(viewer, ", ");CHKERRQ(ierr);}
1255           if (bd)    {ierr = PetscViewerASCIIPrintf(viewer, "%s", DMBoundaryTypes[bd[d]]);CHKERRQ(ierr);}
1256         }
1257         ierr = PetscViewerASCIIPrintf(viewer, ") coordinates %s\n", localized ? "localized" : "not localized");CHKERRQ(ierr);
1258         ierr = PetscViewerASCIIUseTabs(viewer, PETSC_TRUE);CHKERRQ(ierr);
1259       }
1260     }
1261     ierr = DMGetNumLabels(dm, &numLabels);CHKERRQ(ierr);
1262     if (numLabels) {ierr = PetscViewerASCIIPrintf(viewer, "Labels:\n");CHKERRQ(ierr);}
1263     for (l = 0; l < numLabels; ++l) {
1264       DMLabel         label;
1265       const char     *name;
1266       IS              valueIS;
1267       const PetscInt *values;
1268       PetscInt        numValues, v;
1269 
1270       ierr = DMGetLabelName(dm, l, &name);CHKERRQ(ierr);
1271       ierr = DMGetLabel(dm, name, &label);CHKERRQ(ierr);
1272       ierr = DMLabelGetNumValues(label, &numValues);CHKERRQ(ierr);
1273       ierr = PetscViewerASCIIPrintf(viewer, "  %s: %D strata with value/size (", name, numValues);CHKERRQ(ierr);
1274       ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
1275       ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
1276       ierr = PetscViewerASCIIUseTabs(viewer, PETSC_FALSE);CHKERRQ(ierr);
1277       for (v = 0; v < numValues; ++v) {
1278         PetscInt size;
1279 
1280         ierr = DMLabelGetStratumSize(label, values[v], &size);CHKERRQ(ierr);
1281         if (v > 0) {ierr = PetscViewerASCIIPrintf(viewer, ", ");CHKERRQ(ierr);}
1282         ierr = PetscViewerASCIIPrintf(viewer, "%D (%D)", values[v], size);CHKERRQ(ierr);
1283       }
1284       ierr = PetscViewerASCIIPrintf(viewer, ")\n");CHKERRQ(ierr);
1285       ierr = PetscViewerASCIIUseTabs(viewer, PETSC_TRUE);CHKERRQ(ierr);
1286       ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
1287       ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
1288     }
1289     {
1290       char    **labelNames;
1291       PetscInt  Nl = numLabels;
1292       PetscBool flg;
1293 
1294       ierr = PetscMalloc1(Nl, &labelNames);CHKERRQ(ierr);
1295       ierr = PetscOptionsGetStringArray(((PetscObject) dm)->options, ((PetscObject) dm)->prefix, "-dm_plex_view_labels", labelNames, &Nl, &flg);CHKERRQ(ierr);
1296       for (l = 0; l < Nl; ++l) {
1297         DMLabel label;
1298 
1299         ierr = DMHasLabel(dm, labelNames[l], &flg);CHKERRQ(ierr);
1300         if (flg) {
1301           ierr = DMGetLabel(dm, labelNames[l], &label);CHKERRQ(ierr);
1302           ierr = DMLabelView(label, viewer);CHKERRQ(ierr);
1303         }
1304         ierr = PetscFree(labelNames[l]);CHKERRQ(ierr);
1305       }
1306       ierr = PetscFree(labelNames);CHKERRQ(ierr);
1307     }
1308     /* If no fields are specified, people do not want to see adjacency */
1309     if (dm->Nf) {
1310       PetscInt f;
1311 
1312       for (f = 0; f < dm->Nf; ++f) {
1313         const char *name;
1314 
1315         ierr = PetscObjectGetName(dm->fields[f].disc, &name);CHKERRQ(ierr);
1316         if (numLabels) {ierr = PetscViewerASCIIPrintf(viewer, "Field %s:\n", name);CHKERRQ(ierr);}
1317         ierr = PetscViewerASCIIPushTab(viewer);CHKERRQ(ierr);
1318         if (dm->fields[f].label) {ierr = DMLabelView(dm->fields[f].label, viewer);CHKERRQ(ierr);}
1319         if (dm->fields[f].adjacency[0]) {
1320           if (dm->fields[f].adjacency[1]) {ierr = PetscViewerASCIIPrintf(viewer, "adjacency FVM++\n");CHKERRQ(ierr);}
1321           else                            {ierr = PetscViewerASCIIPrintf(viewer, "adjacency FVM\n");CHKERRQ(ierr);}
1322         } else {
1323           if (dm->fields[f].adjacency[1]) {ierr = PetscViewerASCIIPrintf(viewer, "adjacency FEM\n");CHKERRQ(ierr);}
1324           else                            {ierr = PetscViewerASCIIPrintf(viewer, "adjacency FUNKY\n");CHKERRQ(ierr);}
1325         }
1326         ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr);
1327       }
1328     }
1329     ierr = DMGetCoarseDM(dm, &cdm);CHKERRQ(ierr);
1330     if (cdm) {
1331       ierr = PetscViewerASCIIPushTab(viewer);CHKERRQ(ierr);
1332       ierr = DMPlexView_Ascii(cdm, viewer);CHKERRQ(ierr);
1333       ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr);
1334     }
1335   }
1336   PetscFunctionReturn(0);
1337 }
1338 
1339 static PetscErrorCode DMPlexDrawCell(DM dm, PetscDraw draw, PetscInt cell, const PetscScalar coords[])
1340 {
1341   DMPolytopeType ct;
1342   PetscMPIInt    rank;
1343   PetscErrorCode ierr;
1344 
1345   PetscFunctionBegin;
1346   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRMPI(ierr);
1347   ierr = DMPlexGetCellType(dm, cell, &ct);CHKERRQ(ierr);
1348   switch (ct) {
1349   case DM_POLYTOPE_TRIANGLE:
1350     ierr = PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]),
1351                              PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1352                              PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1353                              PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2);CHKERRQ(ierr);
1354     ierr = PetscDrawLine(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1355     ierr = PetscDrawLine(draw, PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1356     ierr = PetscDrawLine(draw, PetscRealPart(coords[4]), PetscRealPart(coords[5]), PetscRealPart(coords[0]), PetscRealPart(coords[1]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1357     break;
1358   case DM_POLYTOPE_QUADRILATERAL:
1359     ierr = PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]),
1360                               PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1361                               PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1362                               PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2);CHKERRQ(ierr);
1363     ierr = PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), PetscRealPart(coords[6]), PetscRealPart(coords[7]),
1364                               PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1365                               PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1366                               PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2);CHKERRQ(ierr);
1367     ierr = PetscDrawLine(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1368     ierr = PetscDrawLine(draw, PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1369     ierr = PetscDrawLine(draw, PetscRealPart(coords[4]), PetscRealPart(coords[5]), PetscRealPart(coords[6]), PetscRealPart(coords[7]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1370     ierr = PetscDrawLine(draw, PetscRealPart(coords[6]), PetscRealPart(coords[7]), PetscRealPart(coords[0]), PetscRealPart(coords[1]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1371     break;
1372   default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_SUP, "Cannot draw cells of type %s", DMPolytopeTypes[ct]);
1373   }
1374   PetscFunctionReturn(0);
1375 }
1376 
1377 static PetscErrorCode DMPlexDrawCellHighOrder(DM dm, PetscDraw draw, PetscInt cell, const PetscScalar coords[], PetscInt edgeDiv, PetscReal refCoords[], PetscReal edgeCoords[])
1378 {
1379   DMPolytopeType ct;
1380   PetscReal      centroid[2] = {0., 0.};
1381   PetscMPIInt    rank;
1382   PetscInt       fillColor, v, e, d;
1383   PetscErrorCode ierr;
1384 
1385   PetscFunctionBegin;
1386   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRMPI(ierr);
1387   ierr = DMPlexGetCellType(dm, cell, &ct);CHKERRQ(ierr);
1388   fillColor = PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2;
1389   switch (ct) {
1390   case DM_POLYTOPE_TRIANGLE:
1391     {
1392       PetscReal refVertices[6] = {-1., -1., 1., -1., -1., 1.};
1393 
1394       for (v = 0; v < 3; ++v) {centroid[0] += PetscRealPart(coords[v*2+0])/3.;centroid[1] += PetscRealPart(coords[v*2+1])/3.;}
1395       for (e = 0; e < 3; ++e) {
1396         refCoords[0] = refVertices[e*2+0];
1397         refCoords[1] = refVertices[e*2+1];
1398         for (d = 1; d <= edgeDiv; ++d) {
1399           refCoords[d*2+0] = refCoords[0] + (refVertices[(e+1)%3 * 2 + 0] - refCoords[0])*d/edgeDiv;
1400           refCoords[d*2+1] = refCoords[1] + (refVertices[(e+1)%3 * 2 + 1] - refCoords[1])*d/edgeDiv;
1401         }
1402         ierr = DMPlexReferenceToCoordinates(dm, cell, edgeDiv+1, refCoords, edgeCoords);CHKERRQ(ierr);
1403         for (d = 0; d < edgeDiv; ++d) {
1404           ierr = PetscDrawTriangle(draw, centroid[0], centroid[1], edgeCoords[d*2+0], edgeCoords[d*2+1], edgeCoords[(d+1)*2+0], edgeCoords[(d+1)*2+1], fillColor, fillColor, fillColor);CHKERRQ(ierr);
1405           ierr = PetscDrawLine(draw, edgeCoords[d*2+0], edgeCoords[d*2+1], edgeCoords[(d+1)*2+0], edgeCoords[(d+1)*2+1], PETSC_DRAW_BLACK);CHKERRQ(ierr);
1406         }
1407       }
1408     }
1409     break;
1410   default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_SUP, "Cannot draw cells of type %s", DMPolytopeTypes[ct]);
1411   }
1412   PetscFunctionReturn(0);
1413 }
1414 
1415 static PetscErrorCode DMPlexView_Draw(DM dm, PetscViewer viewer)
1416 {
1417   PetscDraw          draw;
1418   DM                 cdm;
1419   PetscSection       coordSection;
1420   Vec                coordinates;
1421   const PetscScalar *coords;
1422   PetscReal          xyl[2],xyr[2],bound[4] = {PETSC_MAX_REAL, PETSC_MAX_REAL, PETSC_MIN_REAL, PETSC_MIN_REAL};
1423   PetscReal         *refCoords, *edgeCoords;
1424   PetscBool          isnull, drawAffine = PETSC_TRUE;
1425   PetscInt           dim, vStart, vEnd, cStart, cEnd, c, N, edgeDiv = 4;
1426   PetscErrorCode     ierr;
1427 
1428   PetscFunctionBegin;
1429   ierr = DMGetCoordinateDim(dm, &dim);CHKERRQ(ierr);
1430   if (dim != 2) SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Cannot draw meshes of dimension %D", dim);
1431   ierr = PetscOptionsGetBool(((PetscObject) dm)->options, ((PetscObject) dm)->prefix, "-dm_view_draw_affine", &drawAffine, NULL);CHKERRQ(ierr);
1432   if (!drawAffine) {ierr = PetscMalloc2((edgeDiv+1)*dim, &refCoords, (edgeDiv+1)*dim, &edgeCoords);CHKERRQ(ierr);}
1433   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
1434   ierr = DMGetLocalSection(cdm, &coordSection);CHKERRQ(ierr);
1435   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
1436   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
1437   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
1438 
1439   ierr = PetscViewerDrawGetDraw(viewer, 0, &draw);CHKERRQ(ierr);
1440   ierr = PetscDrawIsNull(draw, &isnull);CHKERRQ(ierr);
1441   if (isnull) PetscFunctionReturn(0);
1442   ierr = PetscDrawSetTitle(draw, "Mesh");CHKERRQ(ierr);
1443 
1444   ierr = VecGetLocalSize(coordinates, &N);CHKERRQ(ierr);
1445   ierr = VecGetArrayRead(coordinates, &coords);CHKERRQ(ierr);
1446   for (c = 0; c < N; c += dim) {
1447     bound[0] = PetscMin(bound[0], PetscRealPart(coords[c]));   bound[2] = PetscMax(bound[2], PetscRealPart(coords[c]));
1448     bound[1] = PetscMin(bound[1], PetscRealPart(coords[c+1])); bound[3] = PetscMax(bound[3], PetscRealPart(coords[c+1]));
1449   }
1450   ierr = VecRestoreArrayRead(coordinates, &coords);CHKERRQ(ierr);
1451   ierr = MPIU_Allreduce(&bound[0],xyl,2,MPIU_REAL,MPIU_MIN,PetscObjectComm((PetscObject)dm));CHKERRMPI(ierr);
1452   ierr = MPIU_Allreduce(&bound[2],xyr,2,MPIU_REAL,MPIU_MAX,PetscObjectComm((PetscObject)dm));CHKERRMPI(ierr);
1453   ierr = PetscDrawSetCoordinates(draw, xyl[0], xyl[1], xyr[0], xyr[1]);CHKERRQ(ierr);
1454   ierr = PetscDrawClear(draw);CHKERRQ(ierr);
1455 
1456   for (c = cStart; c < cEnd; ++c) {
1457     PetscScalar *coords = NULL;
1458     PetscInt     numCoords;
1459 
1460     ierr = DMPlexVecGetClosureAtDepth_Internal(dm, coordSection, coordinates, c, 0, &numCoords, &coords);CHKERRQ(ierr);
1461     if (drawAffine) {
1462       ierr = DMPlexDrawCell(dm, draw, c, coords);CHKERRQ(ierr);
1463     } else {
1464       ierr = DMPlexDrawCellHighOrder(dm, draw, c, coords, edgeDiv, refCoords, edgeCoords);CHKERRQ(ierr);
1465     }
1466     ierr = DMPlexVecRestoreClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
1467   }
1468   if (!drawAffine) {ierr = PetscFree2(refCoords, edgeCoords);CHKERRQ(ierr);}
1469   ierr = PetscDrawFlush(draw);CHKERRQ(ierr);
1470   ierr = PetscDrawPause(draw);CHKERRQ(ierr);
1471   ierr = PetscDrawSave(draw);CHKERRQ(ierr);
1472   PetscFunctionReturn(0);
1473 }
1474 
1475 #if defined(PETSC_HAVE_EXODUSII)
1476 #include <exodusII.h>
1477 #include <petscviewerexodusii.h>
1478 #endif
1479 
1480 PetscErrorCode DMView_Plex(DM dm, PetscViewer viewer)
1481 {
1482   PetscBool      iascii, ishdf5, isvtk, isdraw, flg, isglvis, isexodus;
1483   char           name[PETSC_MAX_PATH_LEN];
1484   PetscErrorCode ierr;
1485 
1486   PetscFunctionBegin;
1487   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1488   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1489   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERASCII,    &iascii);CHKERRQ(ierr);
1490   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,      &isvtk);CHKERRQ(ierr);
1491   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,     &ishdf5);CHKERRQ(ierr);
1492   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERDRAW,     &isdraw);CHKERRQ(ierr);
1493   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERGLVIS,    &isglvis);CHKERRQ(ierr);
1494   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWEREXODUSII, &isexodus);CHKERRQ(ierr);
1495   if (iascii) {
1496     PetscViewerFormat format;
1497     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
1498     if (format == PETSC_VIEWER_ASCII_GLVIS) {
1499       ierr = DMPlexView_GLVis(dm, viewer);CHKERRQ(ierr);
1500     } else {
1501       ierr = DMPlexView_Ascii(dm, viewer);CHKERRQ(ierr);
1502     }
1503   } else if (ishdf5) {
1504 #if defined(PETSC_HAVE_HDF5)
1505     ierr = DMPlexView_HDF5_Internal(dm, viewer);CHKERRQ(ierr);
1506 #else
1507     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1508 #endif
1509   } else if (isvtk) {
1510     ierr = DMPlexVTKWriteAll((PetscObject) dm,viewer);CHKERRQ(ierr);
1511   } else if (isdraw) {
1512     ierr = DMPlexView_Draw(dm, viewer);CHKERRQ(ierr);
1513   } else if (isglvis) {
1514     ierr = DMPlexView_GLVis(dm, viewer);CHKERRQ(ierr);
1515 #if defined(PETSC_HAVE_EXODUSII)
1516   } else if (isexodus) {
1517 /*
1518       exodusII requires that all sets be part of exactly one cell set.
1519       If the dm does not have a "Cell Sets" label defined, we create one
1520       with ID 1, containig all cells.
1521       Note that if the Cell Sets label is defined but does not cover all cells,
1522       we may still have a problem. This should probably be checked here or in the viewer;
1523     */
1524     PetscInt numCS;
1525     ierr = DMGetLabelSize(dm,"Cell Sets",&numCS);CHKERRQ(ierr);
1526     if (!numCS) {
1527       PetscInt cStart, cEnd, c;
1528       ierr = DMCreateLabel(dm, "Cell Sets");CHKERRQ(ierr);
1529       ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
1530       for (c = cStart; c < cEnd; ++c) {ierr = DMSetLabelValue(dm, "Cell Sets", c, 1);CHKERRQ(ierr);}
1531     }
1532     ierr = DMView_PlexExodusII(dm, viewer);CHKERRQ(ierr);
1533 #endif
1534   } else {
1535     SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Viewer type %s not yet supported for DMPlex writing", ((PetscObject)viewer)->type_name);
1536   }
1537   /* Optionally view the partition */
1538   ierr = PetscOptionsHasName(((PetscObject) dm)->options, ((PetscObject) dm)->prefix, "-dm_partition_view", &flg);CHKERRQ(ierr);
1539   if (flg) {
1540     Vec ranks;
1541     ierr = DMPlexCreateRankField(dm, &ranks);CHKERRQ(ierr);
1542     ierr = VecView(ranks, viewer);CHKERRQ(ierr);
1543     ierr = VecDestroy(&ranks);CHKERRQ(ierr);
1544   }
1545   /* Optionally view a label */
1546   ierr = PetscOptionsGetString(((PetscObject) dm)->options, ((PetscObject) dm)->prefix, "-dm_label_view", name, sizeof(name), &flg);CHKERRQ(ierr);
1547   if (flg) {
1548     DMLabel label;
1549     Vec     val;
1550 
1551     ierr = DMGetLabel(dm, name, &label);CHKERRQ(ierr);
1552     if (!label) SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Label %s provided to -dm_label_view does not exist in this DM", name);
1553     ierr = DMPlexCreateLabelField(dm, label, &val);CHKERRQ(ierr);
1554     ierr = VecView(val, viewer);CHKERRQ(ierr);
1555     ierr = VecDestroy(&val);CHKERRQ(ierr);
1556   }
1557   PetscFunctionReturn(0);
1558 }
1559 
1560 /*@
1561   DMPlexTopologyView - Saves a DMPlex topology into a file
1562 
1563   Collective on DM
1564 
1565   Input Parameters:
1566 + dm     - The DM whose topology is to be saved
1567 - viewer - The PetscViewer for saving
1568 
1569   Level: advanced
1570 
1571 .seealso: DMView(), DMPlexCoordinatesView(), DMPlexLabelsView(), DMPlexTopologyLoad()
1572 @*/
1573 PetscErrorCode DMPlexTopologyView(DM dm, PetscViewer viewer)
1574 {
1575   PetscBool      ishdf5;
1576   PetscErrorCode ierr;
1577 
1578   PetscFunctionBegin;
1579   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1580   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1581   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
1582   if (ishdf5) {
1583 #if defined(PETSC_HAVE_HDF5)
1584     PetscViewerFormat format;
1585     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
1586     if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
1587       IS globalPointNumbering;
1588 
1589       ierr = DMPlexCreatePointNumbering(dm, &globalPointNumbering);CHKERRQ(ierr);
1590       ierr = DMPlexTopologyView_HDF5_Internal(dm, globalPointNumbering, viewer);CHKERRQ(ierr);
1591       ierr = ISDestroy(&globalPointNumbering);CHKERRQ(ierr);
1592     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 output.", PetscViewerFormats[format]);
1593 #else
1594     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1595 #endif
1596   }
1597   PetscFunctionReturn(0);
1598 }
1599 
1600 /*@
1601   DMPlexCoordinatesView - Saves DMPlex coordinates into a file
1602 
1603   Collective on DM
1604 
1605   Input Parameters:
1606 + dm     - The DM whose coordinates are to be saved
1607 - viewer - The PetscViewer for saving
1608 
1609   Level: advanced
1610 
1611 .seealso: DMView(), DMPlexTopologyView(), DMPlexLabelsView(), DMPlexCoordinatesLoad()
1612 @*/
1613 PetscErrorCode DMPlexCoordinatesView(DM dm, PetscViewer viewer)
1614 {
1615   PetscBool      ishdf5;
1616   PetscErrorCode ierr;
1617 
1618   PetscFunctionBegin;
1619   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1620   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1621   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
1622   if (ishdf5) {
1623 #if defined(PETSC_HAVE_HDF5)
1624     PetscViewerFormat format;
1625     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
1626     if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
1627       ierr = DMPlexCoordinatesView_HDF5_Internal(dm, viewer);CHKERRQ(ierr);
1628     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 input.", PetscViewerFormats[format]);
1629 #else
1630     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1631 #endif
1632   }
1633   PetscFunctionReturn(0);
1634 }
1635 
1636 /*@
1637   DMPlexLabelsView - Saves DMPlex labels into a file
1638 
1639   Collective on DM
1640 
1641   Input Parameters:
1642 + dm     - The DM whose labels are to be saved
1643 - viewer - The PetscViewer for saving
1644 
1645   Level: advanced
1646 
1647 .seealso: DMView(), DMPlexTopologyView(), DMPlexCoordinatesView(), DMPlexLabelsLoad()
1648 @*/
1649 PetscErrorCode DMPlexLabelsView(DM dm, PetscViewer viewer)
1650 {
1651   PetscBool      ishdf5;
1652   PetscErrorCode ierr;
1653 
1654   PetscFunctionBegin;
1655   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1656   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1657   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
1658   if (ishdf5) {
1659 #if defined(PETSC_HAVE_HDF5)
1660     IS                globalPointNumbering;
1661     PetscViewerFormat format;
1662 
1663     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
1664     if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
1665       ierr = DMPlexCreatePointNumbering(dm, &globalPointNumbering);CHKERRQ(ierr);
1666       ierr = DMPlexLabelsView_HDF5_Internal(dm, globalPointNumbering, viewer);CHKERRQ(ierr);
1667       ierr = ISDestroy(&globalPointNumbering);CHKERRQ(ierr);
1668     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 input.", PetscViewerFormats[format]);
1669 #else
1670     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1671 #endif
1672   }
1673   PetscFunctionReturn(0);
1674 }
1675 
1676 /*@
1677   DMPlexSectionView - Saves a section associated with a DMPlex
1678 
1679   Collective on DM
1680 
1681   Input Parameters:
1682 + dm         - The DM that contains the topology on which the section to be saved is defined
1683 . viewer     - The PetscViewer for saving
1684 - sectiondm  - The DM that contains the section to be saved
1685 
1686   Level: advanced
1687 
1688   Notes:
1689   This function is a wrapper around PetscSectionView(); in addition to the raw section, it saves information that associates the section points to the topology (dm) points. When the topology (dm) and the section are later loaded with DMPlexTopologyLoad() and DMPlexSectionLoad(), respectively, this information is used to match section points with topology points.
1690 
1691   In general dm and sectiondm are two different objects, the former carrying the topology and the latter carrying the section, and have been given a topology name and a section name, respectively, with PetscObjectSetName(). In practice, however, they can be the same object if it carries both topology and section; in that case the name of the object is used as both the topology name and the section name.
1692 
1693 .seealso: DMView(), DMPlexTopologyView(), DMPlexCoordinatesView(), DMPlexLabelsView(), DMPlexGlobalVectorView(), DMPlexLocalVectorView(), PetscSectionView(), DMPlexSectionLoad()
1694 @*/
1695 PetscErrorCode DMPlexSectionView(DM dm, PetscViewer viewer, DM sectiondm)
1696 {
1697   PetscBool      ishdf5;
1698   PetscErrorCode ierr;
1699 
1700   PetscFunctionBegin;
1701   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1702   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1703   PetscValidHeaderSpecific(sectiondm, DM_CLASSID, 3);
1704   ierr = PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERHDF5,&ishdf5);CHKERRQ(ierr);
1705   if (ishdf5) {
1706 #if defined(PETSC_HAVE_HDF5)
1707     ierr = DMPlexSectionView_HDF5_Internal(dm, viewer, sectiondm);CHKERRQ(ierr);
1708 #else
1709     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1710 #endif
1711   }
1712   PetscFunctionReturn(0);
1713 }
1714 
1715 /*@
1716   DMPlexGlobalVectorView - Saves a global vector
1717 
1718   Collective on DM
1719 
1720   Input Parameters:
1721 + dm        - The DM that represents the topology
1722 . viewer    - The PetscViewer to save data with
1723 . sectiondm - The DM that contains the global section on which vec is defined
1724 - vec       - The global vector to be saved
1725 
1726   Level: advanced
1727 
1728   Notes:
1729   In general dm and sectiondm are two different objects, the former carrying the topology and the latter carrying the section, and have been given a topology name and a section name, respectively, with PetscObjectSetName(). In practice, however, they can be the same object if it carries both topology and section; in that case the name of the object is used as both the topology name and the section name.
1730 
1731   Typical calling sequence
1732 $       DMCreate(PETSC_COMM_WORLD, &dm);
1733 $       DMSetType(dm, DMPLEX);
1734 $       PetscObjectSetName((PetscObject)dm, "topologydm_name");
1735 $       DMClone(dm, &sectiondm);
1736 $       PetscObjectSetName((PetscObject)sectiondm, "sectiondm_name");
1737 $       PetscSectionCreate(PETSC_COMM_WORLD, &section);
1738 $       DMPlexGetChart(sectiondm, &pStart, &pEnd);
1739 $       PetscSectionSetChart(section, pStart, pEnd);
1740 $       PetscSectionSetUp(section);
1741 $       DMSetLocalSection(sectiondm, section);
1742 $       PetscSectionDestroy(&section);
1743 $       DMGetGlobalVector(sectiondm, &vec);
1744 $       PetscObjectSetName((PetscObject)vec, "vec_name");
1745 $       DMPlexTopologyView(dm, viewer);
1746 $       DMPlexSectionView(dm, viewer, sectiondm);
1747 $       DMPlexGlobalVectorView(dm, viewer, sectiondm, vec);
1748 $       DMRestoreGlobalVector(sectiondm, &vec);
1749 $       DMDestroy(&sectiondm);
1750 $       DMDestroy(&dm);
1751 
1752 .seealso: DMPlexTopologyView(), DMPlexSectionView(), DMPlexLocalVectorView(), DMPlexGlobalVectorLoad(), DMPlexLocalVectorLoad()
1753 @*/
1754 PetscErrorCode DMPlexGlobalVectorView(DM dm, PetscViewer viewer, DM sectiondm, Vec vec)
1755 {
1756   PetscBool       ishdf5;
1757   PetscErrorCode  ierr;
1758 
1759   PetscFunctionBegin;
1760   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1761   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1762   PetscValidHeaderSpecific(sectiondm, DM_CLASSID, 3);
1763   PetscValidHeaderSpecific(vec, VEC_CLASSID, 4);
1764   /* Check consistency */
1765   {
1766     PetscSection  section;
1767     PetscBool     includesConstraints;
1768     PetscInt      m, m1;
1769 
1770     ierr = VecGetLocalSize(vec, &m1);CHKERRQ(ierr);
1771     ierr = DMGetGlobalSection(sectiondm, &section);CHKERRQ(ierr);
1772     ierr = PetscSectionGetIncludesConstraints(section, &includesConstraints);CHKERRQ(ierr);
1773     if (includesConstraints) {ierr = PetscSectionGetStorageSize(section, &m);CHKERRQ(ierr);}
1774     else {ierr = PetscSectionGetConstrainedStorageSize(section, &m);CHKERRQ(ierr);}
1775     if (m1 != m) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Global vector size (%D) != global section storage size (%D)", m1, m);
1776   }
1777   ierr = PetscObjectTypeCompare((PetscObject)viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
1778   if (ishdf5) {
1779 #if defined(PETSC_HAVE_HDF5)
1780     ierr = DMPlexGlobalVectorView_HDF5_Internal(dm, viewer, sectiondm, vec);CHKERRQ(ierr);
1781 #else
1782     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1783 #endif
1784   }
1785   PetscFunctionReturn(0);
1786 }
1787 
1788 /*@
1789   DMPlexLocalVectorView - Saves a local vector
1790 
1791   Collective on DM
1792 
1793   Input Parameters:
1794 + dm        - The DM that represents the topology
1795 . viewer    - The PetscViewer to save data with
1796 . sectiondm - The DM that contains the local section on which vec is defined; may be the same as dm
1797 - vec       - The local vector to be saved
1798 
1799   Level: advanced
1800 
1801   Notes:
1802   In general dm and sectiondm are two different objects, the former carrying the topology and the latter carrying the section, and have been given a topology name and a section name, respectively, with PetscObjectSetName(). In practice, however, they can be the same object if it carries both topology and section; in that case the name of the object is used as both the topology name and the section name.
1803 
1804   Typical calling sequence
1805 $       DMCreate(PETSC_COMM_WORLD, &dm);
1806 $       DMSetType(dm, DMPLEX);
1807 $       PetscObjectSetName((PetscObject)dm, "topologydm_name");
1808 $       DMClone(dm, &sectiondm);
1809 $       PetscObjectSetName((PetscObject)sectiondm, "sectiondm_name");
1810 $       PetscSectionCreate(PETSC_COMM_WORLD, &section);
1811 $       DMPlexGetChart(sectiondm, &pStart, &pEnd);
1812 $       PetscSectionSetChart(section, pStart, pEnd);
1813 $       PetscSectionSetUp(section);
1814 $       DMSetLocalSection(sectiondm, section);
1815 $       DMGetLocalVector(sectiondm, &vec);
1816 $       PetscObjectSetName((PetscObject)vec, "vec_name");
1817 $       DMPlexTopologyView(dm, viewer);
1818 $       DMPlexSectionView(dm, viewer, sectiondm);
1819 $       DMPlexLocalVectorView(dm, viewer, sectiondm, vec);
1820 $       DMRestoreLocalVector(sectiondm, &vec);
1821 $       DMDestroy(&sectiondm);
1822 $       DMDestroy(&dm);
1823 
1824 .seealso: DMPlexTopologyView(), DMPlexSectionView(), DMPlexGlobalVectorView(), DMPlexGlobalVectorLoad(), DMPlexLocalVectorLoad()
1825 @*/
1826 PetscErrorCode DMPlexLocalVectorView(DM dm, PetscViewer viewer, DM sectiondm, Vec vec)
1827 {
1828   PetscBool       ishdf5;
1829   PetscErrorCode  ierr;
1830 
1831   PetscFunctionBegin;
1832   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1833   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1834   PetscValidHeaderSpecific(sectiondm, DM_CLASSID, 3);
1835   PetscValidHeaderSpecific(vec, VEC_CLASSID, 4);
1836   /* Check consistency */
1837   {
1838     PetscSection  section;
1839     PetscBool     includesConstraints;
1840     PetscInt      m, m1;
1841 
1842     ierr = VecGetLocalSize(vec, &m1);CHKERRQ(ierr);
1843     ierr = DMGetLocalSection(sectiondm, &section);CHKERRQ(ierr);
1844     ierr = PetscSectionGetIncludesConstraints(section, &includesConstraints);CHKERRQ(ierr);
1845     if (includesConstraints) {ierr = PetscSectionGetStorageSize(section, &m);CHKERRQ(ierr);}
1846     else {ierr = PetscSectionGetConstrainedStorageSize(section, &m);CHKERRQ(ierr);}
1847     if (m1 != m) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Local vector size (%D) != local section storage size (%D)", m1, m);
1848   }
1849   ierr = PetscObjectTypeCompare((PetscObject)viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
1850   if (ishdf5) {
1851 #if defined(PETSC_HAVE_HDF5)
1852     ierr = DMPlexLocalVectorView_HDF5_Internal(dm, viewer, sectiondm, vec);CHKERRQ(ierr);
1853 #else
1854     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1855 #endif
1856   }
1857   PetscFunctionReturn(0);
1858 }
1859 
1860 PetscErrorCode DMLoad_Plex(DM dm, PetscViewer viewer)
1861 {
1862   PetscBool      ishdf5;
1863   PetscErrorCode ierr;
1864 
1865   PetscFunctionBegin;
1866   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1867   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1868   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,   &ishdf5);CHKERRQ(ierr);
1869   if (ishdf5) {
1870 #if defined(PETSC_HAVE_HDF5)
1871     PetscViewerFormat format;
1872     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
1873     if (format == PETSC_VIEWER_HDF5_XDMF || format == PETSC_VIEWER_HDF5_VIZ) {
1874       ierr = DMPlexLoad_HDF5_Xdmf_Internal(dm, viewer);CHKERRQ(ierr);
1875     } else if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
1876       ierr = DMPlexLoad_HDF5_Internal(dm, viewer);CHKERRQ(ierr);
1877     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 input.", PetscViewerFormats[format]);
1878     PetscFunctionReturn(0);
1879 #else
1880     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1881 #endif
1882   } else SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Viewer type %s not yet supported for DMPlex loading", ((PetscObject)viewer)->type_name);
1883 }
1884 
1885 /*@
1886   DMPlexTopologyLoad - Loads a topology into a DMPlex
1887 
1888   Collective on DM
1889 
1890   Input Parameters:
1891 + dm     - The DM into which the topology is loaded
1892 - viewer - The PetscViewer for the saved topology
1893 
1894   Output Parameters:
1895 . globalToLocalPointSF - The PetscSF that pushes points in [0, N) to the associated points in the loaded plex, where N is the global number of points; NULL if unneeded
1896 
1897   Level: advanced
1898 
1899 .seealso: DMLoad(), DMPlexCoordinatesLoad(), DMPlexLabelsLoad(), DMView(), PetscViewerHDF5Open(), PetscViewerPushFormat()
1900 @*/
1901 PetscErrorCode DMPlexTopologyLoad(DM dm, PetscViewer viewer, PetscSF *globalToLocalPointSF)
1902 {
1903   PetscBool      ishdf5;
1904   PetscErrorCode ierr;
1905 
1906   PetscFunctionBegin;
1907   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1908   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1909   if (globalToLocalPointSF) PetscValidPointer(globalToLocalPointSF, 3);
1910   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
1911   if (ishdf5) {
1912 #if defined(PETSC_HAVE_HDF5)
1913     PetscViewerFormat format;
1914     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
1915     if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
1916       ierr = DMPlexTopologyLoad_HDF5_Internal(dm, viewer, globalToLocalPointSF);CHKERRQ(ierr);
1917     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 input.", PetscViewerFormats[format]);
1918 #else
1919     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1920 #endif
1921   }
1922   PetscFunctionReturn(0);
1923 }
1924 
1925 /*@
1926   DMPlexCoordinatesLoad - Loads coordinates into a DMPlex
1927 
1928   Collective on DM
1929 
1930   Input Parameters:
1931 + dm     - The DM into which the coordinates are loaded
1932 - viewer - The PetscViewer for the saved coordinates
1933 
1934   Level: advanced
1935 
1936 .seealso: DMLoad(), DMPlexTopologyLoad(), DMPlexLabelsLoad(), DMView(), PetscViewerHDF5Open(), PetscViewerPushFormat()
1937 @*/
1938 PetscErrorCode DMPlexCoordinatesLoad(DM dm, PetscViewer viewer)
1939 {
1940   PetscBool      ishdf5;
1941   PetscErrorCode ierr;
1942 
1943   PetscFunctionBegin;
1944   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1945   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1946   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
1947   if (ishdf5) {
1948 #if defined(PETSC_HAVE_HDF5)
1949     PetscViewerFormat format;
1950     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
1951     if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
1952       ierr = DMPlexCoordinatesLoad_HDF5_Internal(dm, viewer);CHKERRQ(ierr);
1953     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 input.", PetscViewerFormats[format]);
1954 #else
1955     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1956 #endif
1957   }
1958   PetscFunctionReturn(0);
1959 }
1960 
1961 /*@
1962   DMPlexLabelsLoad - Loads labels into a DMPlex
1963 
1964   Collective on DM
1965 
1966   Input Parameters:
1967 + dm     - The DM into which the labels are loaded
1968 - viewer - The PetscViewer for the saved labels
1969 
1970   Level: advanced
1971 
1972 .seealso: DMLoad(), DMPlexTopologyLoad(), DMPlexCoordinatesLoad(), DMView(), PetscViewerHDF5Open(), PetscViewerPushFormat()
1973 @*/
1974 PetscErrorCode DMPlexLabelsLoad(DM dm, PetscViewer viewer)
1975 {
1976   PetscBool      ishdf5;
1977   PetscErrorCode ierr;
1978 
1979   PetscFunctionBegin;
1980   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1981   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1982   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
1983   if (ishdf5) {
1984 #if defined(PETSC_HAVE_HDF5)
1985     PetscViewerFormat format;
1986 
1987     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
1988     if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
1989       ierr = DMPlexLabelsLoad_HDF5_Internal(dm, viewer);CHKERRQ(ierr);
1990     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 input.", PetscViewerFormats[format]);
1991 #else
1992     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1993 #endif
1994   }
1995   PetscFunctionReturn(0);
1996 }
1997 
1998 /*@
1999   DMPlexSectionLoad - Loads section into a DMPlex
2000 
2001   Collective on DM
2002 
2003   Input Parameters:
2004 + dm          - The DM that represents the topology
2005 . viewer      - The PetscViewer that represents the on-disk section (sectionA)
2006 . sectiondm   - The DM into which the on-disk section (sectionA) is migrated
2007 - globalToLocalPointSF - The SF returned by DMPlexTopologyLoad() when loading dm from viewer
2008 
2009   Output Parameters
2010 + globalDofSF - The SF that migrates any on-disk Vec data associated with sectionA into a global Vec associated with the sectiondm's global section (NULL if not needed)
2011 - localDofSF  - The SF that migrates any on-disk Vec data associated with sectionA into a local Vec associated with the sectiondm's local section (NULL if not needed)
2012 
2013   Level: advanced
2014 
2015   Notes:
2016   This function is a wrapper around PetscSectionLoad(); it loads, in addition to the raw section, a list of global point numbers that associates each on-disk section point with a global point number in [0, NX), where NX is the number of topology points in dm. Noting that globalToLocalPointSF associates each topology point in dm with a global number in [0, NX), one can readily establish an association of the on-disk section points with the topology points.
2017 
2018   In general dm and sectiondm are two different objects, the former carrying the topology and the latter carrying the section, and have been given a topology name and a section name, respectively, with PetscObjectSetName(). In practice, however, they can be the same object if it carries both topology and section; in that case the name of the object is used as both the topology name and the section name.
2019 
2020   The output parameter, globalDofSF (localDofSF), can later be used with DMPlexGlobalVectorLoad() (DMPlexLocalVectorLoad()) to load on-disk vectors into global (local) vectors associated with sectiondm's global (local) section.
2021 
2022   Example using 2 processes:
2023 $  NX (number of points on dm): 4
2024 $  sectionA                   : the on-disk section
2025 $  vecA                       : a vector associated with sectionA
2026 $  sectionB                   : sectiondm's local section constructed in this function
2027 $  vecB (local)               : a vector associated with sectiondm's local section
2028 $  vecB (global)              : a vector associated with sectiondm's global section
2029 $
2030 $                                     rank 0    rank 1
2031 $  vecA (global)                  : [.0 .4 .1 | .2 .3]        <- to be loaded in DMPlexGlobalVectorLoad() or DMPlexLocalVectorLoad()
2032 $  sectionA->atlasOff             :       0 2 | 1             <- loaded in PetscSectionLoad()
2033 $  sectionA->atlasDof             :       1 3 | 1             <- loaded in PetscSectionLoad()
2034 $  sectionA's global point numbers:       0 2 | 3             <- loaded in DMPlexSectionLoad()
2035 $  [0, NX)                        :       0 1 | 2 3           <- conceptual partition used in globalToLocalPointSF
2036 $  sectionB's global point numbers:     0 1 3 | 3 2           <- associated with [0, NX) by globalToLocalPointSF
2037 $  sectionB->atlasDof             :     1 0 1 | 1 3
2038 $  sectionB->atlasOff (no perm)   :     0 1 1 | 0 1
2039 $  vecB (local)                   :   [.0 .4] | [.4 .1 .2 .3] <- to be constructed by calling DMPlexLocalVectorLoad() with localDofSF
2040 $  vecB (global)                  :    [.0 .4 | .1 .2 .3]     <- to be constructed by calling DMPlexGlobalVectorLoad() with globalDofSF
2041 $
2042 $  where "|" represents a partition of loaded data, and global point 3 is assumed to be owned by rank 0.
2043 
2044 .seealso: DMLoad(), DMPlexTopologyLoad(), DMPlexCoordinatesLoad(), DMPlexLabelsLoad(), DMPlexGlobalVectorLoad(), DMPlexLocalVectorLoad(), PetscSectionLoad(), DMPlexSectionView()
2045 @*/
2046 PetscErrorCode DMPlexSectionLoad(DM dm, PetscViewer viewer, DM sectiondm, PetscSF globalToLocalPointSF, PetscSF *globalDofSF, PetscSF *localDofSF)
2047 {
2048   PetscBool      ishdf5;
2049   PetscErrorCode ierr;
2050 
2051   PetscFunctionBegin;
2052   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2053   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
2054   PetscValidHeaderSpecific(sectiondm, DM_CLASSID, 3);
2055   PetscValidHeaderSpecific(globalToLocalPointSF, PETSCSF_CLASSID, 4);
2056   if (globalDofSF) PetscValidPointer(globalDofSF, 5);
2057   if (localDofSF) PetscValidPointer(localDofSF, 6);
2058   ierr = PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERHDF5,&ishdf5);CHKERRQ(ierr);
2059   if (ishdf5) {
2060 #if defined(PETSC_HAVE_HDF5)
2061     ierr = DMPlexSectionLoad_HDF5_Internal(dm, viewer, sectiondm, globalToLocalPointSF, globalDofSF, localDofSF);CHKERRQ(ierr);
2062 #else
2063     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
2064 #endif
2065   }
2066   PetscFunctionReturn(0);
2067 }
2068 
2069 /*@
2070   DMPlexGlobalVectorLoad - Loads on-disk vector data into a global vector
2071 
2072   Collective on DM
2073 
2074   Input Parameters:
2075 + dm        - The DM that represents the topology
2076 . viewer    - The PetscViewer that represents the on-disk vector data
2077 . sectiondm - The DM that contains the global section on which vec is defined
2078 . sf        - The SF that migrates the on-disk vector data into vec
2079 - vec       - The global vector to set values of
2080 
2081   Level: advanced
2082 
2083   Notes:
2084   In general dm and sectiondm are two different objects, the former carrying the topology and the latter carrying the section, and have been given a topology name and a section name, respectively, with PetscObjectSetName(). In practice, however, they can be the same object if it carries both topology and section; in that case the name of the object is used as both the topology name and the section name.
2085 
2086   Typical calling sequence
2087 $       DMCreate(PETSC_COMM_WORLD, &dm);
2088 $       DMSetType(dm, DMPLEX);
2089 $       PetscObjectSetName((PetscObject)dm, "topologydm_name");
2090 $       DMPlexTopologyLoad(dm, viewer, &sfX);
2091 $       DMClone(dm, &sectiondm);
2092 $       PetscObjectSetName((PetscObject)sectiondm, "sectiondm_name");
2093 $       DMPlexSectionLoad(dm, viewer, sectiondm, sfX, &gsf, NULL);
2094 $       DMGetGlobalVector(sectiondm, &vec);
2095 $       PetscObjectSetName((PetscObject)vec, "vec_name");
2096 $       DMPlexGlobalVectorLoad(dm, viewer, sectiondm, gsf, vec);
2097 $       DMRestoreGlobalVector(sectiondm, &vec);
2098 $       PetscSFDestroy(&gsf);
2099 $       PetscSFDestroy(&sfX);
2100 $       DMDestroy(&sectiondm);
2101 $       DMDestroy(&dm);
2102 
2103 .seealso: DMPlexTopologyLoad(), DMPlexSectionLoad(), DMPlexLocalVectorLoad(), DMPlexGlobalVectorView(), DMPlexLocalVectorView()
2104 @*/
2105 PetscErrorCode DMPlexGlobalVectorLoad(DM dm, PetscViewer viewer, DM sectiondm, PetscSF sf, Vec vec)
2106 {
2107   PetscBool       ishdf5;
2108   PetscErrorCode  ierr;
2109 
2110   PetscFunctionBegin;
2111   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2112   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
2113   PetscValidHeaderSpecific(sectiondm, DM_CLASSID, 3);
2114   PetscValidHeaderSpecific(sf, PETSCSF_CLASSID, 4);
2115   PetscValidHeaderSpecific(vec, VEC_CLASSID, 5);
2116   /* Check consistency */
2117   {
2118     PetscSection  section;
2119     PetscBool     includesConstraints;
2120     PetscInt      m, m1;
2121 
2122     ierr = VecGetLocalSize(vec, &m1);CHKERRQ(ierr);
2123     ierr = DMGetGlobalSection(sectiondm, &section);CHKERRQ(ierr);
2124     ierr = PetscSectionGetIncludesConstraints(section, &includesConstraints);CHKERRQ(ierr);
2125     if (includesConstraints) {ierr = PetscSectionGetStorageSize(section, &m);CHKERRQ(ierr);}
2126     else {ierr = PetscSectionGetConstrainedStorageSize(section, &m);CHKERRQ(ierr);}
2127     if (m1 != m) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Global vector size (%D) != global section storage size (%D)", m1, m);
2128   }
2129   ierr = PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERHDF5,&ishdf5);CHKERRQ(ierr);
2130   if (ishdf5) {
2131 #if defined(PETSC_HAVE_HDF5)
2132     ierr = DMPlexVecLoad_HDF5_Internal(dm, viewer, sectiondm, sf, vec);CHKERRQ(ierr);
2133 #else
2134     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
2135 #endif
2136   }
2137   PetscFunctionReturn(0);
2138 }
2139 
2140 /*@
2141   DMPlexLocalVectorLoad - Loads on-disk vector data into a local vector
2142 
2143   Collective on DM
2144 
2145   Input Parameters:
2146 + dm        - The DM that represents the topology
2147 . viewer    - The PetscViewer that represents the on-disk vector data
2148 . sectiondm - The DM that contains the local section on which vec is defined
2149 . sf        - The SF that migrates the on-disk vector data into vec
2150 - vec       - The local vector to set values of
2151 
2152   Level: advanced
2153 
2154   Notes:
2155   In general dm and sectiondm are two different objects, the former carrying the topology and the latter carrying the section, and have been given a topology name and a section name, respectively, with PetscObjectSetName(). In practice, however, they can be the same object if it carries both topology and section; in that case the name of the object is used as both the topology name and the section name.
2156 
2157   Typical calling sequence
2158 $       DMCreate(PETSC_COMM_WORLD, &dm);
2159 $       DMSetType(dm, DMPLEX);
2160 $       PetscObjectSetName((PetscObject)dm, "topologydm_name");
2161 $       DMPlexTopologyLoad(dm, viewer, &sfX);
2162 $       DMClone(dm, &sectiondm);
2163 $       PetscObjectSetName((PetscObject)sectiondm, "sectiondm_name");
2164 $       DMPlexSectionLoad(dm, viewer, sectiondm, sfX, NULL, &lsf);
2165 $       DMGetLocalVector(sectiondm, &vec);
2166 $       PetscObjectSetName((PetscObject)vec, "vec_name");
2167 $       DMPlexLocalVectorLoad(dm, viewer, sectiondm, lsf, vec);
2168 $       DMRestoreLocalVector(sectiondm, &vec);
2169 $       PetscSFDestroy(&lsf);
2170 $       PetscSFDestroy(&sfX);
2171 $       DMDestroy(&sectiondm);
2172 $       DMDestroy(&dm);
2173 
2174 .seealso: DMPlexTopologyLoad(), DMPlexSectionLoad(), DMPlexGlobalVectorLoad(), DMPlexGlobalVectorView(), DMPlexLocalVectorView()
2175 @*/
2176 PetscErrorCode DMPlexLocalVectorLoad(DM dm, PetscViewer viewer, DM sectiondm, PetscSF sf, Vec vec)
2177 {
2178   PetscBool       ishdf5;
2179   PetscErrorCode  ierr;
2180 
2181   PetscFunctionBegin;
2182   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2183   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
2184   PetscValidHeaderSpecific(sectiondm, DM_CLASSID, 3);
2185   PetscValidHeaderSpecific(sf, PETSCSF_CLASSID, 4);
2186   PetscValidHeaderSpecific(vec, VEC_CLASSID, 5);
2187   /* Check consistency */
2188   {
2189     PetscSection  section;
2190     PetscBool     includesConstraints;
2191     PetscInt      m, m1;
2192 
2193     ierr = VecGetLocalSize(vec, &m1);CHKERRQ(ierr);
2194     ierr = DMGetLocalSection(sectiondm, &section);CHKERRQ(ierr);
2195     ierr = PetscSectionGetIncludesConstraints(section, &includesConstraints);CHKERRQ(ierr);
2196     if (includesConstraints) {ierr = PetscSectionGetStorageSize(section, &m);CHKERRQ(ierr);}
2197     else {ierr = PetscSectionGetConstrainedStorageSize(section, &m);CHKERRQ(ierr);}
2198     if (m1 != m) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Local vector size (%D) != local section storage size (%D)", m1, m);
2199   }
2200   ierr = PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERHDF5,&ishdf5);CHKERRQ(ierr);
2201   if (ishdf5) {
2202 #if defined(PETSC_HAVE_HDF5)
2203     ierr = DMPlexVecLoad_HDF5_Internal(dm, viewer, sectiondm, sf, vec);CHKERRQ(ierr);
2204 #else
2205     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
2206 #endif
2207   }
2208   PetscFunctionReturn(0);
2209 }
2210 
2211 PetscErrorCode DMDestroy_Plex(DM dm)
2212 {
2213   DM_Plex       *mesh = (DM_Plex*) dm->data;
2214   PetscErrorCode ierr;
2215 
2216   PetscFunctionBegin;
2217   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMSetUpGLVisViewer_C",NULL);CHKERRQ(ierr);
2218   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMPlexInsertBoundaryValues_C", NULL);CHKERRQ(ierr);
2219   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMCreateNeumannOverlap_C", NULL);CHKERRQ(ierr);
2220   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMInterpolateSolution_C", NULL);CHKERRQ(ierr);
2221   if (--mesh->refct > 0) PetscFunctionReturn(0);
2222   ierr = PetscSectionDestroy(&mesh->coneSection);CHKERRQ(ierr);
2223   ierr = PetscFree(mesh->cones);CHKERRQ(ierr);
2224   ierr = PetscFree(mesh->coneOrientations);CHKERRQ(ierr);
2225   ierr = PetscSectionDestroy(&mesh->supportSection);CHKERRQ(ierr);
2226   ierr = PetscSectionDestroy(&mesh->subdomainSection);CHKERRQ(ierr);
2227   ierr = PetscFree(mesh->supports);CHKERRQ(ierr);
2228   ierr = PetscFree(mesh->facesTmp);CHKERRQ(ierr);
2229   ierr = PetscFree(mesh->tetgenOpts);CHKERRQ(ierr);
2230   ierr = PetscFree(mesh->triangleOpts);CHKERRQ(ierr);
2231   ierr = PetscPartitionerDestroy(&mesh->partitioner);CHKERRQ(ierr);
2232   ierr = DMLabelDestroy(&mesh->subpointMap);CHKERRQ(ierr);
2233   ierr = ISDestroy(&mesh->subpointIS);CHKERRQ(ierr);
2234   ierr = ISDestroy(&mesh->globalVertexNumbers);CHKERRQ(ierr);
2235   ierr = ISDestroy(&mesh->globalCellNumbers);CHKERRQ(ierr);
2236   ierr = PetscSectionDestroy(&mesh->anchorSection);CHKERRQ(ierr);
2237   ierr = ISDestroy(&mesh->anchorIS);CHKERRQ(ierr);
2238   ierr = PetscSectionDestroy(&mesh->parentSection);CHKERRQ(ierr);
2239   ierr = PetscFree(mesh->parents);CHKERRQ(ierr);
2240   ierr = PetscFree(mesh->childIDs);CHKERRQ(ierr);
2241   ierr = PetscSectionDestroy(&mesh->childSection);CHKERRQ(ierr);
2242   ierr = PetscFree(mesh->children);CHKERRQ(ierr);
2243   ierr = DMDestroy(&mesh->referenceTree);CHKERRQ(ierr);
2244   ierr = PetscGridHashDestroy(&mesh->lbox);CHKERRQ(ierr);
2245   ierr = PetscFree(mesh->neighbors);CHKERRQ(ierr);
2246   /* This was originally freed in DMDestroy(), but that prevents reference counting of backend objects */
2247   ierr = PetscFree(mesh);CHKERRQ(ierr);
2248   PetscFunctionReturn(0);
2249 }
2250 
2251 PetscErrorCode DMCreateMatrix_Plex(DM dm, Mat *J)
2252 {
2253   PetscSection           sectionGlobal;
2254   PetscInt               bs = -1, mbs;
2255   PetscInt               localSize;
2256   PetscBool              isShell, isBlock, isSeqBlock, isMPIBlock, isSymBlock, isSymSeqBlock, isSymMPIBlock, isMatIS;
2257   PetscErrorCode         ierr;
2258   MatType                mtype;
2259   ISLocalToGlobalMapping ltog;
2260 
2261   PetscFunctionBegin;
2262   ierr = MatInitializePackage();CHKERRQ(ierr);
2263   mtype = dm->mattype;
2264   ierr = DMGetGlobalSection(dm, &sectionGlobal);CHKERRQ(ierr);
2265   /* ierr = PetscSectionGetStorageSize(sectionGlobal, &localSize);CHKERRQ(ierr); */
2266   ierr = PetscSectionGetConstrainedStorageSize(sectionGlobal, &localSize);CHKERRQ(ierr);
2267   ierr = MatCreate(PetscObjectComm((PetscObject)dm), J);CHKERRQ(ierr);
2268   ierr = MatSetSizes(*J, localSize, localSize, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
2269   ierr = MatSetType(*J, mtype);CHKERRQ(ierr);
2270   ierr = MatSetFromOptions(*J);CHKERRQ(ierr);
2271   ierr = MatGetBlockSize(*J, &mbs);CHKERRQ(ierr);
2272   if (mbs > 1) bs = mbs;
2273   ierr = PetscStrcmp(mtype, MATSHELL, &isShell);CHKERRQ(ierr);
2274   ierr = PetscStrcmp(mtype, MATBAIJ, &isBlock);CHKERRQ(ierr);
2275   ierr = PetscStrcmp(mtype, MATSEQBAIJ, &isSeqBlock);CHKERRQ(ierr);
2276   ierr = PetscStrcmp(mtype, MATMPIBAIJ, &isMPIBlock);CHKERRQ(ierr);
2277   ierr = PetscStrcmp(mtype, MATSBAIJ, &isSymBlock);CHKERRQ(ierr);
2278   ierr = PetscStrcmp(mtype, MATSEQSBAIJ, &isSymSeqBlock);CHKERRQ(ierr);
2279   ierr = PetscStrcmp(mtype, MATMPISBAIJ, &isSymMPIBlock);CHKERRQ(ierr);
2280   ierr = PetscStrcmp(mtype, MATIS, &isMatIS);CHKERRQ(ierr);
2281   if (!isShell) {
2282     PetscSection subSection;
2283     PetscBool    fillMatrix = (PetscBool)(!dm->prealloc_only && !isMatIS);
2284     PetscInt    *dnz, *onz, *dnzu, *onzu, bsLocal[2], bsMinMax[2], *ltogidx, lsize;
2285     PetscInt     pStart, pEnd, p, dof, cdof;
2286 
2287     /* Set localtoglobalmapping on the matrix for MatSetValuesLocal() to work (it also creates the local matrices in case of MATIS) */
2288     if (isMatIS) { /* need a different l2g map than the one computed by DMGetLocalToGlobalMapping */
2289       PetscSection section;
2290       PetscInt     size;
2291 
2292       ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);
2293       ierr = PetscSectionGetStorageSize(section, &size);CHKERRQ(ierr);
2294       ierr = PetscMalloc1(size,&ltogidx);CHKERRQ(ierr);
2295       ierr = DMPlexGetSubdomainSection(dm, &subSection);CHKERRQ(ierr);
2296     } else {
2297       ierr = DMGetLocalToGlobalMapping(dm,&ltog);CHKERRQ(ierr);
2298     }
2299     ierr = PetscSectionGetChart(sectionGlobal, &pStart, &pEnd);CHKERRQ(ierr);
2300     for (p = pStart, lsize = 0; p < pEnd; ++p) {
2301       PetscInt bdof;
2302 
2303       ierr = PetscSectionGetDof(sectionGlobal, p, &dof);CHKERRQ(ierr);
2304       ierr = PetscSectionGetConstraintDof(sectionGlobal, p, &cdof);CHKERRQ(ierr);
2305       dof  = dof < 0 ? -(dof+1) : dof;
2306       bdof = cdof && (dof-cdof) ? 1 : dof;
2307       if (dof) {
2308         if (bs < 0)          {bs = bdof;}
2309         else if (bs != bdof) {bs = 1; if (!isMatIS) break;}
2310       }
2311       if (isMatIS) {
2312         PetscInt loff,c,off;
2313         ierr = PetscSectionGetOffset(subSection, p, &loff);CHKERRQ(ierr);
2314         ierr = PetscSectionGetOffset(sectionGlobal, p, &off);CHKERRQ(ierr);
2315         for (c = 0; c < dof-cdof; ++c, ++lsize) ltogidx[loff+c] = off > -1 ? off+c : -(off+1)+c;
2316       }
2317     }
2318     /* Must have same blocksize on all procs (some might have no points) */
2319     bsLocal[0] = bs < 0 ? PETSC_MAX_INT : bs; bsLocal[1] = bs;
2320     ierr = PetscGlobalMinMaxInt(PetscObjectComm((PetscObject) dm), bsLocal, bsMinMax);CHKERRQ(ierr);
2321     if (bsMinMax[0] != bsMinMax[1]) {bs = 1;}
2322     else                            {bs = bsMinMax[0];}
2323     bs = PetscMax(1,bs);
2324     if (isMatIS) { /* Must reduce indices by blocksize */
2325       PetscInt l;
2326 
2327       lsize = lsize/bs;
2328       if (bs > 1) for (l = 0; l < lsize; ++l) ltogidx[l] = ltogidx[l*bs]/bs;
2329       ierr = ISLocalToGlobalMappingCreate(PetscObjectComm((PetscObject)dm), bs, lsize, ltogidx, PETSC_OWN_POINTER, &ltog);CHKERRQ(ierr);
2330     }
2331     ierr = MatSetLocalToGlobalMapping(*J,ltog,ltog);CHKERRQ(ierr);
2332     if (isMatIS) {
2333       ierr = ISLocalToGlobalMappingDestroy(&ltog);CHKERRQ(ierr);
2334     }
2335     ierr = PetscCalloc4(localSize/bs, &dnz, localSize/bs, &onz, localSize/bs, &dnzu, localSize/bs, &onzu);CHKERRQ(ierr);
2336     ierr = DMPlexPreallocateOperator(dm, bs, dnz, onz, dnzu, onzu, *J, fillMatrix);CHKERRQ(ierr);
2337     ierr = PetscFree4(dnz, onz, dnzu, onzu);CHKERRQ(ierr);
2338   }
2339   ierr = MatSetDM(*J, dm);CHKERRQ(ierr);
2340   PetscFunctionReturn(0);
2341 }
2342 
2343 /*@
2344   DMPlexGetSubdomainSection - Returns the section associated with the subdomain
2345 
2346   Not collective
2347 
2348   Input Parameter:
2349 . mesh - The DMPlex
2350 
2351   Output Parameters:
2352 . subsection - The subdomain section
2353 
2354   Level: developer
2355 
2356 .seealso:
2357 @*/
2358 PetscErrorCode DMPlexGetSubdomainSection(DM dm, PetscSection *subsection)
2359 {
2360   DM_Plex       *mesh = (DM_Plex*) dm->data;
2361   PetscErrorCode ierr;
2362 
2363   PetscFunctionBegin;
2364   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2365   if (!mesh->subdomainSection) {
2366     PetscSection section;
2367     PetscSF      sf;
2368 
2369     ierr = PetscSFCreate(PETSC_COMM_SELF,&sf);CHKERRQ(ierr);
2370     ierr = DMGetLocalSection(dm,&section);CHKERRQ(ierr);
2371     ierr = PetscSectionCreateGlobalSection(section,sf,PETSC_FALSE,PETSC_TRUE,&mesh->subdomainSection);CHKERRQ(ierr);
2372     ierr = PetscSFDestroy(&sf);CHKERRQ(ierr);
2373   }
2374   *subsection = mesh->subdomainSection;
2375   PetscFunctionReturn(0);
2376 }
2377 
2378 /*@
2379   DMPlexGetChart - Return the interval for all mesh points [pStart, pEnd)
2380 
2381   Not collective
2382 
2383   Input Parameter:
2384 . mesh - The DMPlex
2385 
2386   Output Parameters:
2387 + pStart - The first mesh point
2388 - pEnd   - The upper bound for mesh points
2389 
2390   Level: beginner
2391 
2392 .seealso: DMPlexCreate(), DMPlexSetChart()
2393 @*/
2394 PetscErrorCode DMPlexGetChart(DM dm, PetscInt *pStart, PetscInt *pEnd)
2395 {
2396   DM_Plex       *mesh = (DM_Plex*) dm->data;
2397   PetscErrorCode ierr;
2398 
2399   PetscFunctionBegin;
2400   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2401   ierr = PetscSectionGetChart(mesh->coneSection, pStart, pEnd);CHKERRQ(ierr);
2402   PetscFunctionReturn(0);
2403 }
2404 
2405 /*@
2406   DMPlexSetChart - Set the interval for all mesh points [pStart, pEnd)
2407 
2408   Not collective
2409 
2410   Input Parameters:
2411 + mesh - The DMPlex
2412 . pStart - The first mesh point
2413 - pEnd   - The upper bound for mesh points
2414 
2415   Output Parameters:
2416 
2417   Level: beginner
2418 
2419 .seealso: DMPlexCreate(), DMPlexGetChart()
2420 @*/
2421 PetscErrorCode DMPlexSetChart(DM dm, PetscInt pStart, PetscInt pEnd)
2422 {
2423   DM_Plex       *mesh = (DM_Plex*) dm->data;
2424   PetscErrorCode ierr;
2425 
2426   PetscFunctionBegin;
2427   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2428   ierr = PetscSectionSetChart(mesh->coneSection, pStart, pEnd);CHKERRQ(ierr);
2429   ierr = PetscSectionSetChart(mesh->supportSection, pStart, pEnd);CHKERRQ(ierr);
2430   PetscFunctionReturn(0);
2431 }
2432 
2433 /*@
2434   DMPlexGetConeSize - Return the number of in-edges for this point in the DAG
2435 
2436   Not collective
2437 
2438   Input Parameters:
2439 + mesh - The DMPlex
2440 - p - The point, which must lie in the chart set with DMPlexSetChart()
2441 
2442   Output Parameter:
2443 . size - The cone size for point p
2444 
2445   Level: beginner
2446 
2447 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart()
2448 @*/
2449 PetscErrorCode DMPlexGetConeSize(DM dm, PetscInt p, PetscInt *size)
2450 {
2451   DM_Plex       *mesh = (DM_Plex*) dm->data;
2452   PetscErrorCode ierr;
2453 
2454   PetscFunctionBegin;
2455   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2456   PetscValidPointer(size, 3);
2457   ierr = PetscSectionGetDof(mesh->coneSection, p, size);CHKERRQ(ierr);
2458   PetscFunctionReturn(0);
2459 }
2460 
2461 /*@
2462   DMPlexSetConeSize - Set the number of in-edges for this point in the DAG
2463 
2464   Not collective
2465 
2466   Input Parameters:
2467 + mesh - The DMPlex
2468 . p - The point, which must lie in the chart set with DMPlexSetChart()
2469 - size - The cone size for point p
2470 
2471   Output Parameter:
2472 
2473   Note:
2474   This should be called after DMPlexSetChart().
2475 
2476   Level: beginner
2477 
2478 .seealso: DMPlexCreate(), DMPlexGetConeSize(), DMPlexSetChart()
2479 @*/
2480 PetscErrorCode DMPlexSetConeSize(DM dm, PetscInt p, PetscInt size)
2481 {
2482   DM_Plex       *mesh = (DM_Plex*) dm->data;
2483   PetscErrorCode ierr;
2484 
2485   PetscFunctionBegin;
2486   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2487   ierr = PetscSectionSetDof(mesh->coneSection, p, size);CHKERRQ(ierr);
2488 
2489   mesh->maxConeSize = PetscMax(mesh->maxConeSize, size);
2490   PetscFunctionReturn(0);
2491 }
2492 
2493 /*@
2494   DMPlexAddConeSize - Add the given number of in-edges to this point in the DAG
2495 
2496   Not collective
2497 
2498   Input Parameters:
2499 + mesh - The DMPlex
2500 . p - The point, which must lie in the chart set with DMPlexSetChart()
2501 - size - The additional cone size for point p
2502 
2503   Output Parameter:
2504 
2505   Note:
2506   This should be called after DMPlexSetChart().
2507 
2508   Level: beginner
2509 
2510 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexGetConeSize(), DMPlexSetChart()
2511 @*/
2512 PetscErrorCode DMPlexAddConeSize(DM dm, PetscInt p, PetscInt size)
2513 {
2514   DM_Plex       *mesh = (DM_Plex*) dm->data;
2515   PetscInt       csize;
2516   PetscErrorCode ierr;
2517 
2518   PetscFunctionBegin;
2519   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2520   ierr = PetscSectionAddDof(mesh->coneSection, p, size);CHKERRQ(ierr);
2521   ierr = PetscSectionGetDof(mesh->coneSection, p, &csize);CHKERRQ(ierr);
2522 
2523   mesh->maxConeSize = PetscMax(mesh->maxConeSize, csize);
2524   PetscFunctionReturn(0);
2525 }
2526 
2527 /*@C
2528   DMPlexGetCone - Return the points on the in-edges for this point in the DAG
2529 
2530   Not collective
2531 
2532   Input Parameters:
2533 + dm - The DMPlex
2534 - p - The point, which must lie in the chart set with DMPlexSetChart()
2535 
2536   Output Parameter:
2537 . cone - An array of points which are on the in-edges for point p
2538 
2539   Level: beginner
2540 
2541   Fortran Notes:
2542   Since it returns an array, this routine is only available in Fortran 90, and you must
2543   include petsc.h90 in your code.
2544   You must also call DMPlexRestoreCone() after you finish using the returned array.
2545   DMPlexRestoreCone() is not needed/available in C.
2546 
2547 .seealso: DMPlexGetConeSize(), DMPlexSetCone(), DMPlexGetConeTuple(), DMPlexSetChart()
2548 @*/
2549 PetscErrorCode DMPlexGetCone(DM dm, PetscInt p, const PetscInt *cone[])
2550 {
2551   DM_Plex       *mesh = (DM_Plex*) dm->data;
2552   PetscInt       off;
2553   PetscErrorCode ierr;
2554 
2555   PetscFunctionBegin;
2556   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2557   PetscValidPointer(cone, 3);
2558   ierr  = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2559   *cone = &mesh->cones[off];
2560   PetscFunctionReturn(0);
2561 }
2562 
2563 /*@C
2564   DMPlexGetConeTuple - Return the points on the in-edges of several points in the DAG
2565 
2566   Not collective
2567 
2568   Input Parameters:
2569 + dm - The DMPlex
2570 - p - The IS of points, which must lie in the chart set with DMPlexSetChart()
2571 
2572   Output Parameter:
2573 + pConesSection - PetscSection describing the layout of pCones
2574 - pCones - An array of points which are on the in-edges for the point set p
2575 
2576   Level: intermediate
2577 
2578 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexGetConeRecursive(), DMPlexSetChart()
2579 @*/
2580 PetscErrorCode DMPlexGetConeTuple(DM dm, IS p, PetscSection *pConesSection, IS *pCones)
2581 {
2582   PetscSection        cs, newcs;
2583   PetscInt            *cones;
2584   PetscInt            *newarr=NULL;
2585   PetscInt            n;
2586   PetscErrorCode      ierr;
2587 
2588   PetscFunctionBegin;
2589   ierr = DMPlexGetCones(dm, &cones);CHKERRQ(ierr);
2590   ierr = DMPlexGetConeSection(dm, &cs);CHKERRQ(ierr);
2591   ierr = PetscSectionExtractDofsFromArray(cs, MPIU_INT, cones, p, &newcs, pCones ? ((void**)&newarr) : NULL);CHKERRQ(ierr);
2592   if (pConesSection) *pConesSection = newcs;
2593   if (pCones) {
2594     ierr = PetscSectionGetStorageSize(newcs, &n);CHKERRQ(ierr);
2595     ierr = ISCreateGeneral(PetscObjectComm((PetscObject)p), n, newarr, PETSC_OWN_POINTER, pCones);CHKERRQ(ierr);
2596   }
2597   PetscFunctionReturn(0);
2598 }
2599 
2600 /*@
2601   DMPlexGetConeRecursiveVertices - Expand each given point into its cone points and do that recursively until we end up just with vertices.
2602 
2603   Not collective
2604 
2605   Input Parameters:
2606 + dm - The DMPlex
2607 - points - The IS of points, which must lie in the chart set with DMPlexSetChart()
2608 
2609   Output Parameter:
2610 . expandedPoints - An array of vertices recursively expanded from input points
2611 
2612   Level: advanced
2613 
2614   Notes:
2615   Like DMPlexGetConeRecursive but returns only the 0-depth IS (i.e. vertices only) and no sections.
2616   There is no corresponding Restore function, just call ISDestroy() on the returned IS to deallocate.
2617 
2618 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexGetConeTuple(), DMPlexGetConeRecursive(), DMPlexRestoreConeRecursive(), DMPlexGetDepth()
2619 @*/
2620 PetscErrorCode DMPlexGetConeRecursiveVertices(DM dm, IS points, IS *expandedPoints)
2621 {
2622   IS                  *expandedPointsAll;
2623   PetscInt            depth;
2624   PetscErrorCode      ierr;
2625 
2626   PetscFunctionBegin;
2627   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2628   PetscValidHeaderSpecific(points, IS_CLASSID, 2);
2629   PetscValidPointer(expandedPoints, 3);
2630   ierr = DMPlexGetConeRecursive(dm, points, &depth, &expandedPointsAll, NULL);CHKERRQ(ierr);
2631   *expandedPoints = expandedPointsAll[0];
2632   ierr = PetscObjectReference((PetscObject)expandedPointsAll[0]);CHKERRQ(ierr);
2633   ierr = DMPlexRestoreConeRecursive(dm, points, &depth, &expandedPointsAll, NULL);CHKERRQ(ierr);
2634   PetscFunctionReturn(0);
2635 }
2636 
2637 /*@
2638   DMPlexGetConeRecursive - Expand each given point into its cone points and do that recursively until we end up just with vertices (DAG points of depth 0, i.e. without cones).
2639 
2640   Not collective
2641 
2642   Input Parameters:
2643 + dm - The DMPlex
2644 - points - The IS of points, which must lie in the chart set with DMPlexSetChart()
2645 
2646   Output Parameter:
2647 + depth - (optional) Size of the output arrays, equal to DMPlex depth, returned by DMPlexGetDepth()
2648 . expandedPoints - (optional) An array of index sets with recursively expanded cones
2649 - sections - (optional) An array of sections which describe mappings from points to their cone points
2650 
2651   Level: advanced
2652 
2653   Notes:
2654   Like DMPlexGetConeTuple() but recursive.
2655 
2656   Array expandedPoints has size equal to depth. Each expandedPoints[d] contains DAG points with maximum depth d, recursively cone-wise expanded from the input points.
2657   For example, for d=0 it contains only vertices, for d=1 it can contain vertices and edges, etc.
2658 
2659   Array section has size equal to depth.  Each PetscSection sections[d] realizes mapping from expandedPoints[d+1] (section points) to expandedPoints[d] (section dofs) as follows:
2660   (1) DAG points in expandedPoints[d+1] with depth d+1 to their cone points in expandedPoints[d];
2661   (2) DAG points in expandedPoints[d+1] with depth in [0,d] to the same points in expandedPoints[d].
2662 
2663 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexGetConeTuple(), DMPlexRestoreConeRecursive(), DMPlexGetConeRecursiveVertices(), DMPlexGetDepth()
2664 @*/
2665 PetscErrorCode DMPlexGetConeRecursive(DM dm, IS points, PetscInt *depth, IS *expandedPoints[], PetscSection *sections[])
2666 {
2667   const PetscInt      *arr0=NULL, *cone=NULL;
2668   PetscInt            *arr=NULL, *newarr=NULL;
2669   PetscInt            d, depth_, i, n, newn, cn, co, start, end;
2670   IS                  *expandedPoints_;
2671   PetscSection        *sections_;
2672   PetscErrorCode      ierr;
2673 
2674   PetscFunctionBegin;
2675   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2676   PetscValidHeaderSpecific(points, IS_CLASSID, 2);
2677   if (depth) PetscValidIntPointer(depth, 3);
2678   if (expandedPoints) PetscValidPointer(expandedPoints, 4);
2679   if (sections) PetscValidPointer(sections, 5);
2680   ierr = ISGetLocalSize(points, &n);CHKERRQ(ierr);
2681   ierr = ISGetIndices(points, &arr0);CHKERRQ(ierr);
2682   ierr = DMPlexGetDepth(dm, &depth_);CHKERRQ(ierr);
2683   ierr = PetscCalloc1(depth_, &expandedPoints_);CHKERRQ(ierr);
2684   ierr = PetscCalloc1(depth_, &sections_);CHKERRQ(ierr);
2685   arr = (PetscInt*) arr0; /* this is ok because first generation of arr is not modified */
2686   for (d=depth_-1; d>=0; d--) {
2687     ierr = PetscSectionCreate(PETSC_COMM_SELF, &sections_[d]);CHKERRQ(ierr);
2688     ierr = PetscSectionSetChart(sections_[d], 0, n);CHKERRQ(ierr);
2689     for (i=0; i<n; i++) {
2690       ierr = DMPlexGetDepthStratum(dm, d+1, &start, &end);CHKERRQ(ierr);
2691       if (arr[i] >= start && arr[i] < end) {
2692         ierr = DMPlexGetConeSize(dm, arr[i], &cn);CHKERRQ(ierr);
2693         ierr = PetscSectionSetDof(sections_[d], i, cn);CHKERRQ(ierr);
2694       } else {
2695         ierr = PetscSectionSetDof(sections_[d], i, 1);CHKERRQ(ierr);
2696       }
2697     }
2698     ierr = PetscSectionSetUp(sections_[d]);CHKERRQ(ierr);
2699     ierr = PetscSectionGetStorageSize(sections_[d], &newn);CHKERRQ(ierr);
2700     ierr = PetscMalloc1(newn, &newarr);CHKERRQ(ierr);
2701     for (i=0; i<n; i++) {
2702       ierr = PetscSectionGetDof(sections_[d], i, &cn);CHKERRQ(ierr);
2703       ierr = PetscSectionGetOffset(sections_[d], i, &co);CHKERRQ(ierr);
2704       if (cn > 1) {
2705         ierr = DMPlexGetCone(dm, arr[i], &cone);CHKERRQ(ierr);
2706         ierr = PetscMemcpy(&newarr[co], cone, cn*sizeof(PetscInt));CHKERRQ(ierr);
2707       } else {
2708         newarr[co] = arr[i];
2709       }
2710     }
2711     ierr = ISCreateGeneral(PETSC_COMM_SELF, newn, newarr, PETSC_OWN_POINTER, &expandedPoints_[d]);CHKERRQ(ierr);
2712     arr = newarr;
2713     n = newn;
2714   }
2715   ierr = ISRestoreIndices(points, &arr0);CHKERRQ(ierr);
2716   *depth = depth_;
2717   if (expandedPoints) *expandedPoints = expandedPoints_;
2718   else {
2719     for (d=0; d<depth_; d++) {ierr = ISDestroy(&expandedPoints_[d]);CHKERRQ(ierr);}
2720     ierr = PetscFree(expandedPoints_);CHKERRQ(ierr);
2721   }
2722   if (sections) *sections = sections_;
2723   else {
2724     for (d=0; d<depth_; d++) {ierr = PetscSectionDestroy(&sections_[d]);CHKERRQ(ierr);}
2725     ierr = PetscFree(sections_);CHKERRQ(ierr);
2726   }
2727   PetscFunctionReturn(0);
2728 }
2729 
2730 /*@
2731   DMPlexRestoreConeRecursive - Deallocates arrays created by DMPlexGetConeRecursive
2732 
2733   Not collective
2734 
2735   Input Parameters:
2736 + dm - The DMPlex
2737 - points - The IS of points, which must lie in the chart set with DMPlexSetChart()
2738 
2739   Output Parameter:
2740 + depth - (optional) Size of the output arrays, equal to DMPlex depth, returned by DMPlexGetDepth()
2741 . expandedPoints - (optional) An array of recursively expanded cones
2742 - sections - (optional) An array of sections which describe mappings from points to their cone points
2743 
2744   Level: advanced
2745 
2746   Notes:
2747   See DMPlexGetConeRecursive() for details.
2748 
2749 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexGetConeTuple(), DMPlexGetConeRecursive(), DMPlexGetConeRecursiveVertices(), DMPlexGetDepth()
2750 @*/
2751 PetscErrorCode DMPlexRestoreConeRecursive(DM dm, IS points, PetscInt *depth, IS *expandedPoints[], PetscSection *sections[])
2752 {
2753   PetscInt            d, depth_;
2754   PetscErrorCode      ierr;
2755 
2756   PetscFunctionBegin;
2757   ierr = DMPlexGetDepth(dm, &depth_);CHKERRQ(ierr);
2758   if (depth && *depth != depth_) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "depth changed since last call to DMPlexGetConeRecursive");
2759   if (depth) *depth = 0;
2760   if (expandedPoints) {
2761     for (d=0; d<depth_; d++) {ierr = ISDestroy(&((*expandedPoints)[d]));CHKERRQ(ierr);}
2762     ierr = PetscFree(*expandedPoints);CHKERRQ(ierr);
2763   }
2764   if (sections)  {
2765     for (d=0; d<depth_; d++) {ierr = PetscSectionDestroy(&((*sections)[d]));CHKERRQ(ierr);}
2766     ierr = PetscFree(*sections);CHKERRQ(ierr);
2767   }
2768   PetscFunctionReturn(0);
2769 }
2770 
2771 /*@
2772   DMPlexSetCone - Set the points on the in-edges for this point in the DAG; that is these are the points that cover the specific point
2773 
2774   Not collective
2775 
2776   Input Parameters:
2777 + mesh - The DMPlex
2778 . p - The point, which must lie in the chart set with DMPlexSetChart()
2779 - cone - An array of points which are on the in-edges for point p
2780 
2781   Output Parameter:
2782 
2783   Note:
2784   This should be called after all calls to DMPlexSetConeSize() and DMSetUp().
2785 
2786   Level: beginner
2787 
2788 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp(), DMPlexSetSupport(), DMPlexSetSupportSize()
2789 @*/
2790 PetscErrorCode DMPlexSetCone(DM dm, PetscInt p, const PetscInt cone[])
2791 {
2792   DM_Plex       *mesh = (DM_Plex*) dm->data;
2793   PetscInt       pStart, pEnd;
2794   PetscInt       dof, off, c;
2795   PetscErrorCode ierr;
2796 
2797   PetscFunctionBegin;
2798   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2799   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
2800   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2801   if (dof) PetscValidPointer(cone, 3);
2802   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2803   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
2804   for (c = 0; c < dof; ++c) {
2805     if ((cone[c] < pStart) || (cone[c] >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone point %D is not in the valid range [%D, %D)", cone[c], pStart, pEnd);
2806     mesh->cones[off+c] = cone[c];
2807   }
2808   PetscFunctionReturn(0);
2809 }
2810 
2811 /*@C
2812   DMPlexGetConeOrientation - Return the orientations on the in-edges for this point in the DAG
2813 
2814   Not collective
2815 
2816   Input Parameters:
2817 + mesh - The DMPlex
2818 - p - The point, which must lie in the chart set with DMPlexSetChart()
2819 
2820   Output Parameter:
2821 . coneOrientation - An array of orientations which are on the in-edges for point p. An orientation is an
2822                     integer giving the prescription for cone traversal.
2823 
2824   Level: beginner
2825 
2826   Notes:
2827   The number indexes the symmetry transformations for the cell type (see manual). Orientation 0 is always
2828   the identity transformation. Negative orientation indicates reflection so that -(o+1) is the reflection
2829   of o, however it is not necessarily the inverse. To get the inverse, use DMPolytopeTypeComposeOrientationInv()
2830   with the identity.
2831 
2832   Fortran Notes:
2833   Since it returns an array, this routine is only available in Fortran 90, and you must
2834   include petsc.h90 in your code.
2835   You must also call DMPlexRestoreConeOrientation() after you finish using the returned array.
2836   DMPlexRestoreConeOrientation() is not needed/available in C.
2837 
2838 .seealso: DMPolytopeTypeComposeOrientation(), DMPolytopeTypeComposeOrientationInv(), DMPlexCreate(), DMPlexGetCone(), DMPlexSetCone(), DMPlexSetChart()
2839 @*/
2840 PetscErrorCode DMPlexGetConeOrientation(DM dm, PetscInt p, const PetscInt *coneOrientation[])
2841 {
2842   DM_Plex       *mesh = (DM_Plex*) dm->data;
2843   PetscInt       off;
2844   PetscErrorCode ierr;
2845 
2846   PetscFunctionBegin;
2847   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2848   if (PetscDefined(USE_DEBUG)) {
2849     PetscInt dof;
2850     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2851     if (dof) PetscValidPointer(coneOrientation, 3);
2852   }
2853   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2854 
2855   *coneOrientation = &mesh->coneOrientations[off];
2856   PetscFunctionReturn(0);
2857 }
2858 
2859 /*@
2860   DMPlexSetConeOrientation - Set the orientations on the in-edges for this point in the DAG
2861 
2862   Not collective
2863 
2864   Input Parameters:
2865 + mesh - The DMPlex
2866 . p - The point, which must lie in the chart set with DMPlexSetChart()
2867 - coneOrientation - An array of orientations
2868   Output Parameter:
2869 
2870   Notes:
2871   This should be called after all calls to DMPlexSetConeSize() and DMSetUp().
2872 
2873   The meaning of coneOrientation is detailed in DMPlexGetConeOrientation().
2874 
2875   Level: beginner
2876 
2877 .seealso: DMPlexCreate(), DMPlexGetConeOrientation(), DMPlexSetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
2878 @*/
2879 PetscErrorCode DMPlexSetConeOrientation(DM dm, PetscInt p, const PetscInt coneOrientation[])
2880 {
2881   DM_Plex       *mesh = (DM_Plex*) dm->data;
2882   PetscInt       pStart, pEnd;
2883   PetscInt       dof, off, c;
2884   PetscErrorCode ierr;
2885 
2886   PetscFunctionBegin;
2887   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2888   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
2889   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2890   if (dof) PetscValidPointer(coneOrientation, 3);
2891   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2892   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
2893   for (c = 0; c < dof; ++c) {
2894     PetscInt cdof, o = coneOrientation[c];
2895 
2896     ierr = PetscSectionGetDof(mesh->coneSection, mesh->cones[off+c], &cdof);CHKERRQ(ierr);
2897     if (o && ((o < -(cdof+1)) || (o >= cdof))) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone orientation %D is not in the valid range [%D. %D)", o, -(cdof+1), cdof);
2898     mesh->coneOrientations[off+c] = o;
2899   }
2900   PetscFunctionReturn(0);
2901 }
2902 
2903 /*@
2904   DMPlexInsertCone - Insert a point into the in-edges for the point p in the DAG
2905 
2906   Not collective
2907 
2908   Input Parameters:
2909 + mesh - The DMPlex
2910 . p - The point, which must lie in the chart set with DMPlexSetChart()
2911 . conePos - The local index in the cone where the point should be put
2912 - conePoint - The mesh point to insert
2913 
2914   Level: beginner
2915 
2916 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
2917 @*/
2918 PetscErrorCode DMPlexInsertCone(DM dm, PetscInt p, PetscInt conePos, PetscInt conePoint)
2919 {
2920   DM_Plex       *mesh = (DM_Plex*) dm->data;
2921   PetscInt       pStart, pEnd;
2922   PetscInt       dof, off;
2923   PetscErrorCode ierr;
2924 
2925   PetscFunctionBegin;
2926   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2927   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
2928   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
2929   if ((conePoint < pStart) || (conePoint >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone point %D is not in the valid range [%D, %D)", conePoint, pStart, pEnd);
2930   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2931   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2932   if ((conePos < 0) || (conePos >= dof)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone position %D of point %D is not in the valid range [0, %D)", conePos, p, dof);
2933   mesh->cones[off+conePos] = conePoint;
2934   PetscFunctionReturn(0);
2935 }
2936 
2937 /*@
2938   DMPlexInsertConeOrientation - Insert a point orientation for the in-edge for the point p in the DAG
2939 
2940   Not collective
2941 
2942   Input Parameters:
2943 + mesh - The DMPlex
2944 . p - The point, which must lie in the chart set with DMPlexSetChart()
2945 . conePos - The local index in the cone where the point should be put
2946 - coneOrientation - The point orientation to insert
2947 
2948   Level: beginner
2949 
2950   Notes:
2951   The meaning of coneOrientation values is detailed in DMPlexGetConeOrientation().
2952 
2953 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
2954 @*/
2955 PetscErrorCode DMPlexInsertConeOrientation(DM dm, PetscInt p, PetscInt conePos, PetscInt coneOrientation)
2956 {
2957   DM_Plex       *mesh = (DM_Plex*) dm->data;
2958   PetscInt       pStart, pEnd;
2959   PetscInt       dof, off;
2960   PetscErrorCode ierr;
2961 
2962   PetscFunctionBegin;
2963   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2964   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
2965   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
2966   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2967   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2968   if ((conePos < 0) || (conePos >= dof)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone position %D of point %D is not in the valid range [0, %D)", conePos, p, dof);
2969   mesh->coneOrientations[off+conePos] = coneOrientation;
2970   PetscFunctionReturn(0);
2971 }
2972 
2973 /*@
2974   DMPlexGetSupportSize - Return the number of out-edges for this point in the DAG
2975 
2976   Not collective
2977 
2978   Input Parameters:
2979 + mesh - The DMPlex
2980 - p - The point, which must lie in the chart set with DMPlexSetChart()
2981 
2982   Output Parameter:
2983 . size - The support size for point p
2984 
2985   Level: beginner
2986 
2987 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart(), DMPlexGetConeSize()
2988 @*/
2989 PetscErrorCode DMPlexGetSupportSize(DM dm, PetscInt p, PetscInt *size)
2990 {
2991   DM_Plex       *mesh = (DM_Plex*) dm->data;
2992   PetscErrorCode ierr;
2993 
2994   PetscFunctionBegin;
2995   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2996   PetscValidPointer(size, 3);
2997   ierr = PetscSectionGetDof(mesh->supportSection, p, size);CHKERRQ(ierr);
2998   PetscFunctionReturn(0);
2999 }
3000 
3001 /*@
3002   DMPlexSetSupportSize - Set the number of out-edges for this point in the DAG
3003 
3004   Not collective
3005 
3006   Input Parameters:
3007 + mesh - The DMPlex
3008 . p - The point, which must lie in the chart set with DMPlexSetChart()
3009 - size - The support size for point p
3010 
3011   Output Parameter:
3012 
3013   Note:
3014   This should be called after DMPlexSetChart().
3015 
3016   Level: beginner
3017 
3018 .seealso: DMPlexCreate(), DMPlexGetSupportSize(), DMPlexSetChart()
3019 @*/
3020 PetscErrorCode DMPlexSetSupportSize(DM dm, PetscInt p, PetscInt size)
3021 {
3022   DM_Plex       *mesh = (DM_Plex*) dm->data;
3023   PetscErrorCode ierr;
3024 
3025   PetscFunctionBegin;
3026   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3027   ierr = PetscSectionSetDof(mesh->supportSection, p, size);CHKERRQ(ierr);
3028 
3029   mesh->maxSupportSize = PetscMax(mesh->maxSupportSize, size);
3030   PetscFunctionReturn(0);
3031 }
3032 
3033 /*@C
3034   DMPlexGetSupport - Return the points on the out-edges for this point in the DAG
3035 
3036   Not collective
3037 
3038   Input Parameters:
3039 + mesh - The DMPlex
3040 - p - The point, which must lie in the chart set with DMPlexSetChart()
3041 
3042   Output Parameter:
3043 . support - An array of points which are on the out-edges for point p
3044 
3045   Level: beginner
3046 
3047   Fortran Notes:
3048   Since it returns an array, this routine is only available in Fortran 90, and you must
3049   include petsc.h90 in your code.
3050   You must also call DMPlexRestoreSupport() after you finish using the returned array.
3051   DMPlexRestoreSupport() is not needed/available in C.
3052 
3053 .seealso: DMPlexGetSupportSize(), DMPlexSetSupport(), DMPlexGetCone(), DMPlexSetChart()
3054 @*/
3055 PetscErrorCode DMPlexGetSupport(DM dm, PetscInt p, const PetscInt *support[])
3056 {
3057   DM_Plex       *mesh = (DM_Plex*) dm->data;
3058   PetscInt       off;
3059   PetscErrorCode ierr;
3060 
3061   PetscFunctionBegin;
3062   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3063   PetscValidPointer(support, 3);
3064   ierr     = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
3065   *support = &mesh->supports[off];
3066   PetscFunctionReturn(0);
3067 }
3068 
3069 /*@
3070   DMPlexSetSupport - Set the points on the out-edges for this point in the DAG, that is the list of points that this point covers
3071 
3072   Not collective
3073 
3074   Input Parameters:
3075 + mesh - The DMPlex
3076 . p - The point, which must lie in the chart set with DMPlexSetChart()
3077 - support - An array of points which are on the out-edges for point p
3078 
3079   Output Parameter:
3080 
3081   Note:
3082   This should be called after all calls to DMPlexSetSupportSize() and DMSetUp().
3083 
3084   Level: beginner
3085 
3086 .seealso: DMPlexSetCone(), DMPlexSetConeSize(), DMPlexCreate(), DMPlexGetSupport(), DMPlexSetChart(), DMPlexSetSupportSize(), DMSetUp()
3087 @*/
3088 PetscErrorCode DMPlexSetSupport(DM dm, PetscInt p, const PetscInt support[])
3089 {
3090   DM_Plex       *mesh = (DM_Plex*) dm->data;
3091   PetscInt       pStart, pEnd;
3092   PetscInt       dof, off, c;
3093   PetscErrorCode ierr;
3094 
3095   PetscFunctionBegin;
3096   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3097   ierr = PetscSectionGetChart(mesh->supportSection, &pStart, &pEnd);CHKERRQ(ierr);
3098   ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
3099   if (dof) PetscValidPointer(support, 3);
3100   ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
3101   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
3102   for (c = 0; c < dof; ++c) {
3103     if ((support[c] < pStart) || (support[c] >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support point %D is not in the valid range [%D, %D)", support[c], pStart, pEnd);
3104     mesh->supports[off+c] = support[c];
3105   }
3106   PetscFunctionReturn(0);
3107 }
3108 
3109 /*@
3110   DMPlexInsertSupport - Insert a point into the out-edges for the point p in the DAG
3111 
3112   Not collective
3113 
3114   Input Parameters:
3115 + mesh - The DMPlex
3116 . p - The point, which must lie in the chart set with DMPlexSetChart()
3117 . supportPos - The local index in the cone where the point should be put
3118 - supportPoint - The mesh point to insert
3119 
3120   Level: beginner
3121 
3122 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
3123 @*/
3124 PetscErrorCode DMPlexInsertSupport(DM dm, PetscInt p, PetscInt supportPos, PetscInt supportPoint)
3125 {
3126   DM_Plex       *mesh = (DM_Plex*) dm->data;
3127   PetscInt       pStart, pEnd;
3128   PetscInt       dof, off;
3129   PetscErrorCode ierr;
3130 
3131   PetscFunctionBegin;
3132   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3133   ierr = PetscSectionGetChart(mesh->supportSection, &pStart, &pEnd);CHKERRQ(ierr);
3134   ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
3135   ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
3136   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
3137   if ((supportPoint < pStart) || (supportPoint >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support point %D is not in the valid range [%D, %D)", supportPoint, pStart, pEnd);
3138   if (supportPos >= dof) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support position %D of point %D is not in the valid range [0, %D)", supportPos, p, dof);
3139   mesh->supports[off+supportPos] = supportPoint;
3140   PetscFunctionReturn(0);
3141 }
3142 
3143 /* Converts an orientation o in the current numbering to the previous scheme used in Plex */
3144 PetscInt DMPolytopeConvertNewOrientation_Internal(DMPolytopeType ct, PetscInt o)
3145 {
3146   switch (ct) {
3147     case DM_POLYTOPE_SEGMENT:
3148       if (o == -1) return -2;
3149       break;
3150     case DM_POLYTOPE_TRIANGLE:
3151       if (o == -3) return -1;
3152       if (o == -2) return -3;
3153       if (o == -1) return -2;
3154       break;
3155     case DM_POLYTOPE_QUADRILATERAL:
3156       if (o == -4) return -2;
3157       if (o == -3) return -1;
3158       if (o == -2) return -4;
3159       if (o == -1) return -3;
3160       break;
3161     default: return o;
3162   }
3163   return o;
3164 }
3165 
3166 /* Converts an orientation o in the previous scheme used in Plex to the current numbering */
3167 PetscInt DMPolytopeConvertOldOrientation_Internal(DMPolytopeType ct, PetscInt o)
3168 {
3169   switch (ct) {
3170     case DM_POLYTOPE_SEGMENT:
3171       if ((o == -2) || (o == 1)) return -1;
3172       if (o == -1) return 0;
3173       break;
3174     case DM_POLYTOPE_TRIANGLE:
3175       if (o == -3) return -2;
3176       if (o == -2) return -1;
3177       if (o == -1) return -3;
3178       break;
3179     case DM_POLYTOPE_QUADRILATERAL:
3180       if (o == -4) return -2;
3181       if (o == -3) return -1;
3182       if (o == -2) return -4;
3183       if (o == -1) return -3;
3184       break;
3185     default: return o;
3186   }
3187   return o;
3188 }
3189 
3190 /* Takes in a mesh whose orientations are in the previous scheme and converts them all to the current numbering */
3191 PetscErrorCode DMPlexConvertOldOrientations_Internal(DM dm)
3192 {
3193   PetscInt       pStart, pEnd, p;
3194   PetscErrorCode ierr;
3195 
3196   PetscFunctionBegin;
3197   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
3198   for (p = pStart; p < pEnd; ++p) {
3199     const PetscInt *cone, *ornt;
3200     PetscInt        coneSize, c;
3201 
3202     ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
3203     ierr = DMPlexGetCone(dm, p, &cone);CHKERRQ(ierr);
3204     ierr = DMPlexGetConeOrientation(dm, p, &ornt);CHKERRQ(ierr);
3205     for (c = 0; c < coneSize; ++c) {
3206       DMPolytopeType ct;
3207       const PetscInt o = ornt[c];
3208 
3209       ierr = DMPlexGetCellType(dm, cone[c], &ct);CHKERRQ(ierr);
3210       switch (ct) {
3211         case DM_POLYTOPE_SEGMENT:
3212           if ((o == -2) || (o == 1)) {ierr = DMPlexInsertConeOrientation(dm, p, c, -1);CHKERRQ(ierr);}
3213           if (o == -1) {ierr = DMPlexInsertConeOrientation(dm, p, c, 0);CHKERRQ(ierr);}
3214           break;
3215         case DM_POLYTOPE_TRIANGLE:
3216           if (o == -3) {ierr = DMPlexInsertConeOrientation(dm, p, c, -2);CHKERRQ(ierr);}
3217           if (o == -2) {ierr = DMPlexInsertConeOrientation(dm, p, c, -1);CHKERRQ(ierr);}
3218           if (o == -1) {ierr = DMPlexInsertConeOrientation(dm, p, c, -3);CHKERRQ(ierr);}
3219           break;
3220         case DM_POLYTOPE_QUADRILATERAL:
3221           if (o == -4) {ierr = DMPlexInsertConeOrientation(dm, p, c, -2);CHKERRQ(ierr);}
3222           if (o == -3) {ierr = DMPlexInsertConeOrientation(dm, p, c, -1);CHKERRQ(ierr);}
3223           if (o == -2) {ierr = DMPlexInsertConeOrientation(dm, p, c, -4);CHKERRQ(ierr);}
3224           if (o == -1) {ierr = DMPlexInsertConeOrientation(dm, p, c, -3);CHKERRQ(ierr);}
3225           break;
3226         default: break;
3227       }
3228     }
3229   }
3230   PetscFunctionReturn(0);
3231 }
3232 
3233 static PetscErrorCode DMPlexGetTransitiveClosure_Depth1_Static(DM dm, PetscInt p, PetscInt ornt, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
3234 {
3235   DMPolytopeType  ct = DM_POLYTOPE_UNKNOWN;
3236   PetscInt       *closure;
3237   const PetscInt *tmp = NULL, *tmpO = NULL;
3238   PetscInt        off = 0, tmpSize, t;
3239   PetscErrorCode  ierr;
3240 
3241   PetscFunctionBeginHot;
3242   if (ornt) {
3243     ierr = DMPlexGetCellType(dm, p, &ct);CHKERRQ(ierr);
3244     if (ct == DM_POLYTOPE_FV_GHOST || ct == DM_POLYTOPE_INTERIOR_GHOST || ct == DM_POLYTOPE_UNKNOWN) ct = DM_POLYTOPE_UNKNOWN;
3245   }
3246   if (*points) {
3247     closure = *points;
3248   } else {
3249     PetscInt maxConeSize, maxSupportSize;
3250     ierr = DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize);CHKERRQ(ierr);
3251     ierr = DMGetWorkArray(dm, 2*(PetscMax(maxConeSize, maxSupportSize)+1), MPIU_INT, &closure);CHKERRQ(ierr);
3252   }
3253   if (useCone) {
3254     ierr = DMPlexGetConeSize(dm, p, &tmpSize);CHKERRQ(ierr);
3255     ierr = DMPlexGetCone(dm, p, &tmp);CHKERRQ(ierr);
3256     ierr = DMPlexGetConeOrientation(dm, p, &tmpO);CHKERRQ(ierr);
3257   } else {
3258     ierr = DMPlexGetSupportSize(dm, p, &tmpSize);CHKERRQ(ierr);
3259     ierr = DMPlexGetSupport(dm, p, &tmp);CHKERRQ(ierr);
3260   }
3261   if (ct == DM_POLYTOPE_UNKNOWN) {
3262     closure[off++] = p;
3263     closure[off++] = 0;
3264     for (t = 0; t < tmpSize; ++t) {
3265       closure[off++] = tmp[t];
3266       closure[off++] = tmpO ? tmpO[t] : 0;
3267     }
3268   } else {
3269     const PetscInt *arr = DMPolytopeTypeGetArrangment(ct, ornt);CHKERRQ(ierr);
3270 
3271     /* We assume that cells with a valid type have faces with a valid type */
3272     closure[off++] = p;
3273     closure[off++] = ornt;
3274     for (t = 0; t < tmpSize; ++t) {
3275       DMPolytopeType ft;
3276 
3277       ierr = DMPlexGetCellType(dm, tmp[t], &ft);CHKERRQ(ierr);
3278       closure[off++] = tmp[arr[t]];
3279       closure[off++] = tmpO ? DMPolytopeTypeComposeOrientation(ft, ornt, tmpO[t]) : 0;
3280     }
3281   }
3282   if (numPoints) *numPoints = tmpSize+1;
3283   if (points)    *points    = closure;
3284   PetscFunctionReturn(0);
3285 }
3286 
3287 /* We need a special tensor verison becasue we want to allow duplicate points in the endcaps for hybrid cells */
3288 static PetscErrorCode DMPlexTransitiveClosure_Tensor_Internal(DM dm, PetscInt point, DMPolytopeType ct, PetscInt o, PetscBool useCone, PetscInt *numPoints, PetscInt **points)
3289 {
3290   const PetscInt *arr = DMPolytopeTypeGetArrangment(ct, o);
3291   const PetscInt *cone, *ornt;
3292   PetscInt       *pts,  *closure = NULL;
3293   DMPolytopeType  ft;
3294   PetscInt        maxConeSize, maxSupportSize, coneSeries, supportSeries, maxSize;
3295   PetscInt        dim, coneSize, c, d, clSize, cl;
3296   PetscErrorCode  ierr;
3297 
3298   PetscFunctionBeginHot;
3299   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
3300   ierr = DMPlexGetConeSize(dm, point, &coneSize);CHKERRQ(ierr);
3301   ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
3302   ierr = DMPlexGetConeOrientation(dm, point, &ornt);CHKERRQ(ierr);
3303   ierr = DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize);CHKERRQ(ierr);
3304   coneSeries    = (maxConeSize    > 1) ? ((PetscPowInt(maxConeSize,    dim+1)-1)/(maxConeSize-1))    : dim+1;
3305   supportSeries = (maxSupportSize > 1) ? ((PetscPowInt(maxSupportSize, dim+1)-1)/(maxSupportSize-1)) : dim+1;
3306   maxSize       = PetscMax(coneSeries, supportSeries);
3307   if (*points) {pts  = *points;}
3308   else         {ierr = DMGetWorkArray(dm, 2*maxSize, MPIU_INT, &pts);CHKERRQ(ierr);}
3309   c    = 0;
3310   pts[c++] = point;
3311   pts[c++] = o;
3312   ierr = DMPlexGetCellType(dm, cone[arr[0*2+0]], &ft);CHKERRQ(ierr);
3313   ierr = DMPlexGetTransitiveClosure_Internal(dm, cone[arr[0*2+0]], DMPolytopeTypeComposeOrientation(ft, arr[0*2+1], ornt[0]), useCone, &clSize, &closure);CHKERRQ(ierr);
3314   for (cl = 0; cl < clSize*2; cl += 2) {pts[c++] = closure[cl]; pts[c++] = closure[cl+1];}
3315   ierr = DMPlexGetTransitiveClosure_Internal(dm, cone[arr[1*2+0]], DMPolytopeTypeComposeOrientation(ft, arr[1*2+1], ornt[1]), useCone, &clSize, &closure);CHKERRQ(ierr);
3316   for (cl = 0; cl < clSize*2; cl += 2) {pts[c++] = closure[cl]; pts[c++] = closure[cl+1];}
3317   ierr = DMPlexRestoreTransitiveClosure(dm, cone[0], useCone, &clSize, &closure);CHKERRQ(ierr);
3318   for (d = 2; d < coneSize; ++d) {
3319     ierr = DMPlexGetCellType(dm, cone[arr[d*2+0]], &ft);CHKERRQ(ierr);
3320     pts[c++] = cone[arr[d*2+0]];
3321     pts[c++] = DMPolytopeTypeComposeOrientation(ft, arr[d*2+1], ornt[d]);
3322   }
3323   if (dim >= 3) {
3324     for (d = 2; d < coneSize; ++d) {
3325       const PetscInt  fpoint = cone[arr[d*2+0]];
3326       const PetscInt *fcone, *fornt;
3327       PetscInt        fconeSize, fc, i;
3328 
3329       ierr = DMPlexGetCellType(dm, fpoint, &ft);CHKERRQ(ierr);
3330       const PetscInt *farr = DMPolytopeTypeGetArrangment(ft, DMPolytopeTypeComposeOrientation(ft, arr[d*2+1], ornt[d]));
3331       ierr = DMPlexGetConeSize(dm, fpoint, &fconeSize);CHKERRQ(ierr);
3332       ierr = DMPlexGetCone(dm, fpoint, &fcone);CHKERRQ(ierr);
3333       ierr = DMPlexGetConeOrientation(dm, fpoint, &fornt);CHKERRQ(ierr);
3334       for (fc = 0; fc < fconeSize; ++fc) {
3335         const PetscInt cp = fcone[farr[fc*2+0]];
3336         const PetscInt co = farr[fc*2+1];
3337 
3338         for (i = 0; i < c; i += 2) if (pts[i] == cp) break;
3339         if (i == c) {
3340           ierr = DMPlexGetCellType(dm, cp, &ft);CHKERRQ(ierr);
3341           pts[c++] = cp;
3342           pts[c++] = DMPolytopeTypeComposeOrientation(ft, co, fornt[farr[fc*2+0]]);
3343         }
3344       }
3345     }
3346   }
3347   *numPoints = c/2;
3348   *points    = pts;
3349   PetscFunctionReturn(0);
3350 }
3351 
3352 PetscErrorCode DMPlexGetTransitiveClosure_Internal(DM dm, PetscInt p, PetscInt ornt, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
3353 {
3354   DMPolytopeType ct;
3355   PetscInt      *closure, *fifo;
3356   PetscInt       closureSize = 0, fifoStart = 0, fifoSize = 0;
3357   PetscInt       maxConeSize, maxSupportSize, coneSeries, supportSeries;
3358   PetscInt       depth, maxSize;
3359   PetscErrorCode ierr;
3360 
3361   PetscFunctionBeginHot;
3362   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3363   if (depth == 1) {
3364     ierr = DMPlexGetTransitiveClosure_Depth1_Static(dm, p, ornt, useCone, numPoints, points);CHKERRQ(ierr);
3365     PetscFunctionReturn(0);
3366   }
3367   ierr = DMPlexGetCellType(dm, p, &ct);CHKERRQ(ierr);
3368   if (ct == DM_POLYTOPE_FV_GHOST || ct == DM_POLYTOPE_INTERIOR_GHOST || ct == DM_POLYTOPE_UNKNOWN) ct = DM_POLYTOPE_UNKNOWN;
3369   if (ct == DM_POLYTOPE_SEG_PRISM_TENSOR || ct == DM_POLYTOPE_TRI_PRISM_TENSOR || ct == DM_POLYTOPE_QUAD_PRISM_TENSOR) {
3370     ierr = DMPlexTransitiveClosure_Tensor_Internal(dm, p, ct, ornt, useCone, numPoints, points);CHKERRQ(ierr);
3371     PetscFunctionReturn(0);
3372   }
3373   ierr = DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize);CHKERRQ(ierr);
3374   coneSeries    = (maxConeSize    > 1) ? ((PetscPowInt(maxConeSize,    depth+1)-1)/(maxConeSize-1))    : depth+1;
3375   supportSeries = (maxSupportSize > 1) ? ((PetscPowInt(maxSupportSize, depth+1)-1)/(maxSupportSize-1)) : depth+1;
3376   maxSize       = PetscMax(coneSeries, supportSeries);
3377   ierr = DMGetWorkArray(dm, 3*maxSize, MPIU_INT, &fifo);CHKERRQ(ierr);
3378   if (*points) {closure = *points;}
3379   else         {ierr = DMGetWorkArray(dm, 2*maxSize, MPIU_INT, &closure);CHKERRQ(ierr);}
3380   closure[closureSize++] = p;
3381   closure[closureSize++] = ornt;
3382   fifo[fifoSize++]       = p;
3383   fifo[fifoSize++]       = ornt;
3384   fifo[fifoSize++]       = ct;
3385   /* Should kick out early when depth is reached, rather than checking all vertices for empty cones */
3386   while (fifoSize - fifoStart) {
3387     const PetscInt       q    = fifo[fifoStart++];
3388     const PetscInt       o    = fifo[fifoStart++];
3389     const DMPolytopeType qt   = (DMPolytopeType) fifo[fifoStart++];
3390     const PetscInt      *qarr = DMPolytopeTypeGetArrangment(qt, o);
3391     const PetscInt      *tmp, *tmpO;
3392     PetscInt             tmpSize, t;
3393 
3394     if (PetscDefined(USE_DEBUG)) {
3395       PetscInt nO = DMPolytopeTypeGetNumArrangments(qt)/2;
3396       if (o && (o >= nO || o < -nO)) SETERRQ5(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid orientation %D not in [%D,%D) for %s %D", o, -nO, nO, DMPolytopeTypes[qt], q);
3397     }
3398     if (useCone) {
3399       ierr = DMPlexGetConeSize(dm, q, &tmpSize);CHKERRQ(ierr);
3400       ierr = DMPlexGetCone(dm, q, &tmp);CHKERRQ(ierr);
3401       ierr = DMPlexGetConeOrientation(dm, q, &tmpO);CHKERRQ(ierr);
3402     } else {
3403       ierr = DMPlexGetSupportSize(dm, q, &tmpSize);CHKERRQ(ierr);
3404       ierr = DMPlexGetSupport(dm, q, &tmp);CHKERRQ(ierr);
3405       tmpO = NULL;
3406     }
3407     for (t = 0; t < tmpSize; ++t) {
3408       const PetscInt ip = useCone && qarr ? qarr[t*2]   : t;
3409       const PetscInt io = useCone && qarr ? qarr[t*2+1] : 0;
3410       const PetscInt cp = tmp[ip];
3411       ierr = DMPlexGetCellType(dm, cp, &ct);CHKERRQ(ierr);
3412       const PetscInt co = tmpO ? DMPolytopeTypeComposeOrientation(ct, io, tmpO[ip]) : 0;
3413       PetscInt       c;
3414 
3415       /* Check for duplicate */
3416       for (c = 0; c < closureSize; c += 2) {
3417         if (closure[c] == cp) break;
3418       }
3419       if (c == closureSize) {
3420         closure[closureSize++] = cp;
3421         closure[closureSize++] = co;
3422         fifo[fifoSize++]       = cp;
3423         fifo[fifoSize++]       = co;
3424         fifo[fifoSize++]       = ct;
3425       }
3426     }
3427   }
3428   ierr = DMRestoreWorkArray(dm, 3*maxSize, MPIU_INT, &fifo);CHKERRQ(ierr);
3429   if (numPoints) *numPoints = closureSize/2;
3430   if (points)    *points    = closure;
3431   PetscFunctionReturn(0);
3432 }
3433 
3434 /*@C
3435   DMPlexGetTransitiveClosure - Return the points on the transitive closure of the in-edges or out-edges for this point in the DAG
3436 
3437   Not collective
3438 
3439   Input Parameters:
3440 + dm      - The DMPlex
3441 . p       - The mesh point
3442 . useCone - PETSC_TRUE for the closure, otherwise return the star
3443 - points  - If points is NULL on input, internal storage will be returned, otherwise the provided array is used
3444 
3445   Output Parameters:
3446 + numPoints - The number of points in the closure, so points[] is of size 2*numPoints
3447 - points    - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...]
3448 
3449   Note:
3450   If using internal storage (points is NULL on input), each call overwrites the last output.
3451 
3452   Fortran Notes:
3453   Since it returns an array, this routine is only available in Fortran 90, and you must include petsc.h90 in your code.
3454 
3455   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3456 
3457   Level: beginner
3458 
3459 .seealso: DMPlexRestoreTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
3460 @*/
3461 PetscErrorCode DMPlexGetTransitiveClosure(DM dm, PetscInt p, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
3462 {
3463   PetscErrorCode ierr;
3464 
3465   PetscFunctionBeginHot;
3466   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3467   if (numPoints) PetscValidIntPointer(numPoints, 4);
3468   if (points)    PetscValidPointer(points, 5);
3469   ierr = DMPlexGetTransitiveClosure_Internal(dm, p, 0, useCone, numPoints, points);CHKERRQ(ierr);
3470   PetscFunctionReturn(0);
3471 }
3472 
3473 /*@C
3474   DMPlexRestoreTransitiveClosure - Restore the array of points on the transitive closure of the in-edges or out-edges for this point in the DAG
3475 
3476   Not collective
3477 
3478   Input Parameters:
3479 + dm        - The DMPlex
3480 . p         - The mesh point
3481 . useCone   - PETSC_TRUE for the closure, otherwise return the star
3482 . numPoints - The number of points in the closure, so points[] is of size 2*numPoints
3483 - points    - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...]
3484 
3485   Note:
3486   If not using internal storage (points is not NULL on input), this call is unnecessary
3487 
3488   Fortran Notes:
3489   Since it returns an array, this routine is only available in Fortran 90, and you must include petsc.h90 in your code.
3490 
3491   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3492 
3493   Level: beginner
3494 
3495 .seealso: DMPlexGetTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
3496 @*/
3497 PetscErrorCode DMPlexRestoreTransitiveClosure(DM dm, PetscInt p, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
3498 {
3499   PetscErrorCode ierr;
3500 
3501   PetscFunctionBeginHot;
3502   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3503   if (numPoints) *numPoints = 0;
3504   ierr = DMRestoreWorkArray(dm, 0, MPIU_INT, points);CHKERRQ(ierr);
3505   PetscFunctionReturn(0);
3506 }
3507 
3508 /*@
3509   DMPlexGetMaxSizes - Return the maximum number of in-edges (cone) and out-edges (support) for any point in the DAG
3510 
3511   Not collective
3512 
3513   Input Parameter:
3514 . mesh - The DMPlex
3515 
3516   Output Parameters:
3517 + maxConeSize - The maximum number of in-edges
3518 - maxSupportSize - The maximum number of out-edges
3519 
3520   Level: beginner
3521 
3522 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart()
3523 @*/
3524 PetscErrorCode DMPlexGetMaxSizes(DM dm, PetscInt *maxConeSize, PetscInt *maxSupportSize)
3525 {
3526   DM_Plex *mesh = (DM_Plex*) dm->data;
3527 
3528   PetscFunctionBegin;
3529   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3530   if (maxConeSize)    *maxConeSize    = mesh->maxConeSize;
3531   if (maxSupportSize) *maxSupportSize = mesh->maxSupportSize;
3532   PetscFunctionReturn(0);
3533 }
3534 
3535 PetscErrorCode DMSetUp_Plex(DM dm)
3536 {
3537   DM_Plex       *mesh = (DM_Plex*) dm->data;
3538   PetscInt       size;
3539   PetscErrorCode ierr;
3540 
3541   PetscFunctionBegin;
3542   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3543   ierr = PetscSectionSetUp(mesh->coneSection);CHKERRQ(ierr);
3544   ierr = PetscSectionGetStorageSize(mesh->coneSection, &size);CHKERRQ(ierr);
3545   ierr = PetscMalloc1(size, &mesh->cones);CHKERRQ(ierr);
3546   ierr = PetscCalloc1(size, &mesh->coneOrientations);CHKERRQ(ierr);
3547   ierr = PetscLogObjectMemory((PetscObject) dm, size*2*sizeof(PetscInt));CHKERRQ(ierr);
3548   if (mesh->maxSupportSize) {
3549     ierr = PetscSectionSetUp(mesh->supportSection);CHKERRQ(ierr);
3550     ierr = PetscSectionGetStorageSize(mesh->supportSection, &size);CHKERRQ(ierr);
3551     ierr = PetscMalloc1(size, &mesh->supports);CHKERRQ(ierr);
3552     ierr = PetscLogObjectMemory((PetscObject) dm, size*sizeof(PetscInt));CHKERRQ(ierr);
3553   }
3554   PetscFunctionReturn(0);
3555 }
3556 
3557 PetscErrorCode DMCreateSubDM_Plex(DM dm, PetscInt numFields, const PetscInt fields[], IS *is, DM *subdm)
3558 {
3559   PetscErrorCode ierr;
3560 
3561   PetscFunctionBegin;
3562   if (subdm) {ierr = DMClone(dm, subdm);CHKERRQ(ierr);}
3563   ierr = DMCreateSectionSubDM(dm, numFields, fields, is, subdm);CHKERRQ(ierr);
3564   if (subdm) {(*subdm)->useNatural = dm->useNatural;}
3565   if (dm->useNatural && dm->sfMigration) {
3566     PetscSF        sfMigrationInv,sfNatural;
3567     PetscSection   section, sectionSeq;
3568 
3569     (*subdm)->sfMigration = dm->sfMigration;
3570     ierr = PetscObjectReference((PetscObject) dm->sfMigration);CHKERRQ(ierr);
3571     ierr = DMGetLocalSection((*subdm), &section);CHKERRQ(ierr);
3572     ierr = PetscSFCreateInverseSF((*subdm)->sfMigration, &sfMigrationInv);CHKERRQ(ierr);
3573     ierr = PetscSectionCreate(PetscObjectComm((PetscObject) (*subdm)), &sectionSeq);CHKERRQ(ierr);
3574     ierr = PetscSFDistributeSection(sfMigrationInv, section, NULL, sectionSeq);CHKERRQ(ierr);
3575 
3576     ierr = DMPlexCreateGlobalToNaturalSF(*subdm, sectionSeq, (*subdm)->sfMigration, &sfNatural);CHKERRQ(ierr);
3577     (*subdm)->sfNatural = sfNatural;
3578     ierr = PetscSectionDestroy(&sectionSeq);CHKERRQ(ierr);
3579     ierr = PetscSFDestroy(&sfMigrationInv);CHKERRQ(ierr);
3580   }
3581   PetscFunctionReturn(0);
3582 }
3583 
3584 PetscErrorCode DMCreateSuperDM_Plex(DM dms[], PetscInt len, IS **is, DM *superdm)
3585 {
3586   PetscErrorCode ierr;
3587   PetscInt       i = 0;
3588 
3589   PetscFunctionBegin;
3590   ierr = DMClone(dms[0], superdm);CHKERRQ(ierr);
3591   ierr = DMCreateSectionSuperDM(dms, len, is, superdm);CHKERRQ(ierr);
3592   (*superdm)->useNatural = PETSC_FALSE;
3593   for (i = 0; i < len; i++) {
3594     if (dms[i]->useNatural && dms[i]->sfMigration) {
3595       PetscSF        sfMigrationInv,sfNatural;
3596       PetscSection   section, sectionSeq;
3597 
3598       (*superdm)->sfMigration = dms[i]->sfMigration;
3599       ierr = PetscObjectReference((PetscObject) dms[i]->sfMigration);CHKERRQ(ierr);
3600       (*superdm)->useNatural = PETSC_TRUE;
3601       ierr = DMGetLocalSection((*superdm), &section);CHKERRQ(ierr);
3602       ierr = PetscSFCreateInverseSF((*superdm)->sfMigration, &sfMigrationInv);CHKERRQ(ierr);
3603       ierr = PetscSectionCreate(PetscObjectComm((PetscObject) (*superdm)), &sectionSeq);CHKERRQ(ierr);
3604       ierr = PetscSFDistributeSection(sfMigrationInv, section, NULL, sectionSeq);CHKERRQ(ierr);
3605 
3606       ierr = DMPlexCreateGlobalToNaturalSF(*superdm, sectionSeq, (*superdm)->sfMigration, &sfNatural);CHKERRQ(ierr);
3607       (*superdm)->sfNatural = sfNatural;
3608       ierr = PetscSectionDestroy(&sectionSeq);CHKERRQ(ierr);
3609       ierr = PetscSFDestroy(&sfMigrationInv);CHKERRQ(ierr);
3610       break;
3611     }
3612   }
3613   PetscFunctionReturn(0);
3614 }
3615 
3616 /*@
3617   DMPlexSymmetrize - Create support (out-edge) information from cone (in-edge) information
3618 
3619   Not collective
3620 
3621   Input Parameter:
3622 . mesh - The DMPlex
3623 
3624   Output Parameter:
3625 
3626   Note:
3627   This should be called after all calls to DMPlexSetCone()
3628 
3629   Level: beginner
3630 
3631 .seealso: DMPlexCreate(), DMPlexSetChart(), DMPlexSetConeSize(), DMPlexSetCone()
3632 @*/
3633 PetscErrorCode DMPlexSymmetrize(DM dm)
3634 {
3635   DM_Plex       *mesh = (DM_Plex*) dm->data;
3636   PetscInt      *offsets;
3637   PetscInt       supportSize;
3638   PetscInt       pStart, pEnd, p;
3639   PetscErrorCode ierr;
3640 
3641   PetscFunctionBegin;
3642   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3643   if (mesh->supports) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONGSTATE, "Supports were already setup in this DMPlex");
3644   ierr = PetscLogEventBegin(DMPLEX_Symmetrize,dm,0,0,0);CHKERRQ(ierr);
3645   /* Calculate support sizes */
3646   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
3647   for (p = pStart; p < pEnd; ++p) {
3648     PetscInt dof, off, c;
3649 
3650     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
3651     ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
3652     for (c = off; c < off+dof; ++c) {
3653       ierr = PetscSectionAddDof(mesh->supportSection, mesh->cones[c], 1);CHKERRQ(ierr);
3654     }
3655   }
3656   for (p = pStart; p < pEnd; ++p) {
3657     PetscInt dof;
3658 
3659     ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
3660 
3661     mesh->maxSupportSize = PetscMax(mesh->maxSupportSize, dof);
3662   }
3663   ierr = PetscSectionSetUp(mesh->supportSection);CHKERRQ(ierr);
3664   /* Calculate supports */
3665   ierr = PetscSectionGetStorageSize(mesh->supportSection, &supportSize);CHKERRQ(ierr);
3666   ierr = PetscMalloc1(supportSize, &mesh->supports);CHKERRQ(ierr);
3667   ierr = PetscCalloc1(pEnd - pStart, &offsets);CHKERRQ(ierr);
3668   for (p = pStart; p < pEnd; ++p) {
3669     PetscInt dof, off, c;
3670 
3671     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
3672     ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
3673     for (c = off; c < off+dof; ++c) {
3674       const PetscInt q = mesh->cones[c];
3675       PetscInt       offS;
3676 
3677       ierr = PetscSectionGetOffset(mesh->supportSection, q, &offS);CHKERRQ(ierr);
3678 
3679       mesh->supports[offS+offsets[q]] = p;
3680       ++offsets[q];
3681     }
3682   }
3683   ierr = PetscFree(offsets);CHKERRQ(ierr);
3684   ierr = PetscLogEventEnd(DMPLEX_Symmetrize,dm,0,0,0);CHKERRQ(ierr);
3685   PetscFunctionReturn(0);
3686 }
3687 
3688 static PetscErrorCode DMPlexCreateDepthStratum(DM dm, DMLabel label, PetscInt depth, PetscInt pStart, PetscInt pEnd)
3689 {
3690   IS             stratumIS;
3691   PetscErrorCode ierr;
3692 
3693   PetscFunctionBegin;
3694   if (pStart >= pEnd) PetscFunctionReturn(0);
3695   if (PetscDefined(USE_DEBUG)) {
3696     PetscInt  qStart, qEnd, numLevels, level;
3697     PetscBool overlap = PETSC_FALSE;
3698     ierr = DMLabelGetNumValues(label, &numLevels);CHKERRQ(ierr);
3699     for (level = 0; level < numLevels; level++) {
3700       ierr = DMLabelGetStratumBounds(label, level, &qStart, &qEnd);CHKERRQ(ierr);
3701       if ((pStart >= qStart && pStart < qEnd) || (pEnd > qStart && pEnd <= qEnd)) {overlap = PETSC_TRUE; break;}
3702     }
3703     if (overlap) SETERRQ6(PETSC_COMM_SELF, PETSC_ERR_PLIB, "New depth %D range [%D,%D) overlaps with depth %D range [%D,%D)", depth, pStart, pEnd, level, qStart, qEnd);
3704   }
3705   ierr = ISCreateStride(PETSC_COMM_SELF, pEnd-pStart, pStart, 1, &stratumIS);CHKERRQ(ierr);
3706   ierr = DMLabelSetStratumIS(label, depth, stratumIS);CHKERRQ(ierr);
3707   ierr = ISDestroy(&stratumIS);CHKERRQ(ierr);
3708   PetscFunctionReturn(0);
3709 }
3710 
3711 /*@
3712   DMPlexStratify - The DAG for most topologies is a graded poset (https://en.wikipedia.org/wiki/Graded_poset), and
3713   can be illustrated by a Hasse Diagram (https://en.wikipedia.org/wiki/Hasse_diagram). The strata group all points of the
3714   same grade, and this function calculates the strata. This grade can be seen as the height (or depth) of the point in
3715   the DAG.
3716 
3717   Collective on dm
3718 
3719   Input Parameter:
3720 . mesh - The DMPlex
3721 
3722   Output Parameter:
3723 
3724   Notes:
3725   Concretely, DMPlexStratify() creates a new label named "depth" containing the depth in the DAG of each point. For cell-vertex
3726   meshes, vertices are depth 0 and cells are depth 1. For fully interpolated meshes, depth 0 for vertices, 1 for edges, and so on
3727   until cells have depth equal to the dimension of the mesh. The depth label can be accessed through DMPlexGetDepthLabel() or DMPlexGetDepthStratum(), or
3728   manually via DMGetLabel().  The height is defined implicitly by height = maxDimension - depth, and can be accessed
3729   via DMPlexGetHeightStratum().  For example, cells have height 0 and faces have height 1.
3730 
3731   The depth of a point is calculated by executing a breadth-first search (BFS) on the DAG. This could produce surprising results
3732   if run on a partially interpolated mesh, meaning one that had some edges and faces, but not others. For example, suppose that
3733   we had a mesh consisting of one triangle (c0) and three vertices (v0, v1, v2), and only one edge is on the boundary so we choose
3734   to interpolate only that one (e0), so that
3735 $  cone(c0) = {e0, v2}
3736 $  cone(e0) = {v0, v1}
3737   If DMPlexStratify() is run on this mesh, it will give depths
3738 $  depth 0 = {v0, v1, v2}
3739 $  depth 1 = {e0, c0}
3740   where the triangle has been given depth 1, instead of 2, because it is reachable from vertex v2.
3741 
3742   DMPlexStratify() should be called after all calls to DMPlexSymmetrize()
3743 
3744   Level: beginner
3745 
3746 .seealso: DMPlexCreate(), DMPlexSymmetrize(), DMPlexComputeCellTypes()
3747 @*/
3748 PetscErrorCode DMPlexStratify(DM dm)
3749 {
3750   DM_Plex       *mesh = (DM_Plex*) dm->data;
3751   DMLabel        label;
3752   PetscInt       pStart, pEnd, p;
3753   PetscInt       numRoots = 0, numLeaves = 0;
3754   PetscErrorCode ierr;
3755 
3756   PetscFunctionBegin;
3757   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3758   ierr = PetscLogEventBegin(DMPLEX_Stratify,dm,0,0,0);CHKERRQ(ierr);
3759 
3760   /* Create depth label */
3761   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
3762   ierr = DMCreateLabel(dm, "depth");CHKERRQ(ierr);
3763   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
3764 
3765   {
3766     /* Initialize roots and count leaves */
3767     PetscInt sMin = PETSC_MAX_INT;
3768     PetscInt sMax = PETSC_MIN_INT;
3769     PetscInt coneSize, supportSize;
3770 
3771     for (p = pStart; p < pEnd; ++p) {
3772       ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
3773       ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
3774       if (!coneSize && supportSize) {
3775         sMin = PetscMin(p, sMin);
3776         sMax = PetscMax(p, sMax);
3777         ++numRoots;
3778       } else if (!supportSize && coneSize) {
3779         ++numLeaves;
3780       } else if (!supportSize && !coneSize) {
3781         /* Isolated points */
3782         sMin = PetscMin(p, sMin);
3783         sMax = PetscMax(p, sMax);
3784       }
3785     }
3786     ierr = DMPlexCreateDepthStratum(dm, label, 0, sMin, sMax+1);CHKERRQ(ierr);
3787   }
3788 
3789   if (numRoots + numLeaves == (pEnd - pStart)) {
3790     PetscInt sMin = PETSC_MAX_INT;
3791     PetscInt sMax = PETSC_MIN_INT;
3792     PetscInt coneSize, supportSize;
3793 
3794     for (p = pStart; p < pEnd; ++p) {
3795       ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
3796       ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
3797       if (!supportSize && coneSize) {
3798         sMin = PetscMin(p, sMin);
3799         sMax = PetscMax(p, sMax);
3800       }
3801     }
3802     ierr = DMPlexCreateDepthStratum(dm, label, 1, sMin, sMax+1);CHKERRQ(ierr);
3803   } else {
3804     PetscInt level = 0;
3805     PetscInt qStart, qEnd, q;
3806 
3807     ierr = DMLabelGetStratumBounds(label, level, &qStart, &qEnd);CHKERRQ(ierr);
3808     while (qEnd > qStart) {
3809       PetscInt sMin = PETSC_MAX_INT;
3810       PetscInt sMax = PETSC_MIN_INT;
3811 
3812       for (q = qStart; q < qEnd; ++q) {
3813         const PetscInt *support;
3814         PetscInt        supportSize, s;
3815 
3816         ierr = DMPlexGetSupportSize(dm, q, &supportSize);CHKERRQ(ierr);
3817         ierr = DMPlexGetSupport(dm, q, &support);CHKERRQ(ierr);
3818         for (s = 0; s < supportSize; ++s) {
3819           sMin = PetscMin(support[s], sMin);
3820           sMax = PetscMax(support[s], sMax);
3821         }
3822       }
3823       ierr = DMLabelGetNumValues(label, &level);CHKERRQ(ierr);
3824       ierr = DMPlexCreateDepthStratum(dm, label, level, sMin, sMax+1);CHKERRQ(ierr);
3825       ierr = DMLabelGetStratumBounds(label, level, &qStart, &qEnd);CHKERRQ(ierr);
3826     }
3827   }
3828   { /* just in case there is an empty process */
3829     PetscInt numValues, maxValues = 0, v;
3830 
3831     ierr = DMLabelGetNumValues(label, &numValues);CHKERRQ(ierr);
3832     ierr = MPI_Allreduce(&numValues,&maxValues,1,MPIU_INT,MPI_MAX,PetscObjectComm((PetscObject)dm));CHKERRMPI(ierr);
3833     for (v = numValues; v < maxValues; v++) {
3834       ierr = DMLabelAddStratum(label, v);CHKERRQ(ierr);
3835     }
3836   }
3837   ierr = PetscObjectStateGet((PetscObject) label, &mesh->depthState);CHKERRQ(ierr);
3838   ierr = PetscLogEventEnd(DMPLEX_Stratify,dm,0,0,0);CHKERRQ(ierr);
3839   PetscFunctionReturn(0);
3840 }
3841 
3842 PetscErrorCode DMPlexComputeCellType_Internal(DM dm, PetscInt p, PetscInt pdepth, DMPolytopeType *pt)
3843 {
3844   DMPolytopeType ct = DM_POLYTOPE_UNKNOWN;
3845   PetscInt       dim, depth, pheight, coneSize;
3846   PetscErrorCode ierr;
3847 
3848   PetscFunctionBeginHot;
3849   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
3850   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3851   ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
3852   pheight = depth - pdepth;
3853   if (depth <= 1) {
3854     switch (pdepth) {
3855       case 0: ct = DM_POLYTOPE_POINT;break;
3856       case 1:
3857         switch (coneSize) {
3858           case 2: ct = DM_POLYTOPE_SEGMENT;break;
3859           case 3: ct = DM_POLYTOPE_TRIANGLE;break;
3860           case 4:
3861           switch (dim) {
3862             case 2: ct = DM_POLYTOPE_QUADRILATERAL;break;
3863             case 3: ct = DM_POLYTOPE_TETRAHEDRON;break;
3864             default: break;
3865           }
3866           break;
3867         case 5: ct = DM_POLYTOPE_PYRAMID;break;
3868         case 6: ct = DM_POLYTOPE_TRI_PRISM_TENSOR;break;
3869         case 8: ct = DM_POLYTOPE_HEXAHEDRON;break;
3870         default: break;
3871       }
3872     }
3873   } else {
3874     if (pdepth == 0) {
3875       ct = DM_POLYTOPE_POINT;
3876     } else if (pheight == 0) {
3877       switch (dim) {
3878         case 1:
3879           switch (coneSize) {
3880             case 2: ct = DM_POLYTOPE_SEGMENT;break;
3881             default: break;
3882           }
3883           break;
3884         case 2:
3885           switch (coneSize) {
3886             case 3: ct = DM_POLYTOPE_TRIANGLE;break;
3887             case 4: ct = DM_POLYTOPE_QUADRILATERAL;break;
3888             default: break;
3889           }
3890           break;
3891         case 3:
3892           switch (coneSize) {
3893             case 4: ct = DM_POLYTOPE_TETRAHEDRON;break;
3894             case 5:
3895             {
3896               const PetscInt *cone;
3897               PetscInt        faceConeSize;
3898 
3899               ierr = DMPlexGetCone(dm, p, &cone);CHKERRQ(ierr);
3900               ierr = DMPlexGetConeSize(dm, cone[0], &faceConeSize);CHKERRQ(ierr);
3901               switch (faceConeSize) {
3902                 case 3: ct = DM_POLYTOPE_TRI_PRISM_TENSOR;break;
3903                 case 4: ct = DM_POLYTOPE_PYRAMID;break;
3904               }
3905             }
3906             break;
3907             case 6: ct = DM_POLYTOPE_HEXAHEDRON;break;
3908             default: break;
3909           }
3910           break;
3911         default: break;
3912       }
3913     } else if (pheight > 0) {
3914       switch (coneSize) {
3915         case 2: ct = DM_POLYTOPE_SEGMENT;break;
3916         case 3: ct = DM_POLYTOPE_TRIANGLE;break;
3917         case 4: ct = DM_POLYTOPE_QUADRILATERAL;break;
3918         default: break;
3919       }
3920     }
3921   }
3922   *pt = ct;
3923   PetscFunctionReturn(0);
3924 }
3925 
3926 /*@
3927   DMPlexComputeCellTypes - Infer the polytope type of every cell using its dimension and cone size.
3928 
3929   Collective on dm
3930 
3931   Input Parameter:
3932 . mesh - The DMPlex
3933 
3934   DMPlexComputeCellTypes() should be called after all calls to DMPlexSymmetrize() and DMPlexStratify()
3935 
3936   Level: developer
3937 
3938   Note: This function is normally called automatically by Plex when a cell type is requested. It creates an
3939   internal DMLabel named "celltype" which can be directly accessed using DMGetLabel(). A user may disable
3940   automatic creation by creating the label manually, using DMCreateLabel(dm, "celltype").
3941 
3942 .seealso: DMPlexCreate(), DMPlexSymmetrize(), DMPlexStratify(), DMGetLabel(), DMCreateLabel()
3943 @*/
3944 PetscErrorCode DMPlexComputeCellTypes(DM dm)
3945 {
3946   DM_Plex       *mesh;
3947   DMLabel        ctLabel;
3948   PetscInt       pStart, pEnd, p;
3949   PetscErrorCode ierr;
3950 
3951   PetscFunctionBegin;
3952   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3953   mesh = (DM_Plex *) dm->data;
3954   ierr = DMCreateLabel(dm, "celltype");CHKERRQ(ierr);
3955   ierr = DMPlexGetCellTypeLabel(dm, &ctLabel);CHKERRQ(ierr);
3956   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
3957   for (p = pStart; p < pEnd; ++p) {
3958     DMPolytopeType ct = DM_POLYTOPE_UNKNOWN;
3959     PetscInt       pdepth;
3960 
3961     ierr = DMPlexGetPointDepth(dm, p, &pdepth);CHKERRQ(ierr);
3962     ierr = DMPlexComputeCellType_Internal(dm, p, pdepth, &ct);CHKERRQ(ierr);
3963     if (ct == DM_POLYTOPE_UNKNOWN) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_SUP, "Point %D is screwed up", p);
3964     ierr = DMLabelSetValue(ctLabel, p, ct);CHKERRQ(ierr);
3965   }
3966   ierr = PetscObjectStateGet((PetscObject) ctLabel, &mesh->celltypeState);CHKERRQ(ierr);
3967   ierr = PetscObjectViewFromOptions((PetscObject) ctLabel, NULL, "-dm_plex_celltypes_view");CHKERRQ(ierr);
3968   PetscFunctionReturn(0);
3969 }
3970 
3971 /*@C
3972   DMPlexGetJoin - Get an array for the join of the set of points
3973 
3974   Not Collective
3975 
3976   Input Parameters:
3977 + dm - The DMPlex object
3978 . numPoints - The number of input points for the join
3979 - points - The input points
3980 
3981   Output Parameters:
3982 + numCoveredPoints - The number of points in the join
3983 - coveredPoints - The points in the join
3984 
3985   Level: intermediate
3986 
3987   Note: Currently, this is restricted to a single level join
3988 
3989   Fortran Notes:
3990   Since it returns an array, this routine is only available in Fortran 90, and you must
3991   include petsc.h90 in your code.
3992 
3993   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3994 
3995 .seealso: DMPlexRestoreJoin(), DMPlexGetMeet()
3996 @*/
3997 PetscErrorCode DMPlexGetJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
3998 {
3999   DM_Plex       *mesh = (DM_Plex*) dm->data;
4000   PetscInt      *join[2];
4001   PetscInt       joinSize, i = 0;
4002   PetscInt       dof, off, p, c, m;
4003   PetscErrorCode ierr;
4004 
4005   PetscFunctionBegin;
4006   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4007   PetscValidIntPointer(points, 3);
4008   PetscValidIntPointer(numCoveredPoints, 4);
4009   PetscValidPointer(coveredPoints, 5);
4010   ierr = DMGetWorkArray(dm, mesh->maxSupportSize, MPIU_INT, &join[0]);CHKERRQ(ierr);
4011   ierr = DMGetWorkArray(dm, mesh->maxSupportSize, MPIU_INT, &join[1]);CHKERRQ(ierr);
4012   /* Copy in support of first point */
4013   ierr = PetscSectionGetDof(mesh->supportSection, points[0], &dof);CHKERRQ(ierr);
4014   ierr = PetscSectionGetOffset(mesh->supportSection, points[0], &off);CHKERRQ(ierr);
4015   for (joinSize = 0; joinSize < dof; ++joinSize) {
4016     join[i][joinSize] = mesh->supports[off+joinSize];
4017   }
4018   /* Check each successive support */
4019   for (p = 1; p < numPoints; ++p) {
4020     PetscInt newJoinSize = 0;
4021 
4022     ierr = PetscSectionGetDof(mesh->supportSection, points[p], &dof);CHKERRQ(ierr);
4023     ierr = PetscSectionGetOffset(mesh->supportSection, points[p], &off);CHKERRQ(ierr);
4024     for (c = 0; c < dof; ++c) {
4025       const PetscInt point = mesh->supports[off+c];
4026 
4027       for (m = 0; m < joinSize; ++m) {
4028         if (point == join[i][m]) {
4029           join[1-i][newJoinSize++] = point;
4030           break;
4031         }
4032       }
4033     }
4034     joinSize = newJoinSize;
4035     i        = 1-i;
4036   }
4037   *numCoveredPoints = joinSize;
4038   *coveredPoints    = join[i];
4039   ierr              = DMRestoreWorkArray(dm, mesh->maxSupportSize, MPIU_INT, &join[1-i]);CHKERRQ(ierr);
4040   PetscFunctionReturn(0);
4041 }
4042 
4043 /*@C
4044   DMPlexRestoreJoin - Restore an array for the join of the set of points
4045 
4046   Not Collective
4047 
4048   Input Parameters:
4049 + dm - The DMPlex object
4050 . numPoints - The number of input points for the join
4051 - points - The input points
4052 
4053   Output Parameters:
4054 + numCoveredPoints - The number of points in the join
4055 - coveredPoints - The points in the join
4056 
4057   Fortran Notes:
4058   Since it returns an array, this routine is only available in Fortran 90, and you must
4059   include petsc.h90 in your code.
4060 
4061   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
4062 
4063   Level: intermediate
4064 
4065 .seealso: DMPlexGetJoin(), DMPlexGetFullJoin(), DMPlexGetMeet()
4066 @*/
4067 PetscErrorCode DMPlexRestoreJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
4068 {
4069   PetscErrorCode ierr;
4070 
4071   PetscFunctionBegin;
4072   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4073   if (points) PetscValidIntPointer(points,3);
4074   if (numCoveredPoints) PetscValidIntPointer(numCoveredPoints,4);
4075   PetscValidPointer(coveredPoints, 5);
4076   ierr = DMRestoreWorkArray(dm, 0, MPIU_INT, (void*) coveredPoints);CHKERRQ(ierr);
4077   if (numCoveredPoints) *numCoveredPoints = 0;
4078   PetscFunctionReturn(0);
4079 }
4080 
4081 /*@C
4082   DMPlexGetFullJoin - Get an array for the join of the set of points
4083 
4084   Not Collective
4085 
4086   Input Parameters:
4087 + dm - The DMPlex object
4088 . numPoints - The number of input points for the join
4089 - points - The input points
4090 
4091   Output Parameters:
4092 + numCoveredPoints - The number of points in the join
4093 - coveredPoints - The points in the join
4094 
4095   Fortran Notes:
4096   Since it returns an array, this routine is only available in Fortran 90, and you must
4097   include petsc.h90 in your code.
4098 
4099   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
4100 
4101   Level: intermediate
4102 
4103 .seealso: DMPlexGetJoin(), DMPlexRestoreJoin(), DMPlexGetMeet()
4104 @*/
4105 PetscErrorCode DMPlexGetFullJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
4106 {
4107   DM_Plex       *mesh = (DM_Plex*) dm->data;
4108   PetscInt      *offsets, **closures;
4109   PetscInt      *join[2];
4110   PetscInt       depth = 0, maxSize, joinSize = 0, i = 0;
4111   PetscInt       p, d, c, m, ms;
4112   PetscErrorCode ierr;
4113 
4114   PetscFunctionBegin;
4115   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4116   PetscValidIntPointer(points, 3);
4117   PetscValidIntPointer(numCoveredPoints, 4);
4118   PetscValidPointer(coveredPoints, 5);
4119 
4120   ierr    = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
4121   ierr    = PetscCalloc1(numPoints, &closures);CHKERRQ(ierr);
4122   ierr    = DMGetWorkArray(dm, numPoints*(depth+2), MPIU_INT, &offsets);CHKERRQ(ierr);
4123   ms      = mesh->maxSupportSize;
4124   maxSize = (ms > 1) ? ((PetscPowInt(ms,depth+1)-1)/(ms-1)) : depth + 1;
4125   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &join[0]);CHKERRQ(ierr);
4126   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &join[1]);CHKERRQ(ierr);
4127 
4128   for (p = 0; p < numPoints; ++p) {
4129     PetscInt closureSize;
4130 
4131     ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_FALSE, &closureSize, &closures[p]);CHKERRQ(ierr);
4132 
4133     offsets[p*(depth+2)+0] = 0;
4134     for (d = 0; d < depth+1; ++d) {
4135       PetscInt pStart, pEnd, i;
4136 
4137       ierr = DMPlexGetDepthStratum(dm, d, &pStart, &pEnd);CHKERRQ(ierr);
4138       for (i = offsets[p*(depth+2)+d]; i < closureSize; ++i) {
4139         if ((pStart > closures[p][i*2]) || (pEnd <= closures[p][i*2])) {
4140           offsets[p*(depth+2)+d+1] = i;
4141           break;
4142         }
4143       }
4144       if (i == closureSize) offsets[p*(depth+2)+d+1] = i;
4145     }
4146     if (offsets[p*(depth+2)+depth+1] != closureSize) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Total size of closure %D should be %D", offsets[p*(depth+2)+depth+1], closureSize);
4147   }
4148   for (d = 0; d < depth+1; ++d) {
4149     PetscInt dof;
4150 
4151     /* Copy in support of first point */
4152     dof = offsets[d+1] - offsets[d];
4153     for (joinSize = 0; joinSize < dof; ++joinSize) {
4154       join[i][joinSize] = closures[0][(offsets[d]+joinSize)*2];
4155     }
4156     /* Check each successive cone */
4157     for (p = 1; p < numPoints && joinSize; ++p) {
4158       PetscInt newJoinSize = 0;
4159 
4160       dof = offsets[p*(depth+2)+d+1] - offsets[p*(depth+2)+d];
4161       for (c = 0; c < dof; ++c) {
4162         const PetscInt point = closures[p][(offsets[p*(depth+2)+d]+c)*2];
4163 
4164         for (m = 0; m < joinSize; ++m) {
4165           if (point == join[i][m]) {
4166             join[1-i][newJoinSize++] = point;
4167             break;
4168           }
4169         }
4170       }
4171       joinSize = newJoinSize;
4172       i        = 1-i;
4173     }
4174     if (joinSize) break;
4175   }
4176   *numCoveredPoints = joinSize;
4177   *coveredPoints    = join[i];
4178   for (p = 0; p < numPoints; ++p) {
4179     ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_FALSE, NULL, &closures[p]);CHKERRQ(ierr);
4180   }
4181   ierr = PetscFree(closures);CHKERRQ(ierr);
4182   ierr = DMRestoreWorkArray(dm, numPoints*(depth+2), MPIU_INT, &offsets);CHKERRQ(ierr);
4183   ierr = DMRestoreWorkArray(dm, mesh->maxSupportSize, MPIU_INT, &join[1-i]);CHKERRQ(ierr);
4184   PetscFunctionReturn(0);
4185 }
4186 
4187 /*@C
4188   DMPlexGetMeet - Get an array for the meet of the set of points
4189 
4190   Not Collective
4191 
4192   Input Parameters:
4193 + dm - The DMPlex object
4194 . numPoints - The number of input points for the meet
4195 - points - The input points
4196 
4197   Output Parameters:
4198 + numCoveredPoints - The number of points in the meet
4199 - coveredPoints - The points in the meet
4200 
4201   Level: intermediate
4202 
4203   Note: Currently, this is restricted to a single level meet
4204 
4205   Fortran Notes:
4206   Since it returns an array, this routine is only available in Fortran 90, and you must
4207   include petsc.h90 in your code.
4208 
4209   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
4210 
4211 .seealso: DMPlexRestoreMeet(), DMPlexGetJoin()
4212 @*/
4213 PetscErrorCode DMPlexGetMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveringPoints, const PetscInt **coveringPoints)
4214 {
4215   DM_Plex       *mesh = (DM_Plex*) dm->data;
4216   PetscInt      *meet[2];
4217   PetscInt       meetSize, i = 0;
4218   PetscInt       dof, off, p, c, m;
4219   PetscErrorCode ierr;
4220 
4221   PetscFunctionBegin;
4222   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4223   PetscValidPointer(points, 3);
4224   PetscValidPointer(numCoveringPoints, 4);
4225   PetscValidPointer(coveringPoints, 5);
4226   ierr = DMGetWorkArray(dm, mesh->maxConeSize, MPIU_INT, &meet[0]);CHKERRQ(ierr);
4227   ierr = DMGetWorkArray(dm, mesh->maxConeSize, MPIU_INT, &meet[1]);CHKERRQ(ierr);
4228   /* Copy in cone of first point */
4229   ierr = PetscSectionGetDof(mesh->coneSection, points[0], &dof);CHKERRQ(ierr);
4230   ierr = PetscSectionGetOffset(mesh->coneSection, points[0], &off);CHKERRQ(ierr);
4231   for (meetSize = 0; meetSize < dof; ++meetSize) {
4232     meet[i][meetSize] = mesh->cones[off+meetSize];
4233   }
4234   /* Check each successive cone */
4235   for (p = 1; p < numPoints; ++p) {
4236     PetscInt newMeetSize = 0;
4237 
4238     ierr = PetscSectionGetDof(mesh->coneSection, points[p], &dof);CHKERRQ(ierr);
4239     ierr = PetscSectionGetOffset(mesh->coneSection, points[p], &off);CHKERRQ(ierr);
4240     for (c = 0; c < dof; ++c) {
4241       const PetscInt point = mesh->cones[off+c];
4242 
4243       for (m = 0; m < meetSize; ++m) {
4244         if (point == meet[i][m]) {
4245           meet[1-i][newMeetSize++] = point;
4246           break;
4247         }
4248       }
4249     }
4250     meetSize = newMeetSize;
4251     i        = 1-i;
4252   }
4253   *numCoveringPoints = meetSize;
4254   *coveringPoints    = meet[i];
4255   ierr               = DMRestoreWorkArray(dm, mesh->maxConeSize, MPIU_INT, &meet[1-i]);CHKERRQ(ierr);
4256   PetscFunctionReturn(0);
4257 }
4258 
4259 /*@C
4260   DMPlexRestoreMeet - Restore an array for the meet of the set of points
4261 
4262   Not Collective
4263 
4264   Input Parameters:
4265 + dm - The DMPlex object
4266 . numPoints - The number of input points for the meet
4267 - points - The input points
4268 
4269   Output Parameters:
4270 + numCoveredPoints - The number of points in the meet
4271 - coveredPoints - The points in the meet
4272 
4273   Level: intermediate
4274 
4275   Fortran Notes:
4276   Since it returns an array, this routine is only available in Fortran 90, and you must
4277   include petsc.h90 in your code.
4278 
4279   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
4280 
4281 .seealso: DMPlexGetMeet(), DMPlexGetFullMeet(), DMPlexGetJoin()
4282 @*/
4283 PetscErrorCode DMPlexRestoreMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
4284 {
4285   PetscErrorCode ierr;
4286 
4287   PetscFunctionBegin;
4288   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4289   if (points) PetscValidIntPointer(points,3);
4290   if (numCoveredPoints) PetscValidIntPointer(numCoveredPoints,4);
4291   PetscValidPointer(coveredPoints,5);
4292   ierr = DMRestoreWorkArray(dm, 0, MPIU_INT, (void*) coveredPoints);CHKERRQ(ierr);
4293   if (numCoveredPoints) *numCoveredPoints = 0;
4294   PetscFunctionReturn(0);
4295 }
4296 
4297 /*@C
4298   DMPlexGetFullMeet - Get an array for the meet of the set of points
4299 
4300   Not Collective
4301 
4302   Input Parameters:
4303 + dm - The DMPlex object
4304 . numPoints - The number of input points for the meet
4305 - points - The input points
4306 
4307   Output Parameters:
4308 + numCoveredPoints - The number of points in the meet
4309 - coveredPoints - The points in the meet
4310 
4311   Level: intermediate
4312 
4313   Fortran Notes:
4314   Since it returns an array, this routine is only available in Fortran 90, and you must
4315   include petsc.h90 in your code.
4316 
4317   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
4318 
4319 .seealso: DMPlexGetMeet(), DMPlexRestoreMeet(), DMPlexGetJoin()
4320 @*/
4321 PetscErrorCode DMPlexGetFullMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
4322 {
4323   DM_Plex       *mesh = (DM_Plex*) dm->data;
4324   PetscInt      *offsets, **closures;
4325   PetscInt      *meet[2];
4326   PetscInt       height = 0, maxSize, meetSize = 0, i = 0;
4327   PetscInt       p, h, c, m, mc;
4328   PetscErrorCode ierr;
4329 
4330   PetscFunctionBegin;
4331   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4332   PetscValidPointer(points, 3);
4333   PetscValidPointer(numCoveredPoints, 4);
4334   PetscValidPointer(coveredPoints, 5);
4335 
4336   ierr    = DMPlexGetDepth(dm, &height);CHKERRQ(ierr);
4337   ierr    = PetscMalloc1(numPoints, &closures);CHKERRQ(ierr);
4338   ierr    = DMGetWorkArray(dm, numPoints*(height+2), MPIU_INT, &offsets);CHKERRQ(ierr);
4339   mc      = mesh->maxConeSize;
4340   maxSize = (mc > 1) ? ((PetscPowInt(mc,height+1)-1)/(mc-1)) : height + 1;
4341   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &meet[0]);CHKERRQ(ierr);
4342   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &meet[1]);CHKERRQ(ierr);
4343 
4344   for (p = 0; p < numPoints; ++p) {
4345     PetscInt closureSize;
4346 
4347     ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_TRUE, &closureSize, &closures[p]);CHKERRQ(ierr);
4348 
4349     offsets[p*(height+2)+0] = 0;
4350     for (h = 0; h < height+1; ++h) {
4351       PetscInt pStart, pEnd, i;
4352 
4353       ierr = DMPlexGetHeightStratum(dm, h, &pStart, &pEnd);CHKERRQ(ierr);
4354       for (i = offsets[p*(height+2)+h]; i < closureSize; ++i) {
4355         if ((pStart > closures[p][i*2]) || (pEnd <= closures[p][i*2])) {
4356           offsets[p*(height+2)+h+1] = i;
4357           break;
4358         }
4359       }
4360       if (i == closureSize) offsets[p*(height+2)+h+1] = i;
4361     }
4362     if (offsets[p*(height+2)+height+1] != closureSize) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Total size of closure %D should be %D", offsets[p*(height+2)+height+1], closureSize);
4363   }
4364   for (h = 0; h < height+1; ++h) {
4365     PetscInt dof;
4366 
4367     /* Copy in cone of first point */
4368     dof = offsets[h+1] - offsets[h];
4369     for (meetSize = 0; meetSize < dof; ++meetSize) {
4370       meet[i][meetSize] = closures[0][(offsets[h]+meetSize)*2];
4371     }
4372     /* Check each successive cone */
4373     for (p = 1; p < numPoints && meetSize; ++p) {
4374       PetscInt newMeetSize = 0;
4375 
4376       dof = offsets[p*(height+2)+h+1] - offsets[p*(height+2)+h];
4377       for (c = 0; c < dof; ++c) {
4378         const PetscInt point = closures[p][(offsets[p*(height+2)+h]+c)*2];
4379 
4380         for (m = 0; m < meetSize; ++m) {
4381           if (point == meet[i][m]) {
4382             meet[1-i][newMeetSize++] = point;
4383             break;
4384           }
4385         }
4386       }
4387       meetSize = newMeetSize;
4388       i        = 1-i;
4389     }
4390     if (meetSize) break;
4391   }
4392   *numCoveredPoints = meetSize;
4393   *coveredPoints    = meet[i];
4394   for (p = 0; p < numPoints; ++p) {
4395     ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_TRUE, NULL, &closures[p]);CHKERRQ(ierr);
4396   }
4397   ierr = PetscFree(closures);CHKERRQ(ierr);
4398   ierr = DMRestoreWorkArray(dm, numPoints*(height+2), MPIU_INT, &offsets);CHKERRQ(ierr);
4399   ierr = DMRestoreWorkArray(dm, mesh->maxConeSize, MPIU_INT, &meet[1-i]);CHKERRQ(ierr);
4400   PetscFunctionReturn(0);
4401 }
4402 
4403 /*@C
4404   DMPlexEqual - Determine if two DMs have the same topology
4405 
4406   Not Collective
4407 
4408   Input Parameters:
4409 + dmA - A DMPlex object
4410 - dmB - A DMPlex object
4411 
4412   Output Parameters:
4413 . equal - PETSC_TRUE if the topologies are identical
4414 
4415   Level: intermediate
4416 
4417   Notes:
4418   We are not solving graph isomorphism, so we do not permutation.
4419 
4420 .seealso: DMPlexGetCone()
4421 @*/
4422 PetscErrorCode DMPlexEqual(DM dmA, DM dmB, PetscBool *equal)
4423 {
4424   PetscInt       depth, depthB, pStart, pEnd, pStartB, pEndB, p;
4425   PetscErrorCode ierr;
4426 
4427   PetscFunctionBegin;
4428   PetscValidHeaderSpecific(dmA, DM_CLASSID, 1);
4429   PetscValidHeaderSpecific(dmB, DM_CLASSID, 2);
4430   PetscValidPointer(equal, 3);
4431 
4432   *equal = PETSC_FALSE;
4433   ierr = DMPlexGetDepth(dmA, &depth);CHKERRQ(ierr);
4434   ierr = DMPlexGetDepth(dmB, &depthB);CHKERRQ(ierr);
4435   if (depth != depthB) PetscFunctionReturn(0);
4436   ierr = DMPlexGetChart(dmA, &pStart,  &pEnd);CHKERRQ(ierr);
4437   ierr = DMPlexGetChart(dmB, &pStartB, &pEndB);CHKERRQ(ierr);
4438   if ((pStart != pStartB) || (pEnd != pEndB)) PetscFunctionReturn(0);
4439   for (p = pStart; p < pEnd; ++p) {
4440     const PetscInt *cone, *coneB, *ornt, *orntB, *support, *supportB;
4441     PetscInt        coneSize, coneSizeB, c, supportSize, supportSizeB, s;
4442 
4443     ierr = DMPlexGetConeSize(dmA, p, &coneSize);CHKERRQ(ierr);
4444     ierr = DMPlexGetCone(dmA, p, &cone);CHKERRQ(ierr);
4445     ierr = DMPlexGetConeOrientation(dmA, p, &ornt);CHKERRQ(ierr);
4446     ierr = DMPlexGetConeSize(dmB, p, &coneSizeB);CHKERRQ(ierr);
4447     ierr = DMPlexGetCone(dmB, p, &coneB);CHKERRQ(ierr);
4448     ierr = DMPlexGetConeOrientation(dmB, p, &orntB);CHKERRQ(ierr);
4449     if (coneSize != coneSizeB) PetscFunctionReturn(0);
4450     for (c = 0; c < coneSize; ++c) {
4451       if (cone[c] != coneB[c]) PetscFunctionReturn(0);
4452       if (ornt[c] != orntB[c]) PetscFunctionReturn(0);
4453     }
4454     ierr = DMPlexGetSupportSize(dmA, p, &supportSize);CHKERRQ(ierr);
4455     ierr = DMPlexGetSupport(dmA, p, &support);CHKERRQ(ierr);
4456     ierr = DMPlexGetSupportSize(dmB, p, &supportSizeB);CHKERRQ(ierr);
4457     ierr = DMPlexGetSupport(dmB, p, &supportB);CHKERRQ(ierr);
4458     if (supportSize != supportSizeB) PetscFunctionReturn(0);
4459     for (s = 0; s < supportSize; ++s) {
4460       if (support[s] != supportB[s]) PetscFunctionReturn(0);
4461     }
4462   }
4463   *equal = PETSC_TRUE;
4464   PetscFunctionReturn(0);
4465 }
4466 
4467 /*@C
4468   DMPlexGetNumFaceVertices - Returns the number of vertices on a face
4469 
4470   Not Collective
4471 
4472   Input Parameters:
4473 + dm         - The DMPlex
4474 . cellDim    - The cell dimension
4475 - numCorners - The number of vertices on a cell
4476 
4477   Output Parameters:
4478 . numFaceVertices - The number of vertices on a face
4479 
4480   Level: developer
4481 
4482   Notes:
4483   Of course this can only work for a restricted set of symmetric shapes
4484 
4485 .seealso: DMPlexGetCone()
4486 @*/
4487 PetscErrorCode DMPlexGetNumFaceVertices(DM dm, PetscInt cellDim, PetscInt numCorners, PetscInt *numFaceVertices)
4488 {
4489   MPI_Comm       comm;
4490   PetscErrorCode ierr;
4491 
4492   PetscFunctionBegin;
4493   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
4494   PetscValidPointer(numFaceVertices,4);
4495   switch (cellDim) {
4496   case 0:
4497     *numFaceVertices = 0;
4498     break;
4499   case 1:
4500     *numFaceVertices = 1;
4501     break;
4502   case 2:
4503     switch (numCorners) {
4504     case 3: /* triangle */
4505       *numFaceVertices = 2; /* Edge has 2 vertices */
4506       break;
4507     case 4: /* quadrilateral */
4508       *numFaceVertices = 2; /* Edge has 2 vertices */
4509       break;
4510     case 6: /* quadratic triangle, tri and quad cohesive Lagrange cells */
4511       *numFaceVertices = 3; /* Edge has 3 vertices */
4512       break;
4513     case 9: /* quadratic quadrilateral, quadratic quad cohesive Lagrange cells */
4514       *numFaceVertices = 3; /* Edge has 3 vertices */
4515       break;
4516     default:
4517       SETERRQ2(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid number of face corners %D for dimension %D", numCorners, cellDim);
4518     }
4519     break;
4520   case 3:
4521     switch (numCorners) {
4522     case 4: /* tetradehdron */
4523       *numFaceVertices = 3; /* Face has 3 vertices */
4524       break;
4525     case 6: /* tet cohesive cells */
4526       *numFaceVertices = 4; /* Face has 4 vertices */
4527       break;
4528     case 8: /* hexahedron */
4529       *numFaceVertices = 4; /* Face has 4 vertices */
4530       break;
4531     case 9: /* tet cohesive Lagrange cells */
4532       *numFaceVertices = 6; /* Face has 6 vertices */
4533       break;
4534     case 10: /* quadratic tetrahedron */
4535       *numFaceVertices = 6; /* Face has 6 vertices */
4536       break;
4537     case 12: /* hex cohesive Lagrange cells */
4538       *numFaceVertices = 6; /* Face has 6 vertices */
4539       break;
4540     case 18: /* quadratic tet cohesive Lagrange cells */
4541       *numFaceVertices = 6; /* Face has 6 vertices */
4542       break;
4543     case 27: /* quadratic hexahedron, quadratic hex cohesive Lagrange cells */
4544       *numFaceVertices = 9; /* Face has 9 vertices */
4545       break;
4546     default:
4547       SETERRQ2(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid number of face corners %D for dimension %D", numCorners, cellDim);
4548     }
4549     break;
4550   default:
4551     SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid cell dimension %D", cellDim);
4552   }
4553   PetscFunctionReturn(0);
4554 }
4555 
4556 /*@
4557   DMPlexGetDepthLabel - Get the DMLabel recording the depth of each point
4558 
4559   Not Collective
4560 
4561   Input Parameter:
4562 . dm    - The DMPlex object
4563 
4564   Output Parameter:
4565 . depthLabel - The DMLabel recording point depth
4566 
4567   Level: developer
4568 
4569 .seealso: DMPlexGetDepth(), DMPlexGetHeightStratum(), DMPlexGetDepthStratum(), DMPlexGetPointDepth(),
4570 @*/
4571 PetscErrorCode DMPlexGetDepthLabel(DM dm, DMLabel *depthLabel)
4572 {
4573   PetscFunctionBegin;
4574   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4575   PetscValidPointer(depthLabel, 2);
4576   *depthLabel = dm->depthLabel;
4577   PetscFunctionReturn(0);
4578 }
4579 
4580 /*@
4581   DMPlexGetDepth - Get the depth of the DAG representing this mesh
4582 
4583   Not Collective
4584 
4585   Input Parameter:
4586 . dm    - The DMPlex object
4587 
4588   Output Parameter:
4589 . depth - The number of strata (breadth first levels) in the DAG
4590 
4591   Level: developer
4592 
4593   Notes:
4594   This returns maximum of point depths over all points, i.e. maximum value of the label returned by DMPlexGetDepthLabel().
4595   The point depth is described more in detail in DMPlexGetDepthStratum().
4596   An empty mesh gives -1.
4597 
4598 .seealso: DMPlexGetDepthLabel(), DMPlexGetDepthStratum(), DMPlexGetPointDepth(), DMPlexSymmetrize()
4599 @*/
4600 PetscErrorCode DMPlexGetDepth(DM dm, PetscInt *depth)
4601 {
4602   DMLabel        label;
4603   PetscInt       d = 0;
4604   PetscErrorCode ierr;
4605 
4606   PetscFunctionBegin;
4607   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4608   PetscValidPointer(depth, 2);
4609   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
4610   if (label) {ierr = DMLabelGetNumValues(label, &d);CHKERRQ(ierr);}
4611   *depth = d-1;
4612   PetscFunctionReturn(0);
4613 }
4614 
4615 /*@
4616   DMPlexGetDepthStratum - Get the bounds [start, end) for all points at a certain depth.
4617 
4618   Not Collective
4619 
4620   Input Parameters:
4621 + dm           - The DMPlex object
4622 - stratumValue - The requested depth
4623 
4624   Output Parameters:
4625 + start - The first point at this depth
4626 - end   - One beyond the last point at this depth
4627 
4628   Notes:
4629   Depth indexing is related to topological dimension.  Depth stratum 0 contains the lowest topological dimension points,
4630   often "vertices".  If the mesh is "interpolated" (see DMPlexInterpolate()), then depth stratum 1 contains the next
4631   higher dimension, e.g., "edges".
4632 
4633   Level: developer
4634 
4635 .seealso: DMPlexGetHeightStratum(), DMPlexGetDepth(), DMPlexGetDepthLabel(), DMPlexGetPointDepth(), DMPlexSymmetrize(), DMPlexInterpolate()
4636 @*/
4637 PetscErrorCode DMPlexGetDepthStratum(DM dm, PetscInt stratumValue, PetscInt *start, PetscInt *end)
4638 {
4639   DMLabel        label;
4640   PetscInt       pStart, pEnd;
4641   PetscErrorCode ierr;
4642 
4643   PetscFunctionBegin;
4644   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4645   if (start) {PetscValidPointer(start, 3); *start = 0;}
4646   if (end)   {PetscValidPointer(end,   4); *end   = 0;}
4647   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
4648   if (pStart == pEnd) PetscFunctionReturn(0);
4649   if (stratumValue < 0) {
4650     if (start) *start = pStart;
4651     if (end)   *end   = pEnd;
4652     PetscFunctionReturn(0);
4653   }
4654   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
4655   if (!label) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "No label named depth was found");
4656   ierr = DMLabelGetStratumBounds(label, stratumValue, start, end);CHKERRQ(ierr);
4657   PetscFunctionReturn(0);
4658 }
4659 
4660 /*@
4661   DMPlexGetHeightStratum - Get the bounds [start, end) for all points at a certain height.
4662 
4663   Not Collective
4664 
4665   Input Parameters:
4666 + dm           - The DMPlex object
4667 - stratumValue - The requested height
4668 
4669   Output Parameters:
4670 + start - The first point at this height
4671 - end   - One beyond the last point at this height
4672 
4673   Notes:
4674   Height indexing is related to topological codimension.  Height stratum 0 contains the highest topological dimension
4675   points, often called "cells" or "elements".  If the mesh is "interpolated" (see DMPlexInterpolate()), then height
4676   stratum 1 contains the boundary of these "cells", often called "faces" or "facets".
4677 
4678   Level: developer
4679 
4680 .seealso: DMPlexGetDepthStratum(), DMPlexGetDepth(), DMPlexGetPointHeight()
4681 @*/
4682 PetscErrorCode DMPlexGetHeightStratum(DM dm, PetscInt stratumValue, PetscInt *start, PetscInt *end)
4683 {
4684   DMLabel        label;
4685   PetscInt       depth, pStart, pEnd;
4686   PetscErrorCode ierr;
4687 
4688   PetscFunctionBegin;
4689   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4690   if (start) {PetscValidPointer(start, 3); *start = 0;}
4691   if (end)   {PetscValidPointer(end,   4); *end   = 0;}
4692   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
4693   if (pStart == pEnd) PetscFunctionReturn(0);
4694   if (stratumValue < 0) {
4695     if (start) *start = pStart;
4696     if (end)   *end   = pEnd;
4697     PetscFunctionReturn(0);
4698   }
4699   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
4700   if (!label) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "No label named depth was found");
4701   ierr = DMLabelGetNumValues(label, &depth);CHKERRQ(ierr);
4702   ierr = DMLabelGetStratumBounds(label, depth-1-stratumValue, start, end);CHKERRQ(ierr);
4703   PetscFunctionReturn(0);
4704 }
4705 
4706 /*@
4707   DMPlexGetPointDepth - Get the depth of a given point
4708 
4709   Not Collective
4710 
4711   Input Parameter:
4712 + dm    - The DMPlex object
4713 - point - The point
4714 
4715   Output Parameter:
4716 . depth - The depth of the point
4717 
4718   Level: intermediate
4719 
4720 .seealso: DMPlexGetCellType(), DMPlexGetDepthLabel(), DMPlexGetDepth(), DMPlexGetPointHeight()
4721 @*/
4722 PetscErrorCode DMPlexGetPointDepth(DM dm, PetscInt point, PetscInt *depth)
4723 {
4724   PetscErrorCode ierr;
4725 
4726   PetscFunctionBegin;
4727   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4728   PetscValidIntPointer(depth, 3);
4729   ierr = DMLabelGetValue(dm->depthLabel, point, depth);CHKERRQ(ierr);
4730   PetscFunctionReturn(0);
4731 }
4732 
4733 /*@
4734   DMPlexGetPointHeight - Get the height of a given point
4735 
4736   Not Collective
4737 
4738   Input Parameter:
4739 + dm    - The DMPlex object
4740 - point - The point
4741 
4742   Output Parameter:
4743 . height - The height of the point
4744 
4745   Level: intermediate
4746 
4747 .seealso: DMPlexGetCellType(), DMPlexGetDepthLabel(), DMPlexGetDepth(), DMPlexGetPointDepth()
4748 @*/
4749 PetscErrorCode DMPlexGetPointHeight(DM dm, PetscInt point, PetscInt *height)
4750 {
4751   PetscInt       n, pDepth;
4752   PetscErrorCode ierr;
4753 
4754   PetscFunctionBegin;
4755   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4756   PetscValidIntPointer(height, 3);
4757   ierr = DMLabelGetNumValues(dm->depthLabel, &n);CHKERRQ(ierr);
4758   ierr = DMLabelGetValue(dm->depthLabel, point, &pDepth);CHKERRQ(ierr);
4759   *height = n - 1 - pDepth;  /* DAG depth is n-1 */
4760   PetscFunctionReturn(0);
4761 }
4762 
4763 /*@
4764   DMPlexGetCellTypeLabel - Get the DMLabel recording the polytope type of each cell
4765 
4766   Not Collective
4767 
4768   Input Parameter:
4769 . dm - The DMPlex object
4770 
4771   Output Parameter:
4772 . celltypeLabel - The DMLabel recording cell polytope type
4773 
4774   Note: This function will trigger automatica computation of cell types. This can be disabled by calling
4775   DMCreateLabel(dm, "celltype") beforehand.
4776 
4777   Level: developer
4778 
4779 .seealso: DMPlexGetCellType(), DMPlexGetDepthLabel(), DMCreateLabel()
4780 @*/
4781 PetscErrorCode DMPlexGetCellTypeLabel(DM dm, DMLabel *celltypeLabel)
4782 {
4783   PetscErrorCode ierr;
4784 
4785   PetscFunctionBegin;
4786   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4787   PetscValidPointer(celltypeLabel, 2);
4788   if (!dm->celltypeLabel) {ierr = DMPlexComputeCellTypes(dm);CHKERRQ(ierr);}
4789   *celltypeLabel = dm->celltypeLabel;
4790   PetscFunctionReturn(0);
4791 }
4792 
4793 /*@
4794   DMPlexGetCellType - Get the polytope type of a given cell
4795 
4796   Not Collective
4797 
4798   Input Parameter:
4799 + dm   - The DMPlex object
4800 - cell - The cell
4801 
4802   Output Parameter:
4803 . celltype - The polytope type of the cell
4804 
4805   Level: intermediate
4806 
4807 .seealso: DMPlexGetCellTypeLabel(), DMPlexGetDepthLabel(), DMPlexGetDepth()
4808 @*/
4809 PetscErrorCode DMPlexGetCellType(DM dm, PetscInt cell, DMPolytopeType *celltype)
4810 {
4811   DMLabel        label;
4812   PetscInt       ct;
4813   PetscErrorCode ierr;
4814 
4815   PetscFunctionBegin;
4816   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4817   PetscValidPointer(celltype, 3);
4818   ierr = DMPlexGetCellTypeLabel(dm, &label);CHKERRQ(ierr);
4819   ierr = DMLabelGetValue(label, cell, &ct);CHKERRQ(ierr);
4820   if (ct < 0) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Cell %D has not been assigned a cell type", cell);
4821   *celltype = (DMPolytopeType) ct;
4822   PetscFunctionReturn(0);
4823 }
4824 
4825 /*@
4826   DMPlexSetCellType - Set the polytope type of a given cell
4827 
4828   Not Collective
4829 
4830   Input Parameters:
4831 + dm   - The DMPlex object
4832 . cell - The cell
4833 - celltype - The polytope type of the cell
4834 
4835   Note: By default, cell types will be automatically computed using DMPlexComputeCellTypes() before this function
4836   is executed. This function will override the computed type. However, if automatic classification will not succeed
4837   and a user wants to manually specify all types, the classification must be disabled by calling
4838   DMCreaateLabel(dm, "celltype") before getting or setting any cell types.
4839 
4840   Level: advanced
4841 
4842 .seealso: DMPlexGetCellTypeLabel(), DMPlexGetDepthLabel(), DMPlexGetDepth(), DMPlexComputeCellTypes(), DMCreateLabel()
4843 @*/
4844 PetscErrorCode DMPlexSetCellType(DM dm, PetscInt cell, DMPolytopeType celltype)
4845 {
4846   DMLabel        label;
4847   PetscErrorCode ierr;
4848 
4849   PetscFunctionBegin;
4850   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4851   ierr = DMPlexGetCellTypeLabel(dm, &label);CHKERRQ(ierr);
4852   ierr = DMLabelSetValue(label, cell, celltype);CHKERRQ(ierr);
4853   PetscFunctionReturn(0);
4854 }
4855 
4856 PetscErrorCode DMCreateCoordinateDM_Plex(DM dm, DM *cdm)
4857 {
4858   PetscSection   section, s;
4859   Mat            m;
4860   PetscInt       maxHeight;
4861   PetscErrorCode ierr;
4862 
4863   PetscFunctionBegin;
4864   ierr = DMClone(dm, cdm);CHKERRQ(ierr);
4865   ierr = DMPlexGetMaxProjectionHeight(dm, &maxHeight);CHKERRQ(ierr);
4866   ierr = DMPlexSetMaxProjectionHeight(*cdm, maxHeight);CHKERRQ(ierr);
4867   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &section);CHKERRQ(ierr);
4868   ierr = DMSetLocalSection(*cdm, section);CHKERRQ(ierr);
4869   ierr = PetscSectionDestroy(&section);CHKERRQ(ierr);
4870   ierr = PetscSectionCreate(PETSC_COMM_SELF, &s);CHKERRQ(ierr);
4871   ierr = MatCreate(PETSC_COMM_SELF, &m);CHKERRQ(ierr);
4872   ierr = DMSetDefaultConstraints(*cdm, s, m);CHKERRQ(ierr);
4873   ierr = PetscSectionDestroy(&s);CHKERRQ(ierr);
4874   ierr = MatDestroy(&m);CHKERRQ(ierr);
4875 
4876   ierr = DMSetNumFields(*cdm, 1);CHKERRQ(ierr);
4877   ierr = DMCreateDS(*cdm);CHKERRQ(ierr);
4878   PetscFunctionReturn(0);
4879 }
4880 
4881 PetscErrorCode DMCreateCoordinateField_Plex(DM dm, DMField *field)
4882 {
4883   Vec            coordsLocal;
4884   DM             coordsDM;
4885   PetscErrorCode ierr;
4886 
4887   PetscFunctionBegin;
4888   *field = NULL;
4889   ierr = DMGetCoordinatesLocal(dm,&coordsLocal);CHKERRQ(ierr);
4890   ierr = DMGetCoordinateDM(dm,&coordsDM);CHKERRQ(ierr);
4891   if (coordsLocal && coordsDM) {
4892     ierr = DMFieldCreateDS(coordsDM, 0, coordsLocal, field);CHKERRQ(ierr);
4893   }
4894   PetscFunctionReturn(0);
4895 }
4896 
4897 /*@C
4898   DMPlexGetConeSection - Return a section which describes the layout of cone data
4899 
4900   Not Collective
4901 
4902   Input Parameters:
4903 . dm        - The DMPlex object
4904 
4905   Output Parameter:
4906 . section - The PetscSection object
4907 
4908   Level: developer
4909 
4910 .seealso: DMPlexGetSupportSection(), DMPlexGetCones(), DMPlexGetConeOrientations()
4911 @*/
4912 PetscErrorCode DMPlexGetConeSection(DM dm, PetscSection *section)
4913 {
4914   DM_Plex *mesh = (DM_Plex*) dm->data;
4915 
4916   PetscFunctionBegin;
4917   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4918   if (section) *section = mesh->coneSection;
4919   PetscFunctionReturn(0);
4920 }
4921 
4922 /*@C
4923   DMPlexGetSupportSection - Return a section which describes the layout of support data
4924 
4925   Not Collective
4926 
4927   Input Parameters:
4928 . dm        - The DMPlex object
4929 
4930   Output Parameter:
4931 . section - The PetscSection object
4932 
4933   Level: developer
4934 
4935 .seealso: DMPlexGetConeSection()
4936 @*/
4937 PetscErrorCode DMPlexGetSupportSection(DM dm, PetscSection *section)
4938 {
4939   DM_Plex *mesh = (DM_Plex*) dm->data;
4940 
4941   PetscFunctionBegin;
4942   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4943   if (section) *section = mesh->supportSection;
4944   PetscFunctionReturn(0);
4945 }
4946 
4947 /*@C
4948   DMPlexGetCones - Return cone data
4949 
4950   Not Collective
4951 
4952   Input Parameters:
4953 . dm        - The DMPlex object
4954 
4955   Output Parameter:
4956 . cones - The cone for each point
4957 
4958   Level: developer
4959 
4960 .seealso: DMPlexGetConeSection()
4961 @*/
4962 PetscErrorCode DMPlexGetCones(DM dm, PetscInt *cones[])
4963 {
4964   DM_Plex *mesh = (DM_Plex*) dm->data;
4965 
4966   PetscFunctionBegin;
4967   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4968   if (cones) *cones = mesh->cones;
4969   PetscFunctionReturn(0);
4970 }
4971 
4972 /*@C
4973   DMPlexGetConeOrientations - Return cone orientation data
4974 
4975   Not Collective
4976 
4977   Input Parameters:
4978 . dm        - The DMPlex object
4979 
4980   Output Parameter:
4981 . coneOrientations - The array of cone orientations for all points
4982 
4983   Level: developer
4984 
4985   Notes:
4986   The PetscSection returned by DMPlexGetConeSection() partitions coneOrientations into cone orientations of particular points as returned by DMPlexGetConeOrientation().
4987 
4988   The meaning of coneOrientations values is detailed in DMPlexGetConeOrientation().
4989 
4990 .seealso: DMPlexGetConeSection(), DMPlexGetConeOrientation()
4991 @*/
4992 PetscErrorCode DMPlexGetConeOrientations(DM dm, PetscInt *coneOrientations[])
4993 {
4994   DM_Plex *mesh = (DM_Plex*) dm->data;
4995 
4996   PetscFunctionBegin;
4997   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4998   if (coneOrientations) *coneOrientations = mesh->coneOrientations;
4999   PetscFunctionReturn(0);
5000 }
5001 
5002 /******************************** FEM Support **********************************/
5003 
5004 /*
5005  Returns number of components and tensor degree for the field.  For interpolated meshes, line should be a point
5006  representing a line in the section.
5007 */
5008 static PetscErrorCode PetscSectionFieldGetTensorDegree_Private(PetscSection section,PetscInt field,PetscInt line,PetscBool vertexchart,PetscInt *Nc,PetscInt *k)
5009 {
5010   PetscErrorCode ierr;
5011 
5012   PetscFunctionBeginHot;
5013   ierr = PetscSectionGetFieldComponents(section, field, Nc);CHKERRQ(ierr);
5014   if (line < 0) {
5015     *k = 0;
5016     *Nc = 0;
5017   } else if (vertexchart) {            /* If we only have a vertex chart, we must have degree k=1 */
5018     *k = 1;
5019   } else {                      /* Assume the full interpolated mesh is in the chart; lines in particular */
5020     /* An order k SEM disc has k-1 dofs on an edge */
5021     ierr = PetscSectionGetFieldDof(section, line, field, k);CHKERRQ(ierr);
5022     *k = *k / *Nc + 1;
5023   }
5024   PetscFunctionReturn(0);
5025 }
5026 
5027 /*@
5028 
5029   DMPlexSetClosurePermutationTensor - Create a permutation from the default (BFS) point ordering in the closure, to a
5030   lexicographic ordering over the tensor product cell (i.e., line, quad, hex, etc.), and set this permutation in the
5031   section provided (or the section of the DM).
5032 
5033   Input Parameters:
5034 + dm      - The DM
5035 . point   - Either a cell (highest dim point) or an edge (dim 1 point), or PETSC_DETERMINE
5036 - section - The PetscSection to reorder, or NULL for the default section
5037 
5038   Note: The point is used to determine the number of dofs/field on an edge. For SEM, this is related to the polynomial
5039   degree of the basis.
5040 
5041   Example:
5042   A typical interpolated single-quad mesh might order points as
5043 .vb
5044   [c0, v1, v2, v3, v4, e5, e6, e7, e8]
5045 
5046   v4 -- e6 -- v3
5047   |           |
5048   e7    c0    e8
5049   |           |
5050   v1 -- e5 -- v2
5051 .ve
5052 
5053   (There is no significance to the ordering described here.)  The default section for a Q3 quad might typically assign
5054   dofs in the order of points, e.g.,
5055 .vb
5056     c0 -> [0,1,2,3]
5057     v1 -> [4]
5058     ...
5059     e5 -> [8, 9]
5060 .ve
5061 
5062   which corresponds to the dofs
5063 .vb
5064     6   10  11  7
5065     13  2   3   15
5066     12  0   1   14
5067     4   8   9   5
5068 .ve
5069 
5070   The closure in BFS ordering works through height strata (cells, edges, vertices) to produce the ordering
5071 .vb
5072   0 1 2 3 8 9 14 15 11 10 13 12 4 5 7 6
5073 .ve
5074 
5075   After calling DMPlexSetClosurePermutationTensor(), the closure will be ordered lexicographically,
5076 .vb
5077    4 8 9 5 12 0 1 14 13 2 3 15 6 10 11 7
5078 .ve
5079 
5080   Level: developer
5081 
5082 .seealso: DMGetLocalSection(), PetscSectionSetClosurePermutation(), DMSetGlobalSection()
5083 @*/
5084 PetscErrorCode DMPlexSetClosurePermutationTensor(DM dm, PetscInt point, PetscSection section)
5085 {
5086   DMLabel        label;
5087   PetscInt       dim, depth = -1, eStart = -1, Nf;
5088   PetscBool      vertexchart;
5089   PetscErrorCode ierr;
5090 
5091   PetscFunctionBegin;
5092   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
5093   if (dim < 1) PetscFunctionReturn(0);
5094   if (point < 0) {
5095     PetscInt sStart,sEnd;
5096 
5097     ierr = DMPlexGetDepthStratum(dm, 1, &sStart, &sEnd);CHKERRQ(ierr);
5098     point = sEnd-sStart ? sStart : point;
5099   }
5100   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
5101   if (point >= 0) { ierr = DMLabelGetValue(label, point, &depth);CHKERRQ(ierr); }
5102   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
5103   if (depth == 1) {eStart = point;}
5104   else if  (depth == dim) {
5105     const PetscInt *cone;
5106 
5107     ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
5108     if (dim == 2) eStart = cone[0];
5109     else if (dim == 3) {
5110       const PetscInt *cone2;
5111       ierr = DMPlexGetCone(dm, cone[0], &cone2);CHKERRQ(ierr);
5112       eStart = cone2[0];
5113     } else SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Point %D of depth %D cannot be used to bootstrap spectral ordering for dim %D", point, depth, dim);
5114   } else if (depth >= 0) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Point %D of depth %D cannot be used to bootstrap spectral ordering for dim %D", point, depth, dim);
5115   {                             /* Determine whether the chart covers all points or just vertices. */
5116     PetscInt pStart,pEnd,cStart,cEnd;
5117     ierr = DMPlexGetDepthStratum(dm,0,&pStart,&pEnd);CHKERRQ(ierr);
5118     ierr = PetscSectionGetChart(section,&cStart,&cEnd);CHKERRQ(ierr);
5119     if (pStart == cStart && pEnd == cEnd) vertexchart = PETSC_TRUE; /* Just vertices */
5120     else vertexchart = PETSC_FALSE;                                 /* Assume all interpolated points are in chart */
5121   }
5122   ierr = PetscSectionGetNumFields(section, &Nf);CHKERRQ(ierr);
5123   for (PetscInt d=1; d<=dim; d++) {
5124     PetscInt k, f, Nc, c, i, j, size = 0, offset = 0, foffset = 0;
5125     PetscInt *perm;
5126 
5127     for (f = 0; f < Nf; ++f) {
5128       ierr = PetscSectionFieldGetTensorDegree_Private(section,f,eStart,vertexchart,&Nc,&k);CHKERRQ(ierr);
5129       size += PetscPowInt(k+1, d)*Nc;
5130     }
5131     ierr = PetscMalloc1(size, &perm);CHKERRQ(ierr);
5132     for (f = 0; f < Nf; ++f) {
5133       switch (d) {
5134       case 1:
5135         ierr = PetscSectionFieldGetTensorDegree_Private(section,f,eStart,vertexchart,&Nc,&k);CHKERRQ(ierr);
5136         /*
5137          Original ordering is [ edge of length k-1; vtx0; vtx1 ]
5138          We want              [ vtx0; edge of length k-1; vtx1 ]
5139          */
5140         for (c=0; c<Nc; c++,offset++) perm[offset] = (k-1)*Nc + c + foffset;
5141         for (i=0; i<k-1; i++) for (c=0; c<Nc; c++,offset++) perm[offset] = i*Nc + c + foffset;
5142         for (c=0; c<Nc; c++,offset++) perm[offset] = k*Nc + c + foffset;
5143         foffset = offset;
5144         break;
5145       case 2:
5146         /* The original quad closure is oriented clockwise, {f, e_b, e_r, e_t, e_l, v_lb, v_rb, v_tr, v_tl} */
5147         ierr = PetscSectionFieldGetTensorDegree_Private(section,f,eStart,vertexchart,&Nc,&k);CHKERRQ(ierr);
5148         /* The SEM order is
5149 
5150          v_lb, {e_b}, v_rb,
5151          e^{(k-1)-i}_l, {f^{i*(k-1)}}, e^i_r,
5152          v_lt, reverse {e_t}, v_rt
5153          */
5154         {
5155           const PetscInt of   = 0;
5156           const PetscInt oeb  = of   + PetscSqr(k-1);
5157           const PetscInt oer  = oeb  + (k-1);
5158           const PetscInt oet  = oer  + (k-1);
5159           const PetscInt oel  = oet  + (k-1);
5160           const PetscInt ovlb = oel  + (k-1);
5161           const PetscInt ovrb = ovlb + 1;
5162           const PetscInt ovrt = ovrb + 1;
5163           const PetscInt ovlt = ovrt + 1;
5164           PetscInt       o;
5165 
5166           /* bottom */
5167           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovlb*Nc + c + foffset;
5168           for (o = oeb; o < oer; ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5169           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovrb*Nc + c + foffset;
5170           /* middle */
5171           for (i = 0; i < k-1; ++i) {
5172             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oel+(k-2)-i)*Nc + c + foffset;
5173             for (o = of+(k-1)*i; o < of+(k-1)*(i+1); ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5174             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oer+i)*Nc + c + foffset;
5175           }
5176           /* top */
5177           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovlt*Nc + c + foffset;
5178           for (o = oel-1; o >= oet; --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5179           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovrt*Nc + c + foffset;
5180           foffset = offset;
5181         }
5182         break;
5183       case 3:
5184         /* The original hex closure is
5185 
5186          {c,
5187          f_b, f_t, f_f, f_b, f_r, f_l,
5188          e_bl, e_bb, e_br, e_bf,  e_tf, e_tr, e_tb, e_tl,  e_rf, e_lf, e_lb, e_rb,
5189          v_blf, v_blb, v_brb, v_brf, v_tlf, v_trf, v_trb, v_tlb}
5190          */
5191         ierr = PetscSectionFieldGetTensorDegree_Private(section,f,eStart,vertexchart,&Nc,&k);CHKERRQ(ierr);
5192         /* The SEM order is
5193          Bottom Slice
5194          v_blf, {e^{(k-1)-n}_bf}, v_brf,
5195          e^{i}_bl, f^{n*(k-1)+(k-1)-i}_b, e^{(k-1)-i}_br,
5196          v_blb, {e_bb}, v_brb,
5197 
5198          Middle Slice (j)
5199          {e^{(k-1)-j}_lf}, {f^{j*(k-1)+n}_f}, e^j_rf,
5200          f^{i*(k-1)+j}_l, {c^{(j*(k-1) + i)*(k-1)+n}_t}, f^{j*(k-1)+i}_r,
5201          e^j_lb, {f^{j*(k-1)+(k-1)-n}_b}, e^{(k-1)-j}_rb,
5202 
5203          Top Slice
5204          v_tlf, {e_tf}, v_trf,
5205          e^{(k-1)-i}_tl, {f^{i*(k-1)}_t}, e^{i}_tr,
5206          v_tlb, {e^{(k-1)-n}_tb}, v_trb,
5207          */
5208         {
5209           const PetscInt oc    = 0;
5210           const PetscInt ofb   = oc    + PetscSqr(k-1)*(k-1);
5211           const PetscInt oft   = ofb   + PetscSqr(k-1);
5212           const PetscInt off   = oft   + PetscSqr(k-1);
5213           const PetscInt ofk   = off   + PetscSqr(k-1);
5214           const PetscInt ofr   = ofk   + PetscSqr(k-1);
5215           const PetscInt ofl   = ofr   + PetscSqr(k-1);
5216           const PetscInt oebl  = ofl   + PetscSqr(k-1);
5217           const PetscInt oebb  = oebl  + (k-1);
5218           const PetscInt oebr  = oebb  + (k-1);
5219           const PetscInt oebf  = oebr  + (k-1);
5220           const PetscInt oetf  = oebf  + (k-1);
5221           const PetscInt oetr  = oetf  + (k-1);
5222           const PetscInt oetb  = oetr  + (k-1);
5223           const PetscInt oetl  = oetb  + (k-1);
5224           const PetscInt oerf  = oetl  + (k-1);
5225           const PetscInt oelf  = oerf  + (k-1);
5226           const PetscInt oelb  = oelf  + (k-1);
5227           const PetscInt oerb  = oelb  + (k-1);
5228           const PetscInt ovblf = oerb  + (k-1);
5229           const PetscInt ovblb = ovblf + 1;
5230           const PetscInt ovbrb = ovblb + 1;
5231           const PetscInt ovbrf = ovbrb + 1;
5232           const PetscInt ovtlf = ovbrf + 1;
5233           const PetscInt ovtrf = ovtlf + 1;
5234           const PetscInt ovtrb = ovtrf + 1;
5235           const PetscInt ovtlb = ovtrb + 1;
5236           PetscInt       o, n;
5237 
5238           /* Bottom Slice */
5239           /*   bottom */
5240           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovblf*Nc + c + foffset;
5241           for (o = oetf-1; o >= oebf; --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5242           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovbrf*Nc + c + foffset;
5243           /*   middle */
5244           for (i = 0; i < k-1; ++i) {
5245             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oebl+i)*Nc + c + foffset;
5246             for (n = 0; n < k-1; ++n) {o = ofb+n*(k-1)+i; for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;}
5247             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oebr+(k-2)-i)*Nc + c + foffset;
5248           }
5249           /*   top */
5250           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovblb*Nc + c + foffset;
5251           for (o = oebb; o < oebr; ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5252           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovbrb*Nc + c + foffset;
5253 
5254           /* Middle Slice */
5255           for (j = 0; j < k-1; ++j) {
5256             /*   bottom */
5257             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oelf+(k-2)-j)*Nc + c + foffset;
5258             for (o = off+j*(k-1); o < off+(j+1)*(k-1); ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5259             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oerf+j)*Nc + c + foffset;
5260             /*   middle */
5261             for (i = 0; i < k-1; ++i) {
5262               for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (ofl+i*(k-1)+j)*Nc + c + foffset;
5263               for (n = 0; n < k-1; ++n) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oc+(j*(k-1)+i)*(k-1)+n)*Nc + c + foffset;
5264               for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (ofr+j*(k-1)+i)*Nc + c + foffset;
5265             }
5266             /*   top */
5267             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oelb+j)*Nc + c + foffset;
5268             for (o = ofk+j*(k-1)+(k-2); o >= ofk+j*(k-1); --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5269             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oerb+(k-2)-j)*Nc + c + foffset;
5270           }
5271 
5272           /* Top Slice */
5273           /*   bottom */
5274           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtlf*Nc + c + foffset;
5275           for (o = oetf; o < oetr; ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5276           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtrf*Nc + c + foffset;
5277           /*   middle */
5278           for (i = 0; i < k-1; ++i) {
5279             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oetl+(k-2)-i)*Nc + c + foffset;
5280             for (n = 0; n < k-1; ++n) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oft+i*(k-1)+n)*Nc + c + foffset;
5281             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oetr+i)*Nc + c + foffset;
5282           }
5283           /*   top */
5284           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtlb*Nc + c + foffset;
5285           for (o = oetl-1; o >= oetb; --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
5286           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtrb*Nc + c + foffset;
5287 
5288           foffset = offset;
5289         }
5290         break;
5291       default: SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_OUTOFRANGE, "No spectral ordering for dimension %D", d);
5292       }
5293     }
5294     if (offset != size) SETERRQ2(PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Number of permutation entries %D != %D", offset, size);
5295     /* Check permutation */
5296     {
5297       PetscInt *check;
5298 
5299       ierr = PetscMalloc1(size, &check);CHKERRQ(ierr);
5300       for (i = 0; i < size; ++i) {check[i] = -1; if (perm[i] < 0 || perm[i] >= size) SETERRQ2(PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Invalid permutation index p[%D] = %D", i, perm[i]);}
5301       for (i = 0; i < size; ++i) check[perm[i]] = i;
5302       for (i = 0; i < size; ++i) {if (check[i] < 0) SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Missing permutation index %D", i);}
5303       ierr = PetscFree(check);CHKERRQ(ierr);
5304     }
5305     ierr = PetscSectionSetClosurePermutation_Internal(section, (PetscObject) dm, d, size, PETSC_OWN_POINTER, perm);CHKERRQ(ierr);
5306   }
5307   PetscFunctionReturn(0);
5308 }
5309 
5310 PetscErrorCode DMPlexGetPointDualSpaceFEM(DM dm, PetscInt point, PetscInt field, PetscDualSpace *dspace)
5311 {
5312   PetscDS        prob;
5313   PetscInt       depth, Nf, h;
5314   DMLabel        label;
5315   PetscErrorCode ierr;
5316 
5317   PetscFunctionBeginHot;
5318   ierr = DMGetDS(dm, &prob);CHKERRQ(ierr);
5319   Nf      = prob->Nf;
5320   label   = dm->depthLabel;
5321   *dspace = NULL;
5322   if (field < Nf) {
5323     PetscObject disc = prob->disc[field];
5324 
5325     if (disc->classid == PETSCFE_CLASSID) {
5326       PetscDualSpace dsp;
5327 
5328       ierr = PetscFEGetDualSpace((PetscFE)disc,&dsp);CHKERRQ(ierr);
5329       ierr = DMLabelGetNumValues(label,&depth);CHKERRQ(ierr);
5330       ierr = DMLabelGetValue(label,point,&h);CHKERRQ(ierr);
5331       h    = depth - 1 - h;
5332       if (h) {
5333         ierr = PetscDualSpaceGetHeightSubspace(dsp,h,dspace);CHKERRQ(ierr);
5334       } else {
5335         *dspace = dsp;
5336       }
5337     }
5338   }
5339   PetscFunctionReturn(0);
5340 }
5341 
5342 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecGetClosure_Depth1_Static(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
5343 {
5344   PetscScalar    *array, *vArray;
5345   const PetscInt *cone, *coneO;
5346   PetscInt        pStart, pEnd, p, numPoints, size = 0, offset = 0;
5347   PetscErrorCode  ierr;
5348 
5349   PetscFunctionBeginHot;
5350   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
5351   ierr = DMPlexGetConeSize(dm, point, &numPoints);CHKERRQ(ierr);
5352   ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
5353   ierr = DMPlexGetConeOrientation(dm, point, &coneO);CHKERRQ(ierr);
5354   if (!values || !*values) {
5355     if ((point >= pStart) && (point < pEnd)) {
5356       PetscInt dof;
5357 
5358       ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
5359       size += dof;
5360     }
5361     for (p = 0; p < numPoints; ++p) {
5362       const PetscInt cp = cone[p];
5363       PetscInt       dof;
5364 
5365       if ((cp < pStart) || (cp >= pEnd)) continue;
5366       ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
5367       size += dof;
5368     }
5369     if (!values) {
5370       if (csize) *csize = size;
5371       PetscFunctionReturn(0);
5372     }
5373     ierr = DMGetWorkArray(dm, size, MPIU_SCALAR, &array);CHKERRQ(ierr);
5374   } else {
5375     array = *values;
5376   }
5377   size = 0;
5378   ierr = VecGetArray(v, &vArray);CHKERRQ(ierr);
5379   if ((point >= pStart) && (point < pEnd)) {
5380     PetscInt     dof, off, d;
5381     PetscScalar *varr;
5382 
5383     ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
5384     ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
5385     varr = &vArray[off];
5386     for (d = 0; d < dof; ++d, ++offset) {
5387       array[offset] = varr[d];
5388     }
5389     size += dof;
5390   }
5391   for (p = 0; p < numPoints; ++p) {
5392     const PetscInt cp = cone[p];
5393     PetscInt       o  = coneO[p];
5394     PetscInt       dof, off, d;
5395     PetscScalar   *varr;
5396 
5397     if ((cp < pStart) || (cp >= pEnd)) continue;
5398     ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
5399     ierr = PetscSectionGetOffset(section, cp, &off);CHKERRQ(ierr);
5400     varr = &vArray[off];
5401     if (o >= 0) {
5402       for (d = 0; d < dof; ++d, ++offset) {
5403         array[offset] = varr[d];
5404       }
5405     } else {
5406       for (d = dof-1; d >= 0; --d, ++offset) {
5407         array[offset] = varr[d];
5408       }
5409     }
5410     size += dof;
5411   }
5412   ierr = VecRestoreArray(v, &vArray);CHKERRQ(ierr);
5413   if (!*values) {
5414     if (csize) *csize = size;
5415     *values = array;
5416   } else {
5417     if (size > *csize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Size of input array %D < actual size %D", *csize, size);
5418     *csize = size;
5419   }
5420   PetscFunctionReturn(0);
5421 }
5422 
5423 /* Compress out points not in the section */
5424 PETSC_STATIC_INLINE PetscErrorCode CompressPoints_Private(PetscSection section, PetscInt *numPoints, PetscInt points[])
5425 {
5426   const PetscInt np = *numPoints;
5427   PetscInt       pStart, pEnd, p, q;
5428   PetscErrorCode ierr;
5429 
5430   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
5431   for (p = 0, q = 0; p < np; ++p) {
5432     const PetscInt r = points[p*2];
5433     if ((r >= pStart) && (r < pEnd)) {
5434       points[q*2]   = r;
5435       points[q*2+1] = points[p*2+1];
5436       ++q;
5437     }
5438   }
5439   *numPoints = q;
5440   return 0;
5441 }
5442 
5443 /* Compressed closure does not apply closure permutation */
5444 PetscErrorCode DMPlexGetCompressedClosure(DM dm, PetscSection section, PetscInt point, PetscInt *numPoints, PetscInt **points, PetscSection *clSec, IS *clPoints, const PetscInt **clp)
5445 {
5446   const PetscInt *cla = NULL;
5447   PetscInt       np, *pts = NULL;
5448   PetscErrorCode ierr;
5449 
5450   PetscFunctionBeginHot;
5451   ierr = PetscSectionGetClosureIndex(section, (PetscObject) dm, clSec, clPoints);CHKERRQ(ierr);
5452   if (*clPoints) {
5453     PetscInt dof, off;
5454 
5455     ierr = PetscSectionGetDof(*clSec, point, &dof);CHKERRQ(ierr);
5456     ierr = PetscSectionGetOffset(*clSec, point, &off);CHKERRQ(ierr);
5457     ierr = ISGetIndices(*clPoints, &cla);CHKERRQ(ierr);
5458     np   = dof/2;
5459     pts  = (PetscInt *) &cla[off];
5460   } else {
5461     ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &np, &pts);CHKERRQ(ierr);
5462     ierr = CompressPoints_Private(section, &np, pts);CHKERRQ(ierr);
5463   }
5464   *numPoints = np;
5465   *points    = pts;
5466   *clp       = cla;
5467   PetscFunctionReturn(0);
5468 }
5469 
5470 PetscErrorCode DMPlexRestoreCompressedClosure(DM dm, PetscSection section, PetscInt point, PetscInt *numPoints, PetscInt **points, PetscSection *clSec, IS *clPoints, const PetscInt **clp)
5471 {
5472   PetscErrorCode ierr;
5473 
5474   PetscFunctionBeginHot;
5475   if (!*clPoints) {
5476     ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, numPoints, points);CHKERRQ(ierr);
5477   } else {
5478     ierr = ISRestoreIndices(*clPoints, clp);CHKERRQ(ierr);
5479   }
5480   *numPoints = 0;
5481   *points    = NULL;
5482   *clSec     = NULL;
5483   *clPoints  = NULL;
5484   *clp       = NULL;
5485   PetscFunctionReturn(0);
5486 }
5487 
5488 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecGetClosure_Static(DM dm, PetscSection section, PetscInt numPoints, const PetscInt points[], const PetscInt clperm[], const PetscScalar vArray[], PetscInt *size, PetscScalar array[])
5489 {
5490   PetscInt          offset = 0, p;
5491   const PetscInt    **perms = NULL;
5492   const PetscScalar **flips = NULL;
5493   PetscErrorCode    ierr;
5494 
5495   PetscFunctionBeginHot;
5496   *size = 0;
5497   ierr = PetscSectionGetPointSyms(section,numPoints,points,&perms,&flips);CHKERRQ(ierr);
5498   for (p = 0; p < numPoints; p++) {
5499     const PetscInt    point = points[2*p];
5500     const PetscInt    *perm = perms ? perms[p] : NULL;
5501     const PetscScalar *flip = flips ? flips[p] : NULL;
5502     PetscInt          dof, off, d;
5503     const PetscScalar *varr;
5504 
5505     ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
5506     ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
5507     varr = &vArray[off];
5508     if (clperm) {
5509       if (perm) {
5510         for (d = 0; d < dof; d++) array[clperm[offset + perm[d]]]  = varr[d];
5511       } else {
5512         for (d = 0; d < dof; d++) array[clperm[offset +      d ]]  = varr[d];
5513       }
5514       if (flip) {
5515         for (d = 0; d < dof; d++) array[clperm[offset +      d ]] *= flip[d];
5516       }
5517     } else {
5518       if (perm) {
5519         for (d = 0; d < dof; d++) array[offset + perm[d]]  = varr[d];
5520       } else {
5521         for (d = 0; d < dof; d++) array[offset +      d ]  = varr[d];
5522       }
5523       if (flip) {
5524         for (d = 0; d < dof; d++) array[offset +      d ] *= flip[d];
5525       }
5526     }
5527     offset += dof;
5528   }
5529   ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms,&flips);CHKERRQ(ierr);
5530   *size = offset;
5531   PetscFunctionReturn(0);
5532 }
5533 
5534 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecGetClosure_Fields_Static(DM dm, PetscSection section, PetscInt numPoints, const PetscInt points[], PetscInt numFields, const PetscInt clperm[], const PetscScalar vArray[], PetscInt *size, PetscScalar array[])
5535 {
5536   PetscInt          offset = 0, f;
5537   PetscErrorCode    ierr;
5538 
5539   PetscFunctionBeginHot;
5540   *size = 0;
5541   for (f = 0; f < numFields; ++f) {
5542     PetscInt          p;
5543     const PetscInt    **perms = NULL;
5544     const PetscScalar **flips = NULL;
5545 
5546     ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
5547     for (p = 0; p < numPoints; p++) {
5548       const PetscInt    point = points[2*p];
5549       PetscInt          fdof, foff, b;
5550       const PetscScalar *varr;
5551       const PetscInt    *perm = perms ? perms[p] : NULL;
5552       const PetscScalar *flip = flips ? flips[p] : NULL;
5553 
5554       ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
5555       ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
5556       varr = &vArray[foff];
5557       if (clperm) {
5558         if (perm) {for (b = 0; b < fdof; b++) {array[clperm[offset + perm[b]]]  = varr[b];}}
5559         else      {for (b = 0; b < fdof; b++) {array[clperm[offset +      b ]]  = varr[b];}}
5560         if (flip) {for (b = 0; b < fdof; b++) {array[clperm[offset +      b ]] *= flip[b];}}
5561       } else {
5562         if (perm) {for (b = 0; b < fdof; b++) {array[offset + perm[b]]  = varr[b];}}
5563         else      {for (b = 0; b < fdof; b++) {array[offset +      b ]  = varr[b];}}
5564         if (flip) {for (b = 0; b < fdof; b++) {array[offset +      b ] *= flip[b];}}
5565       }
5566       offset += fdof;
5567     }
5568     ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
5569   }
5570   *size = offset;
5571   PetscFunctionReturn(0);
5572 }
5573 
5574 /*@C
5575   DMPlexVecGetClosure - Get an array of the values on the closure of 'point'
5576 
5577   Not collective
5578 
5579   Input Parameters:
5580 + dm - The DM
5581 . section - The section describing the layout in v, or NULL to use the default section
5582 . v - The local vector
5583 . point - The point in the DM
5584 . csize - The size of the input values array, or NULL
5585 - values - An array to use for the values, or NULL to have it allocated automatically
5586 
5587   Output Parameters:
5588 + csize - The number of values in the closure
5589 - values - The array of values. If the user provided NULL, it is a borrowed array and should not be freed
5590 
5591 $ Note that DMPlexVecGetClosure/DMPlexVecRestoreClosure only allocates the values array if it set to NULL in the
5592 $ calling function. This is because DMPlexVecGetClosure() is typically called in the inner loop of a Vec or Mat
5593 $ assembly function, and a user may already have allocated storage for this operation.
5594 $
5595 $ A typical use could be
5596 $
5597 $  values = NULL;
5598 $  ierr = DMPlexVecGetClosure(dm, NULL, v, p, &clSize, &values);CHKERRQ(ierr);
5599 $  for (cl = 0; cl < clSize; ++cl) {
5600 $    <Compute on closure>
5601 $  }
5602 $  ierr = DMPlexVecRestoreClosure(dm, NULL, v, p, &clSize, &values);CHKERRQ(ierr);
5603 $
5604 $ or
5605 $
5606 $  PetscMalloc1(clMaxSize, &values);
5607 $  for (p = pStart; p < pEnd; ++p) {
5608 $    clSize = clMaxSize;
5609 $    ierr = DMPlexVecGetClosure(dm, NULL, v, p, &clSize, &values);CHKERRQ(ierr);
5610 $    for (cl = 0; cl < clSize; ++cl) {
5611 $      <Compute on closure>
5612 $    }
5613 $  }
5614 $  PetscFree(values);
5615 
5616   Fortran Notes:
5617   Since it returns an array, this routine is only available in Fortran 90, and you must
5618   include petsc.h90 in your code.
5619 
5620   The csize argument is not present in the Fortran 90 binding since it is internal to the array.
5621 
5622   Level: intermediate
5623 
5624 .seealso DMPlexVecRestoreClosure(), DMPlexVecSetClosure(), DMPlexMatSetClosure()
5625 @*/
5626 PetscErrorCode DMPlexVecGetClosure(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
5627 {
5628   PetscSection       clSection;
5629   IS                 clPoints;
5630   PetscInt          *points = NULL;
5631   const PetscInt    *clp, *perm;
5632   PetscInt           depth, numFields, numPoints, asize;
5633   PetscErrorCode     ierr;
5634 
5635   PetscFunctionBeginHot;
5636   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5637   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
5638   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
5639   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
5640   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
5641   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5642   if (depth == 1 && numFields < 2) {
5643     ierr = DMPlexVecGetClosure_Depth1_Static(dm, section, v, point, csize, values);CHKERRQ(ierr);
5644     PetscFunctionReturn(0);
5645   }
5646   /* Get points */
5647   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5648   /* Get sizes */
5649   asize = 0;
5650   for (PetscInt p = 0; p < numPoints*2; p += 2) {
5651     PetscInt dof;
5652     ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5653     asize += dof;
5654   }
5655   if (values) {
5656     const PetscScalar *vArray;
5657     PetscInt          size;
5658 
5659     if (*values) {
5660       if (PetscUnlikely(*csize < asize)) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Provided array size %D not sufficient to hold closure size %D", *csize, asize);
5661     } else {ierr = DMGetWorkArray(dm, asize, MPIU_SCALAR, values);CHKERRQ(ierr);}
5662     ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, depth, asize, &perm);CHKERRQ(ierr);
5663     ierr = VecGetArrayRead(v, &vArray);CHKERRQ(ierr);
5664     /* Get values */
5665     if (numFields > 0) {ierr = DMPlexVecGetClosure_Fields_Static(dm, section, numPoints, points, numFields, perm, vArray, &size, *values);CHKERRQ(ierr);}
5666     else               {ierr = DMPlexVecGetClosure_Static(dm, section, numPoints, points, perm, vArray, &size, *values);CHKERRQ(ierr);}
5667     if (PetscUnlikely(asize != size)) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Section size %D does not match Vec closure size %D", asize, size);
5668     /* Cleanup array */
5669     ierr = VecRestoreArrayRead(v, &vArray);CHKERRQ(ierr);
5670   }
5671   if (csize) *csize = asize;
5672   /* Cleanup points */
5673   ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5674   PetscFunctionReturn(0);
5675 }
5676 
5677 PetscErrorCode DMPlexVecGetClosureAtDepth_Internal(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt depth, PetscInt *csize, PetscScalar *values[])
5678 {
5679   DMLabel            depthLabel;
5680   PetscSection       clSection;
5681   IS                 clPoints;
5682   PetscScalar       *array;
5683   const PetscScalar *vArray;
5684   PetscInt          *points = NULL;
5685   const PetscInt    *clp, *perm = NULL;
5686   PetscInt           mdepth, numFields, numPoints, Np = 0, p, clsize, size;
5687   PetscErrorCode     ierr;
5688 
5689   PetscFunctionBeginHot;
5690   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5691   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
5692   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
5693   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
5694   ierr = DMPlexGetDepth(dm, &mdepth);CHKERRQ(ierr);
5695   ierr = DMPlexGetDepthLabel(dm, &depthLabel);CHKERRQ(ierr);
5696   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5697   if (mdepth == 1 && numFields < 2) {
5698     ierr = DMPlexVecGetClosure_Depth1_Static(dm, section, v, point, csize, values);CHKERRQ(ierr);
5699     PetscFunctionReturn(0);
5700   }
5701   /* Get points */
5702   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5703   for (clsize=0,p=0; p<Np; p++) {
5704     PetscInt dof;
5705     ierr = PetscSectionGetDof(section, points[2*p], &dof);CHKERRQ(ierr);
5706     clsize += dof;
5707   }
5708   ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, depth, clsize, &perm);CHKERRQ(ierr);
5709   /* Filter points */
5710   for (p = 0; p < numPoints*2; p += 2) {
5711     PetscInt dep;
5712 
5713     ierr = DMLabelGetValue(depthLabel, points[p], &dep);CHKERRQ(ierr);
5714     if (dep != depth) continue;
5715     points[Np*2+0] = points[p];
5716     points[Np*2+1] = points[p+1];
5717     ++Np;
5718   }
5719   /* Get array */
5720   if (!values || !*values) {
5721     PetscInt asize = 0, dof;
5722 
5723     for (p = 0; p < Np*2; p += 2) {
5724       ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5725       asize += dof;
5726     }
5727     if (!values) {
5728       ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5729       if (csize) *csize = asize;
5730       PetscFunctionReturn(0);
5731     }
5732     ierr = DMGetWorkArray(dm, asize, MPIU_SCALAR, &array);CHKERRQ(ierr);
5733   } else {
5734     array = *values;
5735   }
5736   ierr = VecGetArrayRead(v, &vArray);CHKERRQ(ierr);
5737   /* Get values */
5738   if (numFields > 0) {ierr = DMPlexVecGetClosure_Fields_Static(dm, section, Np, points, numFields, perm, vArray, &size, array);CHKERRQ(ierr);}
5739   else               {ierr = DMPlexVecGetClosure_Static(dm, section, Np, points, perm, vArray, &size, array);CHKERRQ(ierr);}
5740   /* Cleanup points */
5741   ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5742   /* Cleanup array */
5743   ierr = VecRestoreArrayRead(v, &vArray);CHKERRQ(ierr);
5744   if (!*values) {
5745     if (csize) *csize = size;
5746     *values = array;
5747   } else {
5748     if (size > *csize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Size of input array %D < actual size %D", *csize, size);
5749     *csize = size;
5750   }
5751   PetscFunctionReturn(0);
5752 }
5753 
5754 /*@C
5755   DMPlexVecRestoreClosure - Restore the array of the values on the closure of 'point'
5756 
5757   Not collective
5758 
5759   Input Parameters:
5760 + dm - The DM
5761 . section - The section describing the layout in v, or NULL to use the default section
5762 . v - The local vector
5763 . point - The point in the DM
5764 . csize - The number of values in the closure, or NULL
5765 - values - The array of values, which is a borrowed array and should not be freed
5766 
5767   Note that the array values are discarded and not copied back into v. In order to copy values back to v, use DMPlexVecSetClosure()
5768 
5769   Fortran Notes:
5770   Since it returns an array, this routine is only available in Fortran 90, and you must
5771   include petsc.h90 in your code.
5772 
5773   The csize argument is not present in the Fortran 90 binding since it is internal to the array.
5774 
5775   Level: intermediate
5776 
5777 .seealso DMPlexVecGetClosure(), DMPlexVecSetClosure(), DMPlexMatSetClosure()
5778 @*/
5779 PetscErrorCode DMPlexVecRestoreClosure(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
5780 {
5781   PetscInt       size = 0;
5782   PetscErrorCode ierr;
5783 
5784   PetscFunctionBegin;
5785   /* Should work without recalculating size */
5786   ierr = DMRestoreWorkArray(dm, size, MPIU_SCALAR, (void*) values);CHKERRQ(ierr);
5787   *values = NULL;
5788   PetscFunctionReturn(0);
5789 }
5790 
5791 PETSC_STATIC_INLINE void add   (PetscScalar *x, PetscScalar y) {*x += y;}
5792 PETSC_STATIC_INLINE void insert(PetscScalar *x, PetscScalar y) {*x  = y;}
5793 
5794 PETSC_STATIC_INLINE PetscErrorCode updatePoint_private(PetscSection section, PetscInt point, PetscInt dof, void (*fuse)(PetscScalar*, PetscScalar), PetscBool setBC, const PetscInt perm[], const PetscScalar flip[], const PetscInt clperm[], const PetscScalar values[], PetscInt offset, PetscScalar array[])
5795 {
5796   PetscInt        cdof;   /* The number of constraints on this point */
5797   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
5798   PetscScalar    *a;
5799   PetscInt        off, cind = 0, k;
5800   PetscErrorCode  ierr;
5801 
5802   PetscFunctionBegin;
5803   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
5804   ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
5805   a    = &array[off];
5806   if (!cdof || setBC) {
5807     if (clperm) {
5808       if (perm) {for (k = 0; k < dof; ++k) {fuse(&a[k], values[clperm[offset+perm[k]]] * (flip ? flip[perm[k]] : 1.));}}
5809       else      {for (k = 0; k < dof; ++k) {fuse(&a[k], values[clperm[offset+     k ]] * (flip ? flip[     k ] : 1.));}}
5810     } else {
5811       if (perm) {for (k = 0; k < dof; ++k) {fuse(&a[k], values[offset+perm[k]] * (flip ? flip[perm[k]] : 1.));}}
5812       else      {for (k = 0; k < dof; ++k) {fuse(&a[k], values[offset+     k ] * (flip ? flip[     k ] : 1.));}}
5813     }
5814   } else {
5815     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
5816     if (clperm) {
5817       if (perm) {for (k = 0; k < dof; ++k) {
5818           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
5819           fuse(&a[k], values[clperm[offset+perm[k]]] * (flip ? flip[perm[k]] : 1.));
5820         }
5821       } else {
5822         for (k = 0; k < dof; ++k) {
5823           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
5824           fuse(&a[k], values[clperm[offset+     k ]] * (flip ? flip[     k ] : 1.));
5825         }
5826       }
5827     } else {
5828       if (perm) {
5829         for (k = 0; k < dof; ++k) {
5830           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
5831           fuse(&a[k], values[offset+perm[k]] * (flip ? flip[perm[k]] : 1.));
5832         }
5833       } else {
5834         for (k = 0; k < dof; ++k) {
5835           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
5836           fuse(&a[k], values[offset+     k ] * (flip ? flip[     k ] : 1.));
5837         }
5838       }
5839     }
5840   }
5841   PetscFunctionReturn(0);
5842 }
5843 
5844 PETSC_STATIC_INLINE PetscErrorCode updatePointBC_private(PetscSection section, PetscInt point, PetscInt dof, void (*fuse)(PetscScalar*, PetscScalar), const PetscInt perm[], const PetscScalar flip[], const PetscInt clperm[], const PetscScalar values[], PetscInt offset, PetscScalar array[])
5845 {
5846   PetscInt        cdof;   /* The number of constraints on this point */
5847   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
5848   PetscScalar    *a;
5849   PetscInt        off, cind = 0, k;
5850   PetscErrorCode  ierr;
5851 
5852   PetscFunctionBegin;
5853   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
5854   ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
5855   a    = &array[off];
5856   if (cdof) {
5857     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
5858     if (clperm) {
5859       if (perm) {
5860         for (k = 0; k < dof; ++k) {
5861           if ((cind < cdof) && (k == cdofs[cind])) {
5862             fuse(&a[k], values[clperm[offset+perm[k]]] * (flip ? flip[perm[k]] : 1.));
5863             cind++;
5864           }
5865         }
5866       } else {
5867         for (k = 0; k < dof; ++k) {
5868           if ((cind < cdof) && (k == cdofs[cind])) {
5869             fuse(&a[k], values[clperm[offset+     k ]] * (flip ? flip[     k ] : 1.));
5870             cind++;
5871           }
5872         }
5873       }
5874     } else {
5875       if (perm) {
5876         for (k = 0; k < dof; ++k) {
5877           if ((cind < cdof) && (k == cdofs[cind])) {
5878             fuse(&a[k], values[offset+perm[k]] * (flip ? flip[perm[k]] : 1.));
5879             cind++;
5880           }
5881         }
5882       } else {
5883         for (k = 0; k < dof; ++k) {
5884           if ((cind < cdof) && (k == cdofs[cind])) {
5885             fuse(&a[k], values[offset+     k ] * (flip ? flip[     k ] : 1.));
5886             cind++;
5887           }
5888         }
5889       }
5890     }
5891   }
5892   PetscFunctionReturn(0);
5893 }
5894 
5895 PETSC_STATIC_INLINE PetscErrorCode updatePointFields_private(PetscSection section, PetscInt point, const PetscInt *perm, const PetscScalar *flip, PetscInt f, void (*fuse)(PetscScalar*, PetscScalar), PetscBool setBC, const PetscInt clperm[], const PetscScalar values[], PetscInt *offset, PetscScalar array[])
5896 {
5897   PetscScalar    *a;
5898   PetscInt        fdof, foff, fcdof, foffset = *offset;
5899   const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
5900   PetscInt        cind = 0, b;
5901   PetscErrorCode  ierr;
5902 
5903   PetscFunctionBegin;
5904   ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
5905   ierr = PetscSectionGetFieldConstraintDof(section, point, f, &fcdof);CHKERRQ(ierr);
5906   ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
5907   a    = &array[foff];
5908   if (!fcdof || setBC) {
5909     if (clperm) {
5910       if (perm) {for (b = 0; b < fdof; b++) {fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));}}
5911       else      {for (b = 0; b < fdof; b++) {fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));}}
5912     } else {
5913       if (perm) {for (b = 0; b < fdof; b++) {fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));}}
5914       else      {for (b = 0; b < fdof; b++) {fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));}}
5915     }
5916   } else {
5917     ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
5918     if (clperm) {
5919       if (perm) {
5920         for (b = 0; b < fdof; b++) {
5921           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
5922           fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));
5923         }
5924       } else {
5925         for (b = 0; b < fdof; b++) {
5926           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
5927           fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));
5928         }
5929       }
5930     } else {
5931       if (perm) {
5932         for (b = 0; b < fdof; b++) {
5933           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
5934           fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));
5935         }
5936       } else {
5937         for (b = 0; b < fdof; b++) {
5938           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
5939           fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));
5940         }
5941       }
5942     }
5943   }
5944   *offset += fdof;
5945   PetscFunctionReturn(0);
5946 }
5947 
5948 PETSC_STATIC_INLINE PetscErrorCode updatePointFieldsBC_private(PetscSection section, PetscInt point, const PetscInt perm[], const PetscScalar flip[], PetscInt f, PetscInt Ncc, const PetscInt comps[], void (*fuse)(PetscScalar*, PetscScalar), const PetscInt clperm[], const PetscScalar values[], PetscInt *offset, PetscScalar array[])
5949 {
5950   PetscScalar    *a;
5951   PetscInt        fdof, foff, fcdof, foffset = *offset;
5952   const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
5953   PetscInt        Nc, cind = 0, ncind = 0, b;
5954   PetscBool       ncSet, fcSet;
5955   PetscErrorCode  ierr;
5956 
5957   PetscFunctionBegin;
5958   ierr = PetscSectionGetFieldComponents(section, f, &Nc);CHKERRQ(ierr);
5959   ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
5960   ierr = PetscSectionGetFieldConstraintDof(section, point, f, &fcdof);CHKERRQ(ierr);
5961   ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
5962   a    = &array[foff];
5963   if (fcdof) {
5964     /* We just override fcdof and fcdofs with Ncc and comps */
5965     ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
5966     if (clperm) {
5967       if (perm) {
5968         if (comps) {
5969           for (b = 0; b < fdof; b++) {
5970             ncSet = fcSet = PETSC_FALSE;
5971             if (b%Nc == comps[ncind]) {ncind = (ncind+1)%Ncc; ncSet = PETSC_TRUE;}
5972             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
5973             if (ncSet && fcSet) {fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));}
5974           }
5975         } else {
5976           for (b = 0; b < fdof; b++) {
5977             if ((cind < fcdof) && (b == fcdofs[cind])) {
5978               fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));
5979               ++cind;
5980             }
5981           }
5982         }
5983       } else {
5984         if (comps) {
5985           for (b = 0; b < fdof; b++) {
5986             ncSet = fcSet = PETSC_FALSE;
5987             if (b%Nc == comps[ncind]) {ncind = (ncind+1)%Ncc; ncSet = PETSC_TRUE;}
5988             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
5989             if (ncSet && fcSet) {fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));}
5990           }
5991         } else {
5992           for (b = 0; b < fdof; b++) {
5993             if ((cind < fcdof) && (b == fcdofs[cind])) {
5994               fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));
5995               ++cind;
5996             }
5997           }
5998         }
5999       }
6000     } else {
6001       if (perm) {
6002         if (comps) {
6003           for (b = 0; b < fdof; b++) {
6004             ncSet = fcSet = PETSC_FALSE;
6005             if (b%Nc == comps[ncind]) {ncind = (ncind+1)%Ncc; ncSet = PETSC_TRUE;}
6006             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
6007             if (ncSet && fcSet) {fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));}
6008           }
6009         } else {
6010           for (b = 0; b < fdof; b++) {
6011             if ((cind < fcdof) && (b == fcdofs[cind])) {
6012               fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));
6013               ++cind;
6014             }
6015           }
6016         }
6017       } else {
6018         if (comps) {
6019           for (b = 0; b < fdof; b++) {
6020             ncSet = fcSet = PETSC_FALSE;
6021             if (b%Nc == comps[ncind]) {ncind = (ncind+1)%Ncc; ncSet = PETSC_TRUE;}
6022             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
6023             if (ncSet && fcSet) {fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));}
6024           }
6025         } else {
6026           for (b = 0; b < fdof; b++) {
6027             if ((cind < fcdof) && (b == fcdofs[cind])) {
6028               fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));
6029               ++cind;
6030             }
6031           }
6032         }
6033       }
6034     }
6035   }
6036   *offset += fdof;
6037   PetscFunctionReturn(0);
6038 }
6039 
6040 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecSetClosure_Depth1_Static(DM dm, PetscSection section, Vec v, PetscInt point, const PetscScalar values[], InsertMode mode)
6041 {
6042   PetscScalar    *array;
6043   const PetscInt *cone, *coneO;
6044   PetscInt        pStart, pEnd, p, numPoints, off, dof;
6045   PetscErrorCode  ierr;
6046 
6047   PetscFunctionBeginHot;
6048   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
6049   ierr = DMPlexGetConeSize(dm, point, &numPoints);CHKERRQ(ierr);
6050   ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
6051   ierr = DMPlexGetConeOrientation(dm, point, &coneO);CHKERRQ(ierr);
6052   ierr = VecGetArray(v, &array);CHKERRQ(ierr);
6053   for (p = 0, off = 0; p <= numPoints; ++p, off += dof) {
6054     const PetscInt cp = !p ? point : cone[p-1];
6055     const PetscInt o  = !p ? 0     : coneO[p-1];
6056 
6057     if ((cp < pStart) || (cp >= pEnd)) {dof = 0; continue;}
6058     ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
6059     /* ADD_VALUES */
6060     {
6061       const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
6062       PetscScalar    *a;
6063       PetscInt        cdof, coff, cind = 0, k;
6064 
6065       ierr = PetscSectionGetConstraintDof(section, cp, &cdof);CHKERRQ(ierr);
6066       ierr = PetscSectionGetOffset(section, cp, &coff);CHKERRQ(ierr);
6067       a    = &array[coff];
6068       if (!cdof) {
6069         if (o >= 0) {
6070           for (k = 0; k < dof; ++k) {
6071             a[k] += values[off+k];
6072           }
6073         } else {
6074           for (k = 0; k < dof; ++k) {
6075             a[k] += values[off+dof-k-1];
6076           }
6077         }
6078       } else {
6079         ierr = PetscSectionGetConstraintIndices(section, cp, &cdofs);CHKERRQ(ierr);
6080         if (o >= 0) {
6081           for (k = 0; k < dof; ++k) {
6082             if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
6083             a[k] += values[off+k];
6084           }
6085         } else {
6086           for (k = 0; k < dof; ++k) {
6087             if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
6088             a[k] += values[off+dof-k-1];
6089           }
6090         }
6091       }
6092     }
6093   }
6094   ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
6095   PetscFunctionReturn(0);
6096 }
6097 
6098 /*@C
6099   DMPlexVecSetClosure - Set an array of the values on the closure of 'point'
6100 
6101   Not collective
6102 
6103   Input Parameters:
6104 + dm - The DM
6105 . section - The section describing the layout in v, or NULL to use the default section
6106 . v - The local vector
6107 . point - The point in the DM
6108 . values - The array of values
6109 - mode - The insert mode. One of INSERT_ALL_VALUES, ADD_ALL_VALUES, INSERT_VALUES, ADD_VALUES, INSERT_BC_VALUES, and ADD_BC_VALUES,
6110          where INSERT_ALL_VALUES and ADD_ALL_VALUES also overwrite boundary conditions.
6111 
6112   Fortran Notes:
6113   This routine is only available in Fortran 90, and you must include petsc.h90 in your code.
6114 
6115   Level: intermediate
6116 
6117 .seealso DMPlexVecGetClosure(), DMPlexMatSetClosure()
6118 @*/
6119 PetscErrorCode DMPlexVecSetClosure(DM dm, PetscSection section, Vec v, PetscInt point, const PetscScalar values[], InsertMode mode)
6120 {
6121   PetscSection    clSection;
6122   IS              clPoints;
6123   PetscScalar    *array;
6124   PetscInt       *points = NULL;
6125   const PetscInt *clp, *clperm = NULL;
6126   PetscInt        depth, numFields, numPoints, p, clsize;
6127   PetscErrorCode  ierr;
6128 
6129   PetscFunctionBeginHot;
6130   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6131   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
6132   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
6133   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
6134   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
6135   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
6136   if (depth == 1 && numFields < 2 && mode == ADD_VALUES) {
6137     ierr = DMPlexVecSetClosure_Depth1_Static(dm, section, v, point, values, mode);CHKERRQ(ierr);
6138     PetscFunctionReturn(0);
6139   }
6140   /* Get points */
6141   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
6142   for (clsize=0,p=0; p<numPoints; p++) {
6143     PetscInt dof;
6144     ierr = PetscSectionGetDof(section, points[2*p], &dof);CHKERRQ(ierr);
6145     clsize += dof;
6146   }
6147   ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, depth, clsize, &clperm);CHKERRQ(ierr);
6148   /* Get array */
6149   ierr = VecGetArray(v, &array);CHKERRQ(ierr);
6150   /* Get values */
6151   if (numFields > 0) {
6152     PetscInt offset = 0, f;
6153     for (f = 0; f < numFields; ++f) {
6154       const PetscInt    **perms = NULL;
6155       const PetscScalar **flips = NULL;
6156 
6157       ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
6158       switch (mode) {
6159       case INSERT_VALUES:
6160         for (p = 0; p < numPoints; p++) {
6161           const PetscInt    point = points[2*p];
6162           const PetscInt    *perm = perms ? perms[p] : NULL;
6163           const PetscScalar *flip = flips ? flips[p] : NULL;
6164           updatePointFields_private(section, point, perm, flip, f, insert, PETSC_FALSE, clperm, values, &offset, array);
6165         } break;
6166       case INSERT_ALL_VALUES:
6167         for (p = 0; p < numPoints; p++) {
6168           const PetscInt    point = points[2*p];
6169           const PetscInt    *perm = perms ? perms[p] : NULL;
6170           const PetscScalar *flip = flips ? flips[p] : NULL;
6171           updatePointFields_private(section, point, perm, flip, f, insert, PETSC_TRUE, clperm, values, &offset, array);
6172         } break;
6173       case INSERT_BC_VALUES:
6174         for (p = 0; p < numPoints; p++) {
6175           const PetscInt    point = points[2*p];
6176           const PetscInt    *perm = perms ? perms[p] : NULL;
6177           const PetscScalar *flip = flips ? flips[p] : NULL;
6178           updatePointFieldsBC_private(section, point, perm, flip, f, -1, NULL, insert, clperm, values, &offset, array);
6179         } break;
6180       case ADD_VALUES:
6181         for (p = 0; p < numPoints; p++) {
6182           const PetscInt    point = points[2*p];
6183           const PetscInt    *perm = perms ? perms[p] : NULL;
6184           const PetscScalar *flip = flips ? flips[p] : NULL;
6185           updatePointFields_private(section, point, perm, flip, f, add, PETSC_FALSE, clperm, values, &offset, array);
6186         } break;
6187       case ADD_ALL_VALUES:
6188         for (p = 0; p < numPoints; p++) {
6189           const PetscInt    point = points[2*p];
6190           const PetscInt    *perm = perms ? perms[p] : NULL;
6191           const PetscScalar *flip = flips ? flips[p] : NULL;
6192           updatePointFields_private(section, point, perm, flip, f, add, PETSC_TRUE, clperm, values, &offset, array);
6193         } break;
6194       case ADD_BC_VALUES:
6195         for (p = 0; p < numPoints; p++) {
6196           const PetscInt    point = points[2*p];
6197           const PetscInt    *perm = perms ? perms[p] : NULL;
6198           const PetscScalar *flip = flips ? flips[p] : NULL;
6199           updatePointFieldsBC_private(section, point, perm, flip, f, -1, NULL, add, clperm, values, &offset, array);
6200         } break;
6201       default:
6202         SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %d", mode);
6203       }
6204       ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
6205     }
6206   } else {
6207     PetscInt dof, off;
6208     const PetscInt    **perms = NULL;
6209     const PetscScalar **flips = NULL;
6210 
6211     ierr = PetscSectionGetPointSyms(section,numPoints,points,&perms,&flips);CHKERRQ(ierr);
6212     switch (mode) {
6213     case INSERT_VALUES:
6214       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
6215         const PetscInt    point = points[2*p];
6216         const PetscInt    *perm = perms ? perms[p] : NULL;
6217         const PetscScalar *flip = flips ? flips[p] : NULL;
6218         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
6219         updatePoint_private(section, point, dof, insert, PETSC_FALSE, perm, flip, clperm, values, off, array);
6220       } break;
6221     case INSERT_ALL_VALUES:
6222       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
6223         const PetscInt    point = points[2*p];
6224         const PetscInt    *perm = perms ? perms[p] : NULL;
6225         const PetscScalar *flip = flips ? flips[p] : NULL;
6226         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
6227         updatePoint_private(section, point, dof, insert, PETSC_TRUE,  perm, flip, clperm, values, off, array);
6228       } break;
6229     case INSERT_BC_VALUES:
6230       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
6231         const PetscInt    point = points[2*p];
6232         const PetscInt    *perm = perms ? perms[p] : NULL;
6233         const PetscScalar *flip = flips ? flips[p] : NULL;
6234         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
6235         updatePointBC_private(section, point, dof, insert,  perm, flip, clperm, values, off, array);
6236       } break;
6237     case ADD_VALUES:
6238       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
6239         const PetscInt    point = points[2*p];
6240         const PetscInt    *perm = perms ? perms[p] : NULL;
6241         const PetscScalar *flip = flips ? flips[p] : NULL;
6242         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
6243         updatePoint_private(section, point, dof, add,    PETSC_FALSE, perm, flip, clperm, values, off, array);
6244       } break;
6245     case ADD_ALL_VALUES:
6246       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
6247         const PetscInt    point = points[2*p];
6248         const PetscInt    *perm = perms ? perms[p] : NULL;
6249         const PetscScalar *flip = flips ? flips[p] : NULL;
6250         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
6251         updatePoint_private(section, point, dof, add,    PETSC_TRUE,  perm, flip, clperm, values, off, array);
6252       } break;
6253     case ADD_BC_VALUES:
6254       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
6255         const PetscInt    point = points[2*p];
6256         const PetscInt    *perm = perms ? perms[p] : NULL;
6257         const PetscScalar *flip = flips ? flips[p] : NULL;
6258         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
6259         updatePointBC_private(section, point, dof, add,  perm, flip, clperm, values, off, array);
6260       } break;
6261     default:
6262       SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %d", mode);
6263     }
6264     ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms,&flips);CHKERRQ(ierr);
6265   }
6266   /* Cleanup points */
6267   ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
6268   /* Cleanup array */
6269   ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
6270   PetscFunctionReturn(0);
6271 }
6272 
6273 /* Check whether the given point is in the label. If not, update the offset to skip this point */
6274 PETSC_STATIC_INLINE PetscErrorCode CheckPoint_Private(DMLabel label, PetscInt labelId, PetscSection section, PetscInt point, PetscInt f, PetscInt *offset)
6275 {
6276   PetscFunctionBegin;
6277   if (label) {
6278     PetscInt       val, fdof;
6279     PetscErrorCode ierr;
6280 
6281     /* There is a problem with this:
6282          Suppose we have two label values, defining surfaces, interecting along a line in 3D. When we add cells to the label, the cells that
6283        touch both surfaces must pick a label value. Thus we miss setting values for the surface with that other value intersecting that cell.
6284        Thus I am only going to check val != -1, not val != labelId
6285     */
6286     ierr = DMLabelGetValue(label, point, &val);CHKERRQ(ierr);
6287     if (val < 0) {
6288       ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
6289       *offset += fdof;
6290       PetscFunctionReturn(1);
6291     }
6292   }
6293   PetscFunctionReturn(0);
6294 }
6295 
6296 /* Unlike DMPlexVecSetClosure(), this uses plex-native closure permutation, not a user-specified permutation such as DMPlexSetClosurePermutationTensor(). */
6297 PetscErrorCode DMPlexVecSetFieldClosure_Internal(DM dm, PetscSection section, Vec v, PetscBool fieldActive[], PetscInt point, PetscInt Ncc, const PetscInt comps[], DMLabel label, PetscInt labelId, const PetscScalar values[], InsertMode mode)
6298 {
6299   PetscSection      clSection;
6300   IS                clPoints;
6301   PetscScalar       *array;
6302   PetscInt          *points = NULL;
6303   const PetscInt    *clp;
6304   PetscInt          numFields, numPoints, p;
6305   PetscInt          offset = 0, f;
6306   PetscErrorCode    ierr;
6307 
6308   PetscFunctionBeginHot;
6309   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6310   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
6311   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
6312   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
6313   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
6314   /* Get points */
6315   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
6316   /* Get array */
6317   ierr = VecGetArray(v, &array);CHKERRQ(ierr);
6318   /* Get values */
6319   for (f = 0; f < numFields; ++f) {
6320     const PetscInt    **perms = NULL;
6321     const PetscScalar **flips = NULL;
6322 
6323     if (!fieldActive[f]) {
6324       for (p = 0; p < numPoints*2; p += 2) {
6325         PetscInt fdof;
6326         ierr = PetscSectionGetFieldDof(section, points[p], f, &fdof);CHKERRQ(ierr);
6327         offset += fdof;
6328       }
6329       continue;
6330     }
6331     ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
6332     switch (mode) {
6333     case INSERT_VALUES:
6334       for (p = 0; p < numPoints; p++) {
6335         const PetscInt    point = points[2*p];
6336         const PetscInt    *perm = perms ? perms[p] : NULL;
6337         const PetscScalar *flip = flips ? flips[p] : NULL;
6338         ierr = CheckPoint_Private(label, labelId, section, point, f, &offset); if (ierr) continue;
6339         updatePointFields_private(section, point, perm, flip, f, insert, PETSC_FALSE, NULL, values, &offset, array);
6340       } break;
6341     case INSERT_ALL_VALUES:
6342       for (p = 0; p < numPoints; p++) {
6343         const PetscInt    point = points[2*p];
6344         const PetscInt    *perm = perms ? perms[p] : NULL;
6345         const PetscScalar *flip = flips ? flips[p] : NULL;
6346         ierr = CheckPoint_Private(label, labelId, section, point, f, &offset); if (ierr) continue;
6347         updatePointFields_private(section, point, perm, flip, f, insert, PETSC_TRUE, NULL, values, &offset, array);
6348       } break;
6349     case INSERT_BC_VALUES:
6350       for (p = 0; p < numPoints; p++) {
6351         const PetscInt    point = points[2*p];
6352         const PetscInt    *perm = perms ? perms[p] : NULL;
6353         const PetscScalar *flip = flips ? flips[p] : NULL;
6354         ierr = CheckPoint_Private(label, labelId, section, point, f, &offset); if (ierr) continue;
6355         updatePointFieldsBC_private(section, point, perm, flip, f, Ncc, comps, insert, NULL, values, &offset, array);
6356       } break;
6357     case ADD_VALUES:
6358       for (p = 0; p < numPoints; p++) {
6359         const PetscInt    point = points[2*p];
6360         const PetscInt    *perm = perms ? perms[p] : NULL;
6361         const PetscScalar *flip = flips ? flips[p] : NULL;
6362         ierr = CheckPoint_Private(label, labelId, section, point, f, &offset); if (ierr) continue;
6363         updatePointFields_private(section, point, perm, flip, f, add, PETSC_FALSE, NULL, values, &offset, array);
6364       } break;
6365     case ADD_ALL_VALUES:
6366       for (p = 0; p < numPoints; p++) {
6367         const PetscInt    point = points[2*p];
6368         const PetscInt    *perm = perms ? perms[p] : NULL;
6369         const PetscScalar *flip = flips ? flips[p] : NULL;
6370         ierr = CheckPoint_Private(label, labelId, section, point, f, &offset); if (ierr) continue;
6371         updatePointFields_private(section, point, perm, flip, f, add, PETSC_TRUE, NULL, values, &offset, array);
6372       } break;
6373     default:
6374       SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %d", mode);
6375     }
6376     ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
6377   }
6378   /* Cleanup points */
6379   ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
6380   /* Cleanup array */
6381   ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
6382   PetscFunctionReturn(0);
6383 }
6384 
6385 static PetscErrorCode DMPlexPrintMatSetValues(PetscViewer viewer, Mat A, PetscInt point, PetscInt numRIndices, const PetscInt rindices[], PetscInt numCIndices, const PetscInt cindices[], const PetscScalar values[])
6386 {
6387   PetscMPIInt    rank;
6388   PetscInt       i, j;
6389   PetscErrorCode ierr;
6390 
6391   PetscFunctionBegin;
6392   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRMPI(ierr);
6393   ierr = PetscViewerASCIIPrintf(viewer, "[%d]mat for point %D\n", rank, point);CHKERRQ(ierr);
6394   for (i = 0; i < numRIndices; i++) {ierr = PetscViewerASCIIPrintf(viewer, "[%d]mat row indices[%D] = %D\n", rank, i, rindices[i]);CHKERRQ(ierr);}
6395   for (i = 0; i < numCIndices; i++) {ierr = PetscViewerASCIIPrintf(viewer, "[%d]mat col indices[%D] = %D\n", rank, i, cindices[i]);CHKERRQ(ierr);}
6396   numCIndices = numCIndices ? numCIndices : numRIndices;
6397   if (!values) PetscFunctionReturn(0);
6398   for (i = 0; i < numRIndices; i++) {
6399     ierr = PetscViewerASCIIPrintf(viewer, "[%d]", rank);CHKERRQ(ierr);
6400     for (j = 0; j < numCIndices; j++) {
6401 #if defined(PETSC_USE_COMPLEX)
6402       ierr = PetscViewerASCIIPrintf(viewer, " (%g,%g)", (double)PetscRealPart(values[i*numCIndices+j]), (double)PetscImaginaryPart(values[i*numCIndices+j]));CHKERRQ(ierr);
6403 #else
6404       ierr = PetscViewerASCIIPrintf(viewer, " %g", (double)values[i*numCIndices+j]);CHKERRQ(ierr);
6405 #endif
6406     }
6407     ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
6408   }
6409   PetscFunctionReturn(0);
6410 }
6411 
6412 /*
6413   DMPlexGetIndicesPoint_Internal - Add the indices for dofs on a point to an index array
6414 
6415   Input Parameters:
6416 + section - The section for this data layout
6417 . islocal - Is the section (and thus indices being requested) local or global?
6418 . point   - The point contributing dofs with these indices
6419 . off     - The global offset of this point
6420 . loff    - The local offset of each field
6421 . setBC   - The flag determining whether to include indices of bounsary values
6422 . perm    - A permutation of the dofs on this point, or NULL
6423 - indperm - A permutation of the entire indices array, or NULL
6424 
6425   Output Parameter:
6426 . indices - Indices for dofs on this point
6427 
6428   Level: developer
6429 
6430   Note: The indices could be local or global, depending on the value of 'off'.
6431 */
6432 PetscErrorCode DMPlexGetIndicesPoint_Internal(PetscSection section, PetscBool islocal,PetscInt point, PetscInt off, PetscInt *loff, PetscBool setBC, const PetscInt perm[], const PetscInt indperm[], PetscInt indices[])
6433 {
6434   PetscInt        dof;   /* The number of unknowns on this point */
6435   PetscInt        cdof;  /* The number of constraints on this point */
6436   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
6437   PetscInt        cind = 0, k;
6438   PetscErrorCode  ierr;
6439 
6440   PetscFunctionBegin;
6441   if (!islocal && setBC) SETERRQ(PetscObjectComm((PetscObject)section),PETSC_ERR_ARG_INCOMP,"setBC incompatible with global indices; use a local section or disable setBC");
6442   ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
6443   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
6444   if (!cdof || setBC) {
6445     for (k = 0; k < dof; ++k) {
6446       const PetscInt preind = perm ? *loff+perm[k] : *loff+k;
6447       const PetscInt ind    = indperm ? indperm[preind] : preind;
6448 
6449       indices[ind] = off + k;
6450     }
6451   } else {
6452     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
6453     for (k = 0; k < dof; ++k) {
6454       const PetscInt preind = perm ? *loff+perm[k] : *loff+k;
6455       const PetscInt ind    = indperm ? indperm[preind] : preind;
6456 
6457       if ((cind < cdof) && (k == cdofs[cind])) {
6458         /* Insert check for returning constrained indices */
6459         indices[ind] = -(off+k+1);
6460         ++cind;
6461       } else {
6462         indices[ind] = off + k - (islocal ? 0 : cind);
6463       }
6464     }
6465   }
6466   *loff += dof;
6467   PetscFunctionReturn(0);
6468 }
6469 
6470 /*
6471  DMPlexGetIndicesPointFields_Internal - gets section indices for a point in its canonical ordering.
6472 
6473  Input Parameters:
6474 + section - a section (global or local)
6475 - islocal - PETSC_TRUE if requesting local indices (i.e., section is local); PETSC_FALSE for global
6476 . point - point within section
6477 . off - The offset of this point in the (local or global) indexed space - should match islocal and (usually) the section
6478 . foffs - array of length numFields containing the offset in canonical point ordering (the location in indices) of each field
6479 . setBC - identify constrained (boundary condition) points via involution.
6480 . perms - perms[f][permsoff][:] is a permutation of dofs within each field
6481 . permsoff - offset
6482 - indperm - index permutation
6483 
6484  Output Parameter:
6485 . foffs - each entry is incremented by the number of (unconstrained if setBC=FALSE) dofs in that field
6486 . indices - array to hold indices (as defined by section) of each dof associated with point
6487 
6488  Notes:
6489  If section is local and setBC=true, there is no distinction between constrained and unconstrained dofs.
6490  If section is local and setBC=false, the indices for constrained points are the involution -(i+1) of their position
6491  in the local vector.
6492 
6493  If section is global and setBC=false, the indices for constrained points are negative (and their value is not
6494  significant).  It is invalid to call with a global section and setBC=true.
6495 
6496  Developer Note:
6497  The section is only used for field layout, so islocal is technically a statement about the offset (off).  At some point
6498  in the future, global sections may have fields set, in which case we could pass the global section and obtain the
6499  offset could be obtained from the section instead of passing it explicitly as we do now.
6500 
6501  Example:
6502  Suppose a point contains one field with three components, and for which the unconstrained indices are {10, 11, 12}.
6503  When the middle component is constrained, we get the array {10, -12, 12} for (islocal=TRUE, setBC=FALSE).
6504  Note that -12 is the involution of 11, so the user can involute negative indices to recover local indices.
6505  The global vector does not store constrained dofs, so when this function returns global indices, say {110, -112, 111}, the value of -112 is an arbitrary flag that should not be interpreted beyond its sign.
6506 
6507  Level: developer
6508 */
6509 PetscErrorCode DMPlexGetIndicesPointFields_Internal(PetscSection section, PetscBool islocal, PetscInt point, PetscInt off, PetscInt foffs[], PetscBool setBC, const PetscInt ***perms, PetscInt permsoff, const PetscInt indperm[], PetscInt indices[])
6510 {
6511   PetscInt       numFields, foff, f;
6512   PetscErrorCode ierr;
6513 
6514   PetscFunctionBegin;
6515   if (!islocal && setBC) SETERRQ(PetscObjectComm((PetscObject)section),PETSC_ERR_ARG_INCOMP,"setBC incompatible with global indices; use a local section or disable setBC");
6516   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
6517   for (f = 0, foff = 0; f < numFields; ++f) {
6518     PetscInt        fdof, cfdof;
6519     const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
6520     PetscInt        cind = 0, b;
6521     const PetscInt  *perm = (perms && perms[f]) ? perms[f][permsoff] : NULL;
6522 
6523     ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
6524     ierr = PetscSectionGetFieldConstraintDof(section, point, f, &cfdof);CHKERRQ(ierr);
6525     if (!cfdof || setBC) {
6526       for (b = 0; b < fdof; ++b) {
6527         const PetscInt preind = perm ? foffs[f]+perm[b] : foffs[f]+b;
6528         const PetscInt ind    = indperm ? indperm[preind] : preind;
6529 
6530         indices[ind] = off+foff+b;
6531       }
6532     } else {
6533       ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
6534       for (b = 0; b < fdof; ++b) {
6535         const PetscInt preind = perm ? foffs[f]+perm[b] : foffs[f]+b;
6536         const PetscInt ind    = indperm ? indperm[preind] : preind;
6537 
6538         if ((cind < cfdof) && (b == fcdofs[cind])) {
6539           indices[ind] = -(off+foff+b+1);
6540           ++cind;
6541         } else {
6542           indices[ind] = off + foff + b - (islocal ? 0 : cind);
6543         }
6544       }
6545     }
6546     foff     += (setBC || islocal ? fdof : (fdof - cfdof));
6547     foffs[f] += fdof;
6548   }
6549   PetscFunctionReturn(0);
6550 }
6551 
6552 /*
6553   This version believes the globalSection offsets for each field, rather than just the point offset
6554 
6555  . foffs - The offset into 'indices' for each field, since it is segregated by field
6556 
6557  Notes:
6558  The semantics of this function relate to that of setBC=FALSE in DMPlexGetIndicesPointFields_Internal.
6559  Since this function uses global indices, setBC=TRUE would be invalid, so no such argument exists.
6560 */
6561 static PetscErrorCode DMPlexGetIndicesPointFieldsSplit_Internal(PetscSection section, PetscSection globalSection, PetscInt point, PetscInt foffs[], const PetscInt ***perms, PetscInt permsoff, const PetscInt indperm[], PetscInt indices[])
6562 {
6563   PetscInt       numFields, foff, f;
6564   PetscErrorCode ierr;
6565 
6566   PetscFunctionBegin;
6567   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
6568   for (f = 0; f < numFields; ++f) {
6569     PetscInt        fdof, cfdof;
6570     const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
6571     PetscInt        cind = 0, b;
6572     const PetscInt  *perm = (perms && perms[f]) ? perms[f][permsoff] : NULL;
6573 
6574     ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
6575     ierr = PetscSectionGetFieldConstraintDof(section, point, f, &cfdof);CHKERRQ(ierr);
6576     ierr = PetscSectionGetFieldOffset(globalSection, point, f, &foff);CHKERRQ(ierr);
6577     if (!cfdof) {
6578       for (b = 0; b < fdof; ++b) {
6579         const PetscInt preind = perm ? foffs[f]+perm[b] : foffs[f]+b;
6580         const PetscInt ind    = indperm ? indperm[preind] : preind;
6581 
6582         indices[ind] = foff+b;
6583       }
6584     } else {
6585       ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
6586       for (b = 0; b < fdof; ++b) {
6587         const PetscInt preind = perm ? foffs[f]+perm[b] : foffs[f]+b;
6588         const PetscInt ind    = indperm ? indperm[preind] : preind;
6589 
6590         if ((cind < cfdof) && (b == fcdofs[cind])) {
6591           indices[ind] = -(foff+b+1);
6592           ++cind;
6593         } else {
6594           indices[ind] = foff+b-cind;
6595         }
6596       }
6597     }
6598     foffs[f] += fdof;
6599   }
6600   PetscFunctionReturn(0);
6601 }
6602 
6603 PetscErrorCode DMPlexAnchorsModifyMat(DM dm, PetscSection section, PetscInt numPoints, PetscInt numIndices, const PetscInt points[], const PetscInt ***perms, const PetscScalar values[], PetscInt *outNumPoints, PetscInt *outNumIndices, PetscInt *outPoints[], PetscScalar *outValues[], PetscInt offsets[], PetscBool multiplyLeft)
6604 {
6605   Mat             cMat;
6606   PetscSection    aSec, cSec;
6607   IS              aIS;
6608   PetscInt        aStart = -1, aEnd = -1;
6609   const PetscInt  *anchors;
6610   PetscInt        numFields, f, p, q, newP = 0;
6611   PetscInt        newNumPoints = 0, newNumIndices = 0;
6612   PetscInt        *newPoints, *indices, *newIndices;
6613   PetscInt        maxAnchor, maxDof;
6614   PetscInt        newOffsets[32];
6615   PetscInt        *pointMatOffsets[32];
6616   PetscInt        *newPointOffsets[32];
6617   PetscScalar     *pointMat[32];
6618   PetscScalar     *newValues=NULL,*tmpValues;
6619   PetscBool       anyConstrained = PETSC_FALSE;
6620   PetscErrorCode  ierr;
6621 
6622   PetscFunctionBegin;
6623   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6624   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
6625   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
6626 
6627   ierr = DMPlexGetAnchors(dm,&aSec,&aIS);CHKERRQ(ierr);
6628   /* if there are point-to-point constraints */
6629   if (aSec) {
6630     ierr = PetscArrayzero(newOffsets, 32);CHKERRQ(ierr);
6631     ierr = ISGetIndices(aIS,&anchors);CHKERRQ(ierr);
6632     ierr = PetscSectionGetChart(aSec,&aStart,&aEnd);CHKERRQ(ierr);
6633     /* figure out how many points are going to be in the new element matrix
6634      * (we allow double counting, because it's all just going to be summed
6635      * into the global matrix anyway) */
6636     for (p = 0; p < 2*numPoints; p+=2) {
6637       PetscInt b    = points[p];
6638       PetscInt bDof = 0, bSecDof;
6639 
6640       ierr = PetscSectionGetDof(section,b,&bSecDof);CHKERRQ(ierr);
6641       if (!bSecDof) {
6642         continue;
6643       }
6644       if (b >= aStart && b < aEnd) {
6645         ierr = PetscSectionGetDof(aSec,b,&bDof);CHKERRQ(ierr);
6646       }
6647       if (bDof) {
6648         /* this point is constrained */
6649         /* it is going to be replaced by its anchors */
6650         PetscInt bOff, q;
6651 
6652         anyConstrained = PETSC_TRUE;
6653         newNumPoints  += bDof;
6654         ierr = PetscSectionGetOffset(aSec,b,&bOff);CHKERRQ(ierr);
6655         for (q = 0; q < bDof; q++) {
6656           PetscInt a = anchors[bOff + q];
6657           PetscInt aDof;
6658 
6659           ierr           = PetscSectionGetDof(section,a,&aDof);CHKERRQ(ierr);
6660           newNumIndices += aDof;
6661           for (f = 0; f < numFields; ++f) {
6662             PetscInt fDof;
6663 
6664             ierr             = PetscSectionGetFieldDof(section, a, f, &fDof);CHKERRQ(ierr);
6665             newOffsets[f+1] += fDof;
6666           }
6667         }
6668       }
6669       else {
6670         /* this point is not constrained */
6671         newNumPoints++;
6672         newNumIndices += bSecDof;
6673         for (f = 0; f < numFields; ++f) {
6674           PetscInt fDof;
6675 
6676           ierr = PetscSectionGetFieldDof(section, b, f, &fDof);CHKERRQ(ierr);
6677           newOffsets[f+1] += fDof;
6678         }
6679       }
6680     }
6681   }
6682   if (!anyConstrained) {
6683     if (outNumPoints)  *outNumPoints  = 0;
6684     if (outNumIndices) *outNumIndices = 0;
6685     if (outPoints)     *outPoints     = NULL;
6686     if (outValues)     *outValues     = NULL;
6687     if (aSec) {ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);}
6688     PetscFunctionReturn(0);
6689   }
6690 
6691   if (outNumPoints)  *outNumPoints  = newNumPoints;
6692   if (outNumIndices) *outNumIndices = newNumIndices;
6693 
6694   for (f = 0; f < numFields; ++f) newOffsets[f+1] += newOffsets[f];
6695 
6696   if (!outPoints && !outValues) {
6697     if (offsets) {
6698       for (f = 0; f <= numFields; f++) {
6699         offsets[f] = newOffsets[f];
6700       }
6701     }
6702     if (aSec) {ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);}
6703     PetscFunctionReturn(0);
6704   }
6705 
6706   if (numFields && newOffsets[numFields] != newNumIndices) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", newOffsets[numFields], newNumIndices);
6707 
6708   ierr = DMGetDefaultConstraints(dm, &cSec, &cMat);CHKERRQ(ierr);
6709 
6710   /* workspaces */
6711   if (numFields) {
6712     for (f = 0; f < numFields; f++) {
6713       ierr = DMGetWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[f]);CHKERRQ(ierr);
6714       ierr = DMGetWorkArray(dm,numPoints+1,MPIU_INT,&newPointOffsets[f]);CHKERRQ(ierr);
6715     }
6716   }
6717   else {
6718     ierr = DMGetWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[0]);CHKERRQ(ierr);
6719     ierr = DMGetWorkArray(dm,numPoints,MPIU_INT,&newPointOffsets[0]);CHKERRQ(ierr);
6720   }
6721 
6722   /* get workspaces for the point-to-point matrices */
6723   if (numFields) {
6724     PetscInt totalOffset, totalMatOffset;
6725 
6726     for (p = 0; p < numPoints; p++) {
6727       PetscInt b    = points[2*p];
6728       PetscInt bDof = 0, bSecDof;
6729 
6730       ierr = PetscSectionGetDof(section,b,&bSecDof);CHKERRQ(ierr);
6731       if (!bSecDof) {
6732         for (f = 0; f < numFields; f++) {
6733           newPointOffsets[f][p + 1] = 0;
6734           pointMatOffsets[f][p + 1] = 0;
6735         }
6736         continue;
6737       }
6738       if (b >= aStart && b < aEnd) {
6739         ierr = PetscSectionGetDof(aSec, b, &bDof);CHKERRQ(ierr);
6740       }
6741       if (bDof) {
6742         for (f = 0; f < numFields; f++) {
6743           PetscInt fDof, q, bOff, allFDof = 0;
6744 
6745           ierr = PetscSectionGetFieldDof(section, b, f, &fDof);CHKERRQ(ierr);
6746           ierr = PetscSectionGetOffset(aSec, b, &bOff);CHKERRQ(ierr);
6747           for (q = 0; q < bDof; q++) {
6748             PetscInt a = anchors[bOff + q];
6749             PetscInt aFDof;
6750 
6751             ierr     = PetscSectionGetFieldDof(section, a, f, &aFDof);CHKERRQ(ierr);
6752             allFDof += aFDof;
6753           }
6754           newPointOffsets[f][p+1] = allFDof;
6755           pointMatOffsets[f][p+1] = fDof * allFDof;
6756         }
6757       }
6758       else {
6759         for (f = 0; f < numFields; f++) {
6760           PetscInt fDof;
6761 
6762           ierr = PetscSectionGetFieldDof(section, b, f, &fDof);CHKERRQ(ierr);
6763           newPointOffsets[f][p+1] = fDof;
6764           pointMatOffsets[f][p+1] = 0;
6765         }
6766       }
6767     }
6768     for (f = 0, totalOffset = 0, totalMatOffset = 0; f < numFields; f++) {
6769       newPointOffsets[f][0] = totalOffset;
6770       pointMatOffsets[f][0] = totalMatOffset;
6771       for (p = 0; p < numPoints; p++) {
6772         newPointOffsets[f][p+1] += newPointOffsets[f][p];
6773         pointMatOffsets[f][p+1] += pointMatOffsets[f][p];
6774       }
6775       totalOffset    = newPointOffsets[f][numPoints];
6776       totalMatOffset = pointMatOffsets[f][numPoints];
6777       ierr = DMGetWorkArray(dm,pointMatOffsets[f][numPoints],MPIU_SCALAR,&pointMat[f]);CHKERRQ(ierr);
6778     }
6779   }
6780   else {
6781     for (p = 0; p < numPoints; p++) {
6782       PetscInt b    = points[2*p];
6783       PetscInt bDof = 0, bSecDof;
6784 
6785       ierr = PetscSectionGetDof(section,b,&bSecDof);CHKERRQ(ierr);
6786       if (!bSecDof) {
6787         newPointOffsets[0][p + 1] = 0;
6788         pointMatOffsets[0][p + 1] = 0;
6789         continue;
6790       }
6791       if (b >= aStart && b < aEnd) {
6792         ierr = PetscSectionGetDof(aSec, b, &bDof);CHKERRQ(ierr);
6793       }
6794       if (bDof) {
6795         PetscInt bOff, q, allDof = 0;
6796 
6797         ierr = PetscSectionGetOffset(aSec, b, &bOff);CHKERRQ(ierr);
6798         for (q = 0; q < bDof; q++) {
6799           PetscInt a = anchors[bOff + q], aDof;
6800 
6801           ierr    = PetscSectionGetDof(section, a, &aDof);CHKERRQ(ierr);
6802           allDof += aDof;
6803         }
6804         newPointOffsets[0][p+1] = allDof;
6805         pointMatOffsets[0][p+1] = bSecDof * allDof;
6806       }
6807       else {
6808         newPointOffsets[0][p+1] = bSecDof;
6809         pointMatOffsets[0][p+1] = 0;
6810       }
6811     }
6812     newPointOffsets[0][0] = 0;
6813     pointMatOffsets[0][0] = 0;
6814     for (p = 0; p < numPoints; p++) {
6815       newPointOffsets[0][p+1] += newPointOffsets[0][p];
6816       pointMatOffsets[0][p+1] += pointMatOffsets[0][p];
6817     }
6818     ierr = DMGetWorkArray(dm,pointMatOffsets[0][numPoints],MPIU_SCALAR,&pointMat[0]);CHKERRQ(ierr);
6819   }
6820 
6821   /* output arrays */
6822   ierr = DMGetWorkArray(dm,2*newNumPoints,MPIU_INT,&newPoints);CHKERRQ(ierr);
6823 
6824   /* get the point-to-point matrices; construct newPoints */
6825   ierr = PetscSectionGetMaxDof(aSec, &maxAnchor);CHKERRQ(ierr);
6826   ierr = PetscSectionGetMaxDof(section, &maxDof);CHKERRQ(ierr);
6827   ierr = DMGetWorkArray(dm,maxDof,MPIU_INT,&indices);CHKERRQ(ierr);
6828   ierr = DMGetWorkArray(dm,maxAnchor*maxDof,MPIU_INT,&newIndices);CHKERRQ(ierr);
6829   if (numFields) {
6830     for (p = 0, newP = 0; p < numPoints; p++) {
6831       PetscInt b    = points[2*p];
6832       PetscInt o    = points[2*p+1];
6833       PetscInt bDof = 0, bSecDof;
6834 
6835       ierr = PetscSectionGetDof(section, b, &bSecDof);CHKERRQ(ierr);
6836       if (!bSecDof) {
6837         continue;
6838       }
6839       if (b >= aStart && b < aEnd) {
6840         ierr = PetscSectionGetDof(aSec, b, &bDof);CHKERRQ(ierr);
6841       }
6842       if (bDof) {
6843         PetscInt fStart[32], fEnd[32], fAnchorStart[32], fAnchorEnd[32], bOff, q;
6844 
6845         fStart[0] = 0;
6846         fEnd[0]   = 0;
6847         for (f = 0; f < numFields; f++) {
6848           PetscInt fDof;
6849 
6850           ierr        = PetscSectionGetFieldDof(cSec, b, f, &fDof);CHKERRQ(ierr);
6851           fStart[f+1] = fStart[f] + fDof;
6852           fEnd[f+1]   = fStart[f+1];
6853         }
6854         ierr = PetscSectionGetOffset(cSec, b, &bOff);CHKERRQ(ierr);
6855         ierr = DMPlexGetIndicesPointFields_Internal(cSec, PETSC_TRUE, b, bOff, fEnd, PETSC_TRUE, perms, p, NULL, indices);CHKERRQ(ierr);
6856 
6857         fAnchorStart[0] = 0;
6858         fAnchorEnd[0]   = 0;
6859         for (f = 0; f < numFields; f++) {
6860           PetscInt fDof = newPointOffsets[f][p + 1] - newPointOffsets[f][p];
6861 
6862           fAnchorStart[f+1] = fAnchorStart[f] + fDof;
6863           fAnchorEnd[f+1]   = fAnchorStart[f + 1];
6864         }
6865         ierr = PetscSectionGetOffset(aSec, b, &bOff);CHKERRQ(ierr);
6866         for (q = 0; q < bDof; q++) {
6867           PetscInt a = anchors[bOff + q], aOff;
6868 
6869           /* we take the orientation of ap into account in the order that we constructed the indices above: the newly added points have no orientation */
6870           newPoints[2*(newP + q)]     = a;
6871           newPoints[2*(newP + q) + 1] = 0;
6872           ierr = PetscSectionGetOffset(section, a, &aOff);CHKERRQ(ierr);
6873           ierr = DMPlexGetIndicesPointFields_Internal(section, PETSC_TRUE, a, aOff, fAnchorEnd, PETSC_TRUE, NULL, -1, NULL, newIndices);CHKERRQ(ierr);
6874         }
6875         newP += bDof;
6876 
6877         if (outValues) {
6878           /* get the point-to-point submatrix */
6879           for (f = 0; f < numFields; f++) {
6880             ierr = MatGetValues(cMat,fEnd[f]-fStart[f],indices + fStart[f],fAnchorEnd[f] - fAnchorStart[f],newIndices + fAnchorStart[f],pointMat[f] + pointMatOffsets[f][p]);CHKERRQ(ierr);
6881           }
6882         }
6883       }
6884       else {
6885         newPoints[2 * newP]     = b;
6886         newPoints[2 * newP + 1] = o;
6887         newP++;
6888       }
6889     }
6890   } else {
6891     for (p = 0; p < numPoints; p++) {
6892       PetscInt b    = points[2*p];
6893       PetscInt o    = points[2*p+1];
6894       PetscInt bDof = 0, bSecDof;
6895 
6896       ierr = PetscSectionGetDof(section, b, &bSecDof);CHKERRQ(ierr);
6897       if (!bSecDof) {
6898         continue;
6899       }
6900       if (b >= aStart && b < aEnd) {
6901         ierr = PetscSectionGetDof(aSec, b, &bDof);CHKERRQ(ierr);
6902       }
6903       if (bDof) {
6904         PetscInt bEnd = 0, bAnchorEnd = 0, bOff;
6905 
6906         ierr = PetscSectionGetOffset(cSec, b, &bOff);CHKERRQ(ierr);
6907         ierr = DMPlexGetIndicesPoint_Internal(cSec, PETSC_TRUE, b, bOff, &bEnd, PETSC_TRUE, (perms && perms[0]) ? perms[0][p] : NULL, NULL, indices);CHKERRQ(ierr);
6908 
6909         ierr = PetscSectionGetOffset (aSec, b, &bOff);CHKERRQ(ierr);
6910         for (q = 0; q < bDof; q++) {
6911           PetscInt a = anchors[bOff + q], aOff;
6912 
6913           /* we take the orientation of ap into account in the order that we constructed the indices above: the newly added points have no orientation */
6914 
6915           newPoints[2*(newP + q)]     = a;
6916           newPoints[2*(newP + q) + 1] = 0;
6917           ierr = PetscSectionGetOffset(section, a, &aOff);CHKERRQ(ierr);
6918           ierr = DMPlexGetIndicesPoint_Internal(section, PETSC_TRUE, a, aOff, &bAnchorEnd, PETSC_TRUE, NULL, NULL, newIndices);CHKERRQ(ierr);
6919         }
6920         newP += bDof;
6921 
6922         /* get the point-to-point submatrix */
6923         if (outValues) {
6924           ierr = MatGetValues(cMat,bEnd,indices,bAnchorEnd,newIndices,pointMat[0] + pointMatOffsets[0][p]);CHKERRQ(ierr);
6925         }
6926       }
6927       else {
6928         newPoints[2 * newP]     = b;
6929         newPoints[2 * newP + 1] = o;
6930         newP++;
6931       }
6932     }
6933   }
6934 
6935   if (outValues) {
6936     ierr = DMGetWorkArray(dm,newNumIndices*numIndices,MPIU_SCALAR,&tmpValues);CHKERRQ(ierr);
6937     ierr = PetscArrayzero(tmpValues,newNumIndices*numIndices);CHKERRQ(ierr);
6938     /* multiply constraints on the right */
6939     if (numFields) {
6940       for (f = 0; f < numFields; f++) {
6941         PetscInt oldOff = offsets[f];
6942 
6943         for (p = 0; p < numPoints; p++) {
6944           PetscInt cStart = newPointOffsets[f][p];
6945           PetscInt b      = points[2 * p];
6946           PetscInt c, r, k;
6947           PetscInt dof;
6948 
6949           ierr = PetscSectionGetFieldDof(section,b,f,&dof);CHKERRQ(ierr);
6950           if (!dof) {
6951             continue;
6952           }
6953           if (pointMatOffsets[f][p] < pointMatOffsets[f][p + 1]) {
6954             PetscInt nCols         = newPointOffsets[f][p+1]-cStart;
6955             const PetscScalar *mat = pointMat[f] + pointMatOffsets[f][p];
6956 
6957             for (r = 0; r < numIndices; r++) {
6958               for (c = 0; c < nCols; c++) {
6959                 for (k = 0; k < dof; k++) {
6960                   tmpValues[r * newNumIndices + cStart + c] += values[r * numIndices + oldOff + k] * mat[k * nCols + c];
6961                 }
6962               }
6963             }
6964           }
6965           else {
6966             /* copy this column as is */
6967             for (r = 0; r < numIndices; r++) {
6968               for (c = 0; c < dof; c++) {
6969                 tmpValues[r * newNumIndices + cStart + c] = values[r * numIndices + oldOff + c];
6970               }
6971             }
6972           }
6973           oldOff += dof;
6974         }
6975       }
6976     }
6977     else {
6978       PetscInt oldOff = 0;
6979       for (p = 0; p < numPoints; p++) {
6980         PetscInt cStart = newPointOffsets[0][p];
6981         PetscInt b      = points[2 * p];
6982         PetscInt c, r, k;
6983         PetscInt dof;
6984 
6985         ierr = PetscSectionGetDof(section,b,&dof);CHKERRQ(ierr);
6986         if (!dof) {
6987           continue;
6988         }
6989         if (pointMatOffsets[0][p] < pointMatOffsets[0][p + 1]) {
6990           PetscInt nCols         = newPointOffsets[0][p+1]-cStart;
6991           const PetscScalar *mat = pointMat[0] + pointMatOffsets[0][p];
6992 
6993           for (r = 0; r < numIndices; r++) {
6994             for (c = 0; c < nCols; c++) {
6995               for (k = 0; k < dof; k++) {
6996                 tmpValues[r * newNumIndices + cStart + c] += mat[k * nCols + c] * values[r * numIndices + oldOff + k];
6997               }
6998             }
6999           }
7000         }
7001         else {
7002           /* copy this column as is */
7003           for (r = 0; r < numIndices; r++) {
7004             for (c = 0; c < dof; c++) {
7005               tmpValues[r * newNumIndices + cStart + c] = values[r * numIndices + oldOff + c];
7006             }
7007           }
7008         }
7009         oldOff += dof;
7010       }
7011     }
7012 
7013     if (multiplyLeft) {
7014       ierr = DMGetWorkArray(dm,newNumIndices*newNumIndices,MPIU_SCALAR,&newValues);CHKERRQ(ierr);
7015       ierr = PetscArrayzero(newValues,newNumIndices*newNumIndices);CHKERRQ(ierr);
7016       /* multiply constraints transpose on the left */
7017       if (numFields) {
7018         for (f = 0; f < numFields; f++) {
7019           PetscInt oldOff = offsets[f];
7020 
7021           for (p = 0; p < numPoints; p++) {
7022             PetscInt rStart = newPointOffsets[f][p];
7023             PetscInt b      = points[2 * p];
7024             PetscInt c, r, k;
7025             PetscInt dof;
7026 
7027             ierr = PetscSectionGetFieldDof(section,b,f,&dof);CHKERRQ(ierr);
7028             if (pointMatOffsets[f][p] < pointMatOffsets[f][p + 1]) {
7029               PetscInt nRows                        = newPointOffsets[f][p+1]-rStart;
7030               const PetscScalar *PETSC_RESTRICT mat = pointMat[f] + pointMatOffsets[f][p];
7031 
7032               for (r = 0; r < nRows; r++) {
7033                 for (c = 0; c < newNumIndices; c++) {
7034                   for (k = 0; k < dof; k++) {
7035                     newValues[(rStart + r) * newNumIndices + c] += mat[k * nRows + r] * tmpValues[(oldOff + k) * newNumIndices + c];
7036                   }
7037                 }
7038               }
7039             }
7040             else {
7041               /* copy this row as is */
7042               for (r = 0; r < dof; r++) {
7043                 for (c = 0; c < newNumIndices; c++) {
7044                   newValues[(rStart + r) * newNumIndices + c] = tmpValues[(oldOff + r) * newNumIndices + c];
7045                 }
7046               }
7047             }
7048             oldOff += dof;
7049           }
7050         }
7051       }
7052       else {
7053         PetscInt oldOff = 0;
7054 
7055         for (p = 0; p < numPoints; p++) {
7056           PetscInt rStart = newPointOffsets[0][p];
7057           PetscInt b      = points[2 * p];
7058           PetscInt c, r, k;
7059           PetscInt dof;
7060 
7061           ierr = PetscSectionGetDof(section,b,&dof);CHKERRQ(ierr);
7062           if (pointMatOffsets[0][p] < pointMatOffsets[0][p + 1]) {
7063             PetscInt nRows                        = newPointOffsets[0][p+1]-rStart;
7064             const PetscScalar *PETSC_RESTRICT mat = pointMat[0] + pointMatOffsets[0][p];
7065 
7066             for (r = 0; r < nRows; r++) {
7067               for (c = 0; c < newNumIndices; c++) {
7068                 for (k = 0; k < dof; k++) {
7069                   newValues[(rStart + r) * newNumIndices + c] += mat[k * nRows + r] * tmpValues[(oldOff + k) * newNumIndices + c];
7070                 }
7071               }
7072             }
7073           }
7074           else {
7075             /* copy this row as is */
7076             for (r = 0; r < dof; r++) {
7077               for (c = 0; c < newNumIndices; c++) {
7078                 newValues[(rStart + r) * newNumIndices + c] = tmpValues[(oldOff + r) * newNumIndices + c];
7079               }
7080             }
7081           }
7082           oldOff += dof;
7083         }
7084       }
7085 
7086       ierr = DMRestoreWorkArray(dm,newNumIndices*numIndices,MPIU_SCALAR,&tmpValues);CHKERRQ(ierr);
7087     }
7088     else {
7089       newValues = tmpValues;
7090     }
7091   }
7092 
7093   /* clean up */
7094   ierr = DMRestoreWorkArray(dm,maxDof,MPIU_INT,&indices);CHKERRQ(ierr);
7095   ierr = DMRestoreWorkArray(dm,maxAnchor*maxDof,MPIU_INT,&newIndices);CHKERRQ(ierr);
7096 
7097   if (numFields) {
7098     for (f = 0; f < numFields; f++) {
7099       ierr = DMRestoreWorkArray(dm,pointMatOffsets[f][numPoints],MPIU_SCALAR,&pointMat[f]);CHKERRQ(ierr);
7100       ierr = DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[f]);CHKERRQ(ierr);
7101       ierr = DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&newPointOffsets[f]);CHKERRQ(ierr);
7102     }
7103   }
7104   else {
7105     ierr = DMRestoreWorkArray(dm,pointMatOffsets[0][numPoints],MPIU_SCALAR,&pointMat[0]);CHKERRQ(ierr);
7106     ierr = DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[0]);CHKERRQ(ierr);
7107     ierr = DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&newPointOffsets[0]);CHKERRQ(ierr);
7108   }
7109   ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);
7110 
7111   /* output */
7112   if (outPoints) {
7113     *outPoints = newPoints;
7114   }
7115   else {
7116     ierr = DMRestoreWorkArray(dm,2*newNumPoints,MPIU_INT,&newPoints);CHKERRQ(ierr);
7117   }
7118   if (outValues) {
7119     *outValues = newValues;
7120   }
7121   for (f = 0; f <= numFields; f++) {
7122     offsets[f] = newOffsets[f];
7123   }
7124   PetscFunctionReturn(0);
7125 }
7126 
7127 /*@C
7128   DMPlexGetClosureIndices - Gets the global dof indices associated with the closure of the given point within the provided sections.
7129 
7130   Not collective
7131 
7132   Input Parameters:
7133 + dm         - The DM
7134 . section    - The PetscSection describing the points (a local section)
7135 . idxSection - The PetscSection from which to obtain indices (may be local or global)
7136 . point      - The point defining the closure
7137 - useClPerm  - Use the closure point permutation if available
7138 
7139   Output Parameters:
7140 + numIndices - The number of dof indices in the closure of point with the input sections
7141 . indices    - The dof indices
7142 . outOffsets - Array to write the field offsets into, or NULL
7143 - values     - The input values, which may be modified if sign flips are induced by the point symmetries, or NULL
7144 
7145   Notes:
7146   Must call DMPlexRestoreClosureIndices() to free allocated memory
7147 
7148   If idxSection is global, any constrained dofs (see DMAddBoundary(), for example) will get negative indices.  The value
7149   of those indices is not significant.  If idxSection is local, the constrained dofs will yield the involution -(idx+1)
7150   of their index in a local vector.  A caller who does not wish to distinguish those points may recover the nonnegative
7151   indices via involution, -(-(idx+1)+1)==idx.  Local indices are provided when idxSection == section, otherwise global
7152   indices (with the above semantics) are implied.
7153 
7154   Level: advanced
7155 
7156 .seealso DMPlexRestoreClosureIndices(), DMPlexVecGetClosure(), DMPlexMatSetClosure(), DMGetLocalSection(), DMGetGlobalSection()
7157 @*/
7158 PetscErrorCode DMPlexGetClosureIndices(DM dm, PetscSection section, PetscSection idxSection, PetscInt point, PetscBool useClPerm,
7159                                        PetscInt *numIndices, PetscInt *indices[], PetscInt outOffsets[], PetscScalar *values[])
7160 {
7161   /* Closure ordering */
7162   PetscSection        clSection;
7163   IS                  clPoints;
7164   const PetscInt     *clp;
7165   PetscInt           *points;
7166   const PetscInt     *clperm = NULL;
7167   /* Dof permutation and sign flips */
7168   const PetscInt    **perms[32] = {NULL};
7169   const PetscScalar **flips[32] = {NULL};
7170   PetscScalar        *valCopy   = NULL;
7171   /* Hanging node constraints */
7172   PetscInt           *pointsC = NULL;
7173   PetscScalar        *valuesC = NULL;
7174   PetscInt            NclC, NiC;
7175 
7176   PetscInt           *idx;
7177   PetscInt            Nf, Ncl, Ni = 0, offsets[32], p, f;
7178   PetscBool           isLocal = (section == idxSection) ? PETSC_TRUE : PETSC_FALSE;
7179   PetscErrorCode      ierr;
7180 
7181   PetscFunctionBeginHot;
7182   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7183   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
7184   PetscValidHeaderSpecific(idxSection, PETSC_SECTION_CLASSID, 3);
7185   if (numIndices) PetscValidPointer(numIndices, 6);
7186   if (indices)    PetscValidPointer(indices, 7);
7187   if (outOffsets) PetscValidPointer(outOffsets, 8);
7188   if (values)     PetscValidPointer(values, 9);
7189   ierr = PetscSectionGetNumFields(section, &Nf);CHKERRQ(ierr);
7190   if (Nf > 31) SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", Nf);
7191   ierr = PetscArrayzero(offsets, 32);CHKERRQ(ierr);
7192   /* 1) Get points in closure */
7193   ierr = DMPlexGetCompressedClosure(dm, section, point, &Ncl, &points, &clSection, &clPoints, &clp);CHKERRQ(ierr);
7194   if (useClPerm) {
7195     PetscInt depth, clsize;
7196     ierr = DMPlexGetPointDepth(dm, point, &depth);CHKERRQ(ierr);
7197     for (clsize=0,p=0; p<Ncl; p++) {
7198       PetscInt dof;
7199       ierr = PetscSectionGetDof(section, points[2*p], &dof);CHKERRQ(ierr);
7200       clsize += dof;
7201     }
7202     ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, depth, clsize, &clperm);CHKERRQ(ierr);
7203   }
7204   /* 2) Get number of indices on these points and field offsets from section */
7205   for (p = 0; p < Ncl*2; p += 2) {
7206     PetscInt dof, fdof;
7207 
7208     ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
7209     for (f = 0; f < Nf; ++f) {
7210       ierr = PetscSectionGetFieldDof(section, points[p], f, &fdof);CHKERRQ(ierr);
7211       offsets[f+1] += fdof;
7212     }
7213     Ni += dof;
7214   }
7215   for (f = 1; f < Nf; ++f) offsets[f+1] += offsets[f];
7216   if (Nf && offsets[Nf] != Ni) SETERRQ2(PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", offsets[Nf], Ni);
7217   /* 3) Get symmetries and sign flips. Apply sign flips to values if passed in (only works for square values matrix) */
7218   for (f = 0; f < PetscMax(1, Nf); ++f) {
7219     if (Nf) {ierr = PetscSectionGetFieldPointSyms(section, f, Ncl, points, &perms[f], &flips[f]);CHKERRQ(ierr);}
7220     else    {ierr = PetscSectionGetPointSyms(section, Ncl, points, &perms[f], &flips[f]);CHKERRQ(ierr);}
7221     /* may need to apply sign changes to the element matrix */
7222     if (values && flips[f]) {
7223       PetscInt foffset = offsets[f];
7224 
7225       for (p = 0; p < Ncl; ++p) {
7226         PetscInt           pnt  = points[2*p], fdof;
7227         const PetscScalar *flip = flips[f] ? flips[f][p] : NULL;
7228 
7229         if (!Nf) {ierr = PetscSectionGetDof(section, pnt, &fdof);CHKERRQ(ierr);}
7230         else     {ierr = PetscSectionGetFieldDof(section, pnt, f, &fdof);CHKERRQ(ierr);}
7231         if (flip) {
7232           PetscInt i, j, k;
7233 
7234           if (!valCopy) {
7235             ierr = DMGetWorkArray(dm, Ni*Ni, MPIU_SCALAR, &valCopy);CHKERRQ(ierr);
7236             for (j = 0; j < Ni * Ni; ++j) valCopy[j] = (*values)[j];
7237             *values = valCopy;
7238           }
7239           for (i = 0; i < fdof; ++i) {
7240             PetscScalar fval = flip[i];
7241 
7242             for (k = 0; k < Ni; ++k) {
7243               valCopy[Ni * (foffset + i) + k] *= fval;
7244               valCopy[Ni * k + (foffset + i)] *= fval;
7245             }
7246           }
7247         }
7248         foffset += fdof;
7249       }
7250     }
7251   }
7252   /* 4) Apply hanging node constraints. Get new symmetries and replace all storage with constrained storage */
7253   ierr = DMPlexAnchorsModifyMat(dm, section, Ncl, Ni, points, perms, values ? *values : NULL, &NclC, &NiC, &pointsC, values ? &valuesC : NULL, offsets, PETSC_TRUE);CHKERRQ(ierr);
7254   if (NclC) {
7255     if (valCopy) {ierr = DMRestoreWorkArray(dm, Ni*Ni, MPIU_SCALAR, &valCopy);CHKERRQ(ierr);}
7256     for (f = 0; f < PetscMax(1, Nf); ++f) {
7257       if (Nf) {ierr = PetscSectionRestoreFieldPointSyms(section, f, Ncl, points, &perms[f], &flips[f]);CHKERRQ(ierr);}
7258       else    {ierr = PetscSectionRestorePointSyms(section, Ncl, points, &perms[f], &flips[f]);CHKERRQ(ierr);}
7259     }
7260     for (f = 0; f < PetscMax(1, Nf); ++f) {
7261       if (Nf) {ierr = PetscSectionGetFieldPointSyms(section, f, NclC, pointsC, &perms[f], &flips[f]);CHKERRQ(ierr);}
7262       else    {ierr = PetscSectionGetPointSyms(section, NclC, pointsC, &perms[f], &flips[f]);CHKERRQ(ierr);}
7263     }
7264     ierr = DMPlexRestoreCompressedClosure(dm, section, point, &Ncl, &points, &clSection, &clPoints, &clp);CHKERRQ(ierr);
7265     Ncl     = NclC;
7266     Ni      = NiC;
7267     points  = pointsC;
7268     if (values) *values = valuesC;
7269   }
7270   /* 5) Calculate indices */
7271   ierr = DMGetWorkArray(dm, Ni, MPIU_INT, &idx);CHKERRQ(ierr);
7272   if (Nf) {
7273     PetscInt  idxOff;
7274     PetscBool useFieldOffsets;
7275 
7276     if (outOffsets) {for (f = 0; f <= Nf; f++) outOffsets[f] = offsets[f];}
7277     ierr = PetscSectionGetUseFieldOffsets(idxSection, &useFieldOffsets);CHKERRQ(ierr);
7278     if (useFieldOffsets) {
7279       for (p = 0; p < Ncl; ++p) {
7280         const PetscInt pnt = points[p*2];
7281 
7282         ierr = DMPlexGetIndicesPointFieldsSplit_Internal(section, idxSection, pnt, offsets, perms, p, clperm, idx);CHKERRQ(ierr);
7283       }
7284     } else {
7285       for (p = 0; p < Ncl; ++p) {
7286         const PetscInt pnt = points[p*2];
7287 
7288         ierr = PetscSectionGetOffset(idxSection, pnt, &idxOff);CHKERRQ(ierr);
7289         /* Note that we pass a local section even though we're using global offsets.  This is because global sections do
7290          * not (at the time of this writing) have fields set. They probably should, in which case we would pass the
7291          * global section. */
7292         ierr = DMPlexGetIndicesPointFields_Internal(section, isLocal, pnt, idxOff < 0 ? -(idxOff+1) : idxOff, offsets, PETSC_FALSE, perms, p, clperm, idx);CHKERRQ(ierr);
7293       }
7294     }
7295   } else {
7296     PetscInt off = 0, idxOff;
7297 
7298     for (p = 0; p < Ncl; ++p) {
7299       const PetscInt  pnt  = points[p*2];
7300       const PetscInt *perm = perms[0] ? perms[0][p] : NULL;
7301 
7302       ierr = PetscSectionGetOffset(idxSection, pnt, &idxOff);CHKERRQ(ierr);
7303       /* Note that we pass a local section even though we're using global offsets.  This is because global sections do
7304        * not (at the time of this writing) have fields set. They probably should, in which case we would pass the global section. */
7305       ierr = DMPlexGetIndicesPoint_Internal(section, isLocal, pnt, idxOff < 0 ? -(idxOff+1) : idxOff, &off, PETSC_FALSE, perm, clperm, idx);CHKERRQ(ierr);
7306     }
7307   }
7308   /* 6) Cleanup */
7309   for (f = 0; f < PetscMax(1, Nf); ++f) {
7310     if (Nf) {ierr = PetscSectionRestoreFieldPointSyms(section, f, Ncl, points, &perms[f], &flips[f]);CHKERRQ(ierr);}
7311     else    {ierr = PetscSectionRestorePointSyms(section, Ncl, points, &perms[f], &flips[f]);CHKERRQ(ierr);}
7312   }
7313   if (NclC) {
7314     ierr = DMRestoreWorkArray(dm, NclC*2, MPIU_INT, &pointsC);CHKERRQ(ierr);
7315   } else {
7316     ierr = DMPlexRestoreCompressedClosure(dm, section, point, &Ncl, &points, &clSection, &clPoints, &clp);CHKERRQ(ierr);
7317   }
7318 
7319   if (numIndices) *numIndices = Ni;
7320   if (indices)    *indices    = idx;
7321   PetscFunctionReturn(0);
7322 }
7323 
7324 /*@C
7325   DMPlexRestoreClosureIndices - Restores the global dof indices associated with the closure of the given point within the provided sections.
7326 
7327   Not collective
7328 
7329   Input Parameters:
7330 + dm         - The DM
7331 . section    - The PetscSection describing the points (a local section)
7332 . idxSection - The PetscSection from which to obtain indices (may be local or global)
7333 . point      - The point defining the closure
7334 - useClPerm  - Use the closure point permutation if available
7335 
7336   Output Parameters:
7337 + numIndices - The number of dof indices in the closure of point with the input sections
7338 . indices    - The dof indices
7339 . outOffsets - Array to write the field offsets into, or NULL
7340 - values     - The input values, which may be modified if sign flips are induced by the point symmetries, or NULL
7341 
7342   Notes:
7343   If values were modified, the user is responsible for calling DMRestoreWorkArray(dm, 0, MPIU_SCALAR, &values).
7344 
7345   If idxSection is global, any constrained dofs (see DMAddBoundary(), for example) will get negative indices.  The value
7346   of those indices is not significant.  If idxSection is local, the constrained dofs will yield the involution -(idx+1)
7347   of their index in a local vector.  A caller who does not wish to distinguish those points may recover the nonnegative
7348   indices via involution, -(-(idx+1)+1)==idx.  Local indices are provided when idxSection == section, otherwise global
7349   indices (with the above semantics) are implied.
7350 
7351   Level: advanced
7352 
7353 .seealso DMPlexGetClosureIndices(), DMPlexVecGetClosure(), DMPlexMatSetClosure(), DMGetLocalSection(), DMGetGlobalSection()
7354 @*/
7355 PetscErrorCode DMPlexRestoreClosureIndices(DM dm, PetscSection section, PetscSection idxSection, PetscInt point, PetscBool useClPerm,
7356                                            PetscInt *numIndices, PetscInt *indices[], PetscInt outOffsets[], PetscScalar *values[])
7357 {
7358   PetscErrorCode ierr;
7359 
7360   PetscFunctionBegin;
7361   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7362   PetscValidPointer(indices, 7);
7363   ierr = DMRestoreWorkArray(dm, 0, MPIU_INT, indices);CHKERRQ(ierr);
7364   PetscFunctionReturn(0);
7365 }
7366 
7367 /*@C
7368   DMPlexMatSetClosure - Set an array of the values on the closure of 'point'
7369 
7370   Not collective
7371 
7372   Input Parameters:
7373 + dm - The DM
7374 . section - The section describing the layout in v, or NULL to use the default section
7375 . globalSection - The section describing the layout in v, or NULL to use the default global section
7376 . A - The matrix
7377 . point - The point in the DM
7378 . values - The array of values
7379 - mode - The insert mode, where INSERT_ALL_VALUES and ADD_ALL_VALUES also overwrite boundary conditions
7380 
7381   Fortran Notes:
7382   This routine is only available in Fortran 90, and you must include petsc.h90 in your code.
7383 
7384   Level: intermediate
7385 
7386 .seealso DMPlexMatSetClosureGeneral(), DMPlexVecGetClosure(), DMPlexVecSetClosure()
7387 @*/
7388 PetscErrorCode DMPlexMatSetClosure(DM dm, PetscSection section, PetscSection globalSection, Mat A, PetscInt point, const PetscScalar values[], InsertMode mode)
7389 {
7390   DM_Plex           *mesh = (DM_Plex*) dm->data;
7391   PetscInt          *indices;
7392   PetscInt           numIndices;
7393   const PetscScalar *valuesOrig = values;
7394   PetscErrorCode     ierr;
7395 
7396   PetscFunctionBegin;
7397   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7398   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
7399   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
7400   if (!globalSection) {ierr = DMGetGlobalSection(dm, &globalSection);CHKERRQ(ierr);}
7401   PetscValidHeaderSpecific(globalSection, PETSC_SECTION_CLASSID, 3);
7402   PetscValidHeaderSpecific(A, MAT_CLASSID, 4);
7403 
7404   ierr = DMPlexGetClosureIndices(dm, section, globalSection, point, PETSC_TRUE, &numIndices, &indices, NULL, (PetscScalar **) &values);CHKERRQ(ierr);
7405 
7406   if (mesh->printSetValues) {ierr = DMPlexPrintMatSetValues(PETSC_VIEWER_STDOUT_SELF, A, point, numIndices, indices, 0, NULL, values);CHKERRQ(ierr);}
7407   ierr = MatSetValues(A, numIndices, indices, numIndices, indices, values, mode);
7408   if (ierr) {
7409     PetscMPIInt    rank;
7410     PetscErrorCode ierr2;
7411 
7412     ierr2 = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRMPI(ierr2);
7413     ierr2 = (*PetscErrorPrintf)("[%d]ERROR in DMPlexMatSetClosure\n", rank);CHKERRQ(ierr2);
7414     ierr2 = DMPlexPrintMatSetValues(PETSC_VIEWER_STDERR_SELF, A, point, numIndices, indices, 0, NULL, values);CHKERRQ(ierr2);
7415     ierr2 = DMPlexRestoreClosureIndices(dm, section, globalSection, point, PETSC_TRUE, &numIndices, &indices, NULL, (PetscScalar **) &values);CHKERRQ(ierr2);
7416     if (values != valuesOrig) {ierr2 = DMRestoreWorkArray(dm, 0, MPIU_SCALAR, &values);CHKERRQ(ierr2);}
7417     CHKERRQ(ierr);
7418   }
7419   if (mesh->printFEM > 1) {
7420     PetscInt i;
7421     ierr = PetscPrintf(PETSC_COMM_SELF, "  Indices:");CHKERRQ(ierr);
7422     for (i = 0; i < numIndices; ++i) {ierr = PetscPrintf(PETSC_COMM_SELF, " %D", indices[i]);CHKERRQ(ierr);}
7423     ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
7424   }
7425 
7426   ierr = DMPlexRestoreClosureIndices(dm, section, globalSection, point, PETSC_TRUE, &numIndices, &indices, NULL, (PetscScalar **) &values);CHKERRQ(ierr);
7427   if (values != valuesOrig) {ierr = DMRestoreWorkArray(dm, 0, MPIU_SCALAR, &values);CHKERRQ(ierr);}
7428   PetscFunctionReturn(0);
7429 }
7430 
7431 /*@C
7432   DMPlexMatSetClosure - Set an array of the values on the closure of 'point' using a different row and column section
7433 
7434   Not collective
7435 
7436   Input Parameters:
7437 + dmRow - The DM for the row fields
7438 . sectionRow - The section describing the layout, or NULL to use the default section in dmRow
7439 . globalSectionRow - The section describing the layout, or NULL to use the default global section in dmRow
7440 . dmCol - The DM for the column fields
7441 . sectionCol - The section describing the layout, or NULL to use the default section in dmCol
7442 . globalSectionCol - The section describing the layout, or NULL to use the default global section in dmCol
7443 . A - The matrix
7444 . point - The point in the DMs
7445 . values - The array of values
7446 - mode - The insert mode, where INSERT_ALL_VALUES and ADD_ALL_VALUES also overwrite boundary conditions
7447 
7448   Level: intermediate
7449 
7450 .seealso DMPlexMatSetClosure(), DMPlexVecGetClosure(), DMPlexVecSetClosure()
7451 @*/
7452 PetscErrorCode DMPlexMatSetClosureGeneral(DM dmRow, PetscSection sectionRow, PetscSection globalSectionRow, DM dmCol, PetscSection sectionCol, PetscSection globalSectionCol, Mat A, PetscInt point, const PetscScalar values[], InsertMode mode)
7453 {
7454   DM_Plex           *mesh = (DM_Plex*) dmRow->data;
7455   PetscInt          *indicesRow, *indicesCol;
7456   PetscInt           numIndicesRow, numIndicesCol;
7457   const PetscScalar *valuesOrig = values;
7458   PetscErrorCode     ierr;
7459 
7460   PetscFunctionBegin;
7461   PetscValidHeaderSpecific(dmRow, DM_CLASSID, 1);
7462   if (!sectionRow) {ierr = DMGetLocalSection(dmRow, &sectionRow);CHKERRQ(ierr);}
7463   PetscValidHeaderSpecific(sectionRow, PETSC_SECTION_CLASSID, 2);
7464   if (!globalSectionRow) {ierr = DMGetGlobalSection(dmRow, &globalSectionRow);CHKERRQ(ierr);}
7465   PetscValidHeaderSpecific(globalSectionRow, PETSC_SECTION_CLASSID, 3);
7466   PetscValidHeaderSpecific(dmCol, DM_CLASSID, 4);
7467   if (!sectionCol) {ierr = DMGetLocalSection(dmCol, &sectionCol);CHKERRQ(ierr);}
7468   PetscValidHeaderSpecific(sectionCol, PETSC_SECTION_CLASSID, 5);
7469   if (!globalSectionCol) {ierr = DMGetGlobalSection(dmCol, &globalSectionCol);CHKERRQ(ierr);}
7470   PetscValidHeaderSpecific(globalSectionCol, PETSC_SECTION_CLASSID, 6);
7471   PetscValidHeaderSpecific(A, MAT_CLASSID, 7);
7472 
7473   ierr = DMPlexGetClosureIndices(dmRow, sectionRow, globalSectionRow, point, PETSC_TRUE, &numIndicesRow, &indicesRow, NULL, (PetscScalar **) &values);CHKERRQ(ierr);
7474   ierr = DMPlexGetClosureIndices(dmCol, sectionCol, globalSectionCol, point, PETSC_TRUE, &numIndicesCol, &indicesCol, NULL, (PetscScalar **) &values);CHKERRQ(ierr);
7475 
7476   if (mesh->printSetValues) {ierr = DMPlexPrintMatSetValues(PETSC_VIEWER_STDOUT_SELF, A, point, numIndicesRow, indicesRow, numIndicesCol, indicesCol, values);CHKERRQ(ierr);}
7477   ierr = MatSetValues(A, numIndicesRow, indicesRow, numIndicesCol, indicesCol, values, mode);
7478   if (ierr) {
7479     PetscMPIInt    rank;
7480     PetscErrorCode ierr2;
7481 
7482     ierr2 = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRMPI(ierr2);
7483     ierr2 = (*PetscErrorPrintf)("[%d]ERROR in DMPlexMatSetClosure\n", rank);CHKERRQ(ierr2);
7484     ierr2 = DMPlexPrintMatSetValues(PETSC_VIEWER_STDERR_SELF, A, point, numIndicesRow, indicesRow, numIndicesCol, indicesCol, values);CHKERRQ(ierr2);
7485     ierr2 = DMPlexRestoreClosureIndices(dmRow, sectionRow, globalSectionRow, point, PETSC_TRUE, &numIndicesRow, &indicesRow, NULL, (PetscScalar **) &values);CHKERRQ(ierr2);
7486     ierr2 = DMPlexRestoreClosureIndices(dmCol, sectionCol, globalSectionCol, point, PETSC_TRUE, &numIndicesCol, &indicesRow, NULL, (PetscScalar **) &values);CHKERRQ(ierr2);
7487     if (values != valuesOrig) {ierr2 = DMRestoreWorkArray(dmRow, 0, MPIU_SCALAR, &values);CHKERRQ(ierr2);}
7488     CHKERRQ(ierr);
7489   }
7490 
7491   ierr = DMPlexRestoreClosureIndices(dmRow, sectionRow, globalSectionRow, point, PETSC_TRUE, &numIndicesRow, &indicesRow, NULL, (PetscScalar **) &values);CHKERRQ(ierr);
7492   ierr = DMPlexRestoreClosureIndices(dmCol, sectionCol, globalSectionCol, point, PETSC_TRUE, &numIndicesCol, &indicesCol, NULL, (PetscScalar **) &values);CHKERRQ(ierr);
7493   if (values != valuesOrig) {ierr = DMRestoreWorkArray(dmRow, 0, MPIU_SCALAR, &values);CHKERRQ(ierr);}
7494   PetscFunctionReturn(0);
7495 }
7496 
7497 PetscErrorCode DMPlexMatSetClosureRefined(DM dmf, PetscSection fsection, PetscSection globalFSection, DM dmc, PetscSection csection, PetscSection globalCSection, Mat A, PetscInt point, const PetscScalar values[], InsertMode mode)
7498 {
7499   DM_Plex        *mesh   = (DM_Plex*) dmf->data;
7500   PetscInt       *fpoints = NULL, *ftotpoints = NULL;
7501   PetscInt       *cpoints = NULL;
7502   PetscInt       *findices, *cindices;
7503   const PetscInt *fclperm = NULL, *cclperm = NULL; /* Closure permutations cannot work here */
7504   PetscInt        foffsets[32], coffsets[32];
7505   DMPolytopeType  ct;
7506   PetscInt        numFields, numSubcells, maxFPoints, numFPoints, numCPoints, numFIndices, numCIndices, dof, off, globalOff, pStart, pEnd, p, q, r, s, f;
7507   PetscErrorCode  ierr;
7508 
7509   PetscFunctionBegin;
7510   PetscValidHeaderSpecific(dmf, DM_CLASSID, 1);
7511   PetscValidHeaderSpecific(dmc, DM_CLASSID, 4);
7512   if (!fsection) {ierr = DMGetLocalSection(dmf, &fsection);CHKERRQ(ierr);}
7513   PetscValidHeaderSpecific(fsection, PETSC_SECTION_CLASSID, 2);
7514   if (!csection) {ierr = DMGetLocalSection(dmc, &csection);CHKERRQ(ierr);}
7515   PetscValidHeaderSpecific(csection, PETSC_SECTION_CLASSID, 5);
7516   if (!globalFSection) {ierr = DMGetGlobalSection(dmf, &globalFSection);CHKERRQ(ierr);}
7517   PetscValidHeaderSpecific(globalFSection, PETSC_SECTION_CLASSID, 3);
7518   if (!globalCSection) {ierr = DMGetGlobalSection(dmc, &globalCSection);CHKERRQ(ierr);}
7519   PetscValidHeaderSpecific(globalCSection, PETSC_SECTION_CLASSID, 6);
7520   PetscValidHeaderSpecific(A, MAT_CLASSID, 7);
7521   ierr = PetscSectionGetNumFields(fsection, &numFields);CHKERRQ(ierr);
7522   if (numFields > 31) SETERRQ1(PetscObjectComm((PetscObject)dmf), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", numFields);
7523   ierr = PetscArrayzero(foffsets, 32);CHKERRQ(ierr);
7524   ierr = PetscArrayzero(coffsets, 32);CHKERRQ(ierr);
7525   /* Column indices */
7526   ierr = DMPlexGetTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
7527   maxFPoints = numCPoints;
7528   /* Compress out points not in the section */
7529   /*   TODO: Squeeze out points with 0 dof as well */
7530   ierr = PetscSectionGetChart(csection, &pStart, &pEnd);CHKERRQ(ierr);
7531   for (p = 0, q = 0; p < numCPoints*2; p += 2) {
7532     if ((cpoints[p] >= pStart) && (cpoints[p] < pEnd)) {
7533       cpoints[q*2]   = cpoints[p];
7534       cpoints[q*2+1] = cpoints[p+1];
7535       ++q;
7536     }
7537   }
7538   numCPoints = q;
7539   for (p = 0, numCIndices = 0; p < numCPoints*2; p += 2) {
7540     PetscInt fdof;
7541 
7542     ierr = PetscSectionGetDof(csection, cpoints[p], &dof);CHKERRQ(ierr);
7543     if (!dof) continue;
7544     for (f = 0; f < numFields; ++f) {
7545       ierr           = PetscSectionGetFieldDof(csection, cpoints[p], f, &fdof);CHKERRQ(ierr);
7546       coffsets[f+1] += fdof;
7547     }
7548     numCIndices += dof;
7549   }
7550   for (f = 1; f < numFields; ++f) coffsets[f+1] += coffsets[f];
7551   /* Row indices */
7552   ierr = DMPlexGetCellType(dmc, point, &ct);CHKERRQ(ierr);
7553   {
7554     DMPlexCellRefiner cr;
7555     ierr = DMPlexCellRefinerCreate(dmc, &cr);CHKERRQ(ierr);
7556     ierr = DMPlexCellRefinerGetAffineTransforms(cr, ct, &numSubcells, NULL, NULL, NULL);CHKERRQ(ierr);
7557     ierr = DMPlexCellRefinerDestroy(&cr);CHKERRQ(ierr);
7558   }
7559   ierr = DMGetWorkArray(dmf, maxFPoints*2*numSubcells, MPIU_INT, &ftotpoints);CHKERRQ(ierr);
7560   for (r = 0, q = 0; r < numSubcells; ++r) {
7561     /* TODO Map from coarse to fine cells */
7562     ierr = DMPlexGetTransitiveClosure(dmf, point*numSubcells + r, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
7563     /* Compress out points not in the section */
7564     ierr = PetscSectionGetChart(fsection, &pStart, &pEnd);CHKERRQ(ierr);
7565     for (p = 0; p < numFPoints*2; p += 2) {
7566       if ((fpoints[p] >= pStart) && (fpoints[p] < pEnd)) {
7567         ierr = PetscSectionGetDof(fsection, fpoints[p], &dof);CHKERRQ(ierr);
7568         if (!dof) continue;
7569         for (s = 0; s < q; ++s) if (fpoints[p] == ftotpoints[s*2]) break;
7570         if (s < q) continue;
7571         ftotpoints[q*2]   = fpoints[p];
7572         ftotpoints[q*2+1] = fpoints[p+1];
7573         ++q;
7574       }
7575     }
7576     ierr = DMPlexRestoreTransitiveClosure(dmf, point, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
7577   }
7578   numFPoints = q;
7579   for (p = 0, numFIndices = 0; p < numFPoints*2; p += 2) {
7580     PetscInt fdof;
7581 
7582     ierr = PetscSectionGetDof(fsection, ftotpoints[p], &dof);CHKERRQ(ierr);
7583     if (!dof) continue;
7584     for (f = 0; f < numFields; ++f) {
7585       ierr           = PetscSectionGetFieldDof(fsection, ftotpoints[p], f, &fdof);CHKERRQ(ierr);
7586       foffsets[f+1] += fdof;
7587     }
7588     numFIndices += dof;
7589   }
7590   for (f = 1; f < numFields; ++f) foffsets[f+1] += foffsets[f];
7591 
7592   if (numFields && foffsets[numFields] != numFIndices) SETERRQ2(PetscObjectComm((PetscObject)dmf), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", foffsets[numFields], numFIndices);
7593   if (numFields && coffsets[numFields] != numCIndices) SETERRQ2(PetscObjectComm((PetscObject)dmc), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", coffsets[numFields], numCIndices);
7594   ierr = DMGetWorkArray(dmf, numFIndices, MPIU_INT, &findices);CHKERRQ(ierr);
7595   ierr = DMGetWorkArray(dmc, numCIndices, MPIU_INT, &cindices);CHKERRQ(ierr);
7596   if (numFields) {
7597     const PetscInt **permsF[32] = {NULL};
7598     const PetscInt **permsC[32] = {NULL};
7599 
7600     for (f = 0; f < numFields; f++) {
7601       ierr = PetscSectionGetFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL);CHKERRQ(ierr);
7602       ierr = PetscSectionGetFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL);CHKERRQ(ierr);
7603     }
7604     for (p = 0; p < numFPoints; p++) {
7605       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff);CHKERRQ(ierr);
7606       ierr = DMPlexGetIndicesPointFields_Internal(fsection, PETSC_FALSE, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, foffsets, PETSC_FALSE, permsF, p, fclperm, findices);CHKERRQ(ierr);
7607     }
7608     for (p = 0; p < numCPoints; p++) {
7609       ierr = PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff);CHKERRQ(ierr);
7610       ierr = DMPlexGetIndicesPointFields_Internal(csection, PETSC_FALSE, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, coffsets, PETSC_FALSE, permsC, p, cclperm, cindices);CHKERRQ(ierr);
7611     }
7612     for (f = 0; f < numFields; f++) {
7613       ierr = PetscSectionRestoreFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL);CHKERRQ(ierr);
7614       ierr = PetscSectionRestoreFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL);CHKERRQ(ierr);
7615     }
7616   } else {
7617     const PetscInt **permsF = NULL;
7618     const PetscInt **permsC = NULL;
7619 
7620     ierr = PetscSectionGetPointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL);CHKERRQ(ierr);
7621     ierr = PetscSectionGetPointSyms(csection,numCPoints,cpoints,&permsC,NULL);CHKERRQ(ierr);
7622     for (p = 0, off = 0; p < numFPoints; p++) {
7623       const PetscInt *perm = permsF ? permsF[p] : NULL;
7624 
7625       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff);CHKERRQ(ierr);
7626       ierr = DMPlexGetIndicesPoint_Internal(fsection, PETSC_FALSE, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, fclperm, findices);CHKERRQ(ierr);
7627     }
7628     for (p = 0, off = 0; p < numCPoints; p++) {
7629       const PetscInt *perm = permsC ? permsC[p] : NULL;
7630 
7631       ierr = PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff);CHKERRQ(ierr);
7632       ierr = DMPlexGetIndicesPoint_Internal(csection, PETSC_FALSE, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, cclperm, cindices);CHKERRQ(ierr);
7633     }
7634     ierr = PetscSectionRestorePointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL);CHKERRQ(ierr);
7635     ierr = PetscSectionRestorePointSyms(csection,numCPoints,cpoints,&permsC,NULL);CHKERRQ(ierr);
7636   }
7637   if (mesh->printSetValues) {ierr = DMPlexPrintMatSetValues(PETSC_VIEWER_STDOUT_SELF, A, point, numFIndices, findices, numCIndices, cindices, values);CHKERRQ(ierr);}
7638   /* TODO: flips */
7639   ierr = MatSetValues(A, numFIndices, findices, numCIndices, cindices, values, mode);
7640   if (ierr) {
7641     PetscMPIInt    rank;
7642     PetscErrorCode ierr2;
7643 
7644     ierr2 = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRMPI(ierr2);
7645     ierr2 = (*PetscErrorPrintf)("[%d]ERROR in DMPlexMatSetClosure\n", rank);CHKERRQ(ierr2);
7646     ierr2 = DMPlexPrintMatSetValues(PETSC_VIEWER_STDERR_SELF, A, point, numFIndices, findices, numCIndices, cindices, values);CHKERRQ(ierr2);
7647     ierr2 = DMRestoreWorkArray(dmf, numFIndices, MPIU_INT, &findices);CHKERRQ(ierr2);
7648     ierr2 = DMRestoreWorkArray(dmc, numCIndices, MPIU_INT, &cindices);CHKERRQ(ierr2);
7649     CHKERRQ(ierr);
7650   }
7651   ierr = DMRestoreWorkArray(dmf, numCPoints*2*4, MPIU_INT, &ftotpoints);CHKERRQ(ierr);
7652   ierr = DMPlexRestoreTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
7653   ierr = DMRestoreWorkArray(dmf, numFIndices, MPIU_INT, &findices);CHKERRQ(ierr);
7654   ierr = DMRestoreWorkArray(dmc, numCIndices, MPIU_INT, &cindices);CHKERRQ(ierr);
7655   PetscFunctionReturn(0);
7656 }
7657 
7658 PetscErrorCode DMPlexMatGetClosureIndicesRefined(DM dmf, PetscSection fsection, PetscSection globalFSection, DM dmc, PetscSection csection, PetscSection globalCSection, PetscInt point, PetscInt cindices[], PetscInt findices[])
7659 {
7660   PetscInt      *fpoints = NULL, *ftotpoints = NULL;
7661   PetscInt      *cpoints = NULL;
7662   PetscInt       foffsets[32], coffsets[32];
7663   const PetscInt *fclperm = NULL, *cclperm = NULL; /* Closure permutations cannot work here */
7664   DMPolytopeType ct;
7665   PetscInt       numFields, numSubcells, maxFPoints, numFPoints, numCPoints, numFIndices, numCIndices, dof, off, globalOff, pStart, pEnd, p, q, r, s, f;
7666   PetscErrorCode ierr;
7667 
7668   PetscFunctionBegin;
7669   PetscValidHeaderSpecific(dmf, DM_CLASSID, 1);
7670   PetscValidHeaderSpecific(dmc, DM_CLASSID, 4);
7671   if (!fsection) {ierr = DMGetLocalSection(dmf, &fsection);CHKERRQ(ierr);}
7672   PetscValidHeaderSpecific(fsection, PETSC_SECTION_CLASSID, 2);
7673   if (!csection) {ierr = DMGetLocalSection(dmc, &csection);CHKERRQ(ierr);}
7674   PetscValidHeaderSpecific(csection, PETSC_SECTION_CLASSID, 5);
7675   if (!globalFSection) {ierr = DMGetGlobalSection(dmf, &globalFSection);CHKERRQ(ierr);}
7676   PetscValidHeaderSpecific(globalFSection, PETSC_SECTION_CLASSID, 3);
7677   if (!globalCSection) {ierr = DMGetGlobalSection(dmc, &globalCSection);CHKERRQ(ierr);}
7678   PetscValidHeaderSpecific(globalCSection, PETSC_SECTION_CLASSID, 6);
7679   ierr = PetscSectionGetNumFields(fsection, &numFields);CHKERRQ(ierr);
7680   if (numFields > 31) SETERRQ1(PetscObjectComm((PetscObject)dmf), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", numFields);
7681   ierr = PetscArrayzero(foffsets, 32);CHKERRQ(ierr);
7682   ierr = PetscArrayzero(coffsets, 32);CHKERRQ(ierr);
7683   /* Column indices */
7684   ierr = DMPlexGetTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
7685   maxFPoints = numCPoints;
7686   /* Compress out points not in the section */
7687   /*   TODO: Squeeze out points with 0 dof as well */
7688   ierr = PetscSectionGetChart(csection, &pStart, &pEnd);CHKERRQ(ierr);
7689   for (p = 0, q = 0; p < numCPoints*2; p += 2) {
7690     if ((cpoints[p] >= pStart) && (cpoints[p] < pEnd)) {
7691       cpoints[q*2]   = cpoints[p];
7692       cpoints[q*2+1] = cpoints[p+1];
7693       ++q;
7694     }
7695   }
7696   numCPoints = q;
7697   for (p = 0, numCIndices = 0; p < numCPoints*2; p += 2) {
7698     PetscInt fdof;
7699 
7700     ierr = PetscSectionGetDof(csection, cpoints[p], &dof);CHKERRQ(ierr);
7701     if (!dof) continue;
7702     for (f = 0; f < numFields; ++f) {
7703       ierr           = PetscSectionGetFieldDof(csection, cpoints[p], f, &fdof);CHKERRQ(ierr);
7704       coffsets[f+1] += fdof;
7705     }
7706     numCIndices += dof;
7707   }
7708   for (f = 1; f < numFields; ++f) coffsets[f+1] += coffsets[f];
7709   /* Row indices */
7710   ierr = DMPlexGetCellType(dmc, point, &ct);CHKERRQ(ierr);
7711   {
7712     DMPlexCellRefiner cr;
7713     ierr = DMPlexCellRefinerCreate(dmc, &cr);CHKERRQ(ierr);
7714     ierr = DMPlexCellRefinerGetAffineTransforms(cr, ct, &numSubcells, NULL, NULL, NULL);CHKERRQ(ierr);
7715     ierr = DMPlexCellRefinerDestroy(&cr);CHKERRQ(ierr);
7716   }
7717   ierr = DMGetWorkArray(dmf, maxFPoints*2*numSubcells, MPIU_INT, &ftotpoints);CHKERRQ(ierr);
7718   for (r = 0, q = 0; r < numSubcells; ++r) {
7719     /* TODO Map from coarse to fine cells */
7720     ierr = DMPlexGetTransitiveClosure(dmf, point*numSubcells + r, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
7721     /* Compress out points not in the section */
7722     ierr = PetscSectionGetChart(fsection, &pStart, &pEnd);CHKERRQ(ierr);
7723     for (p = 0; p < numFPoints*2; p += 2) {
7724       if ((fpoints[p] >= pStart) && (fpoints[p] < pEnd)) {
7725         ierr = PetscSectionGetDof(fsection, fpoints[p], &dof);CHKERRQ(ierr);
7726         if (!dof) continue;
7727         for (s = 0; s < q; ++s) if (fpoints[p] == ftotpoints[s*2]) break;
7728         if (s < q) continue;
7729         ftotpoints[q*2]   = fpoints[p];
7730         ftotpoints[q*2+1] = fpoints[p+1];
7731         ++q;
7732       }
7733     }
7734     ierr = DMPlexRestoreTransitiveClosure(dmf, point, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
7735   }
7736   numFPoints = q;
7737   for (p = 0, numFIndices = 0; p < numFPoints*2; p += 2) {
7738     PetscInt fdof;
7739 
7740     ierr = PetscSectionGetDof(fsection, ftotpoints[p], &dof);CHKERRQ(ierr);
7741     if (!dof) continue;
7742     for (f = 0; f < numFields; ++f) {
7743       ierr           = PetscSectionGetFieldDof(fsection, ftotpoints[p], f, &fdof);CHKERRQ(ierr);
7744       foffsets[f+1] += fdof;
7745     }
7746     numFIndices += dof;
7747   }
7748   for (f = 1; f < numFields; ++f) foffsets[f+1] += foffsets[f];
7749 
7750   if (numFields && foffsets[numFields] != numFIndices) SETERRQ2(PetscObjectComm((PetscObject)dmf), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", foffsets[numFields], numFIndices);
7751   if (numFields && coffsets[numFields] != numCIndices) SETERRQ2(PetscObjectComm((PetscObject)dmc), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", coffsets[numFields], numCIndices);
7752   if (numFields) {
7753     const PetscInt **permsF[32] = {NULL};
7754     const PetscInt **permsC[32] = {NULL};
7755 
7756     for (f = 0; f < numFields; f++) {
7757       ierr = PetscSectionGetFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL);CHKERRQ(ierr);
7758       ierr = PetscSectionGetFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL);CHKERRQ(ierr);
7759     }
7760     for (p = 0; p < numFPoints; p++) {
7761       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff);CHKERRQ(ierr);
7762       ierr = DMPlexGetIndicesPointFields_Internal(fsection, PETSC_FALSE, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, foffsets, PETSC_FALSE, permsF, p, fclperm, findices);CHKERRQ(ierr);
7763     }
7764     for (p = 0; p < numCPoints; p++) {
7765       ierr = PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff);CHKERRQ(ierr);
7766       ierr = DMPlexGetIndicesPointFields_Internal(csection, PETSC_FALSE, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, coffsets, PETSC_FALSE, permsC, p, cclperm, cindices);CHKERRQ(ierr);
7767     }
7768     for (f = 0; f < numFields; f++) {
7769       ierr = PetscSectionRestoreFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL);CHKERRQ(ierr);
7770       ierr = PetscSectionRestoreFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL);CHKERRQ(ierr);
7771     }
7772   } else {
7773     const PetscInt **permsF = NULL;
7774     const PetscInt **permsC = NULL;
7775 
7776     ierr = PetscSectionGetPointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL);CHKERRQ(ierr);
7777     ierr = PetscSectionGetPointSyms(csection,numCPoints,cpoints,&permsC,NULL);CHKERRQ(ierr);
7778     for (p = 0, off = 0; p < numFPoints; p++) {
7779       const PetscInt *perm = permsF ? permsF[p] : NULL;
7780 
7781       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff);CHKERRQ(ierr);
7782       ierr = DMPlexGetIndicesPoint_Internal(fsection, PETSC_FALSE, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, fclperm, findices);CHKERRQ(ierr);
7783     }
7784     for (p = 0, off = 0; p < numCPoints; p++) {
7785       const PetscInt *perm = permsC ? permsC[p] : NULL;
7786 
7787       ierr = PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff);CHKERRQ(ierr);
7788       ierr = DMPlexGetIndicesPoint_Internal(csection, PETSC_FALSE, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, cclperm, cindices);CHKERRQ(ierr);
7789     }
7790     ierr = PetscSectionRestorePointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL);CHKERRQ(ierr);
7791     ierr = PetscSectionRestorePointSyms(csection,numCPoints,cpoints,&permsC,NULL);CHKERRQ(ierr);
7792   }
7793   ierr = DMRestoreWorkArray(dmf, numCPoints*2*4, MPIU_INT, &ftotpoints);CHKERRQ(ierr);
7794   ierr = DMPlexRestoreTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
7795   PetscFunctionReturn(0);
7796 }
7797 
7798 /*@C
7799   DMPlexGetVTKCellHeight - Returns the height in the DAG used to determine which points are cells (normally 0)
7800 
7801   Input Parameter:
7802 . dm   - The DMPlex object
7803 
7804   Output Parameter:
7805 . cellHeight - The height of a cell
7806 
7807   Level: developer
7808 
7809 .seealso DMPlexSetVTKCellHeight()
7810 @*/
7811 PetscErrorCode DMPlexGetVTKCellHeight(DM dm, PetscInt *cellHeight)
7812 {
7813   DM_Plex *mesh = (DM_Plex*) dm->data;
7814 
7815   PetscFunctionBegin;
7816   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7817   PetscValidPointer(cellHeight, 2);
7818   *cellHeight = mesh->vtkCellHeight;
7819   PetscFunctionReturn(0);
7820 }
7821 
7822 /*@C
7823   DMPlexSetVTKCellHeight - Sets the height in the DAG used to determine which points are cells (normally 0)
7824 
7825   Input Parameters:
7826 + dm   - The DMPlex object
7827 - cellHeight - The height of a cell
7828 
7829   Level: developer
7830 
7831 .seealso DMPlexGetVTKCellHeight()
7832 @*/
7833 PetscErrorCode DMPlexSetVTKCellHeight(DM dm, PetscInt cellHeight)
7834 {
7835   DM_Plex *mesh = (DM_Plex*) dm->data;
7836 
7837   PetscFunctionBegin;
7838   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7839   mesh->vtkCellHeight = cellHeight;
7840   PetscFunctionReturn(0);
7841 }
7842 
7843 /*@
7844   DMPlexGetGhostCellStratum - Get the range of cells which are used to enforce FV boundary conditions
7845 
7846   Input Parameter:
7847 . dm - The DMPlex object
7848 
7849   Output Parameters:
7850 + gcStart - The first ghost cell, or NULL
7851 - gcEnd   - The upper bound on ghost cells, or NULL
7852 
7853   Level: advanced
7854 
7855 .seealso DMPlexConstructGhostCells(), DMPlexGetGhostCellStratum()
7856 @*/
7857 PetscErrorCode DMPlexGetGhostCellStratum(DM dm, PetscInt *gcStart, PetscInt *gcEnd)
7858 {
7859   DMLabel        ctLabel;
7860   PetscErrorCode ierr;
7861 
7862   PetscFunctionBegin;
7863   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7864   ierr = DMPlexGetCellTypeLabel(dm, &ctLabel);CHKERRQ(ierr);
7865   ierr = DMLabelGetStratumBounds(ctLabel, DM_POLYTOPE_FV_GHOST, gcStart, gcEnd);CHKERRQ(ierr);
7866   PetscFunctionReturn(0);
7867 }
7868 
7869 /* We can easily have a form that takes an IS instead */
7870 PetscErrorCode DMPlexCreateNumbering_Plex(DM dm, PetscInt pStart, PetscInt pEnd, PetscInt shift, PetscInt *globalSize, PetscSF sf, IS *numbering)
7871 {
7872   PetscSection   section, globalSection;
7873   PetscInt      *numbers, p;
7874   PetscErrorCode ierr;
7875 
7876   PetscFunctionBegin;
7877   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &section);CHKERRQ(ierr);
7878   ierr = PetscSectionSetChart(section, pStart, pEnd);CHKERRQ(ierr);
7879   for (p = pStart; p < pEnd; ++p) {
7880     ierr = PetscSectionSetDof(section, p, 1);CHKERRQ(ierr);
7881   }
7882   ierr = PetscSectionSetUp(section);CHKERRQ(ierr);
7883   ierr = PetscSectionCreateGlobalSection(section, sf, PETSC_FALSE, PETSC_FALSE, &globalSection);CHKERRQ(ierr);
7884   ierr = PetscMalloc1(pEnd - pStart, &numbers);CHKERRQ(ierr);
7885   for (p = pStart; p < pEnd; ++p) {
7886     ierr = PetscSectionGetOffset(globalSection, p, &numbers[p-pStart]);CHKERRQ(ierr);
7887     if (numbers[p-pStart] < 0) numbers[p-pStart] -= shift;
7888     else                       numbers[p-pStart] += shift;
7889   }
7890   ierr = ISCreateGeneral(PetscObjectComm((PetscObject) dm), pEnd - pStart, numbers, PETSC_OWN_POINTER, numbering);CHKERRQ(ierr);
7891   if (globalSize) {
7892     PetscLayout layout;
7893     ierr = PetscSectionGetPointLayout(PetscObjectComm((PetscObject) dm), globalSection, &layout);CHKERRQ(ierr);
7894     ierr = PetscLayoutGetSize(layout, globalSize);CHKERRQ(ierr);
7895     ierr = PetscLayoutDestroy(&layout);CHKERRQ(ierr);
7896   }
7897   ierr = PetscSectionDestroy(&section);CHKERRQ(ierr);
7898   ierr = PetscSectionDestroy(&globalSection);CHKERRQ(ierr);
7899   PetscFunctionReturn(0);
7900 }
7901 
7902 PetscErrorCode DMPlexCreateCellNumbering_Internal(DM dm, PetscBool includeHybrid, IS *globalCellNumbers)
7903 {
7904   PetscInt       cellHeight, cStart, cEnd;
7905   PetscErrorCode ierr;
7906 
7907   PetscFunctionBegin;
7908   ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
7909   if (includeHybrid) {ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);}
7910   else               {ierr = DMPlexGetSimplexOrBoxCells(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);}
7911   ierr = DMPlexCreateNumbering_Plex(dm, cStart, cEnd, 0, NULL, dm->sf, globalCellNumbers);CHKERRQ(ierr);
7912   PetscFunctionReturn(0);
7913 }
7914 
7915 /*@
7916   DMPlexGetCellNumbering - Get a global cell numbering for all cells on this process
7917 
7918   Input Parameter:
7919 . dm   - The DMPlex object
7920 
7921   Output Parameter:
7922 . globalCellNumbers - Global cell numbers for all cells on this process
7923 
7924   Level: developer
7925 
7926 .seealso DMPlexGetVertexNumbering()
7927 @*/
7928 PetscErrorCode DMPlexGetCellNumbering(DM dm, IS *globalCellNumbers)
7929 {
7930   DM_Plex       *mesh = (DM_Plex*) dm->data;
7931   PetscErrorCode ierr;
7932 
7933   PetscFunctionBegin;
7934   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7935   if (!mesh->globalCellNumbers) {ierr = DMPlexCreateCellNumbering_Internal(dm, PETSC_FALSE, &mesh->globalCellNumbers);CHKERRQ(ierr);}
7936   *globalCellNumbers = mesh->globalCellNumbers;
7937   PetscFunctionReturn(0);
7938 }
7939 
7940 PetscErrorCode DMPlexCreateVertexNumbering_Internal(DM dm, PetscBool includeHybrid, IS *globalVertexNumbers)
7941 {
7942   PetscInt       vStart, vEnd;
7943   PetscErrorCode ierr;
7944 
7945   PetscFunctionBegin;
7946   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7947   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
7948   ierr = DMPlexCreateNumbering_Plex(dm, vStart, vEnd, 0, NULL, dm->sf, globalVertexNumbers);CHKERRQ(ierr);
7949   PetscFunctionReturn(0);
7950 }
7951 
7952 /*@
7953   DMPlexGetVertexNumbering - Get a global vertex numbering for all vertices on this process
7954 
7955   Input Parameter:
7956 . dm   - The DMPlex object
7957 
7958   Output Parameter:
7959 . globalVertexNumbers - Global vertex numbers for all vertices on this process
7960 
7961   Level: developer
7962 
7963 .seealso DMPlexGetCellNumbering()
7964 @*/
7965 PetscErrorCode DMPlexGetVertexNumbering(DM dm, IS *globalVertexNumbers)
7966 {
7967   DM_Plex       *mesh = (DM_Plex*) dm->data;
7968   PetscErrorCode ierr;
7969 
7970   PetscFunctionBegin;
7971   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7972   if (!mesh->globalVertexNumbers) {ierr = DMPlexCreateVertexNumbering_Internal(dm, PETSC_FALSE, &mesh->globalVertexNumbers);CHKERRQ(ierr);}
7973   *globalVertexNumbers = mesh->globalVertexNumbers;
7974   PetscFunctionReturn(0);
7975 }
7976 
7977 /*@
7978   DMPlexCreatePointNumbering - Create a global numbering for all points on this process
7979 
7980   Input Parameter:
7981 . dm   - The DMPlex object
7982 
7983   Output Parameter:
7984 . globalPointNumbers - Global numbers for all points on this process
7985 
7986   Level: developer
7987 
7988 .seealso DMPlexGetCellNumbering()
7989 @*/
7990 PetscErrorCode DMPlexCreatePointNumbering(DM dm, IS *globalPointNumbers)
7991 {
7992   IS             nums[4];
7993   PetscInt       depths[4], gdepths[4], starts[4];
7994   PetscInt       depth, d, shift = 0;
7995   PetscErrorCode ierr;
7996 
7997   PetscFunctionBegin;
7998   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7999   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
8000   /* For unstratified meshes use dim instead of depth */
8001   if (depth < 0) {ierr = DMGetDimension(dm, &depth);CHKERRQ(ierr);}
8002   for (d = 0; d <= depth; ++d) {
8003     PetscInt end;
8004 
8005     depths[d] = depth-d;
8006     ierr = DMPlexGetDepthStratum(dm, depths[d], &starts[d], &end);CHKERRQ(ierr);
8007     if (!(starts[d]-end)) { starts[d] = depths[d] = -1; }
8008   }
8009   ierr = PetscSortIntWithArray(depth+1, starts, depths);CHKERRQ(ierr);
8010   ierr = MPIU_Allreduce(depths, gdepths, depth+1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject) dm));CHKERRMPI(ierr);
8011   for (d = 0; d <= depth; ++d) {
8012     if (starts[d] >= 0 && depths[d] != gdepths[d]) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Expected depth %D, found %D",depths[d],gdepths[d]);
8013   }
8014   for (d = 0; d <= depth; ++d) {
8015     PetscInt pStart, pEnd, gsize;
8016 
8017     ierr = DMPlexGetDepthStratum(dm, gdepths[d], &pStart, &pEnd);CHKERRQ(ierr);
8018     ierr = DMPlexCreateNumbering_Plex(dm, pStart, pEnd, shift, &gsize, dm->sf, &nums[d]);CHKERRQ(ierr);
8019     shift += gsize;
8020   }
8021   ierr = ISConcatenate(PetscObjectComm((PetscObject) dm), depth+1, nums, globalPointNumbers);CHKERRQ(ierr);
8022   for (d = 0; d <= depth; ++d) {ierr = ISDestroy(&nums[d]);CHKERRQ(ierr);}
8023   PetscFunctionReturn(0);
8024 }
8025 
8026 /*@
8027   DMPlexCreateRankField - Create a cell field whose value is the rank of the owner
8028 
8029   Input Parameter:
8030 . dm - The DMPlex object
8031 
8032   Output Parameter:
8033 . ranks - The rank field
8034 
8035   Options Database Keys:
8036 . -dm_partition_view - Adds the rank field into the DM output from -dm_view using the same viewer
8037 
8038   Level: intermediate
8039 
8040 .seealso: DMView()
8041 @*/
8042 PetscErrorCode DMPlexCreateRankField(DM dm, Vec *ranks)
8043 {
8044   DM             rdm;
8045   PetscFE        fe;
8046   PetscScalar   *r;
8047   PetscMPIInt    rank;
8048   DMPolytopeType ct;
8049   PetscInt       dim, cStart, cEnd, c;
8050   PetscBool      simplex;
8051   PetscErrorCode ierr;
8052 
8053   PetscFunctionBeginUser;
8054   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8055   PetscValidPointer(ranks, 2);
8056   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRMPI(ierr);
8057   ierr = DMClone(dm, &rdm);CHKERRQ(ierr);
8058   ierr = DMGetDimension(rdm, &dim);CHKERRQ(ierr);
8059   ierr = DMPlexGetHeightStratum(rdm, 0, &cStart, &cEnd);CHKERRQ(ierr);
8060   ierr = DMPlexGetCellType(dm, cStart, &ct);CHKERRQ(ierr);
8061   simplex = DMPolytopeTypeGetNumVertices(ct) == DMPolytopeTypeGetDim(ct)+1 ? PETSC_TRUE : PETSC_FALSE;
8062   ierr = PetscFECreateDefault(PETSC_COMM_SELF, dim, 1, simplex, "PETSc___rank_", -1, &fe);CHKERRQ(ierr);
8063   ierr = PetscObjectSetName((PetscObject) fe, "rank");CHKERRQ(ierr);
8064   ierr = DMSetField(rdm, 0, NULL, (PetscObject) fe);CHKERRQ(ierr);
8065   ierr = PetscFEDestroy(&fe);CHKERRQ(ierr);
8066   ierr = DMCreateDS(rdm);CHKERRQ(ierr);
8067   ierr = DMCreateGlobalVector(rdm, ranks);CHKERRQ(ierr);
8068   ierr = PetscObjectSetName((PetscObject) *ranks, "partition");CHKERRQ(ierr);
8069   ierr = VecGetArray(*ranks, &r);CHKERRQ(ierr);
8070   for (c = cStart; c < cEnd; ++c) {
8071     PetscScalar *lr;
8072 
8073     ierr = DMPlexPointGlobalRef(rdm, c, r, &lr);CHKERRQ(ierr);
8074     if (lr) *lr = rank;
8075   }
8076   ierr = VecRestoreArray(*ranks, &r);CHKERRQ(ierr);
8077   ierr = DMDestroy(&rdm);CHKERRQ(ierr);
8078   PetscFunctionReturn(0);
8079 }
8080 
8081 /*@
8082   DMPlexCreateLabelField - Create a cell field whose value is the label value for that cell
8083 
8084   Input Parameters:
8085 + dm    - The DMPlex
8086 - label - The DMLabel
8087 
8088   Output Parameter:
8089 . val - The label value field
8090 
8091   Options Database Keys:
8092 . -dm_label_view - Adds the label value field into the DM output from -dm_view using the same viewer
8093 
8094   Level: intermediate
8095 
8096 .seealso: DMView()
8097 @*/
8098 PetscErrorCode DMPlexCreateLabelField(DM dm, DMLabel label, Vec *val)
8099 {
8100   DM             rdm;
8101   PetscFE        fe;
8102   PetscScalar   *v;
8103   PetscInt       dim, cStart, cEnd, c;
8104   PetscErrorCode ierr;
8105 
8106   PetscFunctionBeginUser;
8107   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8108   PetscValidPointer(label, 2);
8109   PetscValidPointer(val, 3);
8110   ierr = DMClone(dm, &rdm);CHKERRQ(ierr);
8111   ierr = DMGetDimension(rdm, &dim);CHKERRQ(ierr);
8112   ierr = PetscFECreateDefault(PetscObjectComm((PetscObject) rdm), dim, 1, PETSC_TRUE, "PETSc___label_value_", -1, &fe);CHKERRQ(ierr);
8113   ierr = PetscObjectSetName((PetscObject) fe, "label_value");CHKERRQ(ierr);
8114   ierr = DMSetField(rdm, 0, NULL, (PetscObject) fe);CHKERRQ(ierr);
8115   ierr = PetscFEDestroy(&fe);CHKERRQ(ierr);
8116   ierr = DMCreateDS(rdm);CHKERRQ(ierr);
8117   ierr = DMPlexGetHeightStratum(rdm, 0, &cStart, &cEnd);CHKERRQ(ierr);
8118   ierr = DMCreateGlobalVector(rdm, val);CHKERRQ(ierr);
8119   ierr = PetscObjectSetName((PetscObject) *val, "label_value");CHKERRQ(ierr);
8120   ierr = VecGetArray(*val, &v);CHKERRQ(ierr);
8121   for (c = cStart; c < cEnd; ++c) {
8122     PetscScalar *lv;
8123     PetscInt     cval;
8124 
8125     ierr = DMPlexPointGlobalRef(rdm, c, v, &lv);CHKERRQ(ierr);
8126     ierr = DMLabelGetValue(label, c, &cval);CHKERRQ(ierr);
8127     *lv = cval;
8128   }
8129   ierr = VecRestoreArray(*val, &v);CHKERRQ(ierr);
8130   ierr = DMDestroy(&rdm);CHKERRQ(ierr);
8131   PetscFunctionReturn(0);
8132 }
8133 
8134 /*@
8135   DMPlexCheckSymmetry - Check that the adjacency information in the mesh is symmetric.
8136 
8137   Input Parameter:
8138 . dm - The DMPlex object
8139 
8140   Notes:
8141   This is a useful diagnostic when creating meshes programmatically.
8142 
8143   For the complete list of DMPlexCheck* functions, see DMSetFromOptions().
8144 
8145   Level: developer
8146 
8147 .seealso: DMCreate(), DMSetFromOptions()
8148 @*/
8149 PetscErrorCode DMPlexCheckSymmetry(DM dm)
8150 {
8151   PetscSection    coneSection, supportSection;
8152   const PetscInt *cone, *support;
8153   PetscInt        coneSize, c, supportSize, s;
8154   PetscInt        pStart, pEnd, p, pp, csize, ssize;
8155   PetscBool       storagecheck = PETSC_TRUE;
8156   PetscErrorCode  ierr;
8157 
8158   PetscFunctionBegin;
8159   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8160   ierr = DMViewFromOptions(dm, NULL, "-sym_dm_view");CHKERRQ(ierr);
8161   ierr = DMPlexGetConeSection(dm, &coneSection);CHKERRQ(ierr);
8162   ierr = DMPlexGetSupportSection(dm, &supportSection);CHKERRQ(ierr);
8163   /* Check that point p is found in the support of its cone points, and vice versa */
8164   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
8165   for (p = pStart; p < pEnd; ++p) {
8166     ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
8167     ierr = DMPlexGetCone(dm, p, &cone);CHKERRQ(ierr);
8168     for (c = 0; c < coneSize; ++c) {
8169       PetscBool dup = PETSC_FALSE;
8170       PetscInt  d;
8171       for (d = c-1; d >= 0; --d) {
8172         if (cone[c] == cone[d]) {dup = PETSC_TRUE; break;}
8173       }
8174       ierr = DMPlexGetSupportSize(dm, cone[c], &supportSize);CHKERRQ(ierr);
8175       ierr = DMPlexGetSupport(dm, cone[c], &support);CHKERRQ(ierr);
8176       for (s = 0; s < supportSize; ++s) {
8177         if (support[s] == p) break;
8178       }
8179       if ((s >= supportSize) || (dup && (support[s+1] != p))) {
8180         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %D cone: ", p);CHKERRQ(ierr);
8181         for (s = 0; s < coneSize; ++s) {
8182           ierr = PetscPrintf(PETSC_COMM_SELF, "%D, ", cone[s]);CHKERRQ(ierr);
8183         }
8184         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
8185         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %D support: ", cone[c]);CHKERRQ(ierr);
8186         for (s = 0; s < supportSize; ++s) {
8187           ierr = PetscPrintf(PETSC_COMM_SELF, "%D, ", support[s]);CHKERRQ(ierr);
8188         }
8189         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
8190         if (dup) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %D not repeatedly found in support of repeated cone point %D", p, cone[c]);
8191         else SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %D not found in support of cone point %D", p, cone[c]);
8192       }
8193     }
8194     ierr = DMPlexGetTreeParent(dm, p, &pp, NULL);CHKERRQ(ierr);
8195     if (p != pp) { storagecheck = PETSC_FALSE; continue; }
8196     ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
8197     ierr = DMPlexGetSupport(dm, p, &support);CHKERRQ(ierr);
8198     for (s = 0; s < supportSize; ++s) {
8199       ierr = DMPlexGetConeSize(dm, support[s], &coneSize);CHKERRQ(ierr);
8200       ierr = DMPlexGetCone(dm, support[s], &cone);CHKERRQ(ierr);
8201       for (c = 0; c < coneSize; ++c) {
8202         ierr = DMPlexGetTreeParent(dm, cone[c], &pp, NULL);CHKERRQ(ierr);
8203         if (cone[c] != pp) { c = 0; break; }
8204         if (cone[c] == p) break;
8205       }
8206       if (c >= coneSize) {
8207         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %D support: ", p);CHKERRQ(ierr);
8208         for (c = 0; c < supportSize; ++c) {
8209           ierr = PetscPrintf(PETSC_COMM_SELF, "%D, ", support[c]);CHKERRQ(ierr);
8210         }
8211         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
8212         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %D cone: ", support[s]);CHKERRQ(ierr);
8213         for (c = 0; c < coneSize; ++c) {
8214           ierr = PetscPrintf(PETSC_COMM_SELF, "%D, ", cone[c]);CHKERRQ(ierr);
8215         }
8216         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
8217         SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %D not found in cone of support point %D", p, support[s]);
8218       }
8219     }
8220   }
8221   if (storagecheck) {
8222     ierr = PetscSectionGetStorageSize(coneSection, &csize);CHKERRQ(ierr);
8223     ierr = PetscSectionGetStorageSize(supportSection, &ssize);CHKERRQ(ierr);
8224     if (csize != ssize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "Total cone size %D != Total support size %D", csize, ssize);
8225   }
8226   PetscFunctionReturn(0);
8227 }
8228 
8229 /*
8230   For submeshes with cohesive cells (see DMPlexConstructCohesiveCells()), we allow a special case where some of the boundary of a face (edges and vertices) are not duplicated. We call these special boundary points "unsplit", since the same edge or vertex appears in both copies of the face. These unsplit points throw off our counting, so we have to explicitly account for them here.
8231 */
8232 static PetscErrorCode DMPlexCellUnsplitVertices_Private(DM dm, PetscInt c, DMPolytopeType ct, PetscInt *unsplit)
8233 {
8234   DMPolytopeType  cct;
8235   PetscInt        ptpoints[4];
8236   const PetscInt *cone, *ccone, *ptcone;
8237   PetscInt        coneSize, cp, cconeSize, ccp, npt = 0, pt;
8238   PetscErrorCode  ierr;
8239 
8240   PetscFunctionBegin;
8241   *unsplit = 0;
8242   switch (ct) {
8243     case DM_POLYTOPE_POINT_PRISM_TENSOR:
8244       ptpoints[npt++] = c;
8245       break;
8246     case DM_POLYTOPE_SEG_PRISM_TENSOR:
8247       ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
8248       ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
8249       for (cp = 0; cp < coneSize; ++cp) {
8250         ierr = DMPlexGetCellType(dm, cone[cp], &cct);CHKERRQ(ierr);
8251         if (cct == DM_POLYTOPE_POINT_PRISM_TENSOR) ptpoints[npt++] = cone[cp];
8252       }
8253       break;
8254     case DM_POLYTOPE_TRI_PRISM_TENSOR:
8255     case DM_POLYTOPE_QUAD_PRISM_TENSOR:
8256       ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
8257       ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
8258       for (cp = 0; cp < coneSize; ++cp) {
8259         ierr = DMPlexGetCone(dm, cone[cp], &ccone);CHKERRQ(ierr);
8260         ierr = DMPlexGetConeSize(dm, cone[cp], &cconeSize);CHKERRQ(ierr);
8261         for (ccp = 0; ccp < cconeSize; ++ccp) {
8262           ierr = DMPlexGetCellType(dm, ccone[ccp], &cct);CHKERRQ(ierr);
8263           if (cct == DM_POLYTOPE_POINT_PRISM_TENSOR) {
8264             PetscInt p;
8265             for (p = 0; p < npt; ++p) if (ptpoints[p] == ccone[ccp]) break;
8266             if (p == npt) ptpoints[npt++] = ccone[ccp];
8267           }
8268         }
8269       }
8270       break;
8271     default: break;
8272   }
8273   for (pt = 0; pt < npt; ++pt) {
8274     ierr = DMPlexGetCone(dm, ptpoints[pt], &ptcone);CHKERRQ(ierr);
8275     if (ptcone[0] == ptcone[1]) ++(*unsplit);
8276   }
8277   PetscFunctionReturn(0);
8278 }
8279 
8280 /*@
8281   DMPlexCheckSkeleton - Check that each cell has the correct number of vertices
8282 
8283   Input Parameters:
8284 + dm - The DMPlex object
8285 - cellHeight - Normally 0
8286 
8287   Notes:
8288   This is a useful diagnostic when creating meshes programmatically.
8289   Currently applicable only to homogeneous simplex or tensor meshes.
8290 
8291   For the complete list of DMPlexCheck* functions, see DMSetFromOptions().
8292 
8293   Level: developer
8294 
8295 .seealso: DMCreate(), DMSetFromOptions()
8296 @*/
8297 PetscErrorCode DMPlexCheckSkeleton(DM dm, PetscInt cellHeight)
8298 {
8299   DMPlexInterpolatedFlag interp;
8300   DMPolytopeType         ct;
8301   PetscInt               vStart, vEnd, cStart, cEnd, c;
8302   PetscErrorCode         ierr;
8303 
8304   PetscFunctionBegin;
8305   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8306   ierr = DMPlexIsInterpolated(dm, &interp);CHKERRQ(ierr);
8307   ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
8308   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
8309   for (c = cStart; c < cEnd; ++c) {
8310     PetscInt *closure = NULL;
8311     PetscInt  coneSize, closureSize, cl, Nv = 0;
8312 
8313     ierr = DMPlexGetCellType(dm, c, &ct);CHKERRQ(ierr);
8314     if ((PetscInt) ct < 0) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %D has no cell type", c);
8315     if (ct == DM_POLYTOPE_UNKNOWN) continue;
8316     if (interp == DMPLEX_INTERPOLATED_FULL) {
8317       ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
8318       if (coneSize != DMPolytopeTypeGetConeSize(ct)) SETERRQ4(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %D of type %s has cone size %D != %D", c, DMPolytopeTypes[ct], coneSize, DMPolytopeTypeGetConeSize(ct));
8319     }
8320     ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
8321     for (cl = 0; cl < closureSize*2; cl += 2) {
8322       const PetscInt p = closure[cl];
8323       if ((p >= vStart) && (p < vEnd)) ++Nv;
8324     }
8325     ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
8326     /* Special Case: Tensor faces with identified vertices */
8327     if (Nv < DMPolytopeTypeGetNumVertices(ct)) {
8328       PetscInt unsplit;
8329 
8330       ierr = DMPlexCellUnsplitVertices_Private(dm, c, ct, &unsplit);CHKERRQ(ierr);
8331       if (Nv + unsplit == DMPolytopeTypeGetNumVertices(ct)) continue;
8332     }
8333     if (Nv != DMPolytopeTypeGetNumVertices(ct)) SETERRQ4(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %D of type %s has %D vertices != %D", c, DMPolytopeTypes[ct], Nv, DMPolytopeTypeGetNumVertices(ct));
8334   }
8335   PetscFunctionReturn(0);
8336 }
8337 
8338 /*@
8339   DMPlexCheckFaces - Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type
8340 
8341   Not Collective
8342 
8343   Input Parameters:
8344 + dm - The DMPlex object
8345 - cellHeight - Normally 0
8346 
8347   Notes:
8348   This is a useful diagnostic when creating meshes programmatically.
8349   This routine is only relevant for meshes that are fully interpolated across all ranks.
8350   It will error out if a partially interpolated mesh is given on some rank.
8351   It will do nothing for locally uninterpolated mesh (as there is nothing to check).
8352 
8353   For the complete list of DMPlexCheck* functions, see DMSetFromOptions().
8354 
8355   Level: developer
8356 
8357 .seealso: DMCreate(), DMPlexGetVTKCellHeight(), DMSetFromOptions()
8358 @*/
8359 PetscErrorCode DMPlexCheckFaces(DM dm, PetscInt cellHeight)
8360 {
8361   PetscInt       dim, depth, vStart, vEnd, cStart, cEnd, c, h;
8362   PetscErrorCode ierr;
8363   DMPlexInterpolatedFlag interpEnum;
8364 
8365   PetscFunctionBegin;
8366   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8367   ierr = DMPlexIsInterpolated(dm, &interpEnum);CHKERRQ(ierr);
8368   if (interpEnum == DMPLEX_INTERPOLATED_NONE) PetscFunctionReturn(0);
8369   if (interpEnum == DMPLEX_INTERPOLATED_PARTIAL) {
8370     PetscMPIInt rank;
8371     MPI_Comm    comm;
8372 
8373     ierr = PetscObjectGetComm((PetscObject) dm, &comm);CHKERRQ(ierr);
8374     ierr = MPI_Comm_rank(comm, &rank);CHKERRMPI(ierr);
8375     SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_SUP, "Mesh is only partially interpolated on rank %d, this is currently not supported", rank);
8376   }
8377 
8378   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
8379   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
8380   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
8381   for (h = cellHeight; h < PetscMin(depth, dim); ++h) {
8382     ierr = DMPlexGetHeightStratum(dm, h, &cStart, &cEnd);CHKERRQ(ierr);
8383     for (c = cStart; c < cEnd; ++c) {
8384       const PetscInt      *cone, *ornt, *faceSizes, *faces;
8385       const DMPolytopeType *faceTypes;
8386       DMPolytopeType        ct;
8387       PetscInt              numFaces, coneSize, f;
8388       PetscInt             *closure = NULL, closureSize, cl, numCorners = 0, fOff = 0, unsplit;
8389 
8390       ierr = DMPlexGetCellType(dm, c, &ct);CHKERRQ(ierr);
8391       ierr = DMPlexCellUnsplitVertices_Private(dm, c, ct, &unsplit);CHKERRQ(ierr);
8392       if (unsplit) continue;
8393       ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
8394       ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
8395       ierr = DMPlexGetConeOrientation(dm, c, &ornt);CHKERRQ(ierr);
8396       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
8397       for (cl = 0; cl < closureSize*2; cl += 2) {
8398         const PetscInt p = closure[cl];
8399         if ((p >= vStart) && (p < vEnd)) closure[numCorners++] = p;
8400       }
8401       ierr = DMPlexGetRawFaces_Internal(dm, ct, closure, &numFaces, &faceTypes, &faceSizes, &faces);CHKERRQ(ierr);
8402       if (coneSize != numFaces) SETERRQ4(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %D of type %s has %D faces but should have %D", c, DMPolytopeTypes[ct], coneSize, numFaces);
8403       for (f = 0; f < numFaces; ++f) {
8404         DMPolytopeType fct;
8405         PetscInt       *fclosure = NULL, fclosureSize, cl, fnumCorners = 0, v;
8406 
8407         ierr = DMPlexGetCellType(dm, cone[f], &fct);CHKERRQ(ierr);
8408         ierr = DMPlexGetTransitiveClosure_Internal(dm, cone[f], ornt[f], PETSC_TRUE, &fclosureSize, &fclosure);CHKERRQ(ierr);
8409         for (cl = 0; cl < fclosureSize*2; cl += 2) {
8410           const PetscInt p = fclosure[cl];
8411           if ((p >= vStart) && (p < vEnd)) fclosure[fnumCorners++] = p;
8412         }
8413         if (fnumCorners != faceSizes[f]) SETERRQ7(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %D of type %s (cone idx %D) of cell %D of type %s has %D vertices but should have %D", cone[f], DMPolytopeTypes[fct], f, c, DMPolytopeTypes[ct], fnumCorners, faceSizes[f]);
8414         for (v = 0; v < fnumCorners; ++v) {
8415           if (fclosure[v] != faces[fOff+v]) {
8416             PetscInt v1;
8417 
8418             ierr = PetscPrintf(PETSC_COMM_SELF, "face closure:");CHKERRQ(ierr);
8419             for (v1 = 0; v1 < fnumCorners; ++v1) {ierr = PetscPrintf(PETSC_COMM_SELF, " %D", fclosure[v1]);CHKERRQ(ierr);}
8420             ierr = PetscPrintf(PETSC_COMM_SELF, "\ncell face:");CHKERRQ(ierr);
8421             for (v1 = 0; v1 < fnumCorners; ++v1) {ierr = PetscPrintf(PETSC_COMM_SELF, " %D", faces[fOff+v1]);CHKERRQ(ierr);}
8422             ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
8423             SETERRQ9(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %D of type %s (cone idx %d, ornt %D) of cell %D of type %s vertex %D, %D != %D", cone[f], DMPolytopeTypes[fct], f, ornt[f], c, DMPolytopeTypes[ct], v, fclosure[v], faces[fOff+v]);
8424           }
8425         }
8426         ierr = DMPlexRestoreTransitiveClosure(dm, cone[f], PETSC_TRUE, &fclosureSize, &fclosure);CHKERRQ(ierr);
8427         fOff += faceSizes[f];
8428       }
8429       ierr = DMPlexRestoreRawFaces_Internal(dm, ct, closure, &numFaces, &faceTypes, &faceSizes, &faces);CHKERRQ(ierr);
8430       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
8431     }
8432   }
8433   PetscFunctionReturn(0);
8434 }
8435 
8436 /*@
8437   DMPlexCheckGeometry - Check the geometry of mesh cells
8438 
8439   Input Parameter:
8440 . dm - The DMPlex object
8441 
8442   Notes:
8443   This is a useful diagnostic when creating meshes programmatically.
8444 
8445   For the complete list of DMPlexCheck* functions, see DMSetFromOptions().
8446 
8447   Level: developer
8448 
8449 .seealso: DMCreate(), DMSetFromOptions()
8450 @*/
8451 PetscErrorCode DMPlexCheckGeometry(DM dm)
8452 {
8453   Vec            coordinates;
8454   PetscReal      detJ, J[9], refVol = 1.0;
8455   PetscReal      vol;
8456   PetscBool      periodic;
8457   PetscInt       dim, depth, dE, d, cStart, cEnd, c;
8458   PetscErrorCode ierr;
8459 
8460   PetscFunctionBegin;
8461   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
8462   ierr = DMGetCoordinateDim(dm, &dE);CHKERRQ(ierr);
8463   if (dim != dE) PetscFunctionReturn(0);
8464   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
8465   ierr = DMGetPeriodicity(dm, &periodic, NULL, NULL, NULL);CHKERRQ(ierr);
8466   for (d = 0; d < dim; ++d) refVol *= 2.0;
8467   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
8468   /* Make sure local coordinates are created, because that step is collective */
8469   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
8470   for (c = cStart; c < cEnd; ++c) {
8471     DMPolytopeType ct;
8472     PetscInt       unsplit;
8473     PetscBool      ignoreZeroVol = PETSC_FALSE;
8474 
8475     ierr = DMPlexGetCellType(dm, c, &ct);CHKERRQ(ierr);
8476     switch (ct) {
8477       case DM_POLYTOPE_SEG_PRISM_TENSOR:
8478       case DM_POLYTOPE_TRI_PRISM_TENSOR:
8479       case DM_POLYTOPE_QUAD_PRISM_TENSOR:
8480         ignoreZeroVol = PETSC_TRUE; break;
8481       default: break;
8482     }
8483     switch (ct) {
8484       case DM_POLYTOPE_TRI_PRISM:
8485       case DM_POLYTOPE_TRI_PRISM_TENSOR:
8486       case DM_POLYTOPE_QUAD_PRISM_TENSOR:
8487       case DM_POLYTOPE_PYRAMID:
8488         continue;
8489       default: break;
8490     }
8491     ierr = DMPlexCellUnsplitVertices_Private(dm, c, ct, &unsplit);CHKERRQ(ierr);
8492     if (unsplit) continue;
8493     ierr = DMPlexComputeCellGeometryFEM(dm, c, NULL, NULL, J, NULL, &detJ);CHKERRQ(ierr);
8494     if (detJ < -PETSC_SMALL || (detJ <= 0.0 && !ignoreZeroVol)) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Mesh cell %D of type %s is inverted, |J| = %g", c, DMPolytopeTypes[ct], (double) detJ);
8495     ierr = PetscInfo2(dm, "Cell %D FEM Volume %g\n", c, (double) detJ*refVol);CHKERRQ(ierr);
8496     if (depth > 1 && !periodic) {
8497       ierr = DMPlexComputeCellGeometryFVM(dm, c, &vol, NULL, NULL);CHKERRQ(ierr);
8498       if (vol < -PETSC_SMALL || (vol <= 0.0 && !ignoreZeroVol)) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Mesh cell %D of type %s is inverted, vol = %g", c, DMPolytopeTypes[ct], (double) vol);
8499       ierr = PetscInfo2(dm, "Cell %D FVM Volume %g\n", c, (double) vol);CHKERRQ(ierr);
8500     }
8501   }
8502   PetscFunctionReturn(0);
8503 }
8504 
8505 /*@
8506   DMPlexCheckPointSF - Check that several necessary conditions are met for the point SF of this plex.
8507 
8508   Input Parameters:
8509 . dm - The DMPlex object
8510 
8511   Notes:
8512   This is mainly intended for debugging/testing purposes.
8513   It currently checks only meshes with no partition overlapping.
8514 
8515   For the complete list of DMPlexCheck* functions, see DMSetFromOptions().
8516 
8517   Level: developer
8518 
8519 .seealso: DMGetPointSF(), DMSetFromOptions()
8520 @*/
8521 PetscErrorCode DMPlexCheckPointSF(DM dm)
8522 {
8523   PetscSF         pointSF;
8524   PetscInt        cellHeight, cStart, cEnd, l, nleaves, nroots, overlap;
8525   const PetscInt *locals, *rootdegree;
8526   PetscBool       distributed;
8527   PetscErrorCode  ierr;
8528 
8529   PetscFunctionBegin;
8530   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8531   ierr = DMGetPointSF(dm, &pointSF);CHKERRQ(ierr);
8532   ierr = DMPlexIsDistributed(dm, &distributed);CHKERRQ(ierr);
8533   if (!distributed) PetscFunctionReturn(0);
8534   ierr = DMPlexGetOverlap(dm, &overlap);CHKERRQ(ierr);
8535   if (overlap) {
8536     ierr = PetscPrintf(PetscObjectComm((PetscObject)dm), "Warning: DMPlexCheckPointSF() is currently not implemented for meshes with partition overlapping");CHKERRQ(ierr);
8537     PetscFunctionReturn(0);
8538   }
8539   if (!pointSF) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONGSTATE, "This DMPlex is distributed but does not have PointSF attached");
8540   ierr = PetscSFGetGraph(pointSF, &nroots, &nleaves, &locals, NULL);CHKERRQ(ierr);
8541   if (nroots < 0) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONGSTATE, "This DMPlex is distributed but its PointSF has no graph set");
8542   ierr = PetscSFComputeDegreeBegin(pointSF, &rootdegree);CHKERRQ(ierr);
8543   ierr = PetscSFComputeDegreeEnd(pointSF, &rootdegree);CHKERRQ(ierr);
8544 
8545   /* 1) check there are no faces in 2D, cells in 3D, in interface */
8546   ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
8547   ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
8548   for (l = 0; l < nleaves; ++l) {
8549     const PetscInt point = locals[l];
8550 
8551     if (point >= cStart && point < cEnd) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point SF contains %D which is a cell", point);
8552   }
8553 
8554   /* 2) if some point is in interface, then all its cone points must be also in interface (either as leaves or roots) */
8555   for (l = 0; l < nleaves; ++l) {
8556     const PetscInt  point = locals[l];
8557     const PetscInt *cone;
8558     PetscInt        coneSize, c, idx;
8559 
8560     ierr = DMPlexGetConeSize(dm, point, &coneSize);CHKERRQ(ierr);
8561     ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
8562     for (c = 0; c < coneSize; ++c) {
8563       if (!rootdegree[cone[c]]) {
8564         ierr = PetscFindInt(cone[c], nleaves, locals, &idx);CHKERRQ(ierr);
8565         if (idx < 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point SF contains %D but not %D from its cone", point, cone[c]);
8566       }
8567     }
8568   }
8569   PetscFunctionReturn(0);
8570 }
8571 
8572 PetscErrorCode DMPlexCheckAll_Internal(DM dm, PetscInt cellHeight)
8573 {
8574   PetscErrorCode ierr;
8575 
8576   PetscFunctionBegin;
8577   ierr = DMPlexCheckSymmetry(dm);CHKERRQ(ierr);
8578   ierr = DMPlexCheckSkeleton(dm, cellHeight);CHKERRQ(ierr);
8579   ierr = DMPlexCheckFaces(dm, cellHeight);CHKERRQ(ierr);
8580   ierr = DMPlexCheckGeometry(dm);CHKERRQ(ierr);
8581   ierr = DMPlexCheckPointSF(dm);CHKERRQ(ierr);
8582   ierr = DMPlexCheckInterfaceCones(dm);CHKERRQ(ierr);
8583   PetscFunctionReturn(0);
8584 }
8585 
8586 typedef struct cell_stats
8587 {
8588   PetscReal min, max, sum, squaresum;
8589   PetscInt  count;
8590 } cell_stats_t;
8591 
8592 static void MPIAPI cell_stats_reduce(void *a, void *b, int * len, MPI_Datatype *datatype)
8593 {
8594   PetscInt i, N = *len;
8595 
8596   for (i = 0; i < N; i++) {
8597     cell_stats_t *A = (cell_stats_t *) a;
8598     cell_stats_t *B = (cell_stats_t *) b;
8599 
8600     B->min = PetscMin(A->min,B->min);
8601     B->max = PetscMax(A->max,B->max);
8602     B->sum += A->sum;
8603     B->squaresum += A->squaresum;
8604     B->count += A->count;
8605   }
8606 }
8607 
8608 /*@
8609   DMPlexCheckCellShape - Checks the Jacobian of the mapping from reference to real cells and computes some minimal statistics.
8610 
8611   Collective on dm
8612 
8613   Input Parameters:
8614 + dm        - The DMPlex object
8615 . output    - If true, statistics will be displayed on stdout
8616 - condLimit - Display all cells above this condition number, or PETSC_DETERMINE for no cell output
8617 
8618   Notes:
8619   This is mainly intended for debugging/testing purposes.
8620 
8621   For the complete list of DMPlexCheck* functions, see DMSetFromOptions().
8622 
8623   Level: developer
8624 
8625 .seealso: DMSetFromOptions(), DMPlexComputeOrthogonalQuality()
8626 @*/
8627 PetscErrorCode DMPlexCheckCellShape(DM dm, PetscBool output, PetscReal condLimit)
8628 {
8629   DM             dmCoarse;
8630   cell_stats_t   stats, globalStats;
8631   MPI_Comm       comm = PetscObjectComm((PetscObject)dm);
8632   PetscReal      *J, *invJ, min = 0, max = 0, mean = 0, stdev = 0;
8633   PetscReal      limit = condLimit > 0 ? condLimit : PETSC_MAX_REAL;
8634   PetscInt       cdim, cStart, cEnd, c, eStart, eEnd, count = 0;
8635   PetscMPIInt    rank,size;
8636   PetscErrorCode ierr;
8637 
8638   PetscFunctionBegin;
8639   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8640   stats.min   = PETSC_MAX_REAL;
8641   stats.max   = PETSC_MIN_REAL;
8642   stats.sum   = stats.squaresum = 0.;
8643   stats.count = 0;
8644 
8645   ierr = MPI_Comm_size(comm, &size);CHKERRMPI(ierr);
8646   ierr = MPI_Comm_rank(comm, &rank);CHKERRMPI(ierr);
8647   ierr = DMGetCoordinateDim(dm,&cdim);CHKERRQ(ierr);
8648   ierr = PetscMalloc2(PetscSqr(cdim), &J, PetscSqr(cdim), &invJ);CHKERRQ(ierr);
8649   ierr = DMPlexGetSimplexOrBoxCells(dm,0,&cStart,&cEnd);CHKERRQ(ierr);
8650   ierr = DMPlexGetDepthStratum(dm,1,&eStart,&eEnd);CHKERRQ(ierr);
8651   for (c = cStart; c < cEnd; c++) {
8652     PetscInt  i;
8653     PetscReal frobJ = 0., frobInvJ = 0., cond2, cond, detJ;
8654 
8655     ierr = DMPlexComputeCellGeometryAffineFEM(dm,c,NULL,J,invJ,&detJ);CHKERRQ(ierr);
8656     if (detJ < 0.0) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Mesh cell %D is inverted", c);
8657     for (i = 0; i < PetscSqr(cdim); ++i) {
8658       frobJ    += J[i] * J[i];
8659       frobInvJ += invJ[i] * invJ[i];
8660     }
8661     cond2 = frobJ * frobInvJ;
8662     cond  = PetscSqrtReal(cond2);
8663 
8664     stats.min        = PetscMin(stats.min,cond);
8665     stats.max        = PetscMax(stats.max,cond);
8666     stats.sum       += cond;
8667     stats.squaresum += cond2;
8668     stats.count++;
8669     if (output && cond > limit) {
8670       PetscSection coordSection;
8671       Vec          coordsLocal;
8672       PetscScalar *coords = NULL;
8673       PetscInt     Nv, d, clSize, cl, *closure = NULL;
8674 
8675       ierr = DMGetCoordinatesLocal(dm, &coordsLocal);CHKERRQ(ierr);
8676       ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
8677       ierr = DMPlexVecGetClosure(dm, coordSection, coordsLocal, c, &Nv, &coords);CHKERRQ(ierr);
8678       ierr = PetscSynchronizedPrintf(comm, "[%d] Cell %D cond %g\n", rank, c, (double) cond);CHKERRQ(ierr);
8679       for (i = 0; i < Nv/cdim; ++i) {
8680         ierr = PetscSynchronizedPrintf(comm, "  Vertex %D: (", i);CHKERRQ(ierr);
8681         for (d = 0; d < cdim; ++d) {
8682           if (d > 0) {ierr = PetscSynchronizedPrintf(comm, ", ");CHKERRQ(ierr);}
8683           ierr = PetscSynchronizedPrintf(comm, "%g", (double) PetscRealPart(coords[i*cdim+d]));CHKERRQ(ierr);
8684         }
8685         ierr = PetscSynchronizedPrintf(comm, ")\n");CHKERRQ(ierr);
8686       }
8687       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &clSize, &closure);CHKERRQ(ierr);
8688       for (cl = 0; cl < clSize*2; cl += 2) {
8689         const PetscInt edge = closure[cl];
8690 
8691         if ((edge >= eStart) && (edge < eEnd)) {
8692           PetscReal len;
8693 
8694           ierr = DMPlexComputeCellGeometryFVM(dm, edge, &len, NULL, NULL);CHKERRQ(ierr);
8695           ierr = PetscSynchronizedPrintf(comm, "  Edge %D: length %g\n", edge, (double) len);CHKERRQ(ierr);
8696         }
8697       }
8698       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &clSize, &closure);CHKERRQ(ierr);
8699       ierr = DMPlexVecRestoreClosure(dm, coordSection, coordsLocal, c, &Nv, &coords);CHKERRQ(ierr);
8700     }
8701   }
8702   if (output) {ierr = PetscSynchronizedFlush(comm, NULL);CHKERRQ(ierr);}
8703 
8704   if (size > 1) {
8705     PetscMPIInt   blockLengths[2] = {4,1};
8706     MPI_Aint      blockOffsets[2] = {offsetof(cell_stats_t,min),offsetof(cell_stats_t,count)};
8707     MPI_Datatype  blockTypes[2]   = {MPIU_REAL,MPIU_INT}, statType;
8708     MPI_Op        statReduce;
8709 
8710     ierr = MPI_Type_create_struct(2,blockLengths,blockOffsets,blockTypes,&statType);CHKERRMPI(ierr);
8711     ierr = MPI_Type_commit(&statType);CHKERRMPI(ierr);
8712     ierr = MPI_Op_create(cell_stats_reduce, PETSC_TRUE, &statReduce);CHKERRMPI(ierr);
8713     ierr = MPI_Reduce(&stats,&globalStats,1,statType,statReduce,0,comm);CHKERRMPI(ierr);
8714     ierr = MPI_Op_free(&statReduce);CHKERRMPI(ierr);
8715     ierr = MPI_Type_free(&statType);CHKERRMPI(ierr);
8716   } else {
8717     ierr = PetscArraycpy(&globalStats,&stats,1);CHKERRQ(ierr);
8718   }
8719   if (!rank) {
8720     count = globalStats.count;
8721     min   = globalStats.min;
8722     max   = globalStats.max;
8723     mean  = globalStats.sum / globalStats.count;
8724     stdev = globalStats.count > 1 ? PetscSqrtReal(PetscMax((globalStats.squaresum - globalStats.count * mean * mean) / (globalStats.count - 1),0)) : 0.0;
8725   }
8726 
8727   if (output) {
8728     ierr = PetscPrintf(comm,"Mesh with %D cells, shape condition numbers: min = %g, max = %g, mean = %g, stddev = %g\n", count, (double) min, (double) max, (double) mean, (double) stdev);CHKERRQ(ierr);
8729   }
8730   ierr = PetscFree2(J,invJ);CHKERRQ(ierr);
8731 
8732   ierr = DMGetCoarseDM(dm,&dmCoarse);CHKERRQ(ierr);
8733   if (dmCoarse) {
8734     PetscBool isplex;
8735 
8736     ierr = PetscObjectTypeCompare((PetscObject)dmCoarse,DMPLEX,&isplex);CHKERRQ(ierr);
8737     if (isplex) {
8738       ierr = DMPlexCheckCellShape(dmCoarse,output,condLimit);CHKERRQ(ierr);
8739     }
8740   }
8741   PetscFunctionReturn(0);
8742 }
8743 
8744 /*@
8745   DMPlexComputeOrthogonalQuality - Compute cell-wise orthogonal quality mesh statistic. Optionally tags all cells with
8746   orthogonal quality below given tolerance.
8747 
8748   Collective on dm
8749 
8750   Input Parameters:
8751 + dm   - The DMPlex object
8752 . fv   - Optional PetscFV object for pre-computed cell/face centroid information
8753 - atol - [0, 1] Absolute tolerance for tagging cells.
8754 
8755   Output Parameters:
8756 + OrthQual      - Vec containing orthogonal quality per cell
8757 - OrthQualLabel - DMLabel tagging cells below atol with DM_ADAPT_REFINE
8758 
8759   Options Database Keys:
8760 + -dm_plex_orthogonal_quality_label_view - view OrthQualLabel if label is requested. Currently only PETSCVIEWERASCII is
8761 supported.
8762 - -dm_plex_orthogonal_quality_vec_view - view OrthQual vector.
8763 
8764   Notes:
8765   Orthogonal quality is given by the following formula:
8766 
8767   \min \left[ \frac{A_i \cdot f_i}{\|A_i\| \|f_i\|} , \frac{A_i \cdot c_i}{\|A_i\| \|c_i\|} \right]
8768 
8769   Where A_i is the i'th face-normal vector, f_i is the vector from the cell centroid to the i'th face centroid, and c_i
8770   is the vector from the current cells centroid to the centroid of its i'th neighbor (which shares a face with the
8771   current cell). This computes the vector similarity between each cell face and its corresponding neighbor centroid by
8772   calculating the cosine of the angle between these vectors.
8773 
8774   Orthogonal quality ranges from 1 (best) to 0 (worst).
8775 
8776   This routine is mainly useful for FVM, however is not restricted to only FVM. The PetscFV object is optionally used to check for
8777   pre-computed FVM cell data, but if it is not passed in then this data will be computed.
8778 
8779   Cells are tagged if they have an orthogonal quality less than or equal to the absolute tolerance.
8780 
8781   Level: intermediate
8782 
8783 .seealso: DMPlexCheckCellShape(), DMCreateLabel()
8784 @*/
8785 PetscErrorCode DMPlexComputeOrthogonalQuality(DM dm, PetscFV fv, PetscReal atol, Vec *OrthQual, DMLabel *OrthQualLabel)
8786 {
8787   PetscInt                nc, cellHeight, cStart, cEnd, cell, cellIter = 0;
8788   PetscInt                *idx;
8789   PetscScalar             *oqVals;
8790   const PetscScalar       *cellGeomArr, *faceGeomArr;
8791   PetscReal               *ci, *fi, *Ai;
8792   MPI_Comm                comm;
8793   Vec                     cellgeom, facegeom;
8794   DM                      dmFace, dmCell;
8795   IS                      glob;
8796   ISLocalToGlobalMapping  ltog;
8797   PetscViewer             vwr;
8798   PetscErrorCode          ierr;
8799 
8800   PetscFunctionBegin;
8801   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8802   if (fv) {PetscValidHeaderSpecific(fv, PETSCFV_CLASSID, 2);}
8803   PetscValidPointer(OrthQual, 4);
8804   if (PetscUnlikelyDebug(atol < 0.0 || atol > 1.0)) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"Absolute tolerance %g not in [0,1]",(double)atol);
8805   ierr = PetscObjectGetComm((PetscObject) dm, &comm);CHKERRQ(ierr);
8806   ierr = DMGetDimension(dm, &nc);CHKERRQ(ierr);
8807   if (nc < 2) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONGSTATE, "DM must have dimension >= 2 (current %D)", nc);
8808   {
8809     DMPlexInterpolatedFlag interpFlag;
8810 
8811     ierr = DMPlexIsInterpolated(dm, &interpFlag);CHKERRQ(ierr);
8812     if (interpFlag != DMPLEX_INTERPOLATED_FULL) {
8813       PetscMPIInt rank;
8814 
8815       ierr = MPI_Comm_rank(comm, &rank);CHKERRMPI(ierr);
8816       SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONGSTATE, "DM must be fully interpolated, DM on rank %d is not fully interpolated", rank);
8817     }
8818   }
8819   if (OrthQualLabel) {
8820     PetscValidPointer(OrthQualLabel, 5);
8821     ierr = DMCreateLabel(dm, "Orthogonal_Quality");CHKERRQ(ierr);
8822     ierr = DMGetLabel(dm, "Orthogonal_Quality", OrthQualLabel);CHKERRQ(ierr);
8823   } else {*OrthQualLabel = NULL;}
8824   ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
8825   ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
8826   ierr = DMPlexCreateCellNumbering_Internal(dm, PETSC_TRUE, &glob);CHKERRQ(ierr);
8827   ierr = ISLocalToGlobalMappingCreateIS(glob, &ltog);CHKERRQ(ierr);
8828   ierr = ISLocalToGlobalMappingSetType(ltog, ISLOCALTOGLOBALMAPPINGHASH);CHKERRQ(ierr);
8829   ierr = VecCreate(comm, OrthQual);CHKERRQ(ierr);
8830   ierr = VecSetType(*OrthQual, VECSTANDARD);CHKERRQ(ierr);
8831   ierr = VecSetSizes(*OrthQual, cEnd-cStart, PETSC_DETERMINE);CHKERRQ(ierr);
8832   ierr = VecSetLocalToGlobalMapping(*OrthQual, ltog);CHKERRQ(ierr);
8833   ierr = VecSetUp(*OrthQual);CHKERRQ(ierr);
8834   ierr = ISDestroy(&glob);CHKERRQ(ierr);
8835   ierr = ISLocalToGlobalMappingDestroy(&ltog);CHKERRQ(ierr);
8836   ierr = DMPlexGetDataFVM(dm, fv, &cellgeom, &facegeom, NULL);CHKERRQ(ierr);
8837   ierr = VecGetArrayRead(cellgeom, &cellGeomArr);CHKERRQ(ierr);
8838   ierr = VecGetArrayRead(facegeom, &faceGeomArr);CHKERRQ(ierr);
8839   ierr = VecGetDM(cellgeom, &dmCell);CHKERRQ(ierr);
8840   ierr = VecGetDM(facegeom, &dmFace);CHKERRQ(ierr);
8841   ierr = PetscMalloc5(cEnd-cStart, &idx, cEnd-cStart, &oqVals, nc, &ci, nc, &fi, nc, &Ai);CHKERRQ(ierr);
8842   for (cell = cStart; cell < cEnd; cellIter++,cell++) {
8843     PetscInt           cellneigh, cellneighiter = 0, adjSize = PETSC_DETERMINE;
8844     PetscInt           cellarr[2], *adj = NULL;
8845     PetscScalar        *cArr, *fArr;
8846     PetscReal          minvalc = 1.0, minvalf = 1.0;
8847     PetscFVCellGeom    *cg;
8848 
8849     idx[cellIter] = cell-cStart;
8850     cellarr[0] = cell;
8851     /* Make indexing into cellGeom easier */
8852     ierr = DMPlexPointLocalRead(dmCell, cell, cellGeomArr, &cg);CHKERRQ(ierr);
8853     ierr = DMPlexGetAdjacency_Internal(dm, cell, PETSC_TRUE, PETSC_FALSE, PETSC_FALSE, &adjSize, &adj);CHKERRQ(ierr);
8854     /* Technically 1 too big, but easier than fiddling with empty adjacency array */
8855     ierr = PetscCalloc2(adjSize, &cArr, adjSize, &fArr);CHKERRQ(ierr);
8856     for (cellneigh = 0; cellneigh < adjSize; cellneighiter++,cellneigh++) {
8857       PetscInt         i;
8858       const PetscInt   neigh = adj[cellneigh];
8859       PetscReal        normci = 0, normfi = 0, normai = 0;
8860       PetscFVCellGeom  *cgneigh;
8861       PetscFVFaceGeom  *fg;
8862 
8863       /* Don't count ourselves in the neighbor list */
8864       if (neigh == cell) continue;
8865       ierr = DMPlexPointLocalRead(dmCell, neigh, cellGeomArr, &cgneigh);CHKERRQ(ierr);
8866       cellarr[1] = neigh;
8867       {
8868         PetscInt       numcovpts;
8869         const PetscInt *covpts;
8870 
8871         ierr = DMPlexGetMeet(dm, 2, cellarr, &numcovpts, &covpts);CHKERRQ(ierr);
8872         ierr = DMPlexPointLocalRead(dmFace, covpts[0], faceGeomArr, &fg);CHKERRQ(ierr);
8873         ierr = DMPlexRestoreMeet(dm, 2, cellarr, &numcovpts, &covpts);CHKERRQ(ierr);
8874       }
8875 
8876       /* Compute c_i, f_i and their norms */
8877       for (i = 0; i < nc; i++) {
8878         ci[i] = cgneigh->centroid[i] - cg->centroid[i];
8879         fi[i] = fg->centroid[i] - cg->centroid[i];
8880         Ai[i] = fg->normal[i];
8881         normci += PetscPowReal(ci[i], 2);
8882         normfi += PetscPowReal(fi[i], 2);
8883         normai += PetscPowReal(Ai[i], 2);
8884       }
8885       normci = PetscSqrtReal(normci);
8886       normfi = PetscSqrtReal(normfi);
8887       normai = PetscSqrtReal(normai);
8888 
8889       /* Normalize and compute for each face-cell-normal pair */
8890       for (i = 0; i < nc; i++) {
8891         ci[i] = ci[i]/normci;
8892         fi[i] = fi[i]/normfi;
8893         Ai[i] = Ai[i]/normai;
8894         /* PetscAbs because I don't know if normals are guaranteed to point out */
8895         cArr[cellneighiter] += PetscAbs(Ai[i]*ci[i]);
8896         fArr[cellneighiter] += PetscAbs(Ai[i]*fi[i]);
8897       }
8898       if (PetscRealPart(cArr[cellneighiter]) < minvalc) {
8899         minvalc = PetscRealPart(cArr[cellneighiter]);
8900       }
8901       if (PetscRealPart(fArr[cellneighiter]) < minvalf) {
8902         minvalf = PetscRealPart(fArr[cellneighiter]);
8903       }
8904     }
8905     ierr = PetscFree(adj);CHKERRQ(ierr);
8906     ierr = PetscFree2(cArr, fArr);CHKERRQ(ierr);
8907     /* Defer to cell if they're equal */
8908     oqVals[cellIter] = PetscMin(minvalf, minvalc);
8909     if (OrthQualLabel) {
8910       if (PetscRealPart(oqVals[cellIter]) <= atol) {ierr = DMLabelSetValue(*OrthQualLabel, cell, DM_ADAPT_REFINE);CHKERRQ(ierr);}
8911     }
8912   }
8913   ierr = VecSetValuesLocal(*OrthQual, cEnd-cStart, idx, oqVals, INSERT_VALUES);CHKERRQ(ierr);
8914   ierr = VecAssemblyBegin(*OrthQual);CHKERRQ(ierr);
8915   ierr = VecAssemblyEnd(*OrthQual);CHKERRQ(ierr);
8916   ierr = VecRestoreArrayRead(cellgeom, &cellGeomArr);CHKERRQ(ierr);
8917   ierr = VecRestoreArrayRead(facegeom, &faceGeomArr);CHKERRQ(ierr);
8918   ierr = PetscOptionsGetViewer(comm, NULL, NULL, "-dm_plex_orthogonal_quality_label_view", &vwr, NULL, NULL);CHKERRQ(ierr);
8919   if (OrthQualLabel) {
8920     if (vwr) {ierr = DMLabelView(*OrthQualLabel, vwr);CHKERRQ(ierr);}
8921   }
8922   ierr = PetscFree5(idx, oqVals, ci, fi, Ai);CHKERRQ(ierr);
8923   ierr = PetscViewerDestroy(&vwr);CHKERRQ(ierr);
8924   ierr = VecViewFromOptions(*OrthQual, NULL, "-dm_plex_orthogonal_quality_vec_view");CHKERRQ(ierr);
8925   PetscFunctionReturn(0);
8926 }
8927 
8928 /* this is here insead of DMGetOutputDM because output DM still has constraints in the local indices that affect
8929  * interpolator construction */
8930 static PetscErrorCode DMGetFullDM(DM dm, DM *odm)
8931 {
8932   PetscSection   section, newSection, gsection;
8933   PetscSF        sf;
8934   PetscBool      hasConstraints, ghasConstraints;
8935   PetscErrorCode ierr;
8936 
8937   PetscFunctionBegin;
8938   PetscValidHeaderSpecific(dm,DM_CLASSID,1);
8939   PetscValidPointer(odm,2);
8940   ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);
8941   ierr = PetscSectionHasConstraints(section, &hasConstraints);CHKERRQ(ierr);
8942   ierr = MPI_Allreduce(&hasConstraints, &ghasConstraints, 1, MPIU_BOOL, MPI_LOR, PetscObjectComm((PetscObject) dm));CHKERRMPI(ierr);
8943   if (!ghasConstraints) {
8944     ierr = PetscObjectReference((PetscObject)dm);CHKERRQ(ierr);
8945     *odm = dm;
8946     PetscFunctionReturn(0);
8947   }
8948   ierr = DMClone(dm, odm);CHKERRQ(ierr);
8949   ierr = DMCopyFields(dm, *odm);CHKERRQ(ierr);
8950   ierr = DMGetLocalSection(*odm, &newSection);CHKERRQ(ierr);
8951   ierr = DMGetPointSF(*odm, &sf);CHKERRQ(ierr);
8952   ierr = PetscSectionCreateGlobalSection(newSection, sf, PETSC_TRUE, PETSC_FALSE, &gsection);CHKERRQ(ierr);
8953   ierr = DMSetGlobalSection(*odm, gsection);CHKERRQ(ierr);
8954   ierr = PetscSectionDestroy(&gsection);CHKERRQ(ierr);
8955   PetscFunctionReturn(0);
8956 }
8957 
8958 static PetscErrorCode DMCreateAffineInterpolationCorrection_Plex(DM dmc, DM dmf, Vec *shift)
8959 {
8960   DM             dmco, dmfo;
8961   Mat            interpo;
8962   Vec            rscale;
8963   Vec            cglobalo, clocal;
8964   Vec            fglobal, fglobalo, flocal;
8965   PetscBool      regular;
8966   PetscErrorCode ierr;
8967 
8968   PetscFunctionBegin;
8969   ierr = DMGetFullDM(dmc, &dmco);CHKERRQ(ierr);
8970   ierr = DMGetFullDM(dmf, &dmfo);CHKERRQ(ierr);
8971   ierr = DMSetCoarseDM(dmfo, dmco);CHKERRQ(ierr);
8972   ierr = DMPlexGetRegularRefinement(dmf, &regular);CHKERRQ(ierr);
8973   ierr = DMPlexSetRegularRefinement(dmfo, regular);CHKERRQ(ierr);
8974   ierr = DMCreateInterpolation(dmco, dmfo, &interpo, &rscale);CHKERRQ(ierr);
8975   ierr = DMCreateGlobalVector(dmco, &cglobalo);CHKERRQ(ierr);
8976   ierr = DMCreateLocalVector(dmc, &clocal);CHKERRQ(ierr);
8977   ierr = VecSet(cglobalo, 0.);CHKERRQ(ierr);
8978   ierr = VecSet(clocal, 0.);CHKERRQ(ierr);
8979   ierr = DMCreateGlobalVector(dmf, &fglobal);CHKERRQ(ierr);
8980   ierr = DMCreateGlobalVector(dmfo, &fglobalo);CHKERRQ(ierr);
8981   ierr = DMCreateLocalVector(dmf, &flocal);CHKERRQ(ierr);
8982   ierr = VecSet(fglobal, 0.);CHKERRQ(ierr);
8983   ierr = VecSet(fglobalo, 0.);CHKERRQ(ierr);
8984   ierr = VecSet(flocal, 0.);CHKERRQ(ierr);
8985   ierr = DMPlexInsertBoundaryValues(dmc, PETSC_TRUE, clocal, 0., NULL, NULL, NULL);CHKERRQ(ierr);
8986   ierr = DMLocalToGlobalBegin(dmco, clocal, INSERT_VALUES, cglobalo);CHKERRQ(ierr);
8987   ierr = DMLocalToGlobalEnd(dmco, clocal, INSERT_VALUES, cglobalo);CHKERRQ(ierr);
8988   ierr = MatMult(interpo, cglobalo, fglobalo);CHKERRQ(ierr);
8989   ierr = DMGlobalToLocalBegin(dmfo, fglobalo, INSERT_VALUES, flocal);CHKERRQ(ierr);
8990   ierr = DMGlobalToLocalEnd(dmfo, fglobalo, INSERT_VALUES, flocal);CHKERRQ(ierr);
8991   ierr = DMLocalToGlobalBegin(dmf, flocal, INSERT_VALUES, fglobal);CHKERRQ(ierr);
8992   ierr = DMLocalToGlobalEnd(dmf, flocal, INSERT_VALUES, fglobal);CHKERRQ(ierr);
8993   *shift = fglobal;
8994   ierr = VecDestroy(&flocal);CHKERRQ(ierr);
8995   ierr = VecDestroy(&fglobalo);CHKERRQ(ierr);
8996   ierr = VecDestroy(&clocal);CHKERRQ(ierr);
8997   ierr = VecDestroy(&cglobalo);CHKERRQ(ierr);
8998   ierr = VecDestroy(&rscale);CHKERRQ(ierr);
8999   ierr = MatDestroy(&interpo);CHKERRQ(ierr);
9000   ierr = DMDestroy(&dmfo);CHKERRQ(ierr);
9001   ierr = DMDestroy(&dmco);CHKERRQ(ierr);
9002   PetscFunctionReturn(0);
9003 }
9004 
9005 PETSC_INTERN PetscErrorCode DMInterpolateSolution_Plex(DM coarse, DM fine, Mat interp, Vec coarseSol, Vec fineSol)
9006 {
9007   PetscObject    shifto;
9008   Vec            shift;
9009 
9010   PetscErrorCode ierr;
9011 
9012   PetscFunctionBegin;
9013   if (!interp) {
9014     Vec rscale;
9015 
9016     ierr = DMCreateInterpolation(coarse, fine, &interp, &rscale);CHKERRQ(ierr);
9017     ierr = VecDestroy(&rscale);CHKERRQ(ierr);
9018   } else {
9019     ierr = PetscObjectReference((PetscObject)interp);CHKERRQ(ierr);
9020   }
9021   ierr = PetscObjectQuery((PetscObject)interp, "_DMInterpolateSolution_Plex_Vec", &shifto);CHKERRQ(ierr);
9022   if (!shifto) {
9023     ierr = DMCreateAffineInterpolationCorrection_Plex(coarse, fine, &shift);CHKERRQ(ierr);
9024     ierr = PetscObjectCompose((PetscObject)interp, "_DMInterpolateSolution_Plex_Vec", (PetscObject) shift);CHKERRQ(ierr);
9025     shifto = (PetscObject) shift;
9026     ierr = VecDestroy(&shift);CHKERRQ(ierr);
9027   }
9028   shift = (Vec) shifto;
9029   ierr = MatInterpolate(interp, coarseSol, fineSol);CHKERRQ(ierr);
9030   ierr = VecAXPY(fineSol, 1.0, shift);CHKERRQ(ierr);
9031   ierr = MatDestroy(&interp);CHKERRQ(ierr);
9032   PetscFunctionReturn(0);
9033 }
9034 
9035 /* Pointwise interpolation
9036      Just code FEM for now
9037      u^f = I u^c
9038      sum_k u^f_k phi^f_k = I sum_j u^c_j phi^c_j
9039      u^f_i = sum_j psi^f_i I phi^c_j u^c_j
9040      I_{ij} = psi^f_i phi^c_j
9041 */
9042 PetscErrorCode DMCreateInterpolation_Plex(DM dmCoarse, DM dmFine, Mat *interpolation, Vec *scaling)
9043 {
9044   PetscSection   gsc, gsf;
9045   PetscInt       m, n;
9046   void          *ctx;
9047   DM             cdm;
9048   PetscBool      regular, ismatis, isRefined = dmCoarse->data == dmFine->data ? PETSC_FALSE : PETSC_TRUE;
9049   PetscErrorCode ierr;
9050 
9051   PetscFunctionBegin;
9052   ierr = DMGetGlobalSection(dmFine, &gsf);CHKERRQ(ierr);
9053   ierr = PetscSectionGetConstrainedStorageSize(gsf, &m);CHKERRQ(ierr);
9054   ierr = DMGetGlobalSection(dmCoarse, &gsc);CHKERRQ(ierr);
9055   ierr = PetscSectionGetConstrainedStorageSize(gsc, &n);CHKERRQ(ierr);
9056 
9057   ierr = PetscStrcmp(dmCoarse->mattype, MATIS, &ismatis);CHKERRQ(ierr);
9058   ierr = MatCreate(PetscObjectComm((PetscObject) dmCoarse), interpolation);CHKERRQ(ierr);
9059   ierr = MatSetSizes(*interpolation, m, n, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
9060   ierr = MatSetType(*interpolation, ismatis ? MATAIJ : dmCoarse->mattype);CHKERRQ(ierr);
9061   ierr = DMGetApplicationContext(dmFine, &ctx);CHKERRQ(ierr);
9062 
9063   ierr = DMGetCoarseDM(dmFine, &cdm);CHKERRQ(ierr);
9064   ierr = DMPlexGetRegularRefinement(dmFine, &regular);CHKERRQ(ierr);
9065   if (!isRefined || (regular && cdm == dmCoarse)) {ierr = DMPlexComputeInterpolatorNested(dmCoarse, dmFine, isRefined, *interpolation, ctx);CHKERRQ(ierr);}
9066   else                                            {ierr = DMPlexComputeInterpolatorGeneral(dmCoarse, dmFine, *interpolation, ctx);CHKERRQ(ierr);}
9067   ierr = MatViewFromOptions(*interpolation, NULL, "-interp_mat_view");CHKERRQ(ierr);
9068   if (scaling) {
9069     /* Use naive scaling */
9070     ierr = DMCreateInterpolationScale(dmCoarse, dmFine, *interpolation, scaling);CHKERRQ(ierr);
9071   }
9072   PetscFunctionReturn(0);
9073 }
9074 
9075 PetscErrorCode DMCreateInjection_Plex(DM dmCoarse, DM dmFine, Mat *mat)
9076 {
9077   PetscErrorCode ierr;
9078   VecScatter     ctx;
9079 
9080   PetscFunctionBegin;
9081   ierr = DMPlexComputeInjectorFEM(dmCoarse, dmFine, &ctx, NULL);CHKERRQ(ierr);
9082   ierr = MatCreateScatter(PetscObjectComm((PetscObject)ctx), ctx, mat);CHKERRQ(ierr);
9083   ierr = VecScatterDestroy(&ctx);CHKERRQ(ierr);
9084   PetscFunctionReturn(0);
9085 }
9086 
9087 static void g0_identity_private(PetscInt dim, PetscInt Nf, PetscInt NfAux,
9088                                 const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[],
9089                                 const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[],
9090                                 PetscReal t, PetscReal u_tShift, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar g0[])
9091 {
9092   g0[0] = 1.0;
9093 }
9094 
9095 PetscErrorCode DMCreateMassMatrix_Plex(DM dmCoarse, DM dmFine, Mat *mass)
9096 {
9097   PetscSection   gsc, gsf;
9098   PetscInt       m, n;
9099   void          *ctx;
9100   DM             cdm;
9101   PetscBool      regular;
9102   PetscErrorCode ierr;
9103 
9104   PetscFunctionBegin;
9105   if (dmFine == dmCoarse) {
9106     DM       dmc;
9107     PetscDS  ds;
9108     Vec      u;
9109     IS       cellIS;
9110     PetscFormKey key;
9111     PetscInt depth;
9112 
9113     ierr = DMClone(dmFine, &dmc);CHKERRQ(ierr);
9114     ierr = DMCopyDisc(dmFine, dmc);CHKERRQ(ierr);
9115     ierr = DMGetDS(dmc, &ds);CHKERRQ(ierr);
9116     ierr = PetscDSSetJacobian(ds, 0, 0, g0_identity_private, NULL, NULL, NULL);CHKERRQ(ierr);
9117     ierr = DMCreateMatrix(dmc, mass);CHKERRQ(ierr);
9118     ierr = DMGetGlobalVector(dmc, &u);CHKERRQ(ierr);
9119     ierr = DMPlexGetDepth(dmc, &depth);CHKERRQ(ierr);
9120     ierr = DMGetStratumIS(dmc, "depth", depth, &cellIS);CHKERRQ(ierr);
9121     ierr = MatZeroEntries(*mass);CHKERRQ(ierr);
9122     key.label = NULL;
9123     key.value = 0;
9124     key.field = 0;
9125     key.part  = 0;
9126     ierr = DMPlexComputeJacobian_Internal(dmc, key, cellIS, 0.0, 0.0, u, NULL, *mass, *mass, NULL);CHKERRQ(ierr);
9127     ierr = ISDestroy(&cellIS);CHKERRQ(ierr);
9128     ierr = DMRestoreGlobalVector(dmc, &u);CHKERRQ(ierr);
9129     ierr = DMDestroy(&dmc);CHKERRQ(ierr);
9130   } else {
9131     ierr = DMGetGlobalSection(dmFine, &gsf);CHKERRQ(ierr);
9132     ierr = PetscSectionGetConstrainedStorageSize(gsf, &m);CHKERRQ(ierr);
9133     ierr = DMGetGlobalSection(dmCoarse, &gsc);CHKERRQ(ierr);
9134     ierr = PetscSectionGetConstrainedStorageSize(gsc, &n);CHKERRQ(ierr);
9135 
9136     ierr = MatCreate(PetscObjectComm((PetscObject) dmCoarse), mass);CHKERRQ(ierr);
9137     ierr = MatSetSizes(*mass, m, n, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
9138     ierr = MatSetType(*mass, dmCoarse->mattype);CHKERRQ(ierr);
9139     ierr = DMGetApplicationContext(dmFine, &ctx);CHKERRQ(ierr);
9140 
9141     ierr = DMGetCoarseDM(dmFine, &cdm);CHKERRQ(ierr);
9142     ierr = DMPlexGetRegularRefinement(dmFine, &regular);CHKERRQ(ierr);
9143     if (regular && cdm == dmCoarse) {ierr = DMPlexComputeMassMatrixNested(dmCoarse, dmFine, *mass, ctx);CHKERRQ(ierr);}
9144     else                            {ierr = DMPlexComputeMassMatrixGeneral(dmCoarse, dmFine, *mass, ctx);CHKERRQ(ierr);}
9145   }
9146   ierr = MatViewFromOptions(*mass, NULL, "-mass_mat_view");CHKERRQ(ierr);
9147   PetscFunctionReturn(0);
9148 }
9149 
9150 /*@
9151   DMPlexGetRegularRefinement - Get the flag indicating that this mesh was obtained by regular refinement from its coarse mesh
9152 
9153   Input Parameter:
9154 . dm - The DMPlex object
9155 
9156   Output Parameter:
9157 . regular - The flag
9158 
9159   Level: intermediate
9160 
9161 .seealso: DMPlexSetRegularRefinement()
9162 @*/
9163 PetscErrorCode DMPlexGetRegularRefinement(DM dm, PetscBool *regular)
9164 {
9165   PetscFunctionBegin;
9166   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9167   PetscValidPointer(regular, 2);
9168   *regular = ((DM_Plex *) dm->data)->regularRefinement;
9169   PetscFunctionReturn(0);
9170 }
9171 
9172 /*@
9173   DMPlexSetRegularRefinement - Set the flag indicating that this mesh was obtained by regular refinement from its coarse mesh
9174 
9175   Input Parameters:
9176 + dm - The DMPlex object
9177 - regular - The flag
9178 
9179   Level: intermediate
9180 
9181 .seealso: DMPlexGetRegularRefinement()
9182 @*/
9183 PetscErrorCode DMPlexSetRegularRefinement(DM dm, PetscBool regular)
9184 {
9185   PetscFunctionBegin;
9186   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9187   ((DM_Plex *) dm->data)->regularRefinement = regular;
9188   PetscFunctionReturn(0);
9189 }
9190 
9191 /*@
9192   DMPlexGetCellRefinerType - Get the strategy for refining a cell
9193 
9194   Input Parameter:
9195 . dm - The DMPlex object
9196 
9197   Output Parameter:
9198 . cr - The strategy number
9199 
9200   Level: intermediate
9201 
9202 .seealso: DMPlexSetCellRefinerType(), DMPlexSetRegularRefinement()
9203 @*/
9204 PetscErrorCode DMPlexGetCellRefinerType(DM dm, DMPlexCellRefinerType *cr)
9205 {
9206   PetscFunctionBegin;
9207   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9208   PetscValidPointer(cr, 2);
9209   *cr = ((DM_Plex *) dm->data)->cellRefiner;
9210   PetscFunctionReturn(0);
9211 }
9212 
9213 /*@
9214   DMPlexSetCellRefinerType - Set the strategy for refining a cell
9215 
9216   Input Parameters:
9217 + dm - The DMPlex object
9218 - cr - The strategy number
9219 
9220   Level: intermediate
9221 
9222 .seealso: DMPlexGetCellRefinerType(), DMPlexGetRegularRefinement()
9223 @*/
9224 PetscErrorCode DMPlexSetCellRefinerType(DM dm, DMPlexCellRefinerType cr)
9225 {
9226   PetscFunctionBegin;
9227   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9228   ((DM_Plex *) dm->data)->cellRefiner = cr;
9229   PetscFunctionReturn(0);
9230 }
9231 
9232 /* anchors */
9233 /*@
9234   DMPlexGetAnchors - Get the layout of the anchor (point-to-point) constraints.  Typically, the user will not have to
9235   call DMPlexGetAnchors() directly: if there are anchors, then DMPlexGetAnchors() is called during DMGetConstraints().
9236 
9237   not collective
9238 
9239   Input Parameters:
9240 . dm - The DMPlex object
9241 
9242   Output Parameters:
9243 + anchorSection - If not NULL, set to the section describing which points anchor the constrained points.
9244 - anchorIS - If not NULL, set to the list of anchors indexed by anchorSection
9245 
9246   Level: intermediate
9247 
9248 .seealso: DMPlexSetAnchors(), DMGetConstraints(), DMSetConstraints()
9249 @*/
9250 PetscErrorCode DMPlexGetAnchors(DM dm, PetscSection *anchorSection, IS *anchorIS)
9251 {
9252   DM_Plex *plex = (DM_Plex *)dm->data;
9253   PetscErrorCode ierr;
9254 
9255   PetscFunctionBegin;
9256   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9257   if (!plex->anchorSection && !plex->anchorIS && plex->createanchors) {ierr = (*plex->createanchors)(dm);CHKERRQ(ierr);}
9258   if (anchorSection) *anchorSection = plex->anchorSection;
9259   if (anchorIS) *anchorIS = plex->anchorIS;
9260   PetscFunctionReturn(0);
9261 }
9262 
9263 /*@
9264   DMPlexSetAnchors - Set the layout of the local anchor (point-to-point) constraints.  Unlike boundary conditions,
9265   when a point's degrees of freedom in a section are constrained to an outside value, the anchor constraints set a
9266   point's degrees of freedom to be a linear combination of other points' degrees of freedom.
9267 
9268   After specifying the layout of constraints with DMPlexSetAnchors(), one specifies the constraints by calling
9269   DMGetConstraints() and filling in the entries in the constraint matrix.
9270 
9271   collective on dm
9272 
9273   Input Parameters:
9274 + dm - The DMPlex object
9275 . anchorSection - The section that describes the mapping from constrained points to the anchor points listed in anchorIS.  Must have a local communicator (PETSC_COMM_SELF or derivative).
9276 - anchorIS - The list of all anchor points.  Must have a local communicator (PETSC_COMM_SELF or derivative).
9277 
9278   The reference counts of anchorSection and anchorIS are incremented.
9279 
9280   Level: intermediate
9281 
9282 .seealso: DMPlexGetAnchors(), DMGetConstraints(), DMSetConstraints()
9283 @*/
9284 PetscErrorCode DMPlexSetAnchors(DM dm, PetscSection anchorSection, IS anchorIS)
9285 {
9286   DM_Plex        *plex = (DM_Plex *)dm->data;
9287   PetscMPIInt    result;
9288   PetscErrorCode ierr;
9289 
9290   PetscFunctionBegin;
9291   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9292   if (anchorSection) {
9293     PetscValidHeaderSpecific(anchorSection,PETSC_SECTION_CLASSID,2);
9294     ierr = MPI_Comm_compare(PETSC_COMM_SELF,PetscObjectComm((PetscObject)anchorSection),&result);CHKERRMPI(ierr);
9295     if (result != MPI_CONGRUENT && result != MPI_IDENT) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_NOTSAMECOMM,"anchor section must have local communicator");
9296   }
9297   if (anchorIS) {
9298     PetscValidHeaderSpecific(anchorIS,IS_CLASSID,3);
9299     ierr = MPI_Comm_compare(PETSC_COMM_SELF,PetscObjectComm((PetscObject)anchorIS),&result);CHKERRMPI(ierr);
9300     if (result != MPI_CONGRUENT && result != MPI_IDENT) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_NOTSAMECOMM,"anchor IS must have local communicator");
9301   }
9302 
9303   ierr = PetscObjectReference((PetscObject)anchorSection);CHKERRQ(ierr);
9304   ierr = PetscSectionDestroy(&plex->anchorSection);CHKERRQ(ierr);
9305   plex->anchorSection = anchorSection;
9306 
9307   ierr = PetscObjectReference((PetscObject)anchorIS);CHKERRQ(ierr);
9308   ierr = ISDestroy(&plex->anchorIS);CHKERRQ(ierr);
9309   plex->anchorIS = anchorIS;
9310 
9311   if (PetscUnlikelyDebug(anchorIS && anchorSection)) {
9312     PetscInt size, a, pStart, pEnd;
9313     const PetscInt *anchors;
9314 
9315     ierr = PetscSectionGetChart(anchorSection,&pStart,&pEnd);CHKERRQ(ierr);
9316     ierr = ISGetLocalSize(anchorIS,&size);CHKERRQ(ierr);
9317     ierr = ISGetIndices(anchorIS,&anchors);CHKERRQ(ierr);
9318     for (a = 0; a < size; a++) {
9319       PetscInt p;
9320 
9321       p = anchors[a];
9322       if (p >= pStart && p < pEnd) {
9323         PetscInt dof;
9324 
9325         ierr = PetscSectionGetDof(anchorSection,p,&dof);CHKERRQ(ierr);
9326         if (dof) {
9327           PetscErrorCode ierr2;
9328 
9329           ierr2 = ISRestoreIndices(anchorIS,&anchors);CHKERRQ(ierr2);
9330           SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_INCOMP,"Point %D cannot be constrained and an anchor",p);
9331         }
9332       }
9333     }
9334     ierr = ISRestoreIndices(anchorIS,&anchors);CHKERRQ(ierr);
9335   }
9336   /* reset the generic constraints */
9337   ierr = DMSetDefaultConstraints(dm,NULL,NULL);CHKERRQ(ierr);
9338   PetscFunctionReturn(0);
9339 }
9340 
9341 static PetscErrorCode DMPlexCreateConstraintSection_Anchors(DM dm, PetscSection section, PetscSection *cSec)
9342 {
9343   PetscSection anchorSection;
9344   PetscInt pStart, pEnd, sStart, sEnd, p, dof, numFields, f;
9345   PetscErrorCode ierr;
9346 
9347   PetscFunctionBegin;
9348   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9349   ierr = DMPlexGetAnchors(dm,&anchorSection,NULL);CHKERRQ(ierr);
9350   ierr = PetscSectionCreate(PETSC_COMM_SELF,cSec);CHKERRQ(ierr);
9351   ierr = PetscSectionGetNumFields(section,&numFields);CHKERRQ(ierr);
9352   if (numFields) {
9353     PetscInt f;
9354     ierr = PetscSectionSetNumFields(*cSec,numFields);CHKERRQ(ierr);
9355 
9356     for (f = 0; f < numFields; f++) {
9357       PetscInt numComp;
9358 
9359       ierr = PetscSectionGetFieldComponents(section,f,&numComp);CHKERRQ(ierr);
9360       ierr = PetscSectionSetFieldComponents(*cSec,f,numComp);CHKERRQ(ierr);
9361     }
9362   }
9363   ierr = PetscSectionGetChart(anchorSection,&pStart,&pEnd);CHKERRQ(ierr);
9364   ierr = PetscSectionGetChart(section,&sStart,&sEnd);CHKERRQ(ierr);
9365   pStart = PetscMax(pStart,sStart);
9366   pEnd   = PetscMin(pEnd,sEnd);
9367   pEnd   = PetscMax(pStart,pEnd);
9368   ierr = PetscSectionSetChart(*cSec,pStart,pEnd);CHKERRQ(ierr);
9369   for (p = pStart; p < pEnd; p++) {
9370     ierr = PetscSectionGetDof(anchorSection,p,&dof);CHKERRQ(ierr);
9371     if (dof) {
9372       ierr = PetscSectionGetDof(section,p,&dof);CHKERRQ(ierr);
9373       ierr = PetscSectionSetDof(*cSec,p,dof);CHKERRQ(ierr);
9374       for (f = 0; f < numFields; f++) {
9375         ierr = PetscSectionGetFieldDof(section,p,f,&dof);CHKERRQ(ierr);
9376         ierr = PetscSectionSetFieldDof(*cSec,p,f,dof);CHKERRQ(ierr);
9377       }
9378     }
9379   }
9380   ierr = PetscSectionSetUp(*cSec);CHKERRQ(ierr);
9381   ierr = PetscObjectSetName((PetscObject) *cSec, "Constraint Section");CHKERRQ(ierr);
9382   PetscFunctionReturn(0);
9383 }
9384 
9385 static PetscErrorCode DMPlexCreateConstraintMatrix_Anchors(DM dm, PetscSection section, PetscSection cSec, Mat *cMat)
9386 {
9387   PetscSection   aSec;
9388   PetscInt       pStart, pEnd, p, sStart, sEnd, dof, aDof, aOff, off, nnz, annz, m, n, q, a, offset, *i, *j;
9389   const PetscInt *anchors;
9390   PetscInt       numFields, f;
9391   IS             aIS;
9392   PetscErrorCode ierr;
9393   MatType        mtype;
9394   PetscBool      iscuda,iskokkos;
9395 
9396   PetscFunctionBegin;
9397   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9398   ierr = PetscSectionGetStorageSize(cSec, &m);CHKERRQ(ierr);
9399   ierr = PetscSectionGetStorageSize(section, &n);CHKERRQ(ierr);
9400   ierr = MatCreate(PETSC_COMM_SELF,cMat);CHKERRQ(ierr);
9401   ierr = MatSetSizes(*cMat,m,n,m,n);CHKERRQ(ierr);
9402   ierr = PetscStrcmp(dm->mattype,MATSEQAIJCUSPARSE,&iscuda);CHKERRQ(ierr);
9403   if (!iscuda) { ierr = PetscStrcmp(dm->mattype,MATMPIAIJCUSPARSE,&iscuda);CHKERRQ(ierr); }
9404   ierr = PetscStrcmp(dm->mattype,MATSEQAIJKOKKOS,&iskokkos);CHKERRQ(ierr);
9405   if (!iskokkos) { ierr = PetscStrcmp(dm->mattype,MATMPIAIJKOKKOS,&iskokkos);CHKERRQ(ierr); }
9406   if (iscuda) mtype = MATSEQAIJCUSPARSE;
9407   else if (iskokkos) mtype = MATSEQAIJKOKKOS;
9408   else mtype = MATSEQAIJ;
9409   ierr = MatSetType(*cMat,mtype);CHKERRQ(ierr);
9410   ierr = DMPlexGetAnchors(dm,&aSec,&aIS);CHKERRQ(ierr);
9411   ierr = ISGetIndices(aIS,&anchors);CHKERRQ(ierr);
9412   /* cSec will be a subset of aSec and section */
9413   ierr = PetscSectionGetChart(cSec,&pStart,&pEnd);CHKERRQ(ierr);
9414   ierr = PetscSectionGetChart(section,&sStart,&sEnd);CHKERRQ(ierr);
9415   ierr = PetscMalloc1(m+1,&i);CHKERRQ(ierr);
9416   i[0] = 0;
9417   ierr = PetscSectionGetNumFields(section,&numFields);CHKERRQ(ierr);
9418   for (p = pStart; p < pEnd; p++) {
9419     PetscInt rDof, rOff, r;
9420 
9421     ierr = PetscSectionGetDof(aSec,p,&rDof);CHKERRQ(ierr);
9422     if (!rDof) continue;
9423     ierr = PetscSectionGetOffset(aSec,p,&rOff);CHKERRQ(ierr);
9424     if (numFields) {
9425       for (f = 0; f < numFields; f++) {
9426         annz = 0;
9427         for (r = 0; r < rDof; r++) {
9428           a = anchors[rOff + r];
9429           if (a < sStart || a >= sEnd) continue;
9430           ierr = PetscSectionGetFieldDof(section,a,f,&aDof);CHKERRQ(ierr);
9431           annz += aDof;
9432         }
9433         ierr = PetscSectionGetFieldDof(cSec,p,f,&dof);CHKERRQ(ierr);
9434         ierr = PetscSectionGetFieldOffset(cSec,p,f,&off);CHKERRQ(ierr);
9435         for (q = 0; q < dof; q++) {
9436           i[off + q + 1] = i[off + q] + annz;
9437         }
9438       }
9439     }
9440     else {
9441       annz = 0;
9442       ierr = PetscSectionGetDof(cSec,p,&dof);CHKERRQ(ierr);
9443       for (q = 0; q < dof; q++) {
9444         a = anchors[rOff + q];
9445         if (a < sStart || a >= sEnd) continue;
9446         ierr = PetscSectionGetDof(section,a,&aDof);CHKERRQ(ierr);
9447         annz += aDof;
9448       }
9449       ierr = PetscSectionGetDof(cSec,p,&dof);CHKERRQ(ierr);
9450       ierr = PetscSectionGetOffset(cSec,p,&off);CHKERRQ(ierr);
9451       for (q = 0; q < dof; q++) {
9452         i[off + q + 1] = i[off + q] + annz;
9453       }
9454     }
9455   }
9456   nnz = i[m];
9457   ierr = PetscMalloc1(nnz,&j);CHKERRQ(ierr);
9458   offset = 0;
9459   for (p = pStart; p < pEnd; p++) {
9460     if (numFields) {
9461       for (f = 0; f < numFields; f++) {
9462         ierr = PetscSectionGetFieldDof(cSec,p,f,&dof);CHKERRQ(ierr);
9463         for (q = 0; q < dof; q++) {
9464           PetscInt rDof, rOff, r;
9465           ierr = PetscSectionGetDof(aSec,p,&rDof);CHKERRQ(ierr);
9466           ierr = PetscSectionGetOffset(aSec,p,&rOff);CHKERRQ(ierr);
9467           for (r = 0; r < rDof; r++) {
9468             PetscInt s;
9469 
9470             a = anchors[rOff + r];
9471             if (a < sStart || a >= sEnd) continue;
9472             ierr = PetscSectionGetFieldDof(section,a,f,&aDof);CHKERRQ(ierr);
9473             ierr = PetscSectionGetFieldOffset(section,a,f,&aOff);CHKERRQ(ierr);
9474             for (s = 0; s < aDof; s++) {
9475               j[offset++] = aOff + s;
9476             }
9477           }
9478         }
9479       }
9480     }
9481     else {
9482       ierr = PetscSectionGetDof(cSec,p,&dof);CHKERRQ(ierr);
9483       for (q = 0; q < dof; q++) {
9484         PetscInt rDof, rOff, r;
9485         ierr = PetscSectionGetDof(aSec,p,&rDof);CHKERRQ(ierr);
9486         ierr = PetscSectionGetOffset(aSec,p,&rOff);CHKERRQ(ierr);
9487         for (r = 0; r < rDof; r++) {
9488           PetscInt s;
9489 
9490           a = anchors[rOff + r];
9491           if (a < sStart || a >= sEnd) continue;
9492           ierr = PetscSectionGetDof(section,a,&aDof);CHKERRQ(ierr);
9493           ierr = PetscSectionGetOffset(section,a,&aOff);CHKERRQ(ierr);
9494           for (s = 0; s < aDof; s++) {
9495             j[offset++] = aOff + s;
9496           }
9497         }
9498       }
9499     }
9500   }
9501   ierr = MatSeqAIJSetPreallocationCSR(*cMat,i,j,NULL);CHKERRQ(ierr);
9502   ierr = PetscFree(i);CHKERRQ(ierr);
9503   ierr = PetscFree(j);CHKERRQ(ierr);
9504   ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);
9505   PetscFunctionReturn(0);
9506 }
9507 
9508 PetscErrorCode DMCreateDefaultConstraints_Plex(DM dm)
9509 {
9510   DM_Plex        *plex = (DM_Plex *)dm->data;
9511   PetscSection   anchorSection, section, cSec;
9512   Mat            cMat;
9513   PetscErrorCode ierr;
9514 
9515   PetscFunctionBegin;
9516   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9517   ierr = DMPlexGetAnchors(dm,&anchorSection,NULL);CHKERRQ(ierr);
9518   if (anchorSection) {
9519     PetscInt Nf;
9520 
9521     ierr = DMGetLocalSection(dm,&section);CHKERRQ(ierr);
9522     ierr = DMPlexCreateConstraintSection_Anchors(dm,section,&cSec);CHKERRQ(ierr);
9523     ierr = DMPlexCreateConstraintMatrix_Anchors(dm,section,cSec,&cMat);CHKERRQ(ierr);
9524     ierr = DMGetNumFields(dm,&Nf);CHKERRQ(ierr);
9525     if (Nf && plex->computeanchormatrix) {ierr = (*plex->computeanchormatrix)(dm,section,cSec,cMat);CHKERRQ(ierr);}
9526     ierr = DMSetDefaultConstraints(dm,cSec,cMat);CHKERRQ(ierr);
9527     ierr = PetscSectionDestroy(&cSec);CHKERRQ(ierr);
9528     ierr = MatDestroy(&cMat);CHKERRQ(ierr);
9529   }
9530   PetscFunctionReturn(0);
9531 }
9532 
9533 PetscErrorCode DMCreateSubDomainDM_Plex(DM dm, DMLabel label, PetscInt value, IS *is, DM *subdm)
9534 {
9535   IS             subis;
9536   PetscSection   section, subsection;
9537   PetscErrorCode ierr;
9538 
9539   PetscFunctionBegin;
9540   ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);
9541   if (!section) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Must set default section for DM before splitting subdomain");
9542   if (!subdm)   SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Must set output subDM for splitting subdomain");
9543   /* Create subdomain */
9544   ierr = DMPlexFilter(dm, label, value, subdm);CHKERRQ(ierr);
9545   /* Create submodel */
9546   ierr = DMPlexGetSubpointIS(*subdm, &subis);CHKERRQ(ierr);
9547   ierr = PetscSectionCreateSubmeshSection(section, subis, &subsection);CHKERRQ(ierr);
9548   ierr = DMSetLocalSection(*subdm, subsection);CHKERRQ(ierr);
9549   ierr = PetscSectionDestroy(&subsection);CHKERRQ(ierr);
9550   ierr = DMCopyDisc(dm, *subdm);CHKERRQ(ierr);
9551   /* Create map from submodel to global model */
9552   if (is) {
9553     PetscSection    sectionGlobal, subsectionGlobal;
9554     IS              spIS;
9555     const PetscInt *spmap;
9556     PetscInt       *subIndices;
9557     PetscInt        subSize = 0, subOff = 0, pStart, pEnd, p;
9558     PetscInt        Nf, f, bs = -1, bsLocal[2], bsMinMax[2];
9559 
9560     ierr = DMPlexGetSubpointIS(*subdm, &spIS);CHKERRQ(ierr);
9561     ierr = ISGetIndices(spIS, &spmap);CHKERRQ(ierr);
9562     ierr = PetscSectionGetNumFields(section, &Nf);CHKERRQ(ierr);
9563     ierr = DMGetGlobalSection(dm, &sectionGlobal);CHKERRQ(ierr);
9564     ierr = DMGetGlobalSection(*subdm, &subsectionGlobal);CHKERRQ(ierr);
9565     ierr = PetscSectionGetChart(subsection, &pStart, &pEnd);CHKERRQ(ierr);
9566     for (p = pStart; p < pEnd; ++p) {
9567       PetscInt gdof, pSubSize  = 0;
9568 
9569       ierr = PetscSectionGetDof(sectionGlobal, p, &gdof);CHKERRQ(ierr);
9570       if (gdof > 0) {
9571         for (f = 0; f < Nf; ++f) {
9572           PetscInt fdof, fcdof;
9573 
9574           ierr     = PetscSectionGetFieldDof(subsection, p, f, &fdof);CHKERRQ(ierr);
9575           ierr     = PetscSectionGetFieldConstraintDof(subsection, p, f, &fcdof);CHKERRQ(ierr);
9576           pSubSize += fdof-fcdof;
9577         }
9578         subSize += pSubSize;
9579         if (pSubSize) {
9580           if (bs < 0) {
9581             bs = pSubSize;
9582           } else if (bs != pSubSize) {
9583             /* Layout does not admit a pointwise block size */
9584             bs = 1;
9585           }
9586         }
9587       }
9588     }
9589     /* Must have same blocksize on all procs (some might have no points) */
9590     bsLocal[0] = bs < 0 ? PETSC_MAX_INT : bs; bsLocal[1] = bs;
9591     ierr = PetscGlobalMinMaxInt(PetscObjectComm((PetscObject) dm), bsLocal, bsMinMax);CHKERRQ(ierr);
9592     if (bsMinMax[0] != bsMinMax[1]) {bs = 1;}
9593     else                            {bs = bsMinMax[0];}
9594     ierr = PetscMalloc1(subSize, &subIndices);CHKERRQ(ierr);
9595     for (p = pStart; p < pEnd; ++p) {
9596       PetscInt gdof, goff;
9597 
9598       ierr = PetscSectionGetDof(subsectionGlobal, p, &gdof);CHKERRQ(ierr);
9599       if (gdof > 0) {
9600         const PetscInt point = spmap[p];
9601 
9602         ierr = PetscSectionGetOffset(sectionGlobal, point, &goff);CHKERRQ(ierr);
9603         for (f = 0; f < Nf; ++f) {
9604           PetscInt fdof, fcdof, fc, f2, poff = 0;
9605 
9606           /* Can get rid of this loop by storing field information in the global section */
9607           for (f2 = 0; f2 < f; ++f2) {
9608             ierr  = PetscSectionGetFieldDof(section, p, f2, &fdof);CHKERRQ(ierr);
9609             ierr  = PetscSectionGetFieldConstraintDof(section, p, f2, &fcdof);CHKERRQ(ierr);
9610             poff += fdof-fcdof;
9611           }
9612           ierr = PetscSectionGetFieldDof(section, p, f, &fdof);CHKERRQ(ierr);
9613           ierr = PetscSectionGetFieldConstraintDof(section, p, f, &fcdof);CHKERRQ(ierr);
9614           for (fc = 0; fc < fdof-fcdof; ++fc, ++subOff) {
9615             subIndices[subOff] = goff+poff+fc;
9616           }
9617         }
9618       }
9619     }
9620     ierr = ISRestoreIndices(spIS, &spmap);CHKERRQ(ierr);
9621     ierr = ISCreateGeneral(PetscObjectComm((PetscObject)dm), subSize, subIndices, PETSC_OWN_POINTER, is);CHKERRQ(ierr);
9622     if (bs > 1) {
9623       /* We need to check that the block size does not come from non-contiguous fields */
9624       PetscInt i, j, set = 1;
9625       for (i = 0; i < subSize; i += bs) {
9626         for (j = 0; j < bs; ++j) {
9627           if (subIndices[i+j] != subIndices[i]+j) {set = 0; break;}
9628         }
9629       }
9630       if (set) {ierr = ISSetBlockSize(*is, bs);CHKERRQ(ierr);}
9631     }
9632     /* Attach nullspace */
9633     for (f = 0; f < Nf; ++f) {
9634       (*subdm)->nullspaceConstructors[f] = dm->nullspaceConstructors[f];
9635       if ((*subdm)->nullspaceConstructors[f]) break;
9636     }
9637     if (f < Nf) {
9638       MatNullSpace nullSpace;
9639       ierr = (*(*subdm)->nullspaceConstructors[f])(*subdm, f, f, &nullSpace);CHKERRQ(ierr);
9640 
9641       ierr = PetscObjectCompose((PetscObject) *is, "nullspace", (PetscObject) nullSpace);CHKERRQ(ierr);
9642       ierr = MatNullSpaceDestroy(&nullSpace);CHKERRQ(ierr);
9643     }
9644   }
9645   PetscFunctionReturn(0);
9646 }
9647 
9648 /*@
9649   DMPlexMonitorThroughput - Report the cell throughput of FE integration
9650 
9651   Input Parameter:
9652 - dm - The DM
9653 
9654   Level: developer
9655 
9656   Options Database Keys:
9657 . -dm_plex_monitor_throughput - Activate the monitor
9658 
9659 .seealso: DMSetFromOptions(), DMPlexCreate()
9660 @*/
9661 PetscErrorCode DMPlexMonitorThroughput(DM dm, void *dummy)
9662 {
9663 #if defined(PETSC_USE_LOG)
9664   PetscStageLog      stageLog;
9665   PetscLogEvent      event;
9666   PetscLogStage      stage;
9667   PetscEventPerfInfo eventInfo;
9668   PetscReal          cellRate, flopRate;
9669   PetscInt           cStart, cEnd, Nf, N;
9670   const char        *name;
9671   PetscErrorCode     ierr;
9672 #endif
9673 
9674   PetscFunctionBegin;
9675   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9676 #if defined(PETSC_USE_LOG)
9677   ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
9678   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
9679   ierr = DMGetNumFields(dm, &Nf);CHKERRQ(ierr);
9680   ierr = PetscLogGetStageLog(&stageLog);CHKERRQ(ierr);
9681   ierr = PetscStageLogGetCurrent(stageLog, &stage);CHKERRQ(ierr);
9682   ierr = PetscLogEventGetId("DMPlexResidualFE", &event);CHKERRQ(ierr);
9683   ierr = PetscLogEventGetPerfInfo(stage, event, &eventInfo);CHKERRQ(ierr);
9684   N        = (cEnd - cStart)*Nf*eventInfo.count;
9685   flopRate = eventInfo.flops/eventInfo.time;
9686   cellRate = N/eventInfo.time;
9687   ierr = PetscPrintf(PetscObjectComm((PetscObject) dm), "DM (%s) FE Residual Integration: %D integrals %D reps\n  Cell rate: %.2g/s flop rate: %.2g MF/s\n", name ? name : "unknown", N, eventInfo.count, (double) cellRate, (double) (flopRate/1.e6));CHKERRQ(ierr);
9688 #else
9689   SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Plex Throughput Monitor is not supported if logging is turned off. Reconfigure using --with-log.");
9690 #endif
9691   PetscFunctionReturn(0);
9692 }
9693