xref: /petsc/src/dm/impls/plex/plex.c (revision 5ab264f3a2c5f7b4402bc8da2ae9fefa99c8eabd)
1 #include <petsc/private/dmpleximpl.h>   /*I      "petscdmplex.h"   I*/
2 #include <petsc/private/isimpl.h>
3 #include <petsc/private/vecimpl.h>
4 #include <petsc/private/glvisvecimpl.h>
5 #include <petscsf.h>
6 #include <petscds.h>
7 #include <petscdraw.h>
8 
9 /* Logging support */
10 PetscLogEvent DMPLEX_Interpolate, PETSCPARTITIONER_Partition, DMPLEX_Distribute, DMPLEX_DistributeCones, DMPLEX_DistributeLabels, DMPLEX_DistributeSF, DMPLEX_DistributeOverlap, DMPLEX_DistributeField, DMPLEX_DistributeData, DMPLEX_Migrate, DMPLEX_InterpolateSF, DMPLEX_GlobalToNaturalBegin, DMPLEX_GlobalToNaturalEnd, DMPLEX_NaturalToGlobalBegin, DMPLEX_NaturalToGlobalEnd, DMPLEX_Stratify, DMPLEX_Preallocate, DMPLEX_ResidualFEM, DMPLEX_JacobianFEM, DMPLEX_InterpolatorFEM, DMPLEX_InjectorFEM, DMPLEX_IntegralFEM, DMPLEX_CreateGmsh;
11 
12 PETSC_EXTERN PetscErrorCode VecView_MPI(Vec, PetscViewer);
13 
14 /*@
15   DMPlexRefineSimplexToTensor - Uniformly refines simplicial cells into tensor product cells.
16   3 quadrilaterals per triangle in 2D and 4 hexahedra per tetrahedron in 3D.
17 
18   Collective
19 
20   Input Parameters:
21 . dm - The DMPlex object
22 
23   Output Parameters:
24 . dmRefined - The refined DMPlex object
25 
26   Note: Returns NULL if the mesh is already a tensor product mesh.
27 
28   Level: intermediate
29 
30 .seealso: DMPlexCreate(), DMPlexSetRefinementUniform()
31 @*/
32 PetscErrorCode DMPlexRefineSimplexToTensor(DM dm, DM *dmRefined)
33 {
34   PetscInt         dim, cMax, fMax, cStart, cEnd, coneSize;
35   CellRefiner      cellRefiner;
36   PetscBool        lop, allnoop, localized;
37   PetscErrorCode   ierr;
38 
39   PetscFunctionBegin;
40   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
41   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
42   ierr = DMPlexGetHybridBounds(dm,&cMax,&fMax,NULL,NULL);CHKERRQ(ierr);
43   if (cMax >= 0 || fMax >= 0) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"Cannot handle hybrid meshes yet");
44   ierr = DMPlexGetHeightStratum(dm,0,&cStart,&cEnd);CHKERRQ(ierr);
45   if (!(cEnd - cStart)) cellRefiner = REFINER_NOOP;
46   else {
47     ierr = DMPlexGetConeSize(dm,cStart,&coneSize);CHKERRQ(ierr);
48     switch (dim) {
49     case 1:
50       cellRefiner = REFINER_NOOP;
51     break;
52     case 2:
53       switch (coneSize) {
54       case 3:
55         cellRefiner = REFINER_SIMPLEX_TO_HEX_2D;
56       break;
57       case 4:
58         cellRefiner = REFINER_NOOP;
59       break;
60       default: SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_SUP,"Cannot handle coneSize %D with dimension %D",coneSize,dim);
61       }
62     break;
63     case 3:
64       switch (coneSize) {
65       case 4:
66         cellRefiner = REFINER_SIMPLEX_TO_HEX_3D;
67       break;
68       case 6:
69         cellRefiner = REFINER_NOOP;
70       break;
71       default: SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_SUP,"Cannot handle coneSize %D with dimension %D",coneSize,dim);
72       }
73     break;
74     default: SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SUP,"Cannot handle dimension %D",dim);
75     }
76   }
77   /* return if we don't need to refine */
78   lop = (cellRefiner == REFINER_NOOP) ? PETSC_TRUE : PETSC_FALSE;
79   ierr = MPIU_Allreduce(&lop,&allnoop,1,MPIU_BOOL,MPI_LAND,PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
80   if (allnoop) {
81     *dmRefined = NULL;
82     PetscFunctionReturn(0);
83   }
84   ierr = DMPlexRefineUniform_Internal(dm, cellRefiner, dmRefined);CHKERRQ(ierr);
85   ierr = DMCopyBoundary(dm, *dmRefined);CHKERRQ(ierr);
86   ierr = DMGetCoordinatesLocalized(dm, &localized);CHKERRQ(ierr);
87   if (localized) {
88     ierr = DMLocalizeCoordinates(*dmRefined);CHKERRQ(ierr);
89   }
90   PetscFunctionReturn(0);
91 }
92 
93 PetscErrorCode DMPlexGetFieldType_Internal(DM dm, PetscSection section, PetscInt field, PetscInt *sStart, PetscInt *sEnd, PetscViewerVTKFieldType *ft)
94 {
95   PetscInt       dim, pStart, pEnd, vStart, vEnd, cStart, cEnd, cEndInterior, vdof = 0, cdof = 0;
96   PetscErrorCode ierr;
97 
98   PetscFunctionBegin;
99   *ft  = PETSC_VTK_POINT_FIELD;
100   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
101   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
102   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
103   ierr = DMPlexGetHybridBounds(dm, &cEndInterior, NULL, NULL, NULL);CHKERRQ(ierr);
104   cEnd = cEndInterior < 0 ? cEnd : cEndInterior;
105   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
106   if (field >= 0) {
107     if ((vStart >= pStart) && (vStart < pEnd)) {ierr = PetscSectionGetFieldDof(section, vStart, field, &vdof);CHKERRQ(ierr);}
108     if ((cStart >= pStart) && (cStart < pEnd)) {ierr = PetscSectionGetFieldDof(section, cStart, field, &cdof);CHKERRQ(ierr);}
109   } else {
110     if ((vStart >= pStart) && (vStart < pEnd)) {ierr = PetscSectionGetDof(section, vStart, &vdof);CHKERRQ(ierr);}
111     if ((cStart >= pStart) && (cStart < pEnd)) {ierr = PetscSectionGetDof(section, cStart, &cdof);CHKERRQ(ierr);}
112   }
113   if (vdof) {
114     *sStart = vStart;
115     *sEnd   = vEnd;
116     if (vdof == dim) *ft = PETSC_VTK_POINT_VECTOR_FIELD;
117     else             *ft = PETSC_VTK_POINT_FIELD;
118   } else if (cdof) {
119     *sStart = cStart;
120     *sEnd   = cEnd;
121     if (cdof == dim) *ft = PETSC_VTK_CELL_VECTOR_FIELD;
122     else             *ft = PETSC_VTK_CELL_FIELD;
123   } else SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Could not classify input Vec for VTK");
124   PetscFunctionReturn(0);
125 }
126 
127 static PetscErrorCode VecView_Plex_Local_Draw(Vec v, PetscViewer viewer)
128 {
129   DM                 dm;
130   PetscSection       s;
131   PetscDraw          draw, popup;
132   DM                 cdm;
133   PetscSection       coordSection;
134   Vec                coordinates;
135   const PetscScalar *coords, *array;
136   PetscReal          bound[4] = {PETSC_MAX_REAL, PETSC_MAX_REAL, PETSC_MIN_REAL, PETSC_MIN_REAL};
137   PetscReal          vbound[2], time;
138   PetscBool          isnull, flg;
139   PetscInt           dim, Nf, f, Nc, comp, vStart, vEnd, cStart, cEnd, c, N, level, step, w = 0;
140   const char        *name;
141   char               title[PETSC_MAX_PATH_LEN];
142   PetscErrorCode     ierr;
143 
144   PetscFunctionBegin;
145   ierr = PetscViewerDrawGetDraw(viewer, 0, &draw);CHKERRQ(ierr);
146   ierr = PetscDrawIsNull(draw, &isnull);CHKERRQ(ierr);
147   if (isnull) PetscFunctionReturn(0);
148 
149   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
150   ierr = DMGetCoordinateDim(dm, &dim);CHKERRQ(ierr);
151   if (dim != 2) SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Cannot draw meshes of dimension %D. Use PETSCVIEWERGLVIS", dim);
152   ierr = DMGetDefaultSection(dm, &s);CHKERRQ(ierr);
153   ierr = PetscSectionGetNumFields(s, &Nf);CHKERRQ(ierr);
154   ierr = DMGetCoarsenLevel(dm, &level);CHKERRQ(ierr);
155   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
156   ierr = DMGetDefaultSection(cdm, &coordSection);CHKERRQ(ierr);
157   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
158   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
159   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
160 
161   ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
162   ierr = DMGetOutputSequenceNumber(dm, &step, &time);CHKERRQ(ierr);
163 
164   ierr = VecGetLocalSize(coordinates, &N);CHKERRQ(ierr);
165   ierr = VecGetArrayRead(coordinates, &coords);CHKERRQ(ierr);
166   for (c = 0; c < N; c += dim) {
167     bound[0] = PetscMin(bound[0], PetscRealPart(coords[c]));   bound[2] = PetscMax(bound[2], PetscRealPart(coords[c]));
168     bound[1] = PetscMin(bound[1], PetscRealPart(coords[c+1])); bound[3] = PetscMax(bound[3], PetscRealPart(coords[c+1]));
169   }
170   ierr = VecRestoreArrayRead(coordinates, &coords);CHKERRQ(ierr);
171   ierr = PetscDrawClear(draw);CHKERRQ(ierr);
172 
173   /* Could implement something like DMDASelectFields() */
174   for (f = 0; f < Nf; ++f) {
175     DM   fdm = dm;
176     Vec  fv  = v;
177     IS   fis;
178     char prefix[PETSC_MAX_PATH_LEN];
179     const char *fname;
180 
181     ierr = PetscSectionGetFieldComponents(s, f, &Nc);CHKERRQ(ierr);
182     ierr = PetscSectionGetFieldName(s, f, &fname);CHKERRQ(ierr);
183 
184     if (v->hdr.prefix) {ierr = PetscStrcpy(prefix, v->hdr.prefix);CHKERRQ(ierr);}
185     else               {prefix[0] = '\0';}
186     if (Nf > 1) {
187       ierr = DMCreateSubDM(dm, 1, &f, &fis, &fdm);CHKERRQ(ierr);
188       ierr = VecGetSubVector(v, fis, &fv);CHKERRQ(ierr);
189       ierr = PetscStrcat(prefix, fname);CHKERRQ(ierr);
190       ierr = PetscStrcat(prefix, "_");CHKERRQ(ierr);
191     }
192     for (comp = 0; comp < Nc; ++comp, ++w) {
193       PetscInt nmax = 2;
194 
195       ierr = PetscViewerDrawGetDraw(viewer, w, &draw);CHKERRQ(ierr);
196       if (Nc > 1) {ierr = PetscSNPrintf(title, sizeof(title), "%s:%s_%D Step: %D Time: %.4g", name, fname, comp, step, time);CHKERRQ(ierr);}
197       else        {ierr = PetscSNPrintf(title, sizeof(title), "%s:%s Step: %D Time: %.4g", name, fname, step, time);CHKERRQ(ierr);}
198       ierr = PetscDrawSetTitle(draw, title);CHKERRQ(ierr);
199 
200       /* TODO Get max and min only for this component */
201       ierr = PetscOptionsGetRealArray(NULL, prefix, "-vec_view_bounds", vbound, &nmax, &flg);CHKERRQ(ierr);
202       if (!flg) {
203         ierr = VecMin(fv, NULL, &vbound[0]);CHKERRQ(ierr);
204         ierr = VecMax(fv, NULL, &vbound[1]);CHKERRQ(ierr);
205         if (vbound[1] <= vbound[0]) vbound[1] = vbound[0] + 1.0;
206       }
207       ierr = PetscDrawGetPopup(draw, &popup);CHKERRQ(ierr);
208       ierr = PetscDrawScalePopup(popup, vbound[0], vbound[1]);CHKERRQ(ierr);
209       ierr = PetscDrawSetCoordinates(draw, bound[0], bound[1], bound[2], bound[3]);CHKERRQ(ierr);
210 
211       ierr = VecGetArrayRead(fv, &array);CHKERRQ(ierr);
212       for (c = cStart; c < cEnd; ++c) {
213         PetscScalar *coords = NULL, *a = NULL;
214         PetscInt     numCoords, color[4] = {-1,-1,-1,-1};
215 
216         ierr = DMPlexPointLocalRead(fdm, c, array, &a);CHKERRQ(ierr);
217         if (a) {
218           color[0] = PetscDrawRealToColor(PetscRealPart(a[comp]), vbound[0], vbound[1]);
219           color[1] = color[2] = color[3] = color[0];
220         } else {
221           PetscScalar *vals = NULL;
222           PetscInt     numVals, va;
223 
224           ierr = DMPlexVecGetClosure(fdm, NULL, fv, c, &numVals, &vals);CHKERRQ(ierr);
225           if (numVals % Nc) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "The number of components %D does not divide the number of values in the closure %D", Nc, numVals);
226           switch (numVals/Nc) {
227           case 3: /* P1 Triangle */
228           case 4: /* P1 Quadrangle */
229             for (va = 0; va < numVals/Nc; ++va) color[va] = PetscDrawRealToColor(PetscRealPart(vals[va*Nc+comp]), vbound[0], vbound[1]);
230             break;
231           case 6: /* P2 Triangle */
232           case 8: /* P2 Quadrangle */
233             for (va = 0; va < numVals/(Nc*2); ++va) color[va] = PetscDrawRealToColor(PetscRealPart(vals[va*Nc+comp + numVals/(Nc*2)]), vbound[0], vbound[1]);
234             break;
235           default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of values for cell closure %D cannot be handled", numVals/Nc);
236           }
237           ierr = DMPlexVecRestoreClosure(fdm, NULL, fv, c, &numVals, &vals);CHKERRQ(ierr);
238         }
239         ierr = DMPlexVecGetClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
240         switch (numCoords) {
241         case 6:
242           ierr = PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), color[0], color[1], color[2]);CHKERRQ(ierr);
243           break;
244         case 8:
245           ierr = PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), color[0], color[1], color[2]);CHKERRQ(ierr);
246           ierr = PetscDrawTriangle(draw, PetscRealPart(coords[4]), PetscRealPart(coords[5]), PetscRealPart(coords[6]), PetscRealPart(coords[7]), PetscRealPart(coords[0]), PetscRealPart(coords[1]), color[2], color[3], color[0]);CHKERRQ(ierr);
247           break;
248         default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_SUP, "Cannot draw cells with %D coordinates", numCoords);
249         }
250         ierr = DMPlexVecRestoreClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
251       }
252       ierr = VecRestoreArrayRead(fv, &array);CHKERRQ(ierr);
253       ierr = PetscDrawFlush(draw);CHKERRQ(ierr);
254       ierr = PetscDrawPause(draw);CHKERRQ(ierr);
255       ierr = PetscDrawSave(draw);CHKERRQ(ierr);
256     }
257     if (Nf > 1) {
258       ierr = VecRestoreSubVector(v, fis, &fv);CHKERRQ(ierr);
259       ierr = ISDestroy(&fis);CHKERRQ(ierr);
260       ierr = DMDestroy(&fdm);CHKERRQ(ierr);
261     }
262   }
263   PetscFunctionReturn(0);
264 }
265 
266 PetscErrorCode VecView_Plex_Local(Vec v, PetscViewer viewer)
267 {
268   DM             dm;
269   PetscBool      isvtk, ishdf5, isdraw, isseq, isglvis;
270   PetscErrorCode ierr;
271 
272   PetscFunctionBegin;
273   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
274   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
275   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,   &isvtk);CHKERRQ(ierr);
276   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,  &ishdf5);CHKERRQ(ierr);
277   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERDRAW,  &isdraw);CHKERRQ(ierr);
278   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERGLVIS, &isglvis);CHKERRQ(ierr);
279   ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
280   if (isvtk || ishdf5 || isglvis) {
281     PetscInt  numFields;
282     PetscBool fem = PETSC_FALSE;
283 
284     ierr = DMGetNumFields(dm, &numFields);CHKERRQ(ierr);
285     if (numFields) {
286       PetscObject fe;
287 
288       ierr = DMGetField(dm, 0, &fe);CHKERRQ(ierr);
289       if (fe->classid == PETSCFE_CLASSID) fem = PETSC_TRUE;
290     }
291     if (fem) {ierr = DMPlexInsertBoundaryValues(dm, PETSC_TRUE, v, 0.0, NULL, NULL, NULL);CHKERRQ(ierr);}
292   }
293   if (isvtk) {
294     PetscSection            section;
295     PetscViewerVTKFieldType ft;
296     PetscInt                pStart, pEnd;
297 
298     ierr = DMGetDefaultSection(dm, &section);CHKERRQ(ierr);
299     ierr = DMPlexGetFieldType_Internal(dm, section, PETSC_DETERMINE, &pStart, &pEnd, &ft);CHKERRQ(ierr);
300     ierr = PetscObjectReference((PetscObject) v);CHKERRQ(ierr);  /* viewer drops reference */
301     ierr = PetscViewerVTKAddField(viewer, (PetscObject) dm, DMPlexVTKWriteAll, ft, (PetscObject) v);CHKERRQ(ierr);
302   } else if (ishdf5) {
303 #if defined(PETSC_HAVE_HDF5)
304     ierr = VecView_Plex_Local_HDF5_Internal(v, viewer);CHKERRQ(ierr);
305 #else
306     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
307 #endif
308   } else if (isglvis) {
309     ierr = VecView_GLVis(v, viewer);CHKERRQ(ierr);
310   } else if (isdraw) {
311     ierr = VecView_Plex_Local_Draw(v, viewer);CHKERRQ(ierr);
312   } else {
313     if (isseq) {ierr = VecView_Seq(v, viewer);CHKERRQ(ierr);}
314     else       {ierr = VecView_MPI(v, viewer);CHKERRQ(ierr);}
315   }
316   PetscFunctionReturn(0);
317 }
318 
319 PetscErrorCode VecView_Plex(Vec v, PetscViewer viewer)
320 {
321   DM             dm;
322   PetscReal      time = 0.0;
323   PetscBool      isvtk, ishdf5, isdraw, isseq, isglvis;
324   PetscErrorCode ierr;
325 
326   PetscFunctionBegin;
327   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
328   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
329   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,   &isvtk);CHKERRQ(ierr);
330   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,  &ishdf5);CHKERRQ(ierr);
331   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERDRAW,  &isdraw);CHKERRQ(ierr);
332   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERDRAW,  &isdraw);CHKERRQ(ierr);
333   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERGLVIS, &isglvis);CHKERRQ(ierr);
334   ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
335   if (isvtk || isglvis) {
336     PetscInt    num;
337     Vec         locv;
338     const char *name;
339 
340     ierr = DMCreateLocalVector(dm, &locv);CHKERRQ(ierr);
341     ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
342     ierr = PetscObjectSetName((PetscObject) locv, name);CHKERRQ(ierr);
343     ierr = DMGlobalToLocalBegin(dm, v, INSERT_VALUES, locv);CHKERRQ(ierr);
344     ierr = DMGlobalToLocalEnd(dm, v, INSERT_VALUES, locv);CHKERRQ(ierr);
345     ierr = DMGetOutputSequenceNumber(dm, &num, &time);CHKERRQ(ierr);
346     ierr = DMPlexInsertBoundaryValues(dm, PETSC_TRUE, locv, time, NULL, NULL, NULL);CHKERRQ(ierr);
347     ierr = PetscViewerGLVisSetSnapId(viewer, num);CHKERRQ(ierr);
348     ierr = VecView_Plex_Local(locv, viewer);CHKERRQ(ierr);
349   } else if (ishdf5) {
350 #if defined(PETSC_HAVE_HDF5)
351     ierr = VecView_Plex_HDF5_Internal(v, viewer);CHKERRQ(ierr);
352 #else
353     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
354 #endif
355   } else if (isdraw) {
356     Vec         locv;
357     const char *name;
358 
359     ierr = DMCreateLocalVector(dm, &locv);CHKERRQ(ierr);
360     ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
361     ierr = PetscObjectSetName((PetscObject) locv, name);CHKERRQ(ierr);
362     ierr = DMGlobalToLocalBegin(dm, v, INSERT_VALUES, locv);CHKERRQ(ierr);
363     ierr = DMGlobalToLocalEnd(dm, v, INSERT_VALUES, locv);CHKERRQ(ierr);
364     ierr = DMGetOutputSequenceNumber(dm, NULL, &time);CHKERRQ(ierr);
365     ierr = DMPlexInsertBoundaryValues(dm, PETSC_TRUE, locv, time, NULL, NULL, NULL);CHKERRQ(ierr);
366     ierr = VecView_Plex_Local(locv, viewer);CHKERRQ(ierr);
367   } else {
368     if (isseq) {ierr = VecView_Seq(v, viewer);CHKERRQ(ierr);}
369     else       {ierr = VecView_MPI(v, viewer);CHKERRQ(ierr);}
370   }
371   PetscFunctionReturn(0);
372 }
373 
374 PetscErrorCode VecView_Plex_Native(Vec originalv, PetscViewer viewer)
375 {
376   DM                dm;
377   MPI_Comm          comm;
378   PetscViewerFormat format;
379   Vec               v;
380   PetscBool         isvtk, ishdf5;
381   PetscErrorCode    ierr;
382 
383   PetscFunctionBegin;
384   ierr = VecGetDM(originalv, &dm);CHKERRQ(ierr);
385   ierr = PetscObjectGetComm((PetscObject) originalv, &comm);CHKERRQ(ierr);
386   if (!dm) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
387   ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
388   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
389   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,  &isvtk);CHKERRQ(ierr);
390   if (format == PETSC_VIEWER_NATIVE) {
391     const char *vecname;
392     PetscInt    n, nroots;
393 
394     if (dm->sfNatural) {
395       ierr = VecGetLocalSize(originalv, &n);CHKERRQ(ierr);
396       ierr = PetscSFGetGraph(dm->sfNatural, &nroots, NULL, NULL, NULL);CHKERRQ(ierr);
397       if (n == nroots) {
398         ierr = DMGetGlobalVector(dm, &v);CHKERRQ(ierr);
399         ierr = DMPlexGlobalToNaturalBegin(dm, originalv, v);CHKERRQ(ierr);
400         ierr = DMPlexGlobalToNaturalEnd(dm, originalv, v);CHKERRQ(ierr);
401         ierr = PetscObjectGetName((PetscObject) originalv, &vecname);CHKERRQ(ierr);
402         ierr = PetscObjectSetName((PetscObject) v, vecname);CHKERRQ(ierr);
403       } else SETERRQ(comm, PETSC_ERR_ARG_WRONG, "DM global to natural SF only handles global vectors");
404     } else SETERRQ(comm, PETSC_ERR_ARG_WRONGSTATE, "DM global to natural SF was not created");
405   } else {
406     /* we are viewing a natural DMPlex vec. */
407     v = originalv;
408   }
409   if (ishdf5) {
410 #if defined(PETSC_HAVE_HDF5)
411     ierr = VecView_Plex_HDF5_Native_Internal(v, viewer);CHKERRQ(ierr);
412 #else
413     SETERRQ(comm, PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
414 #endif
415   } else if (isvtk) {
416     SETERRQ(comm, PETSC_ERR_SUP, "VTK format does not support viewing in natural order. Please switch to HDF5.");
417   } else {
418     PetscBool isseq;
419 
420     ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
421     if (isseq) {ierr = VecView_Seq(v, viewer);CHKERRQ(ierr);}
422     else       {ierr = VecView_MPI(v, viewer);CHKERRQ(ierr);}
423   }
424   if (format == PETSC_VIEWER_NATIVE) {ierr = DMRestoreGlobalVector(dm, &v);CHKERRQ(ierr);}
425   PetscFunctionReturn(0);
426 }
427 
428 PetscErrorCode VecLoad_Plex_Local(Vec v, PetscViewer viewer)
429 {
430   DM             dm;
431   PetscBool      ishdf5;
432   PetscErrorCode ierr;
433 
434   PetscFunctionBegin;
435   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
436   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
437   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
438   if (ishdf5) {
439     DM          dmBC;
440     Vec         gv;
441     const char *name;
442 
443     ierr = DMGetOutputDM(dm, &dmBC);CHKERRQ(ierr);
444     ierr = DMGetGlobalVector(dmBC, &gv);CHKERRQ(ierr);
445     ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
446     ierr = PetscObjectSetName((PetscObject) gv, name);CHKERRQ(ierr);
447     ierr = VecLoad_Default(gv, viewer);CHKERRQ(ierr);
448     ierr = DMGlobalToLocalBegin(dmBC, gv, INSERT_VALUES, v);CHKERRQ(ierr);
449     ierr = DMGlobalToLocalEnd(dmBC, gv, INSERT_VALUES, v);CHKERRQ(ierr);
450     ierr = DMRestoreGlobalVector(dmBC, &gv);CHKERRQ(ierr);
451   } else {
452     ierr = VecLoad_Default(v, viewer);CHKERRQ(ierr);
453   }
454   PetscFunctionReturn(0);
455 }
456 
457 PetscErrorCode VecLoad_Plex(Vec v, PetscViewer viewer)
458 {
459   DM             dm;
460   PetscBool      ishdf5;
461   PetscErrorCode ierr;
462 
463   PetscFunctionBegin;
464   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
465   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
466   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
467   if (ishdf5) {
468 #if defined(PETSC_HAVE_HDF5)
469     ierr = VecLoad_Plex_HDF5_Internal(v, viewer);CHKERRQ(ierr);
470 #else
471     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
472 #endif
473   } else {
474     ierr = VecLoad_Default(v, viewer);CHKERRQ(ierr);
475   }
476   PetscFunctionReturn(0);
477 }
478 
479 PetscErrorCode VecLoad_Plex_Native(Vec originalv, PetscViewer viewer)
480 {
481   DM                dm;
482   PetscViewerFormat format;
483   PetscBool         ishdf5;
484   PetscErrorCode    ierr;
485 
486   PetscFunctionBegin;
487   ierr = VecGetDM(originalv, &dm);CHKERRQ(ierr);
488   if (!dm) SETERRQ(PetscObjectComm((PetscObject) originalv), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
489   ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
490   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
491   if (format == PETSC_VIEWER_NATIVE) {
492     if (dm->sfNatural) {
493       if (ishdf5) {
494 #if defined(PETSC_HAVE_HDF5)
495         Vec         v;
496         const char *vecname;
497 
498         ierr = DMGetGlobalVector(dm, &v);CHKERRQ(ierr);
499         ierr = PetscObjectGetName((PetscObject) originalv, &vecname);CHKERRQ(ierr);
500         ierr = PetscObjectSetName((PetscObject) v, vecname);CHKERRQ(ierr);
501         ierr = VecLoad_Plex_HDF5_Native_Internal(v, viewer);CHKERRQ(ierr);
502         ierr = DMPlexNaturalToGlobalBegin(dm, v, originalv);CHKERRQ(ierr);
503         ierr = DMPlexNaturalToGlobalEnd(dm, v, originalv);CHKERRQ(ierr);
504         ierr = DMRestoreGlobalVector(dm, &v);CHKERRQ(ierr);
505 #else
506         SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
507 #endif
508       } else SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Reading in natural order is not supported for anything but HDF5.");
509     }
510   }
511   PetscFunctionReturn(0);
512 }
513 
514 PETSC_UNUSED static PetscErrorCode DMPlexView_Ascii_Geometry(DM dm, PetscViewer viewer)
515 {
516   PetscSection       coordSection;
517   Vec                coordinates;
518   DMLabel            depthLabel;
519   const char        *name[4];
520   const PetscScalar *a;
521   PetscInt           dim, pStart, pEnd, cStart, cEnd, c;
522   PetscErrorCode     ierr;
523 
524   PetscFunctionBegin;
525   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
526   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
527   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
528   ierr = DMPlexGetDepthLabel(dm, &depthLabel);CHKERRQ(ierr);
529   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
530   ierr = PetscSectionGetChart(coordSection, &pStart, &pEnd);CHKERRQ(ierr);
531   ierr = VecGetArrayRead(coordinates, &a);CHKERRQ(ierr);
532   name[0]     = "vertex";
533   name[1]     = "edge";
534   name[dim-1] = "face";
535   name[dim]   = "cell";
536   for (c = cStart; c < cEnd; ++c) {
537     PetscInt *closure = NULL;
538     PetscInt  closureSize, cl;
539 
540     ierr = PetscViewerASCIIPrintf(viewer, "Geometry for cell %D:\n", c);CHKERRQ(ierr);
541     ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
542     ierr = PetscViewerASCIIPushTab(viewer);CHKERRQ(ierr);
543     for (cl = 0; cl < closureSize*2; cl += 2) {
544       PetscInt point = closure[cl], depth, dof, off, d, p;
545 
546       if ((point < pStart) || (point >= pEnd)) continue;
547       ierr = PetscSectionGetDof(coordSection, point, &dof);CHKERRQ(ierr);
548       if (!dof) continue;
549       ierr = DMLabelGetValue(depthLabel, point, &depth);CHKERRQ(ierr);
550       ierr = PetscSectionGetOffset(coordSection, point, &off);CHKERRQ(ierr);
551       ierr = PetscViewerASCIIPrintf(viewer, "%s %D coords:", name[depth], point);CHKERRQ(ierr);
552       for (p = 0; p < dof/dim; ++p) {
553         ierr = PetscViewerASCIIPrintf(viewer, " (");CHKERRQ(ierr);
554         for (d = 0; d < dim; ++d) {
555           if (d > 0) {ierr = PetscViewerASCIIPrintf(viewer, ", ");CHKERRQ(ierr);}
556           ierr = PetscViewerASCIIPrintf(viewer, "%g", PetscRealPart(a[off+p*dim+d]));CHKERRQ(ierr);
557         }
558         ierr = PetscViewerASCIIPrintf(viewer, ")");CHKERRQ(ierr);
559       }
560       ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
561     }
562     ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
563     ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr);
564   }
565   ierr = VecRestoreArrayRead(coordinates, &a);CHKERRQ(ierr);
566   PetscFunctionReturn(0);
567 }
568 
569 static PetscErrorCode DMPlexView_Ascii(DM dm, PetscViewer viewer)
570 {
571   DM_Plex          *mesh = (DM_Plex*) dm->data;
572   DM                cdm;
573   DMLabel           markers;
574   PetscSection      coordSection;
575   Vec               coordinates;
576   PetscViewerFormat format;
577   PetscErrorCode    ierr;
578 
579   PetscFunctionBegin;
580   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
581   ierr = DMGetDefaultSection(cdm, &coordSection);CHKERRQ(ierr);
582   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
583   ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
584   if (format == PETSC_VIEWER_ASCII_INFO_DETAIL) {
585     const char *name;
586     PetscInt    dim, cellHeight, maxConeSize, maxSupportSize;
587     PetscInt    pStart, pEnd, p;
588     PetscMPIInt rank, size;
589 
590     ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRQ(ierr);
591     ierr = MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size);CHKERRQ(ierr);
592     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
593     ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
594     ierr = DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize);CHKERRQ(ierr);
595     ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
596     ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
597     if (name) {ierr = PetscViewerASCIIPrintf(viewer, "%s in %D dimension%s:\n", name, dim, dim == 1 ? "" : "s");CHKERRQ(ierr);}
598     else      {ierr = PetscViewerASCIIPrintf(viewer, "Mesh in %D dimension%s:\n", dim, dim == 1 ? "" : "s");CHKERRQ(ierr);}
599     if (cellHeight) {ierr = PetscViewerASCIIPrintf(viewer, "  Cells are at height %D\n", cellHeight);CHKERRQ(ierr);}
600     ierr = PetscViewerASCIIPrintf(viewer, "orientation is missing\n", name);CHKERRQ(ierr);
601     ierr = PetscViewerASCIIPrintf(viewer, "cap --> base:\n", name);CHKERRQ(ierr);
602     ierr = PetscViewerASCIIPushSynchronized(viewer);CHKERRQ(ierr);
603     ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%d] Max sizes cone: %D support: %D\n", rank,maxConeSize, maxSupportSize);CHKERRQ(ierr);
604     for (p = pStart; p < pEnd; ++p) {
605       PetscInt dof, off, s;
606 
607       ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
608       ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
609       for (s = off; s < off+dof; ++s) {
610         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%d]: %D ----> %D\n", rank, p, mesh->supports[s]);CHKERRQ(ierr);
611       }
612     }
613     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
614     ierr = PetscViewerASCIIPrintf(viewer, "base <-- cap:\n", name);CHKERRQ(ierr);
615     for (p = pStart; p < pEnd; ++p) {
616       PetscInt dof, off, c;
617 
618       ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
619       ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
620       for (c = off; c < off+dof; ++c) {
621         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%d]: %D <---- %D (%D)\n", rank, p, mesh->cones[c], mesh->coneOrientations[c]);CHKERRQ(ierr);
622       }
623     }
624     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
625     ierr = PetscViewerASCIIPopSynchronized(viewer);CHKERRQ(ierr);
626     ierr = PetscSectionGetChart(coordSection, &pStart, NULL);CHKERRQ(ierr);
627     if (pStart >= 0) {ierr = PetscSectionVecView(coordSection, coordinates, viewer);CHKERRQ(ierr);}
628     ierr = DMGetLabel(dm, "marker", &markers);CHKERRQ(ierr);
629     ierr = DMLabelView(markers,viewer);CHKERRQ(ierr);
630     if (size > 1) {
631       PetscSF sf;
632 
633       ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr);
634       ierr = PetscSFView(sf, viewer);CHKERRQ(ierr);
635     }
636     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
637   } else if (format == PETSC_VIEWER_ASCII_LATEX) {
638     const char  *name, *color;
639     const char  *defcolors[3]  = {"gray", "orange", "green"};
640     const char  *deflcolors[4] = {"blue", "cyan", "red", "magenta"};
641     PetscReal    scale         = 2.0;
642     PetscBool    useNumbers    = PETSC_TRUE, useLabels, useColors;
643     double       tcoords[3];
644     PetscScalar *coords;
645     PetscInt     numLabels, l, numColors, numLColors, dim, depth, cStart, cEnd, c, vStart, vEnd, v, eStart = 0, eEnd = 0, e, p;
646     PetscMPIInt  rank, size;
647     char         **names, **colors, **lcolors;
648 
649     ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
650     ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
651     ierr = DMGetNumLabels(dm, &numLabels);CHKERRQ(ierr);
652     numLabels  = PetscMax(numLabels, 10);
653     numColors  = 10;
654     numLColors = 10;
655     ierr = PetscCalloc3(numLabels, &names, numColors, &colors, numLColors, &lcolors);CHKERRQ(ierr);
656     ierr = PetscOptionsGetReal(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_scale", &scale, NULL);CHKERRQ(ierr);
657     ierr = PetscOptionsGetBool(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_numbers", &useNumbers, NULL);CHKERRQ(ierr);
658     ierr = PetscOptionsGetStringArray(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_labels", names, &numLabels, &useLabels);CHKERRQ(ierr);
659     if (!useLabels) numLabels = 0;
660     ierr = PetscOptionsGetStringArray(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_colors", colors, &numColors, &useColors);CHKERRQ(ierr);
661     if (!useColors) {
662       numColors = 3;
663       for (c = 0; c < numColors; ++c) {ierr = PetscStrallocpy(defcolors[c], &colors[c]);CHKERRQ(ierr);}
664     }
665     ierr = PetscOptionsGetStringArray(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_lcolors", lcolors, &numLColors, &useColors);CHKERRQ(ierr);
666     if (!useColors) {
667       numLColors = 4;
668       for (c = 0; c < numLColors; ++c) {ierr = PetscStrallocpy(deflcolors[c], &lcolors[c]);CHKERRQ(ierr);}
669     }
670     ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRQ(ierr);
671     ierr = MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size);CHKERRQ(ierr);
672     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
673     ierr = PetscViewerASCIIPrintf(viewer, "\
674 \\documentclass[tikz]{standalone}\n\n\
675 \\usepackage{pgflibraryshapes}\n\
676 \\usetikzlibrary{backgrounds}\n\
677 \\usetikzlibrary{arrows}\n\
678 \\begin{document}\n");CHKERRQ(ierr);
679     if (size > 1) {
680       ierr = PetscViewerASCIIPrintf(viewer, "%s for process ", name);CHKERRQ(ierr);
681       for (p = 0; p < size; ++p) {
682         if (p > 0 && p == size-1) {
683           ierr = PetscViewerASCIIPrintf(viewer, ", and ", colors[p%numColors], p);CHKERRQ(ierr);
684         } else if (p > 0) {
685           ierr = PetscViewerASCIIPrintf(viewer, ", ", colors[p%numColors], p);CHKERRQ(ierr);
686         }
687         ierr = PetscViewerASCIIPrintf(viewer, "{\\textcolor{%s}%D}", colors[p%numColors], p);CHKERRQ(ierr);
688       }
689       ierr = PetscViewerASCIIPrintf(viewer, ".\n\n\n");CHKERRQ(ierr);
690     }
691     ierr = PetscViewerASCIIPrintf(viewer, "\\begin{tikzpicture}[scale = %g,font=\\fontsize{8}{8}\\selectfont]\n", 1.0);CHKERRQ(ierr);
692     /* Plot vertices */
693     ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
694     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
695     ierr = PetscViewerASCIIPushSynchronized(viewer);CHKERRQ(ierr);
696     for (v = vStart; v < vEnd; ++v) {
697       PetscInt  off, dof, d;
698       PetscBool isLabeled = PETSC_FALSE;
699 
700       ierr = PetscSectionGetDof(coordSection, v, &dof);CHKERRQ(ierr);
701       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
702       ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\path (");CHKERRQ(ierr);
703       if (PetscUnlikely(dof > 3)) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"coordSection vertex %D has dof %D > 3",v,dof);
704       for (d = 0; d < dof; ++d) {
705         tcoords[d] = (double) (scale*PetscRealPart(coords[off+d]));
706         tcoords[d] = PetscAbs(tcoords[d]) < 1e-10 ? 0.0 : tcoords[d];
707       }
708       /* Rotate coordinates since PGF makes z point out of the page instead of up */
709       if (dim == 3) {PetscReal tmp = tcoords[1]; tcoords[1] = tcoords[2]; tcoords[2] = -tmp;}
710       for (d = 0; d < dof; ++d) {
711         if (d > 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, ",");CHKERRQ(ierr);}
712         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "%g", tcoords[d]);CHKERRQ(ierr);
713       }
714       color = colors[rank%numColors];
715       for (l = 0; l < numLabels; ++l) {
716         PetscInt val;
717         ierr = DMGetLabelValue(dm, names[l], v, &val);CHKERRQ(ierr);
718         if (val >= 0) {color = lcolors[l%numLColors]; isLabeled = PETSC_TRUE; break;}
719       }
720       if (useNumbers) {
721         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [draw,shape=circle,color=%s] {%D};\n", v, rank, color, v);CHKERRQ(ierr);
722       } else {
723         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [fill,inner sep=%dpt,shape=circle,color=%s] {};\n", v, rank, !isLabeled ? 1 : 2, color);CHKERRQ(ierr);
724       }
725     }
726     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
727     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
728     /* Plot edges */
729     if (depth > 1) {ierr = DMPlexGetDepthStratum(dm, 1, &eStart, &eEnd);CHKERRQ(ierr);}
730     if (dim < 3 && useNumbers) {
731       ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
732       ierr = PetscViewerASCIIPrintf(viewer, "\\path\n");CHKERRQ(ierr);
733       for (e = eStart; e < eEnd; ++e) {
734         const PetscInt *cone;
735         PetscInt        coneSize, offA, offB, dof, d;
736 
737         ierr = DMPlexGetConeSize(dm, e, &coneSize);CHKERRQ(ierr);
738         if (coneSize != 2) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Edge %D cone should have two vertices, not %D", e, coneSize);
739         ierr = DMPlexGetCone(dm, e, &cone);CHKERRQ(ierr);
740         ierr = PetscSectionGetDof(coordSection, cone[0], &dof);CHKERRQ(ierr);
741         ierr = PetscSectionGetOffset(coordSection, cone[0], &offA);CHKERRQ(ierr);
742         ierr = PetscSectionGetOffset(coordSection, cone[1], &offB);CHKERRQ(ierr);
743         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "(");CHKERRQ(ierr);
744         for (d = 0; d < dof; ++d) {
745           tcoords[d] = (double) (scale*PetscRealPart(coords[offA+d]+coords[offB+d]));
746           tcoords[d] = PetscAbs(tcoords[d]) < 1e-10 ? 0.0 : tcoords[d];
747         }
748         /* Rotate coordinates since PGF makes z point out of the page instead of up */
749         if (dim == 3) {PetscReal tmp = tcoords[1]; tcoords[1] = tcoords[2]; tcoords[2] = -tmp;}
750         for (d = 0; d < dof; ++d) {
751           if (d > 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, ",");CHKERRQ(ierr);}
752           ierr = PetscViewerASCIISynchronizedPrintf(viewer, "%g", (double)tcoords[d]);CHKERRQ(ierr);
753         }
754         color = colors[rank%numColors];
755         for (l = 0; l < numLabels; ++l) {
756           PetscInt val;
757           ierr = DMGetLabelValue(dm, names[l], v, &val);CHKERRQ(ierr);
758           if (val >= 0) {color = lcolors[l%numLColors]; break;}
759         }
760         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [draw,shape=circle,color=%s] {%D} --\n", e, rank, color, e);CHKERRQ(ierr);
761       }
762       ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
763       ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
764       ierr = PetscViewerASCIIPrintf(viewer, "(0,0);\n");CHKERRQ(ierr);
765     }
766     /* Plot cells */
767     if (dim == 3 || !useNumbers) {
768       for (e = eStart; e < eEnd; ++e) {
769         const PetscInt *cone;
770 
771         color = colors[rank%numColors];
772         for (l = 0; l < numLabels; ++l) {
773           PetscInt val;
774           ierr = DMGetLabelValue(dm, names[l], e, &val);CHKERRQ(ierr);
775           if (val >= 0) {color = lcolors[l%numLColors]; break;}
776         }
777         ierr = DMPlexGetCone(dm, e, &cone);CHKERRQ(ierr);
778         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\draw[color=%s] (%D_%d) -- (%D_%d);\n", color, cone[0], rank, cone[1], rank);CHKERRQ(ierr);
779       }
780     } else {
781       ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
782       for (c = cStart; c < cEnd; ++c) {
783         PetscInt *closure = NULL;
784         PetscInt  closureSize, firstPoint = -1;
785 
786         ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
787         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\draw[color=%s] ", colors[rank%numColors]);CHKERRQ(ierr);
788         for (p = 0; p < closureSize*2; p += 2) {
789           const PetscInt point = closure[p];
790 
791           if ((point < vStart) || (point >= vEnd)) continue;
792           if (firstPoint >= 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- ");CHKERRQ(ierr);}
793           ierr = PetscViewerASCIISynchronizedPrintf(viewer, "(%D_%d)", point, rank);CHKERRQ(ierr);
794           if (firstPoint < 0) firstPoint = point;
795         }
796         /* Why doesn't this work? ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- cycle;\n");CHKERRQ(ierr); */
797         ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- (%D_%d);\n", firstPoint, rank);CHKERRQ(ierr);
798         ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
799       }
800     }
801     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
802     ierr = PetscViewerASCIIPopSynchronized(viewer);CHKERRQ(ierr);
803     ierr = PetscViewerASCIIPrintf(viewer, "\\end{tikzpicture}\n");CHKERRQ(ierr);
804     ierr = PetscViewerASCIIPrintf(viewer, "\\end{document}\n", name);CHKERRQ(ierr);
805     for (l = 0; l < numLabels;  ++l) {ierr = PetscFree(names[l]);CHKERRQ(ierr);}
806     for (c = 0; c < numColors;  ++c) {ierr = PetscFree(colors[c]);CHKERRQ(ierr);}
807     for (c = 0; c < numLColors; ++c) {ierr = PetscFree(lcolors[c]);CHKERRQ(ierr);}
808     ierr = PetscFree3(names, colors, lcolors);CHKERRQ(ierr);
809   } else {
810     MPI_Comm    comm;
811     PetscInt   *sizes, *hybsizes;
812     PetscInt    locDepth, depth, cellHeight, dim, d, pMax[4];
813     PetscInt    pStart, pEnd, p;
814     PetscInt    numLabels, l;
815     const char *name;
816     PetscMPIInt size;
817 
818     ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
819     ierr = MPI_Comm_size(comm, &size);CHKERRQ(ierr);
820     ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
821     ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
822     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
823     if (name) {ierr = PetscViewerASCIIPrintf(viewer, "%s in %D dimension%s:\n", name, dim, dim == 1 ? "" : "s");CHKERRQ(ierr);}
824     else      {ierr = PetscViewerASCIIPrintf(viewer, "Mesh in %D dimension%s:\n", dim, dim == 1 ? "" : "s");CHKERRQ(ierr);}
825     if (cellHeight) {ierr = PetscViewerASCIIPrintf(viewer, "  Cells are at height %D\n", cellHeight);CHKERRQ(ierr);}
826     ierr = DMPlexGetDepth(dm, &locDepth);CHKERRQ(ierr);
827     ierr = MPIU_Allreduce(&locDepth, &depth, 1, MPIU_INT, MPI_MAX, comm);CHKERRQ(ierr);
828     ierr = DMPlexGetHybridBounds(dm, &pMax[depth], depth > 0 ? &pMax[depth-1] : NULL, &pMax[1], &pMax[0]);CHKERRQ(ierr);
829     ierr = PetscMalloc2(size,&sizes,size,&hybsizes);CHKERRQ(ierr);
830     if (depth == 1) {
831       ierr = DMPlexGetDepthStratum(dm, 0, &pStart, &pEnd);CHKERRQ(ierr);
832       pEnd = pEnd - pStart;
833       ierr = MPI_Gather(&pEnd, 1, MPIU_INT, sizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
834       ierr = PetscViewerASCIIPrintf(viewer, "  %d-cells:", 0);CHKERRQ(ierr);
835       for (p = 0; p < size; ++p) {ierr = PetscViewerASCIIPrintf(viewer, " %D", sizes[p]);CHKERRQ(ierr);}
836       ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
837       ierr = DMPlexGetHeightStratum(dm, 0, &pStart, &pEnd);CHKERRQ(ierr);
838       pEnd = pEnd - pStart;
839       ierr = MPI_Gather(&pEnd, 1, MPIU_INT, sizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
840       ierr = PetscViewerASCIIPrintf(viewer, "  %D-cells:", dim);CHKERRQ(ierr);
841       for (p = 0; p < size; ++p) {ierr = PetscViewerASCIIPrintf(viewer, " %D", sizes[p]);CHKERRQ(ierr);}
842       ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
843     } else {
844       PetscMPIInt rank;
845       ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
846       for (d = 0; d <= dim; d++) {
847         ierr = DMPlexGetDepthStratum(dm, d, &pStart, &pEnd);CHKERRQ(ierr);
848         pEnd    -= pStart;
849         pMax[d] -= pStart;
850         ierr = MPI_Gather(&pEnd, 1, MPIU_INT, sizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
851         ierr = MPI_Gather(&pMax[d], 1, MPIU_INT, hybsizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
852         ierr = PetscViewerASCIIPrintf(viewer, "  %D-cells:", d);CHKERRQ(ierr);
853         for (p = 0; p < size; ++p) {
854           if (!rank) {
855             if (hybsizes[p] >= 0) {ierr = PetscViewerASCIIPrintf(viewer, " %D (%D)", sizes[p], sizes[p] - hybsizes[p]);CHKERRQ(ierr);}
856             else                  {ierr = PetscViewerASCIIPrintf(viewer, " %D", sizes[p]);CHKERRQ(ierr);}
857           }
858         }
859         ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
860       }
861     }
862     ierr = PetscFree2(sizes,hybsizes);CHKERRQ(ierr);
863     ierr = DMGetNumLabels(dm, &numLabels);CHKERRQ(ierr);
864     if (numLabels) {ierr = PetscViewerASCIIPrintf(viewer, "Labels:\n");CHKERRQ(ierr);}
865     for (l = 0; l < numLabels; ++l) {
866       DMLabel         label;
867       const char     *name;
868       IS              valueIS;
869       const PetscInt *values;
870       PetscInt        numValues, v;
871 
872       ierr = DMGetLabelName(dm, l, &name);CHKERRQ(ierr);
873       ierr = DMGetLabel(dm, name, &label);CHKERRQ(ierr);
874       ierr = DMLabelGetNumValues(label, &numValues);CHKERRQ(ierr);
875       ierr = PetscViewerASCIIPrintf(viewer, "  %s: %D strata with value/size (", name, numValues);CHKERRQ(ierr);
876       ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
877       ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
878       ierr = PetscViewerASCIIUseTabs(viewer, PETSC_FALSE);CHKERRQ(ierr);
879       for (v = 0; v < numValues; ++v) {
880         PetscInt size;
881 
882         ierr = DMLabelGetStratumSize(label, values[v], &size);CHKERRQ(ierr);
883         if (v > 0) {ierr = PetscViewerASCIIPrintf(viewer, ", ");CHKERRQ(ierr);}
884         ierr = PetscViewerASCIIPrintf(viewer, "%D (%D)", values[v], size);CHKERRQ(ierr);
885       }
886       ierr = PetscViewerASCIIPrintf(viewer, ")\n");CHKERRQ(ierr);
887       ierr = PetscViewerASCIIUseTabs(viewer, PETSC_TRUE);CHKERRQ(ierr);
888       ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
889       ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
890     }
891     ierr = DMGetCoarseDM(dm, &cdm);CHKERRQ(ierr);
892     if (cdm) {
893       ierr = PetscViewerASCIIPushTab(viewer);CHKERRQ(ierr);
894       ierr = DMPlexView_Ascii(cdm, viewer);CHKERRQ(ierr);
895       ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr);
896     }
897   }
898   PetscFunctionReturn(0);
899 }
900 
901 static PetscErrorCode DMPlexView_Draw(DM dm, PetscViewer viewer)
902 {
903   PetscDraw          draw;
904   DM                 cdm;
905   PetscSection       coordSection;
906   Vec                coordinates;
907   const PetscScalar *coords;
908   PetscReal          xyl[2],xyr[2],bound[4] = {PETSC_MAX_REAL, PETSC_MAX_REAL, PETSC_MIN_REAL, PETSC_MIN_REAL};
909   PetscBool          isnull;
910   PetscInt           dim, vStart, vEnd, cStart, cEnd, c, N;
911   PetscErrorCode     ierr;
912 
913   PetscFunctionBegin;
914   ierr = DMGetCoordinateDim(dm, &dim);CHKERRQ(ierr);
915   if (dim != 2) SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Cannot draw meshes of dimension %D", dim);
916   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
917   ierr = DMGetDefaultSection(cdm, &coordSection);CHKERRQ(ierr);
918   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
919   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
920   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
921 
922   ierr = PetscViewerDrawGetDraw(viewer, 0, &draw);CHKERRQ(ierr);
923   ierr = PetscDrawIsNull(draw, &isnull);CHKERRQ(ierr);
924   if (isnull) PetscFunctionReturn(0);
925   ierr = PetscDrawSetTitle(draw, "Mesh");CHKERRQ(ierr);
926 
927   ierr = VecGetLocalSize(coordinates, &N);CHKERRQ(ierr);
928   ierr = VecGetArrayRead(coordinates, &coords);CHKERRQ(ierr);
929   for (c = 0; c < N; c += dim) {
930     bound[0] = PetscMin(bound[0], PetscRealPart(coords[c]));   bound[2] = PetscMax(bound[2], PetscRealPart(coords[c]));
931     bound[1] = PetscMin(bound[1], PetscRealPart(coords[c+1])); bound[3] = PetscMax(bound[3], PetscRealPart(coords[c+1]));
932   }
933   ierr = VecRestoreArrayRead(coordinates, &coords);CHKERRQ(ierr);
934   ierr = MPIU_Allreduce(&bound[0],xyl,2,MPIU_REAL,MPIU_MIN,PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
935   ierr = MPIU_Allreduce(&bound[2],xyr,2,MPIU_REAL,MPIU_MAX,PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
936   ierr = PetscDrawSetCoordinates(draw, xyl[0], xyl[1], xyr[0], xyr[1]);CHKERRQ(ierr);
937   ierr = PetscDrawClear(draw);CHKERRQ(ierr);
938 
939   for (c = cStart; c < cEnd; ++c) {
940     PetscScalar *coords = NULL;
941     PetscInt     numCoords,coneSize;
942 
943     ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
944     ierr = DMPlexVecGetClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
945     switch (coneSize) {
946     case 3:
947       ierr = PetscDrawLine(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
948       ierr = PetscDrawLine(draw, PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
949       ierr = PetscDrawLine(draw, PetscRealPart(coords[4]), PetscRealPart(coords[5]), PetscRealPart(coords[0]), PetscRealPart(coords[1]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
950       break;
951     case 4:
952       ierr = PetscDrawLine(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
953       ierr = PetscDrawLine(draw, PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
954       ierr = PetscDrawLine(draw, PetscRealPart(coords[4]), PetscRealPart(coords[5]), PetscRealPart(coords[6]), PetscRealPart(coords[7]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
955       ierr = PetscDrawLine(draw, PetscRealPart(coords[6]), PetscRealPart(coords[7]), PetscRealPart(coords[0]), PetscRealPart(coords[1]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
956       break;
957     default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_SUP, "Cannot draw cells with %D facets", coneSize);
958     }
959     ierr = DMPlexVecRestoreClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
960   }
961   ierr = PetscDrawFlush(draw);CHKERRQ(ierr);
962   ierr = PetscDrawPause(draw);CHKERRQ(ierr);
963   ierr = PetscDrawSave(draw);CHKERRQ(ierr);
964   PetscFunctionReturn(0);
965 }
966 
967 PetscErrorCode DMView_Plex(DM dm, PetscViewer viewer)
968 {
969   PetscBool      iascii, ishdf5, isvtk, isdraw, flg, isglvis;
970   PetscErrorCode    ierr;
971 
972   PetscFunctionBegin;
973   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
974   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
975   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERASCII, &iascii);CHKERRQ(ierr);
976   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,   &isvtk);CHKERRQ(ierr);
977   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,  &ishdf5);CHKERRQ(ierr);
978   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERDRAW,  &isdraw);CHKERRQ(ierr);
979   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERGLVIS, &isglvis);CHKERRQ(ierr);
980   if (iascii) {
981     PetscViewerFormat format;
982     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
983     if (format == PETSC_VIEWER_ASCII_GLVIS) {
984       ierr = DMPlexView_GLVis(dm, viewer);CHKERRQ(ierr);
985     } else {
986       ierr = DMPlexView_Ascii(dm, viewer);CHKERRQ(ierr);
987     }
988   } else if (ishdf5) {
989 #if defined(PETSC_HAVE_HDF5)
990     ierr = PetscViewerPushFormat(viewer, PETSC_VIEWER_HDF5_VIZ);CHKERRQ(ierr);
991     ierr = DMPlexView_HDF5_Internal(dm, viewer);CHKERRQ(ierr);
992     ierr = PetscViewerPopFormat(viewer);CHKERRQ(ierr);
993 #else
994     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
995 #endif
996   } else if (isvtk) {
997     ierr = DMPlexVTKWriteAll((PetscObject) dm,viewer);CHKERRQ(ierr);
998   } else if (isdraw) {
999     ierr = DMPlexView_Draw(dm, viewer);CHKERRQ(ierr);
1000   } else if (isglvis) {
1001     ierr = DMPlexView_GLVis(dm, viewer);CHKERRQ(ierr);
1002   }
1003   /* Optionally view the partition */
1004   ierr = PetscOptionsHasName(((PetscObject) dm)->options, ((PetscObject) dm)->prefix, "-dm_partition_view", &flg);CHKERRQ(ierr);
1005   if (flg) {
1006     Vec ranks;
1007     ierr = DMPlexCreateRankField(dm, &ranks);CHKERRQ(ierr);
1008     ierr = VecView(ranks, viewer);CHKERRQ(ierr);
1009     ierr = VecDestroy(&ranks);CHKERRQ(ierr);
1010   }
1011   PetscFunctionReturn(0);
1012 }
1013 
1014 PetscErrorCode DMLoad_Plex(DM dm, PetscViewer viewer)
1015 {
1016   PetscBool      isbinary, ishdf5;
1017   PetscErrorCode ierr;
1018 
1019   PetscFunctionBegin;
1020   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1021   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1022   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERBINARY, &isbinary);CHKERRQ(ierr);
1023   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,   &ishdf5);CHKERRQ(ierr);
1024   if (isbinary) {SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Do not yet support binary viewers");}
1025   else if (ishdf5) {
1026 #if defined(PETSC_HAVE_HDF5)
1027     ierr = DMPlexLoad_HDF5_Internal(dm, viewer);CHKERRQ(ierr);
1028 #else
1029     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1030 #endif
1031   }
1032   PetscFunctionReturn(0);
1033 }
1034 
1035 PetscErrorCode DMDestroy_Plex(DM dm)
1036 {
1037   DM_Plex       *mesh = (DM_Plex*) dm->data;
1038   PetscErrorCode ierr;
1039 
1040   PetscFunctionBegin;
1041   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMSetUpGLVisViewer_C",NULL);CHKERRQ(ierr);
1042   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMPlexInsertBoundaryValues_C", NULL);CHKERRQ(ierr);
1043   if (--mesh->refct > 0) PetscFunctionReturn(0);
1044   ierr = PetscSectionDestroy(&mesh->coneSection);CHKERRQ(ierr);
1045   ierr = PetscFree(mesh->cones);CHKERRQ(ierr);
1046   ierr = PetscFree(mesh->coneOrientations);CHKERRQ(ierr);
1047   ierr = PetscSectionDestroy(&mesh->supportSection);CHKERRQ(ierr);
1048   ierr = PetscSectionDestroy(&mesh->subdomainSection);CHKERRQ(ierr);
1049   ierr = PetscFree(mesh->supports);CHKERRQ(ierr);
1050   ierr = PetscFree(mesh->facesTmp);CHKERRQ(ierr);
1051   ierr = PetscFree(mesh->tetgenOpts);CHKERRQ(ierr);
1052   ierr = PetscFree(mesh->triangleOpts);CHKERRQ(ierr);
1053   ierr = PetscPartitionerDestroy(&mesh->partitioner);CHKERRQ(ierr);
1054   ierr = DMLabelDestroy(&mesh->subpointMap);CHKERRQ(ierr);
1055   ierr = ISDestroy(&mesh->globalVertexNumbers);CHKERRQ(ierr);
1056   ierr = ISDestroy(&mesh->globalCellNumbers);CHKERRQ(ierr);
1057   ierr = PetscSectionDestroy(&mesh->anchorSection);CHKERRQ(ierr);
1058   ierr = ISDestroy(&mesh->anchorIS);CHKERRQ(ierr);
1059   ierr = PetscSectionDestroy(&mesh->parentSection);CHKERRQ(ierr);
1060   ierr = PetscFree(mesh->parents);CHKERRQ(ierr);
1061   ierr = PetscFree(mesh->childIDs);CHKERRQ(ierr);
1062   ierr = PetscSectionDestroy(&mesh->childSection);CHKERRQ(ierr);
1063   ierr = PetscFree(mesh->children);CHKERRQ(ierr);
1064   ierr = DMDestroy(&mesh->referenceTree);CHKERRQ(ierr);
1065   ierr = PetscGridHashDestroy(&mesh->lbox);CHKERRQ(ierr);
1066   /* This was originally freed in DMDestroy(), but that prevents reference counting of backend objects */
1067   ierr = PetscFree(mesh);CHKERRQ(ierr);
1068   PetscFunctionReturn(0);
1069 }
1070 
1071 PetscErrorCode DMCreateMatrix_Plex(DM dm, Mat *J)
1072 {
1073   PetscSection           sectionGlobal;
1074   PetscInt               bs = -1, mbs;
1075   PetscInt               localSize;
1076   PetscBool              isShell, isBlock, isSeqBlock, isMPIBlock, isSymBlock, isSymSeqBlock, isSymMPIBlock, isMatIS;
1077   PetscErrorCode         ierr;
1078   MatType                mtype;
1079   ISLocalToGlobalMapping ltog;
1080 
1081   PetscFunctionBegin;
1082   ierr = MatInitializePackage();CHKERRQ(ierr);
1083   mtype = dm->mattype;
1084   ierr = DMGetDefaultGlobalSection(dm, &sectionGlobal);CHKERRQ(ierr);
1085   /* ierr = PetscSectionGetStorageSize(sectionGlobal, &localSize);CHKERRQ(ierr); */
1086   ierr = PetscSectionGetConstrainedStorageSize(sectionGlobal, &localSize);CHKERRQ(ierr);
1087   ierr = MatCreate(PetscObjectComm((PetscObject)dm), J);CHKERRQ(ierr);
1088   ierr = MatSetSizes(*J, localSize, localSize, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
1089   ierr = MatSetType(*J, mtype);CHKERRQ(ierr);
1090   ierr = MatSetFromOptions(*J);CHKERRQ(ierr);
1091   ierr = MatGetBlockSize(*J, &mbs);CHKERRQ(ierr);
1092   if (mbs > 1) bs = mbs;
1093   ierr = PetscStrcmp(mtype, MATSHELL, &isShell);CHKERRQ(ierr);
1094   ierr = PetscStrcmp(mtype, MATBAIJ, &isBlock);CHKERRQ(ierr);
1095   ierr = PetscStrcmp(mtype, MATSEQBAIJ, &isSeqBlock);CHKERRQ(ierr);
1096   ierr = PetscStrcmp(mtype, MATMPIBAIJ, &isMPIBlock);CHKERRQ(ierr);
1097   ierr = PetscStrcmp(mtype, MATSBAIJ, &isSymBlock);CHKERRQ(ierr);
1098   ierr = PetscStrcmp(mtype, MATSEQSBAIJ, &isSymSeqBlock);CHKERRQ(ierr);
1099   ierr = PetscStrcmp(mtype, MATMPISBAIJ, &isSymMPIBlock);CHKERRQ(ierr);
1100   ierr = PetscStrcmp(mtype, MATIS, &isMatIS);CHKERRQ(ierr);
1101   if (!isShell) {
1102     PetscSection subSection;
1103     PetscBool    fillMatrix = (PetscBool)(!dm->prealloc_only && !isMatIS);
1104     PetscInt    *dnz, *onz, *dnzu, *onzu, bsLocal[2], bsMinMax[2], *ltogidx, lsize;
1105     PetscInt     pStart, pEnd, p, dof, cdof;
1106 
1107     /* Set localtoglobalmapping on the matrix for MatSetValuesLocal() to work (it also creates the local matrices in case of MATIS) */
1108     if (isMatIS) { /* need a different l2g map than the one computed by DMGetLocalToGlobalMapping */
1109       PetscSection section;
1110       PetscInt     size;
1111 
1112       ierr = DMGetDefaultSection(dm, &section);CHKERRQ(ierr);
1113       ierr = PetscSectionGetStorageSize(section, &size);CHKERRQ(ierr);
1114       ierr = PetscMalloc1(size,&ltogidx);CHKERRQ(ierr);
1115       ierr = DMPlexGetSubdomainSection(dm, &subSection);CHKERRQ(ierr);
1116     } else {
1117       ierr = DMGetLocalToGlobalMapping(dm,&ltog);CHKERRQ(ierr);
1118     }
1119     ierr = PetscSectionGetChart(sectionGlobal, &pStart, &pEnd);CHKERRQ(ierr);
1120     for (p = pStart, lsize = 0; p < pEnd; ++p) {
1121       PetscInt bdof;
1122 
1123       ierr = PetscSectionGetDof(sectionGlobal, p, &dof);CHKERRQ(ierr);
1124       ierr = PetscSectionGetConstraintDof(sectionGlobal, p, &cdof);CHKERRQ(ierr);
1125       dof  = dof < 0 ? -(dof+1) : dof;
1126       bdof = cdof && (dof-cdof) ? 1 : dof;
1127       if (dof) {
1128         if (bs < 0)          {bs = bdof;}
1129         else if (bs != bdof) {bs = 1; if (!isMatIS) break;}
1130       }
1131       if (isMatIS) {
1132         PetscInt loff,c,off;
1133         ierr = PetscSectionGetOffset(subSection, p, &loff);CHKERRQ(ierr);
1134         ierr = PetscSectionGetOffset(sectionGlobal, p, &off);CHKERRQ(ierr);
1135         for (c = 0; c < dof-cdof; ++c, ++lsize) ltogidx[loff+c] = off > -1 ? off+c : -(off+1)+c;
1136       }
1137     }
1138     /* Must have same blocksize on all procs (some might have no points) */
1139     bsLocal[0] = bs < 0 ? PETSC_MAX_INT : bs; bsLocal[1] = bs;
1140     ierr = PetscGlobalMinMaxInt(PetscObjectComm((PetscObject) dm), bsLocal, bsMinMax);CHKERRQ(ierr);
1141     if (bsMinMax[0] != bsMinMax[1]) {bs = 1;}
1142     else                            {bs = bsMinMax[0];}
1143     bs = bs < 0 ? 1 : bs;
1144     if (isMatIS) {
1145       PetscInt l;
1146       /* Must reduce indices by blocksize */
1147       if (bs > 1) for (l = 0; l < lsize; ++l) ltogidx[l] /= bs;
1148       ierr = ISLocalToGlobalMappingCreate(PetscObjectComm((PetscObject)dm), bs, lsize, ltogidx, PETSC_OWN_POINTER, &ltog);CHKERRQ(ierr);
1149     }
1150     ierr = MatSetLocalToGlobalMapping(*J,ltog,ltog);CHKERRQ(ierr);
1151     if (isMatIS) {
1152       ierr = ISLocalToGlobalMappingDestroy(&ltog);CHKERRQ(ierr);
1153     }
1154     ierr = PetscCalloc4(localSize/bs, &dnz, localSize/bs, &onz, localSize/bs, &dnzu, localSize/bs, &onzu);CHKERRQ(ierr);
1155     ierr = DMPlexPreallocateOperator(dm, bs, dnz, onz, dnzu, onzu, *J, fillMatrix);CHKERRQ(ierr);
1156     ierr = PetscFree4(dnz, onz, dnzu, onzu);CHKERRQ(ierr);
1157   }
1158   ierr = MatSetDM(*J, dm);CHKERRQ(ierr);
1159   PetscFunctionReturn(0);
1160 }
1161 
1162 /*@
1163   DMPlexGetSubdomainSection - Returns the section associated with the subdomain
1164 
1165   Not collective
1166 
1167   Input Parameter:
1168 . mesh - The DMPlex
1169 
1170   Output Parameters:
1171 . subsection - The subdomain section
1172 
1173   Level: developer
1174 
1175 .seealso:
1176 @*/
1177 PetscErrorCode DMPlexGetSubdomainSection(DM dm, PetscSection *subsection)
1178 {
1179   DM_Plex       *mesh = (DM_Plex*) dm->data;
1180   PetscErrorCode ierr;
1181 
1182   PetscFunctionBegin;
1183   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1184   if (!mesh->subdomainSection) {
1185     PetscSection section;
1186     PetscSF      sf;
1187 
1188     ierr = PetscSFCreate(PETSC_COMM_SELF,&sf);CHKERRQ(ierr);
1189     ierr = DMGetDefaultSection(dm,&section);CHKERRQ(ierr);
1190     ierr = PetscSectionCreateGlobalSection(section,sf,PETSC_FALSE,PETSC_TRUE,&mesh->subdomainSection);CHKERRQ(ierr);
1191     ierr = PetscSFDestroy(&sf);CHKERRQ(ierr);
1192   }
1193   *subsection = mesh->subdomainSection;
1194   PetscFunctionReturn(0);
1195 }
1196 
1197 /*@
1198   DMPlexGetChart - Return the interval for all mesh points [pStart, pEnd)
1199 
1200   Not collective
1201 
1202   Input Parameter:
1203 . mesh - The DMPlex
1204 
1205   Output Parameters:
1206 + pStart - The first mesh point
1207 - pEnd   - The upper bound for mesh points
1208 
1209   Level: beginner
1210 
1211 .seealso: DMPlexCreate(), DMPlexSetChart()
1212 @*/
1213 PetscErrorCode DMPlexGetChart(DM dm, PetscInt *pStart, PetscInt *pEnd)
1214 {
1215   DM_Plex       *mesh = (DM_Plex*) dm->data;
1216   PetscErrorCode ierr;
1217 
1218   PetscFunctionBegin;
1219   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1220   ierr = PetscSectionGetChart(mesh->coneSection, pStart, pEnd);CHKERRQ(ierr);
1221   PetscFunctionReturn(0);
1222 }
1223 
1224 /*@
1225   DMPlexSetChart - Set the interval for all mesh points [pStart, pEnd)
1226 
1227   Not collective
1228 
1229   Input Parameters:
1230 + mesh - The DMPlex
1231 . pStart - The first mesh point
1232 - pEnd   - The upper bound for mesh points
1233 
1234   Output Parameters:
1235 
1236   Level: beginner
1237 
1238 .seealso: DMPlexCreate(), DMPlexGetChart()
1239 @*/
1240 PetscErrorCode DMPlexSetChart(DM dm, PetscInt pStart, PetscInt pEnd)
1241 {
1242   DM_Plex       *mesh = (DM_Plex*) dm->data;
1243   PetscErrorCode ierr;
1244 
1245   PetscFunctionBegin;
1246   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1247   ierr = PetscSectionSetChart(mesh->coneSection, pStart, pEnd);CHKERRQ(ierr);
1248   ierr = PetscSectionSetChart(mesh->supportSection, pStart, pEnd);CHKERRQ(ierr);
1249   PetscFunctionReturn(0);
1250 }
1251 
1252 /*@
1253   DMPlexGetConeSize - Return the number of in-edges for this point in the DAG
1254 
1255   Not collective
1256 
1257   Input Parameters:
1258 + mesh - The DMPlex
1259 - p - The point, which must lie in the chart set with DMPlexSetChart()
1260 
1261   Output Parameter:
1262 . size - The cone size for point p
1263 
1264   Level: beginner
1265 
1266 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart()
1267 @*/
1268 PetscErrorCode DMPlexGetConeSize(DM dm, PetscInt p, PetscInt *size)
1269 {
1270   DM_Plex       *mesh = (DM_Plex*) dm->data;
1271   PetscErrorCode ierr;
1272 
1273   PetscFunctionBegin;
1274   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1275   PetscValidPointer(size, 3);
1276   ierr = PetscSectionGetDof(mesh->coneSection, p, size);CHKERRQ(ierr);
1277   PetscFunctionReturn(0);
1278 }
1279 
1280 /*@
1281   DMPlexSetConeSize - Set the number of in-edges for this point in the DAG
1282 
1283   Not collective
1284 
1285   Input Parameters:
1286 + mesh - The DMPlex
1287 . p - The point, which must lie in the chart set with DMPlexSetChart()
1288 - size - The cone size for point p
1289 
1290   Output Parameter:
1291 
1292   Note:
1293   This should be called after DMPlexSetChart().
1294 
1295   Level: beginner
1296 
1297 .seealso: DMPlexCreate(), DMPlexGetConeSize(), DMPlexSetChart()
1298 @*/
1299 PetscErrorCode DMPlexSetConeSize(DM dm, PetscInt p, PetscInt size)
1300 {
1301   DM_Plex       *mesh = (DM_Plex*) dm->data;
1302   PetscErrorCode ierr;
1303 
1304   PetscFunctionBegin;
1305   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1306   ierr = PetscSectionSetDof(mesh->coneSection, p, size);CHKERRQ(ierr);
1307 
1308   mesh->maxConeSize = PetscMax(mesh->maxConeSize, size);
1309   PetscFunctionReturn(0);
1310 }
1311 
1312 /*@
1313   DMPlexAddConeSize - Add the given number of in-edges to this point in the DAG
1314 
1315   Not collective
1316 
1317   Input Parameters:
1318 + mesh - The DMPlex
1319 . p - The point, which must lie in the chart set with DMPlexSetChart()
1320 - size - The additional cone size for point p
1321 
1322   Output Parameter:
1323 
1324   Note:
1325   This should be called after DMPlexSetChart().
1326 
1327   Level: beginner
1328 
1329 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexGetConeSize(), DMPlexSetChart()
1330 @*/
1331 PetscErrorCode DMPlexAddConeSize(DM dm, PetscInt p, PetscInt size)
1332 {
1333   DM_Plex       *mesh = (DM_Plex*) dm->data;
1334   PetscInt       csize;
1335   PetscErrorCode ierr;
1336 
1337   PetscFunctionBegin;
1338   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1339   ierr = PetscSectionAddDof(mesh->coneSection, p, size);CHKERRQ(ierr);
1340   ierr = PetscSectionGetDof(mesh->coneSection, p, &csize);CHKERRQ(ierr);
1341 
1342   mesh->maxConeSize = PetscMax(mesh->maxConeSize, csize);
1343   PetscFunctionReturn(0);
1344 }
1345 
1346 /*@C
1347   DMPlexGetCone - Return the points on the in-edges for this point in the DAG
1348 
1349   Not collective
1350 
1351   Input Parameters:
1352 + mesh - The DMPlex
1353 - p - The point, which must lie in the chart set with DMPlexSetChart()
1354 
1355   Output Parameter:
1356 . cone - An array of points which are on the in-edges for point p
1357 
1358   Level: beginner
1359 
1360   Fortran Notes:
1361   Since it returns an array, this routine is only available in Fortran 90, and you must
1362   include petsc.h90 in your code.
1363 
1364   You must also call DMPlexRestoreCone() after you finish using the returned array.
1365 
1366 .seealso: DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart()
1367 @*/
1368 PetscErrorCode DMPlexGetCone(DM dm, PetscInt p, const PetscInt *cone[])
1369 {
1370   DM_Plex       *mesh = (DM_Plex*) dm->data;
1371   PetscInt       off;
1372   PetscErrorCode ierr;
1373 
1374   PetscFunctionBegin;
1375   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1376   PetscValidPointer(cone, 3);
1377   ierr  = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1378   *cone = &mesh->cones[off];
1379   PetscFunctionReturn(0);
1380 }
1381 
1382 /*@
1383   DMPlexSetCone - Set the points on the in-edges for this point in the DAG
1384 
1385   Not collective
1386 
1387   Input Parameters:
1388 + mesh - The DMPlex
1389 . p - The point, which must lie in the chart set with DMPlexSetChart()
1390 - cone - An array of points which are on the in-edges for point p
1391 
1392   Output Parameter:
1393 
1394   Note:
1395   This should be called after all calls to DMPlexSetConeSize() and DMSetUp().
1396 
1397   Level: beginner
1398 
1399 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
1400 @*/
1401 PetscErrorCode DMPlexSetCone(DM dm, PetscInt p, const PetscInt cone[])
1402 {
1403   DM_Plex       *mesh = (DM_Plex*) dm->data;
1404   PetscInt       pStart, pEnd;
1405   PetscInt       dof, off, c;
1406   PetscErrorCode ierr;
1407 
1408   PetscFunctionBegin;
1409   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1410   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
1411   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
1412   if (dof) PetscValidPointer(cone, 3);
1413   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1414   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
1415   for (c = 0; c < dof; ++c) {
1416     if ((cone[c] < pStart) || (cone[c] >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone point %D is not in the valid range [%D, %D)", cone[c], pStart, pEnd);
1417     mesh->cones[off+c] = cone[c];
1418   }
1419   PetscFunctionReturn(0);
1420 }
1421 
1422 /*@C
1423   DMPlexGetConeOrientation - Return the orientations on the in-edges for this point in the DAG
1424 
1425   Not collective
1426 
1427   Input Parameters:
1428 + mesh - The DMPlex
1429 - p - The point, which must lie in the chart set with DMPlexSetChart()
1430 
1431   Output Parameter:
1432 . coneOrientation - An array of orientations which are on the in-edges for point p. An orientation is an
1433                     integer giving the prescription for cone traversal. If it is negative, the cone is
1434                     traversed in the opposite direction. Its value 'o', or if negative '-(o+1)', gives
1435                     the index of the cone point on which to start.
1436 
1437   Level: beginner
1438 
1439   Fortran Notes:
1440   Since it returns an array, this routine is only available in Fortran 90, and you must
1441   include petsc.h90 in your code.
1442 
1443   You must also call DMPlexRestoreConeOrientation() after you finish using the returned array.
1444 
1445 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetCone(), DMPlexSetChart()
1446 @*/
1447 PetscErrorCode DMPlexGetConeOrientation(DM dm, PetscInt p, const PetscInt *coneOrientation[])
1448 {
1449   DM_Plex       *mesh = (DM_Plex*) dm->data;
1450   PetscInt       off;
1451   PetscErrorCode ierr;
1452 
1453   PetscFunctionBegin;
1454   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1455 #if defined(PETSC_USE_DEBUG)
1456   {
1457     PetscInt dof;
1458     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
1459     if (dof) PetscValidPointer(coneOrientation, 3);
1460   }
1461 #endif
1462   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1463 
1464   *coneOrientation = &mesh->coneOrientations[off];
1465   PetscFunctionReturn(0);
1466 }
1467 
1468 /*@
1469   DMPlexSetConeOrientation - Set the orientations on the in-edges for this point in the DAG
1470 
1471   Not collective
1472 
1473   Input Parameters:
1474 + mesh - The DMPlex
1475 . p - The point, which must lie in the chart set with DMPlexSetChart()
1476 - coneOrientation - An array of orientations which are on the in-edges for point p. An orientation is an
1477                     integer giving the prescription for cone traversal. If it is negative, the cone is
1478                     traversed in the opposite direction. Its value 'o', or if negative '-(o+1)', gives
1479                     the index of the cone point on which to start.
1480 
1481   Output Parameter:
1482 
1483   Note:
1484   This should be called after all calls to DMPlexSetConeSize() and DMSetUp().
1485 
1486   Level: beginner
1487 
1488 .seealso: DMPlexCreate(), DMPlexGetConeOrientation(), DMPlexSetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
1489 @*/
1490 PetscErrorCode DMPlexSetConeOrientation(DM dm, PetscInt p, const PetscInt coneOrientation[])
1491 {
1492   DM_Plex       *mesh = (DM_Plex*) dm->data;
1493   PetscInt       pStart, pEnd;
1494   PetscInt       dof, off, c;
1495   PetscErrorCode ierr;
1496 
1497   PetscFunctionBegin;
1498   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1499   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
1500   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
1501   if (dof) PetscValidPointer(coneOrientation, 3);
1502   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1503   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
1504   for (c = 0; c < dof; ++c) {
1505     PetscInt cdof, o = coneOrientation[c];
1506 
1507     ierr = PetscSectionGetDof(mesh->coneSection, mesh->cones[off+c], &cdof);CHKERRQ(ierr);
1508     if (o && ((o < -(cdof+1)) || (o >= cdof))) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone orientation %D is not in the valid range [%D. %D)", o, -(cdof+1), cdof);
1509     mesh->coneOrientations[off+c] = o;
1510   }
1511   PetscFunctionReturn(0);
1512 }
1513 
1514 /*@
1515   DMPlexInsertCone - Insert a point into the in-edges for the point p in the DAG
1516 
1517   Not collective
1518 
1519   Input Parameters:
1520 + mesh - The DMPlex
1521 . p - The point, which must lie in the chart set with DMPlexSetChart()
1522 . conePos - The local index in the cone where the point should be put
1523 - conePoint - The mesh point to insert
1524 
1525   Level: beginner
1526 
1527 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
1528 @*/
1529 PetscErrorCode DMPlexInsertCone(DM dm, PetscInt p, PetscInt conePos, PetscInt conePoint)
1530 {
1531   DM_Plex       *mesh = (DM_Plex*) dm->data;
1532   PetscInt       pStart, pEnd;
1533   PetscInt       dof, off;
1534   PetscErrorCode ierr;
1535 
1536   PetscFunctionBegin;
1537   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1538   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
1539   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
1540   if ((conePoint < pStart) || (conePoint >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone point %D is not in the valid range [%D, %D)", conePoint, pStart, pEnd);
1541   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
1542   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1543   if ((conePos < 0) || (conePos >= dof)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone position %D of point %D is not in the valid range [0, %D)", conePos, p, dof);
1544   mesh->cones[off+conePos] = conePoint;
1545   PetscFunctionReturn(0);
1546 }
1547 
1548 /*@
1549   DMPlexInsertConeOrientation - Insert a point orientation for the in-edge for the point p in the DAG
1550 
1551   Not collective
1552 
1553   Input Parameters:
1554 + mesh - The DMPlex
1555 . p - The point, which must lie in the chart set with DMPlexSetChart()
1556 . conePos - The local index in the cone where the point should be put
1557 - coneOrientation - The point orientation to insert
1558 
1559   Level: beginner
1560 
1561 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
1562 @*/
1563 PetscErrorCode DMPlexInsertConeOrientation(DM dm, PetscInt p, PetscInt conePos, PetscInt coneOrientation)
1564 {
1565   DM_Plex       *mesh = (DM_Plex*) dm->data;
1566   PetscInt       pStart, pEnd;
1567   PetscInt       dof, off;
1568   PetscErrorCode ierr;
1569 
1570   PetscFunctionBegin;
1571   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1572   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
1573   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
1574   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
1575   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1576   if ((conePos < 0) || (conePos >= dof)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone position %D of point %D is not in the valid range [0, %D)", conePos, p, dof);
1577   mesh->coneOrientations[off+conePos] = coneOrientation;
1578   PetscFunctionReturn(0);
1579 }
1580 
1581 /*@
1582   DMPlexGetSupportSize - Return the number of out-edges for this point in the DAG
1583 
1584   Not collective
1585 
1586   Input Parameters:
1587 + mesh - The DMPlex
1588 - p - The point, which must lie in the chart set with DMPlexSetChart()
1589 
1590   Output Parameter:
1591 . size - The support size for point p
1592 
1593   Level: beginner
1594 
1595 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart(), DMPlexGetConeSize()
1596 @*/
1597 PetscErrorCode DMPlexGetSupportSize(DM dm, PetscInt p, PetscInt *size)
1598 {
1599   DM_Plex       *mesh = (DM_Plex*) dm->data;
1600   PetscErrorCode ierr;
1601 
1602   PetscFunctionBegin;
1603   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1604   PetscValidPointer(size, 3);
1605   ierr = PetscSectionGetDof(mesh->supportSection, p, size);CHKERRQ(ierr);
1606   PetscFunctionReturn(0);
1607 }
1608 
1609 /*@
1610   DMPlexSetSupportSize - Set the number of out-edges for this point in the DAG
1611 
1612   Not collective
1613 
1614   Input Parameters:
1615 + mesh - The DMPlex
1616 . p - The point, which must lie in the chart set with DMPlexSetChart()
1617 - size - The support size for point p
1618 
1619   Output Parameter:
1620 
1621   Note:
1622   This should be called after DMPlexSetChart().
1623 
1624   Level: beginner
1625 
1626 .seealso: DMPlexCreate(), DMPlexGetSupportSize(), DMPlexSetChart()
1627 @*/
1628 PetscErrorCode DMPlexSetSupportSize(DM dm, PetscInt p, PetscInt size)
1629 {
1630   DM_Plex       *mesh = (DM_Plex*) dm->data;
1631   PetscErrorCode ierr;
1632 
1633   PetscFunctionBegin;
1634   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1635   ierr = PetscSectionSetDof(mesh->supportSection, p, size);CHKERRQ(ierr);
1636 
1637   mesh->maxSupportSize = PetscMax(mesh->maxSupportSize, size);
1638   PetscFunctionReturn(0);
1639 }
1640 
1641 /*@C
1642   DMPlexGetSupport - Return the points on the out-edges for this point in the DAG
1643 
1644   Not collective
1645 
1646   Input Parameters:
1647 + mesh - The DMPlex
1648 - p - The point, which must lie in the chart set with DMPlexSetChart()
1649 
1650   Output Parameter:
1651 . support - An array of points which are on the out-edges for point p
1652 
1653   Level: beginner
1654 
1655   Fortran Notes:
1656   Since it returns an array, this routine is only available in Fortran 90, and you must
1657   include petsc.h90 in your code.
1658 
1659   You must also call DMPlexRestoreSupport() after you finish using the returned array.
1660 
1661 .seealso: DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
1662 @*/
1663 PetscErrorCode DMPlexGetSupport(DM dm, PetscInt p, const PetscInt *support[])
1664 {
1665   DM_Plex       *mesh = (DM_Plex*) dm->data;
1666   PetscInt       off;
1667   PetscErrorCode ierr;
1668 
1669   PetscFunctionBegin;
1670   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1671   PetscValidPointer(support, 3);
1672   ierr     = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
1673   *support = &mesh->supports[off];
1674   PetscFunctionReturn(0);
1675 }
1676 
1677 /*@
1678   DMPlexSetSupport - Set the points on the out-edges for this point in the DAG
1679 
1680   Not collective
1681 
1682   Input Parameters:
1683 + mesh - The DMPlex
1684 . p - The point, which must lie in the chart set with DMPlexSetChart()
1685 - support - An array of points which are on the in-edges for point p
1686 
1687   Output Parameter:
1688 
1689   Note:
1690   This should be called after all calls to DMPlexSetSupportSize() and DMSetUp().
1691 
1692   Level: beginner
1693 
1694 .seealso: DMPlexCreate(), DMPlexGetSupport(), DMPlexSetChart(), DMPlexSetSupportSize(), DMSetUp()
1695 @*/
1696 PetscErrorCode DMPlexSetSupport(DM dm, PetscInt p, const PetscInt support[])
1697 {
1698   DM_Plex       *mesh = (DM_Plex*) dm->data;
1699   PetscInt       pStart, pEnd;
1700   PetscInt       dof, off, c;
1701   PetscErrorCode ierr;
1702 
1703   PetscFunctionBegin;
1704   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1705   ierr = PetscSectionGetChart(mesh->supportSection, &pStart, &pEnd);CHKERRQ(ierr);
1706   ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
1707   if (dof) PetscValidPointer(support, 3);
1708   ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
1709   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
1710   for (c = 0; c < dof; ++c) {
1711     if ((support[c] < pStart) || (support[c] >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support point %D is not in the valid range [%D, %D)", support[c], pStart, pEnd);
1712     mesh->supports[off+c] = support[c];
1713   }
1714   PetscFunctionReturn(0);
1715 }
1716 
1717 /*@
1718   DMPlexInsertSupport - Insert a point into the out-edges for the point p in the DAG
1719 
1720   Not collective
1721 
1722   Input Parameters:
1723 + mesh - The DMPlex
1724 . p - The point, which must lie in the chart set with DMPlexSetChart()
1725 . supportPos - The local index in the cone where the point should be put
1726 - supportPoint - The mesh point to insert
1727 
1728   Level: beginner
1729 
1730 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
1731 @*/
1732 PetscErrorCode DMPlexInsertSupport(DM dm, PetscInt p, PetscInt supportPos, PetscInt supportPoint)
1733 {
1734   DM_Plex       *mesh = (DM_Plex*) dm->data;
1735   PetscInt       pStart, pEnd;
1736   PetscInt       dof, off;
1737   PetscErrorCode ierr;
1738 
1739   PetscFunctionBegin;
1740   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1741   ierr = PetscSectionGetChart(mesh->supportSection, &pStart, &pEnd);CHKERRQ(ierr);
1742   ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
1743   ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
1744   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
1745   if ((supportPoint < pStart) || (supportPoint >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support point %D is not in the valid range [%D, %D)", supportPoint, pStart, pEnd);
1746   if (supportPos >= dof) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support position %D of point %D is not in the valid range [0, %D)", supportPos, p, dof);
1747   mesh->supports[off+supportPos] = supportPoint;
1748   PetscFunctionReturn(0);
1749 }
1750 
1751 /*@C
1752   DMPlexGetTransitiveClosure - Return the points on the transitive closure of the in-edges or out-edges for this point in the DAG
1753 
1754   Not collective
1755 
1756   Input Parameters:
1757 + mesh - The DMPlex
1758 . p - The point, which must lie in the chart set with DMPlexSetChart()
1759 . useCone - PETSC_TRUE for in-edges,  otherwise use out-edges
1760 - points - If points is NULL on input, internal storage will be returned, otherwise the provided array is used
1761 
1762   Output Parameters:
1763 + numPoints - The number of points in the closure, so points[] is of size 2*numPoints
1764 - points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...]
1765 
1766   Note:
1767   If using internal storage (points is NULL on input), each call overwrites the last output.
1768 
1769   Fortran Notes:
1770   Since it returns an array, this routine is only available in Fortran 90, and you must
1771   include petsc.h90 in your code.
1772 
1773   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
1774 
1775   Level: beginner
1776 
1777 .seealso: DMPlexRestoreTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
1778 @*/
1779 PetscErrorCode DMPlexGetTransitiveClosure(DM dm, PetscInt p, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
1780 {
1781   DM_Plex        *mesh = (DM_Plex*) dm->data;
1782   PetscInt       *closure, *fifo;
1783   const PetscInt *tmp = NULL, *tmpO = NULL;
1784   PetscInt        tmpSize, t;
1785   PetscInt        depth       = 0, maxSize;
1786   PetscInt        closureSize = 2, fifoSize = 0, fifoStart = 0;
1787   PetscErrorCode  ierr;
1788 
1789   PetscFunctionBegin;
1790   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1791   ierr    = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
1792   /* This is only 1-level */
1793   if (useCone) {
1794     ierr = DMPlexGetConeSize(dm, p, &tmpSize);CHKERRQ(ierr);
1795     ierr = DMPlexGetCone(dm, p, &tmp);CHKERRQ(ierr);
1796     ierr = DMPlexGetConeOrientation(dm, p, &tmpO);CHKERRQ(ierr);
1797   } else {
1798     ierr = DMPlexGetSupportSize(dm, p, &tmpSize);CHKERRQ(ierr);
1799     ierr = DMPlexGetSupport(dm, p, &tmp);CHKERRQ(ierr);
1800   }
1801   if (depth == 1) {
1802     if (*points) {
1803       closure = *points;
1804     } else {
1805       maxSize = 2*(PetscMax(mesh->maxConeSize, mesh->maxSupportSize)+1);
1806       ierr = DMGetWorkArray(dm, maxSize, MPIU_INT, &closure);CHKERRQ(ierr);
1807     }
1808     closure[0] = p; closure[1] = 0;
1809     for (t = 0; t < tmpSize; ++t, closureSize += 2) {
1810       closure[closureSize]   = tmp[t];
1811       closure[closureSize+1] = tmpO ? tmpO[t] : 0;
1812     }
1813     if (numPoints) *numPoints = closureSize/2;
1814     if (points)    *points    = closure;
1815     PetscFunctionReturn(0);
1816   }
1817   {
1818     PetscInt c, coneSeries, s,supportSeries;
1819 
1820     c = mesh->maxConeSize;
1821     coneSeries = (c > 1) ? ((PetscPowInt(c,depth+1)-1)/(c-1)) : depth+1;
1822     s = mesh->maxSupportSize;
1823     supportSeries = (s > 1) ? ((PetscPowInt(s,depth+1)-1)/(s-1)) : depth+1;
1824     maxSize = 2*PetscMax(coneSeries,supportSeries);
1825   }
1826   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &fifo);CHKERRQ(ierr);
1827   if (*points) {
1828     closure = *points;
1829   } else {
1830     ierr = DMGetWorkArray(dm, maxSize, MPIU_INT, &closure);CHKERRQ(ierr);
1831   }
1832   closure[0] = p; closure[1] = 0;
1833   for (t = 0; t < tmpSize; ++t, closureSize += 2, fifoSize += 2) {
1834     const PetscInt cp = tmp[t];
1835     const PetscInt co = tmpO ? tmpO[t] : 0;
1836 
1837     closure[closureSize]   = cp;
1838     closure[closureSize+1] = co;
1839     fifo[fifoSize]         = cp;
1840     fifo[fifoSize+1]       = co;
1841   }
1842   /* Should kick out early when depth is reached, rather than checking all vertices for empty cones */
1843   while (fifoSize - fifoStart) {
1844     const PetscInt q   = fifo[fifoStart];
1845     const PetscInt o   = fifo[fifoStart+1];
1846     const PetscInt rev = o >= 0 ? 0 : 1;
1847     const PetscInt off = rev ? -(o+1) : o;
1848 
1849     if (useCone) {
1850       ierr = DMPlexGetConeSize(dm, q, &tmpSize);CHKERRQ(ierr);
1851       ierr = DMPlexGetCone(dm, q, &tmp);CHKERRQ(ierr);
1852       ierr = DMPlexGetConeOrientation(dm, q, &tmpO);CHKERRQ(ierr);
1853     } else {
1854       ierr = DMPlexGetSupportSize(dm, q, &tmpSize);CHKERRQ(ierr);
1855       ierr = DMPlexGetSupport(dm, q, &tmp);CHKERRQ(ierr);
1856       tmpO = NULL;
1857     }
1858     for (t = 0; t < tmpSize; ++t) {
1859       const PetscInt i  = ((rev ? tmpSize-t : t) + off)%tmpSize;
1860       const PetscInt cp = tmp[i];
1861       /* Must propogate orientation: When we reverse orientation, we both reverse the direction of iteration and start at the other end of the chain. */
1862       /* HACK: It is worse to get the size here, than to change the interpretation of -(*+1)
1863        const PetscInt co = tmpO ? (rev ? -(tmpO[i]+1) : tmpO[i]) : 0; */
1864       PetscInt       co = tmpO ? tmpO[i] : 0;
1865       PetscInt       c;
1866 
1867       if (rev) {
1868         PetscInt childSize, coff;
1869         ierr = DMPlexGetConeSize(dm, cp, &childSize);CHKERRQ(ierr);
1870         coff = tmpO[i] < 0 ? -(tmpO[i]+1) : tmpO[i];
1871         co   = childSize ? -(((coff+childSize-1)%childSize)+1) : 0;
1872       }
1873       /* Check for duplicate */
1874       for (c = 0; c < closureSize; c += 2) {
1875         if (closure[c] == cp) break;
1876       }
1877       if (c == closureSize) {
1878         closure[closureSize]   = cp;
1879         closure[closureSize+1] = co;
1880         fifo[fifoSize]         = cp;
1881         fifo[fifoSize+1]       = co;
1882         closureSize           += 2;
1883         fifoSize              += 2;
1884       }
1885     }
1886     fifoStart += 2;
1887   }
1888   if (numPoints) *numPoints = closureSize/2;
1889   if (points)    *points    = closure;
1890   ierr = DMRestoreWorkArray(dm, maxSize, MPIU_INT, &fifo);CHKERRQ(ierr);
1891   PetscFunctionReturn(0);
1892 }
1893 
1894 /*@C
1895   DMPlexGetTransitiveClosure_Internal - Return the points on the transitive closure of the in-edges or out-edges for this point in the DAG with a specified initial orientation
1896 
1897   Not collective
1898 
1899   Input Parameters:
1900 + mesh - The DMPlex
1901 . p - The point, which must lie in the chart set with DMPlexSetChart()
1902 . orientation - The orientation of the point
1903 . useCone - PETSC_TRUE for in-edges,  otherwise use out-edges
1904 - points - If points is NULL on input, internal storage will be returned, otherwise the provided array is used
1905 
1906   Output Parameters:
1907 + numPoints - The number of points in the closure, so points[] is of size 2*numPoints
1908 - points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...]
1909 
1910   Note:
1911   If using internal storage (points is NULL on input), each call overwrites the last output.
1912 
1913   Fortran Notes:
1914   Since it returns an array, this routine is only available in Fortran 90, and you must
1915   include petsc.h90 in your code.
1916 
1917   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
1918 
1919   Level: beginner
1920 
1921 .seealso: DMPlexRestoreTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
1922 @*/
1923 PetscErrorCode DMPlexGetTransitiveClosure_Internal(DM dm, PetscInt p, PetscInt ornt, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
1924 {
1925   DM_Plex        *mesh = (DM_Plex*) dm->data;
1926   PetscInt       *closure, *fifo;
1927   const PetscInt *tmp = NULL, *tmpO = NULL;
1928   PetscInt        tmpSize, t;
1929   PetscInt        depth       = 0, maxSize;
1930   PetscInt        closureSize = 2, fifoSize = 0, fifoStart = 0;
1931   PetscErrorCode  ierr;
1932 
1933   PetscFunctionBegin;
1934   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1935   ierr    = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
1936   /* This is only 1-level */
1937   if (useCone) {
1938     ierr = DMPlexGetConeSize(dm, p, &tmpSize);CHKERRQ(ierr);
1939     ierr = DMPlexGetCone(dm, p, &tmp);CHKERRQ(ierr);
1940     ierr = DMPlexGetConeOrientation(dm, p, &tmpO);CHKERRQ(ierr);
1941   } else {
1942     ierr = DMPlexGetSupportSize(dm, p, &tmpSize);CHKERRQ(ierr);
1943     ierr = DMPlexGetSupport(dm, p, &tmp);CHKERRQ(ierr);
1944   }
1945   if (depth == 1) {
1946     if (*points) {
1947       closure = *points;
1948     } else {
1949       maxSize = 2*(PetscMax(mesh->maxConeSize, mesh->maxSupportSize)+1);
1950       ierr = DMGetWorkArray(dm, maxSize, MPIU_INT, &closure);CHKERRQ(ierr);
1951     }
1952     closure[0] = p; closure[1] = ornt;
1953     for (t = 0; t < tmpSize; ++t, closureSize += 2) {
1954       const PetscInt i = ornt >= 0 ? (t+ornt)%tmpSize : (-(ornt+1) + tmpSize-t)%tmpSize;
1955       closure[closureSize]   = tmp[i];
1956       closure[closureSize+1] = tmpO ? tmpO[i] : 0;
1957     }
1958     if (numPoints) *numPoints = closureSize/2;
1959     if (points)    *points    = closure;
1960     PetscFunctionReturn(0);
1961   }
1962   {
1963     PetscInt c, coneSeries, s,supportSeries;
1964 
1965     c = mesh->maxConeSize;
1966     coneSeries = (c > 1) ? ((PetscPowInt(c,depth+1)-1)/(c-1)) : depth+1;
1967     s = mesh->maxSupportSize;
1968     supportSeries = (s > 1) ? ((PetscPowInt(s,depth+1)-1)/(s-1)) : depth+1;
1969     maxSize = 2*PetscMax(coneSeries,supportSeries);
1970   }
1971   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &fifo);CHKERRQ(ierr);
1972   if (*points) {
1973     closure = *points;
1974   } else {
1975     ierr = DMGetWorkArray(dm, maxSize, MPIU_INT, &closure);CHKERRQ(ierr);
1976   }
1977   closure[0] = p; closure[1] = ornt;
1978   for (t = 0; t < tmpSize; ++t, closureSize += 2, fifoSize += 2) {
1979     const PetscInt i  = ornt >= 0 ? (t+ornt)%tmpSize : (-(ornt+1) + tmpSize-t)%tmpSize;
1980     const PetscInt cp = tmp[i];
1981     PetscInt       co = tmpO ? tmpO[i] : 0;
1982 
1983     if (ornt < 0) {
1984       PetscInt childSize, coff;
1985       ierr = DMPlexGetConeSize(dm, cp, &childSize);CHKERRQ(ierr);
1986       coff = co < 0 ? -(tmpO[i]+1) : tmpO[i];
1987       co   = childSize ? -(((coff+childSize-1)%childSize)+1) : 0;
1988     }
1989     closure[closureSize]   = cp;
1990     closure[closureSize+1] = co;
1991     fifo[fifoSize]         = cp;
1992     fifo[fifoSize+1]       = co;
1993   }
1994   /* Should kick out early when depth is reached, rather than checking all vertices for empty cones */
1995   while (fifoSize - fifoStart) {
1996     const PetscInt q   = fifo[fifoStart];
1997     const PetscInt o   = fifo[fifoStart+1];
1998     const PetscInt rev = o >= 0 ? 0 : 1;
1999     const PetscInt off = rev ? -(o+1) : o;
2000 
2001     if (useCone) {
2002       ierr = DMPlexGetConeSize(dm, q, &tmpSize);CHKERRQ(ierr);
2003       ierr = DMPlexGetCone(dm, q, &tmp);CHKERRQ(ierr);
2004       ierr = DMPlexGetConeOrientation(dm, q, &tmpO);CHKERRQ(ierr);
2005     } else {
2006       ierr = DMPlexGetSupportSize(dm, q, &tmpSize);CHKERRQ(ierr);
2007       ierr = DMPlexGetSupport(dm, q, &tmp);CHKERRQ(ierr);
2008       tmpO = NULL;
2009     }
2010     for (t = 0; t < tmpSize; ++t) {
2011       const PetscInt i  = ((rev ? tmpSize-t : t) + off)%tmpSize;
2012       const PetscInt cp = tmp[i];
2013       /* Must propogate orientation: When we reverse orientation, we both reverse the direction of iteration and start at the other end of the chain. */
2014       /* HACK: It is worse to get the size here, than to change the interpretation of -(*+1)
2015        const PetscInt co = tmpO ? (rev ? -(tmpO[i]+1) : tmpO[i]) : 0; */
2016       PetscInt       co = tmpO ? tmpO[i] : 0;
2017       PetscInt       c;
2018 
2019       if (rev) {
2020         PetscInt childSize, coff;
2021         ierr = DMPlexGetConeSize(dm, cp, &childSize);CHKERRQ(ierr);
2022         coff = tmpO[i] < 0 ? -(tmpO[i]+1) : tmpO[i];
2023         co   = childSize ? -(((coff+childSize-1)%childSize)+1) : 0;
2024       }
2025       /* Check for duplicate */
2026       for (c = 0; c < closureSize; c += 2) {
2027         if (closure[c] == cp) break;
2028       }
2029       if (c == closureSize) {
2030         closure[closureSize]   = cp;
2031         closure[closureSize+1] = co;
2032         fifo[fifoSize]         = cp;
2033         fifo[fifoSize+1]       = co;
2034         closureSize           += 2;
2035         fifoSize              += 2;
2036       }
2037     }
2038     fifoStart += 2;
2039   }
2040   if (numPoints) *numPoints = closureSize/2;
2041   if (points)    *points    = closure;
2042   ierr = DMRestoreWorkArray(dm, maxSize, MPIU_INT, &fifo);CHKERRQ(ierr);
2043   PetscFunctionReturn(0);
2044 }
2045 
2046 /*@C
2047   DMPlexRestoreTransitiveClosure - Restore the array of points on the transitive closure of the in-edges or out-edges for this point in the DAG
2048 
2049   Not collective
2050 
2051   Input Parameters:
2052 + mesh - The DMPlex
2053 . p - The point, which must lie in the chart set with DMPlexSetChart()
2054 . useCone - PETSC_TRUE for in-edges,  otherwise use out-edges
2055 . numPoints - The number of points in the closure, so points[] is of size 2*numPoints, zeroed on exit
2056 - points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...], zeroed on exit
2057 
2058   Note:
2059   If not using internal storage (points is not NULL on input), this call is unnecessary
2060 
2061   Fortran Notes:
2062   Since it returns an array, this routine is only available in Fortran 90, and you must
2063   include petsc.h90 in your code.
2064 
2065   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2066 
2067   Level: beginner
2068 
2069 .seealso: DMPlexGetTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
2070 @*/
2071 PetscErrorCode DMPlexRestoreTransitiveClosure(DM dm, PetscInt p, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
2072 {
2073   PetscErrorCode ierr;
2074 
2075   PetscFunctionBegin;
2076   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2077   if (numPoints) PetscValidIntPointer(numPoints,4);
2078   if (points) PetscValidPointer(points,5);
2079   ierr = DMRestoreWorkArray(dm, 0, MPIU_INT, points);CHKERRQ(ierr);
2080   if (numPoints) *numPoints = 0;
2081   PetscFunctionReturn(0);
2082 }
2083 
2084 /*@
2085   DMPlexGetMaxSizes - Return the maximum number of in-edges (cone) and out-edges (support) for any point in the DAG
2086 
2087   Not collective
2088 
2089   Input Parameter:
2090 . mesh - The DMPlex
2091 
2092   Output Parameters:
2093 + maxConeSize - The maximum number of in-edges
2094 - maxSupportSize - The maximum number of out-edges
2095 
2096   Level: beginner
2097 
2098 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart()
2099 @*/
2100 PetscErrorCode DMPlexGetMaxSizes(DM dm, PetscInt *maxConeSize, PetscInt *maxSupportSize)
2101 {
2102   DM_Plex *mesh = (DM_Plex*) dm->data;
2103 
2104   PetscFunctionBegin;
2105   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2106   if (maxConeSize)    *maxConeSize    = mesh->maxConeSize;
2107   if (maxSupportSize) *maxSupportSize = mesh->maxSupportSize;
2108   PetscFunctionReturn(0);
2109 }
2110 
2111 PetscErrorCode DMSetUp_Plex(DM dm)
2112 {
2113   DM_Plex       *mesh = (DM_Plex*) dm->data;
2114   PetscInt       size;
2115   PetscErrorCode ierr;
2116 
2117   PetscFunctionBegin;
2118   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2119   ierr = PetscSectionSetUp(mesh->coneSection);CHKERRQ(ierr);
2120   ierr = PetscSectionGetStorageSize(mesh->coneSection, &size);CHKERRQ(ierr);
2121   ierr = PetscMalloc1(size, &mesh->cones);CHKERRQ(ierr);
2122   ierr = PetscCalloc1(size, &mesh->coneOrientations);CHKERRQ(ierr);
2123   if (mesh->maxSupportSize) {
2124     ierr = PetscSectionSetUp(mesh->supportSection);CHKERRQ(ierr);
2125     ierr = PetscSectionGetStorageSize(mesh->supportSection, &size);CHKERRQ(ierr);
2126     ierr = PetscMalloc1(size, &mesh->supports);CHKERRQ(ierr);
2127   }
2128   PetscFunctionReturn(0);
2129 }
2130 
2131 PetscErrorCode DMCreateSubDM_Plex(DM dm, PetscInt numFields, const PetscInt fields[], IS *is, DM *subdm)
2132 {
2133   PetscErrorCode ierr;
2134 
2135   PetscFunctionBegin;
2136   if (subdm) {ierr = DMClone(dm, subdm);CHKERRQ(ierr);}
2137   ierr = DMCreateSubDM_Section_Private(dm, numFields, fields, is, subdm);CHKERRQ(ierr);
2138   PetscFunctionReturn(0);
2139 }
2140 
2141 PetscErrorCode DMCreateSuperDM_Plex(DM dms[], PetscInt len, IS **is, DM *superdm)
2142 {
2143   PetscErrorCode ierr;
2144 
2145   PetscFunctionBegin;
2146   if (superdm) {ierr = DMClone(dms[0], superdm);CHKERRQ(ierr);}
2147   ierr = DMCreateSuperDM_Section_Private(dms, len, is, superdm);CHKERRQ(ierr);
2148   PetscFunctionReturn(0);
2149 }
2150 
2151 /*@
2152   DMPlexSymmetrize - Create support (out-edge) information from cone (in-edge) information
2153 
2154   Not collective
2155 
2156   Input Parameter:
2157 . mesh - The DMPlex
2158 
2159   Output Parameter:
2160 
2161   Note:
2162   This should be called after all calls to DMPlexSetCone()
2163 
2164   Level: beginner
2165 
2166 .seealso: DMPlexCreate(), DMPlexSetChart(), DMPlexSetConeSize(), DMPlexSetCone()
2167 @*/
2168 PetscErrorCode DMPlexSymmetrize(DM dm)
2169 {
2170   DM_Plex       *mesh = (DM_Plex*) dm->data;
2171   PetscInt      *offsets;
2172   PetscInt       supportSize;
2173   PetscInt       pStart, pEnd, p;
2174   PetscErrorCode ierr;
2175 
2176   PetscFunctionBegin;
2177   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2178   if (mesh->supports) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONGSTATE, "Supports were already setup in this DMPlex");
2179   /* Calculate support sizes */
2180   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
2181   for (p = pStart; p < pEnd; ++p) {
2182     PetscInt dof, off, c;
2183 
2184     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2185     ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2186     for (c = off; c < off+dof; ++c) {
2187       ierr = PetscSectionAddDof(mesh->supportSection, mesh->cones[c], 1);CHKERRQ(ierr);
2188     }
2189   }
2190   for (p = pStart; p < pEnd; ++p) {
2191     PetscInt dof;
2192 
2193     ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
2194 
2195     mesh->maxSupportSize = PetscMax(mesh->maxSupportSize, dof);
2196   }
2197   ierr = PetscSectionSetUp(mesh->supportSection);CHKERRQ(ierr);
2198   /* Calculate supports */
2199   ierr = PetscSectionGetStorageSize(mesh->supportSection, &supportSize);CHKERRQ(ierr);
2200   ierr = PetscMalloc1(supportSize, &mesh->supports);CHKERRQ(ierr);
2201   ierr = PetscCalloc1(pEnd - pStart, &offsets);CHKERRQ(ierr);
2202   for (p = pStart; p < pEnd; ++p) {
2203     PetscInt dof, off, c;
2204 
2205     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2206     ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2207     for (c = off; c < off+dof; ++c) {
2208       const PetscInt q = mesh->cones[c];
2209       PetscInt       offS;
2210 
2211       ierr = PetscSectionGetOffset(mesh->supportSection, q, &offS);CHKERRQ(ierr);
2212 
2213       mesh->supports[offS+offsets[q]] = p;
2214       ++offsets[q];
2215     }
2216   }
2217   ierr = PetscFree(offsets);CHKERRQ(ierr);
2218   PetscFunctionReturn(0);
2219 }
2220 
2221 /*@
2222   DMPlexStratify - The DAG for most topologies is a graded poset (http://en.wikipedia.org/wiki/Graded_poset), and
2223   can be illustrated by a Hasse Diagram (a http://en.wikipedia.org/wiki/Hasse_diagram). The strata group all points of the
2224   same grade, and this function calculates the strata. This grade can be seen as the height (or depth) of the point in
2225   the DAG.
2226 
2227   Collective on dm
2228 
2229   Input Parameter:
2230 . mesh - The DMPlex
2231 
2232   Output Parameter:
2233 
2234   Notes:
2235   Concretely, DMPlexStratify() creates a new label named "depth" containing the dimension of each element: 0 for vertices,
2236   1 for edges, and so on.  The depth label can be accessed through DMPlexGetDepthLabel() or DMPlexGetDepthStratum(), or
2237   manually via DMGetLabel().  The height is defined implicitly by height = maxDimension - depth, and can be accessed
2238   via DMPlexGetHeightStratum().  For example, cells have height 0 and faces have height 1.
2239 
2240   DMPlexStratify() should be called after all calls to DMPlexSymmetrize()
2241 
2242   Level: beginner
2243 
2244 .seealso: DMPlexCreate(), DMPlexSymmetrize()
2245 @*/
2246 PetscErrorCode DMPlexStratify(DM dm)
2247 {
2248   DM_Plex       *mesh = (DM_Plex*) dm->data;
2249   DMLabel        label;
2250   PetscInt       pStart, pEnd, p;
2251   PetscInt       numRoots = 0, numLeaves = 0;
2252   PetscErrorCode ierr;
2253 
2254   PetscFunctionBegin;
2255   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2256   ierr = PetscLogEventBegin(DMPLEX_Stratify,dm,0,0,0);CHKERRQ(ierr);
2257   /* Calculate depth */
2258   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
2259   ierr = DMCreateLabel(dm, "depth");CHKERRQ(ierr);
2260   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
2261   /* Initialize roots and count leaves */
2262   for (p = pStart; p < pEnd; ++p) {
2263     PetscInt coneSize, supportSize;
2264 
2265     ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
2266     ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
2267     if (!coneSize && supportSize) {
2268       ++numRoots;
2269       ierr = DMLabelSetValue(label, p, 0);CHKERRQ(ierr);
2270     } else if (!supportSize && coneSize) {
2271       ++numLeaves;
2272     } else if (!supportSize && !coneSize) {
2273       /* Isolated points */
2274       ierr = DMLabelSetValue(label, p, 0);CHKERRQ(ierr);
2275     }
2276   }
2277   if (numRoots + numLeaves == (pEnd - pStart)) {
2278     for (p = pStart; p < pEnd; ++p) {
2279       PetscInt coneSize, supportSize;
2280 
2281       ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
2282       ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
2283       if (!supportSize && coneSize) {
2284         ierr = DMLabelSetValue(label, p, 1);CHKERRQ(ierr);
2285       }
2286     }
2287   } else {
2288     IS       pointIS;
2289     PetscInt numPoints = 0, level = 0;
2290 
2291     ierr = DMLabelGetStratumIS(label, level, &pointIS);CHKERRQ(ierr);
2292     if (pointIS) {ierr = ISGetLocalSize(pointIS, &numPoints);CHKERRQ(ierr);}
2293     while (numPoints) {
2294       const PetscInt *points;
2295       const PetscInt  newLevel = level+1;
2296 
2297       ierr = ISGetIndices(pointIS, &points);CHKERRQ(ierr);
2298       for (p = 0; p < numPoints; ++p) {
2299         const PetscInt  point = points[p];
2300         const PetscInt *support;
2301         PetscInt        supportSize, s;
2302 
2303         ierr = DMPlexGetSupportSize(dm, point, &supportSize);CHKERRQ(ierr);
2304         ierr = DMPlexGetSupport(dm, point, &support);CHKERRQ(ierr);
2305         for (s = 0; s < supportSize; ++s) {
2306           ierr = DMLabelSetValue(label, support[s], newLevel);CHKERRQ(ierr);
2307         }
2308       }
2309       ierr = ISRestoreIndices(pointIS, &points);CHKERRQ(ierr);
2310       ++level;
2311       ierr = ISDestroy(&pointIS);CHKERRQ(ierr);
2312       ierr = DMLabelGetStratumIS(label, level, &pointIS);CHKERRQ(ierr);
2313       if (pointIS) {ierr = ISGetLocalSize(pointIS, &numPoints);CHKERRQ(ierr);}
2314       else         {numPoints = 0;}
2315     }
2316     ierr = ISDestroy(&pointIS);CHKERRQ(ierr);
2317   }
2318   { /* just in case there is an empty process */
2319     PetscInt numValues, maxValues = 0, v;
2320 
2321     ierr = DMLabelGetNumValues(label,&numValues);CHKERRQ(ierr);
2322     for (v = 0; v < numValues; v++) {
2323       IS pointIS;
2324 
2325       ierr = DMLabelGetStratumIS(label, v, &pointIS);CHKERRQ(ierr);
2326       if (pointIS) {
2327         PetscInt  min, max, numPoints;
2328         PetscInt  start;
2329         PetscBool contig;
2330 
2331         ierr = ISGetLocalSize(pointIS, &numPoints);CHKERRQ(ierr);
2332         ierr = ISGetMinMax(pointIS, &min, &max);CHKERRQ(ierr);
2333         ierr = ISContiguousLocal(pointIS,min,max+1,&start,&contig);CHKERRQ(ierr);
2334         if (start == 0 && contig) {
2335           ierr = ISDestroy(&pointIS);CHKERRQ(ierr);
2336           ierr = ISCreateStride(PETSC_COMM_SELF,numPoints,min,1,&pointIS);CHKERRQ(ierr);
2337           ierr = DMLabelSetStratumIS(label, v, pointIS);CHKERRQ(ierr);
2338         }
2339       }
2340       ierr = ISDestroy(&pointIS);CHKERRQ(ierr);
2341     }
2342     ierr = MPI_Allreduce(&numValues,&maxValues,1,MPIU_INT,MPI_MAX,PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
2343     for (v = numValues; v < maxValues; v++) {
2344       DMLabelAddStratum(label,v);CHKERRQ(ierr);
2345     }
2346   }
2347 
2348   ierr = DMLabelGetState(label, &mesh->depthState);CHKERRQ(ierr);
2349   ierr = PetscLogEventEnd(DMPLEX_Stratify,dm,0,0,0);CHKERRQ(ierr);
2350   PetscFunctionReturn(0);
2351 }
2352 
2353 /*@C
2354   DMPlexGetJoin - Get an array for the join of the set of points
2355 
2356   Not Collective
2357 
2358   Input Parameters:
2359 + dm - The DMPlex object
2360 . numPoints - The number of input points for the join
2361 - points - The input points
2362 
2363   Output Parameters:
2364 + numCoveredPoints - The number of points in the join
2365 - coveredPoints - The points in the join
2366 
2367   Level: intermediate
2368 
2369   Note: Currently, this is restricted to a single level join
2370 
2371   Fortran Notes:
2372   Since it returns an array, this routine is only available in Fortran 90, and you must
2373   include petsc.h90 in your code.
2374 
2375   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2376 
2377 .keywords: mesh
2378 .seealso: DMPlexRestoreJoin(), DMPlexGetMeet()
2379 @*/
2380 PetscErrorCode DMPlexGetJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
2381 {
2382   DM_Plex       *mesh = (DM_Plex*) dm->data;
2383   PetscInt      *join[2];
2384   PetscInt       joinSize, i = 0;
2385   PetscInt       dof, off, p, c, m;
2386   PetscErrorCode ierr;
2387 
2388   PetscFunctionBegin;
2389   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2390   PetscValidPointer(points, 2);
2391   PetscValidPointer(numCoveredPoints, 3);
2392   PetscValidPointer(coveredPoints, 4);
2393   ierr = DMGetWorkArray(dm, mesh->maxSupportSize, MPIU_INT, &join[0]);CHKERRQ(ierr);
2394   ierr = DMGetWorkArray(dm, mesh->maxSupportSize, MPIU_INT, &join[1]);CHKERRQ(ierr);
2395   /* Copy in support of first point */
2396   ierr = PetscSectionGetDof(mesh->supportSection, points[0], &dof);CHKERRQ(ierr);
2397   ierr = PetscSectionGetOffset(mesh->supportSection, points[0], &off);CHKERRQ(ierr);
2398   for (joinSize = 0; joinSize < dof; ++joinSize) {
2399     join[i][joinSize] = mesh->supports[off+joinSize];
2400   }
2401   /* Check each successive support */
2402   for (p = 1; p < numPoints; ++p) {
2403     PetscInt newJoinSize = 0;
2404 
2405     ierr = PetscSectionGetDof(mesh->supportSection, points[p], &dof);CHKERRQ(ierr);
2406     ierr = PetscSectionGetOffset(mesh->supportSection, points[p], &off);CHKERRQ(ierr);
2407     for (c = 0; c < dof; ++c) {
2408       const PetscInt point = mesh->supports[off+c];
2409 
2410       for (m = 0; m < joinSize; ++m) {
2411         if (point == join[i][m]) {
2412           join[1-i][newJoinSize++] = point;
2413           break;
2414         }
2415       }
2416     }
2417     joinSize = newJoinSize;
2418     i        = 1-i;
2419   }
2420   *numCoveredPoints = joinSize;
2421   *coveredPoints    = join[i];
2422   ierr              = DMRestoreWorkArray(dm, mesh->maxSupportSize, MPIU_INT, &join[1-i]);CHKERRQ(ierr);
2423   PetscFunctionReturn(0);
2424 }
2425 
2426 /*@C
2427   DMPlexRestoreJoin - Restore an array for the join of the set of points
2428 
2429   Not Collective
2430 
2431   Input Parameters:
2432 + dm - The DMPlex object
2433 . numPoints - The number of input points for the join
2434 - points - The input points
2435 
2436   Output Parameters:
2437 + numCoveredPoints - The number of points in the join
2438 - coveredPoints - The points in the join
2439 
2440   Fortran Notes:
2441   Since it returns an array, this routine is only available in Fortran 90, and you must
2442   include petsc.h90 in your code.
2443 
2444   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2445 
2446   Level: intermediate
2447 
2448 .keywords: mesh
2449 .seealso: DMPlexGetJoin(), DMPlexGetFullJoin(), DMPlexGetMeet()
2450 @*/
2451 PetscErrorCode DMPlexRestoreJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
2452 {
2453   PetscErrorCode ierr;
2454 
2455   PetscFunctionBegin;
2456   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2457   if (points) PetscValidIntPointer(points,3);
2458   if (numCoveredPoints) PetscValidIntPointer(numCoveredPoints,4);
2459   PetscValidPointer(coveredPoints, 5);
2460   ierr = DMRestoreWorkArray(dm, 0, MPIU_INT, (void*) coveredPoints);CHKERRQ(ierr);
2461   if (numCoveredPoints) *numCoveredPoints = 0;
2462   PetscFunctionReturn(0);
2463 }
2464 
2465 /*@C
2466   DMPlexGetFullJoin - Get an array for the join of the set of points
2467 
2468   Not Collective
2469 
2470   Input Parameters:
2471 + dm - The DMPlex object
2472 . numPoints - The number of input points for the join
2473 - points - The input points
2474 
2475   Output Parameters:
2476 + numCoveredPoints - The number of points in the join
2477 - coveredPoints - The points in the join
2478 
2479   Fortran Notes:
2480   Since it returns an array, this routine is only available in Fortran 90, and you must
2481   include petsc.h90 in your code.
2482 
2483   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2484 
2485   Level: intermediate
2486 
2487 .keywords: mesh
2488 .seealso: DMPlexGetJoin(), DMPlexRestoreJoin(), DMPlexGetMeet()
2489 @*/
2490 PetscErrorCode DMPlexGetFullJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
2491 {
2492   DM_Plex       *mesh = (DM_Plex*) dm->data;
2493   PetscInt      *offsets, **closures;
2494   PetscInt      *join[2];
2495   PetscInt       depth = 0, maxSize, joinSize = 0, i = 0;
2496   PetscInt       p, d, c, m, ms;
2497   PetscErrorCode ierr;
2498 
2499   PetscFunctionBegin;
2500   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2501   PetscValidPointer(points, 2);
2502   PetscValidPointer(numCoveredPoints, 3);
2503   PetscValidPointer(coveredPoints, 4);
2504 
2505   ierr    = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
2506   ierr    = PetscCalloc1(numPoints, &closures);CHKERRQ(ierr);
2507   ierr    = DMGetWorkArray(dm, numPoints*(depth+2), MPIU_INT, &offsets);CHKERRQ(ierr);
2508   ms      = mesh->maxSupportSize;
2509   maxSize = (ms > 1) ? ((PetscPowInt(ms,depth+1)-1)/(ms-1)) : depth + 1;
2510   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &join[0]);CHKERRQ(ierr);
2511   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &join[1]);CHKERRQ(ierr);
2512 
2513   for (p = 0; p < numPoints; ++p) {
2514     PetscInt closureSize;
2515 
2516     ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_FALSE, &closureSize, &closures[p]);CHKERRQ(ierr);
2517 
2518     offsets[p*(depth+2)+0] = 0;
2519     for (d = 0; d < depth+1; ++d) {
2520       PetscInt pStart, pEnd, i;
2521 
2522       ierr = DMPlexGetDepthStratum(dm, d, &pStart, &pEnd);CHKERRQ(ierr);
2523       for (i = offsets[p*(depth+2)+d]; i < closureSize; ++i) {
2524         if ((pStart > closures[p][i*2]) || (pEnd <= closures[p][i*2])) {
2525           offsets[p*(depth+2)+d+1] = i;
2526           break;
2527         }
2528       }
2529       if (i == closureSize) offsets[p*(depth+2)+d+1] = i;
2530     }
2531     if (offsets[p*(depth+2)+depth+1] != closureSize) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Total size of closure %D should be %D", offsets[p*(depth+2)+depth+1], closureSize);
2532   }
2533   for (d = 0; d < depth+1; ++d) {
2534     PetscInt dof;
2535 
2536     /* Copy in support of first point */
2537     dof = offsets[d+1] - offsets[d];
2538     for (joinSize = 0; joinSize < dof; ++joinSize) {
2539       join[i][joinSize] = closures[0][(offsets[d]+joinSize)*2];
2540     }
2541     /* Check each successive cone */
2542     for (p = 1; p < numPoints && joinSize; ++p) {
2543       PetscInt newJoinSize = 0;
2544 
2545       dof = offsets[p*(depth+2)+d+1] - offsets[p*(depth+2)+d];
2546       for (c = 0; c < dof; ++c) {
2547         const PetscInt point = closures[p][(offsets[p*(depth+2)+d]+c)*2];
2548 
2549         for (m = 0; m < joinSize; ++m) {
2550           if (point == join[i][m]) {
2551             join[1-i][newJoinSize++] = point;
2552             break;
2553           }
2554         }
2555       }
2556       joinSize = newJoinSize;
2557       i        = 1-i;
2558     }
2559     if (joinSize) break;
2560   }
2561   *numCoveredPoints = joinSize;
2562   *coveredPoints    = join[i];
2563   for (p = 0; p < numPoints; ++p) {
2564     ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_FALSE, NULL, &closures[p]);CHKERRQ(ierr);
2565   }
2566   ierr = PetscFree(closures);CHKERRQ(ierr);
2567   ierr = DMRestoreWorkArray(dm, numPoints*(depth+2), MPIU_INT, &offsets);CHKERRQ(ierr);
2568   ierr = DMRestoreWorkArray(dm, mesh->maxSupportSize, MPIU_INT, &join[1-i]);CHKERRQ(ierr);
2569   PetscFunctionReturn(0);
2570 }
2571 
2572 /*@C
2573   DMPlexGetMeet - Get an array for the meet of the set of points
2574 
2575   Not Collective
2576 
2577   Input Parameters:
2578 + dm - The DMPlex object
2579 . numPoints - The number of input points for the meet
2580 - points - The input points
2581 
2582   Output Parameters:
2583 + numCoveredPoints - The number of points in the meet
2584 - coveredPoints - The points in the meet
2585 
2586   Level: intermediate
2587 
2588   Note: Currently, this is restricted to a single level meet
2589 
2590   Fortran Notes:
2591   Since it returns an array, this routine is only available in Fortran 90, and you must
2592   include petsc.h90 in your code.
2593 
2594   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2595 
2596 .keywords: mesh
2597 .seealso: DMPlexRestoreMeet(), DMPlexGetJoin()
2598 @*/
2599 PetscErrorCode DMPlexGetMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveringPoints, const PetscInt **coveringPoints)
2600 {
2601   DM_Plex       *mesh = (DM_Plex*) dm->data;
2602   PetscInt      *meet[2];
2603   PetscInt       meetSize, i = 0;
2604   PetscInt       dof, off, p, c, m;
2605   PetscErrorCode ierr;
2606 
2607   PetscFunctionBegin;
2608   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2609   PetscValidPointer(points, 2);
2610   PetscValidPointer(numCoveringPoints, 3);
2611   PetscValidPointer(coveringPoints, 4);
2612   ierr = DMGetWorkArray(dm, mesh->maxConeSize, MPIU_INT, &meet[0]);CHKERRQ(ierr);
2613   ierr = DMGetWorkArray(dm, mesh->maxConeSize, MPIU_INT, &meet[1]);CHKERRQ(ierr);
2614   /* Copy in cone of first point */
2615   ierr = PetscSectionGetDof(mesh->coneSection, points[0], &dof);CHKERRQ(ierr);
2616   ierr = PetscSectionGetOffset(mesh->coneSection, points[0], &off);CHKERRQ(ierr);
2617   for (meetSize = 0; meetSize < dof; ++meetSize) {
2618     meet[i][meetSize] = mesh->cones[off+meetSize];
2619   }
2620   /* Check each successive cone */
2621   for (p = 1; p < numPoints; ++p) {
2622     PetscInt newMeetSize = 0;
2623 
2624     ierr = PetscSectionGetDof(mesh->coneSection, points[p], &dof);CHKERRQ(ierr);
2625     ierr = PetscSectionGetOffset(mesh->coneSection, points[p], &off);CHKERRQ(ierr);
2626     for (c = 0; c < dof; ++c) {
2627       const PetscInt point = mesh->cones[off+c];
2628 
2629       for (m = 0; m < meetSize; ++m) {
2630         if (point == meet[i][m]) {
2631           meet[1-i][newMeetSize++] = point;
2632           break;
2633         }
2634       }
2635     }
2636     meetSize = newMeetSize;
2637     i        = 1-i;
2638   }
2639   *numCoveringPoints = meetSize;
2640   *coveringPoints    = meet[i];
2641   ierr               = DMRestoreWorkArray(dm, mesh->maxConeSize, MPIU_INT, &meet[1-i]);CHKERRQ(ierr);
2642   PetscFunctionReturn(0);
2643 }
2644 
2645 /*@C
2646   DMPlexRestoreMeet - Restore an array for the meet of the set of points
2647 
2648   Not Collective
2649 
2650   Input Parameters:
2651 + dm - The DMPlex object
2652 . numPoints - The number of input points for the meet
2653 - points - The input points
2654 
2655   Output Parameters:
2656 + numCoveredPoints - The number of points in the meet
2657 - coveredPoints - The points in the meet
2658 
2659   Level: intermediate
2660 
2661   Fortran Notes:
2662   Since it returns an array, this routine is only available in Fortran 90, and you must
2663   include petsc.h90 in your code.
2664 
2665   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2666 
2667 .keywords: mesh
2668 .seealso: DMPlexGetMeet(), DMPlexGetFullMeet(), DMPlexGetJoin()
2669 @*/
2670 PetscErrorCode DMPlexRestoreMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
2671 {
2672   PetscErrorCode ierr;
2673 
2674   PetscFunctionBegin;
2675   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2676   if (points) PetscValidIntPointer(points,3);
2677   if (numCoveredPoints) PetscValidIntPointer(numCoveredPoints,4);
2678   PetscValidPointer(coveredPoints,5);
2679   ierr = DMRestoreWorkArray(dm, 0, MPIU_INT, (void*) coveredPoints);CHKERRQ(ierr);
2680   if (numCoveredPoints) *numCoveredPoints = 0;
2681   PetscFunctionReturn(0);
2682 }
2683 
2684 /*@C
2685   DMPlexGetFullMeet - Get an array for the meet of the set of points
2686 
2687   Not Collective
2688 
2689   Input Parameters:
2690 + dm - The DMPlex object
2691 . numPoints - The number of input points for the meet
2692 - points - The input points
2693 
2694   Output Parameters:
2695 + numCoveredPoints - The number of points in the meet
2696 - coveredPoints - The points in the meet
2697 
2698   Level: intermediate
2699 
2700   Fortran Notes:
2701   Since it returns an array, this routine is only available in Fortran 90, and you must
2702   include petsc.h90 in your code.
2703 
2704   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2705 
2706 .keywords: mesh
2707 .seealso: DMPlexGetMeet(), DMPlexRestoreMeet(), DMPlexGetJoin()
2708 @*/
2709 PetscErrorCode DMPlexGetFullMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
2710 {
2711   DM_Plex       *mesh = (DM_Plex*) dm->data;
2712   PetscInt      *offsets, **closures;
2713   PetscInt      *meet[2];
2714   PetscInt       height = 0, maxSize, meetSize = 0, i = 0;
2715   PetscInt       p, h, c, m, mc;
2716   PetscErrorCode ierr;
2717 
2718   PetscFunctionBegin;
2719   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2720   PetscValidPointer(points, 2);
2721   PetscValidPointer(numCoveredPoints, 3);
2722   PetscValidPointer(coveredPoints, 4);
2723 
2724   ierr    = DMPlexGetDepth(dm, &height);CHKERRQ(ierr);
2725   ierr    = PetscMalloc1(numPoints, &closures);CHKERRQ(ierr);
2726   ierr    = DMGetWorkArray(dm, numPoints*(height+2), MPIU_INT, &offsets);CHKERRQ(ierr);
2727   mc      = mesh->maxConeSize;
2728   maxSize = (mc > 1) ? ((PetscPowInt(mc,height+1)-1)/(mc-1)) : height + 1;
2729   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &meet[0]);CHKERRQ(ierr);
2730   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &meet[1]);CHKERRQ(ierr);
2731 
2732   for (p = 0; p < numPoints; ++p) {
2733     PetscInt closureSize;
2734 
2735     ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_TRUE, &closureSize, &closures[p]);CHKERRQ(ierr);
2736 
2737     offsets[p*(height+2)+0] = 0;
2738     for (h = 0; h < height+1; ++h) {
2739       PetscInt pStart, pEnd, i;
2740 
2741       ierr = DMPlexGetHeightStratum(dm, h, &pStart, &pEnd);CHKERRQ(ierr);
2742       for (i = offsets[p*(height+2)+h]; i < closureSize; ++i) {
2743         if ((pStart > closures[p][i*2]) || (pEnd <= closures[p][i*2])) {
2744           offsets[p*(height+2)+h+1] = i;
2745           break;
2746         }
2747       }
2748       if (i == closureSize) offsets[p*(height+2)+h+1] = i;
2749     }
2750     if (offsets[p*(height+2)+height+1] != closureSize) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Total size of closure %D should be %D", offsets[p*(height+2)+height+1], closureSize);
2751   }
2752   for (h = 0; h < height+1; ++h) {
2753     PetscInt dof;
2754 
2755     /* Copy in cone of first point */
2756     dof = offsets[h+1] - offsets[h];
2757     for (meetSize = 0; meetSize < dof; ++meetSize) {
2758       meet[i][meetSize] = closures[0][(offsets[h]+meetSize)*2];
2759     }
2760     /* Check each successive cone */
2761     for (p = 1; p < numPoints && meetSize; ++p) {
2762       PetscInt newMeetSize = 0;
2763 
2764       dof = offsets[p*(height+2)+h+1] - offsets[p*(height+2)+h];
2765       for (c = 0; c < dof; ++c) {
2766         const PetscInt point = closures[p][(offsets[p*(height+2)+h]+c)*2];
2767 
2768         for (m = 0; m < meetSize; ++m) {
2769           if (point == meet[i][m]) {
2770             meet[1-i][newMeetSize++] = point;
2771             break;
2772           }
2773         }
2774       }
2775       meetSize = newMeetSize;
2776       i        = 1-i;
2777     }
2778     if (meetSize) break;
2779   }
2780   *numCoveredPoints = meetSize;
2781   *coveredPoints    = meet[i];
2782   for (p = 0; p < numPoints; ++p) {
2783     ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_TRUE, NULL, &closures[p]);CHKERRQ(ierr);
2784   }
2785   ierr = PetscFree(closures);CHKERRQ(ierr);
2786   ierr = DMRestoreWorkArray(dm, numPoints*(height+2), MPIU_INT, &offsets);CHKERRQ(ierr);
2787   ierr = DMRestoreWorkArray(dm, mesh->maxConeSize, MPIU_INT, &meet[1-i]);CHKERRQ(ierr);
2788   PetscFunctionReturn(0);
2789 }
2790 
2791 /*@C
2792   DMPlexEqual - Determine if two DMs have the same topology
2793 
2794   Not Collective
2795 
2796   Input Parameters:
2797 + dmA - A DMPlex object
2798 - dmB - A DMPlex object
2799 
2800   Output Parameters:
2801 . equal - PETSC_TRUE if the topologies are identical
2802 
2803   Level: intermediate
2804 
2805   Notes:
2806   We are not solving graph isomorphism, so we do not permutation.
2807 
2808 .keywords: mesh
2809 .seealso: DMPlexGetCone()
2810 @*/
2811 PetscErrorCode DMPlexEqual(DM dmA, DM dmB, PetscBool *equal)
2812 {
2813   PetscInt       depth, depthB, pStart, pEnd, pStartB, pEndB, p;
2814   PetscErrorCode ierr;
2815 
2816   PetscFunctionBegin;
2817   PetscValidHeaderSpecific(dmA, DM_CLASSID, 1);
2818   PetscValidHeaderSpecific(dmB, DM_CLASSID, 2);
2819   PetscValidPointer(equal, 3);
2820 
2821   *equal = PETSC_FALSE;
2822   ierr = DMPlexGetDepth(dmA, &depth);CHKERRQ(ierr);
2823   ierr = DMPlexGetDepth(dmB, &depthB);CHKERRQ(ierr);
2824   if (depth != depthB) PetscFunctionReturn(0);
2825   ierr = DMPlexGetChart(dmA, &pStart,  &pEnd);CHKERRQ(ierr);
2826   ierr = DMPlexGetChart(dmB, &pStartB, &pEndB);CHKERRQ(ierr);
2827   if ((pStart != pStartB) || (pEnd != pEndB)) PetscFunctionReturn(0);
2828   for (p = pStart; p < pEnd; ++p) {
2829     const PetscInt *cone, *coneB, *ornt, *orntB, *support, *supportB;
2830     PetscInt        coneSize, coneSizeB, c, supportSize, supportSizeB, s;
2831 
2832     ierr = DMPlexGetConeSize(dmA, p, &coneSize);CHKERRQ(ierr);
2833     ierr = DMPlexGetCone(dmA, p, &cone);CHKERRQ(ierr);
2834     ierr = DMPlexGetConeOrientation(dmA, p, &ornt);CHKERRQ(ierr);
2835     ierr = DMPlexGetConeSize(dmB, p, &coneSizeB);CHKERRQ(ierr);
2836     ierr = DMPlexGetCone(dmB, p, &coneB);CHKERRQ(ierr);
2837     ierr = DMPlexGetConeOrientation(dmB, p, &orntB);CHKERRQ(ierr);
2838     if (coneSize != coneSizeB) PetscFunctionReturn(0);
2839     for (c = 0; c < coneSize; ++c) {
2840       if (cone[c] != coneB[c]) PetscFunctionReturn(0);
2841       if (ornt[c] != orntB[c]) PetscFunctionReturn(0);
2842     }
2843     ierr = DMPlexGetSupportSize(dmA, p, &supportSize);CHKERRQ(ierr);
2844     ierr = DMPlexGetSupport(dmA, p, &support);CHKERRQ(ierr);
2845     ierr = DMPlexGetSupportSize(dmB, p, &supportSizeB);CHKERRQ(ierr);
2846     ierr = DMPlexGetSupport(dmB, p, &supportB);CHKERRQ(ierr);
2847     if (supportSize != supportSizeB) PetscFunctionReturn(0);
2848     for (s = 0; s < supportSize; ++s) {
2849       if (support[s] != supportB[s]) PetscFunctionReturn(0);
2850     }
2851   }
2852   *equal = PETSC_TRUE;
2853   PetscFunctionReturn(0);
2854 }
2855 
2856 /*@C
2857   DMPlexGetNumFaceVertices - Returns the number of vertices on a face
2858 
2859   Not Collective
2860 
2861   Input Parameters:
2862 + dm         - The DMPlex
2863 . cellDim    - The cell dimension
2864 - numCorners - The number of vertices on a cell
2865 
2866   Output Parameters:
2867 . numFaceVertices - The number of vertices on a face
2868 
2869   Level: developer
2870 
2871   Notes:
2872   Of course this can only work for a restricted set of symmetric shapes
2873 
2874 .seealso: DMPlexGetCone()
2875 @*/
2876 PetscErrorCode DMPlexGetNumFaceVertices(DM dm, PetscInt cellDim, PetscInt numCorners, PetscInt *numFaceVertices)
2877 {
2878   MPI_Comm       comm;
2879   PetscErrorCode ierr;
2880 
2881   PetscFunctionBegin;
2882   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
2883   PetscValidPointer(numFaceVertices,3);
2884   switch (cellDim) {
2885   case 0:
2886     *numFaceVertices = 0;
2887     break;
2888   case 1:
2889     *numFaceVertices = 1;
2890     break;
2891   case 2:
2892     switch (numCorners) {
2893     case 3: /* triangle */
2894       *numFaceVertices = 2; /* Edge has 2 vertices */
2895       break;
2896     case 4: /* quadrilateral */
2897       *numFaceVertices = 2; /* Edge has 2 vertices */
2898       break;
2899     case 6: /* quadratic triangle, tri and quad cohesive Lagrange cells */
2900       *numFaceVertices = 3; /* Edge has 3 vertices */
2901       break;
2902     case 9: /* quadratic quadrilateral, quadratic quad cohesive Lagrange cells */
2903       *numFaceVertices = 3; /* Edge has 3 vertices */
2904       break;
2905     default:
2906       SETERRQ2(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid number of face corners %D for dimension %D", numCorners, cellDim);
2907     }
2908     break;
2909   case 3:
2910     switch (numCorners) {
2911     case 4: /* tetradehdron */
2912       *numFaceVertices = 3; /* Face has 3 vertices */
2913       break;
2914     case 6: /* tet cohesive cells */
2915       *numFaceVertices = 4; /* Face has 4 vertices */
2916       break;
2917     case 8: /* hexahedron */
2918       *numFaceVertices = 4; /* Face has 4 vertices */
2919       break;
2920     case 9: /* tet cohesive Lagrange cells */
2921       *numFaceVertices = 6; /* Face has 6 vertices */
2922       break;
2923     case 10: /* quadratic tetrahedron */
2924       *numFaceVertices = 6; /* Face has 6 vertices */
2925       break;
2926     case 12: /* hex cohesive Lagrange cells */
2927       *numFaceVertices = 6; /* Face has 6 vertices */
2928       break;
2929     case 18: /* quadratic tet cohesive Lagrange cells */
2930       *numFaceVertices = 6; /* Face has 6 vertices */
2931       break;
2932     case 27: /* quadratic hexahedron, quadratic hex cohesive Lagrange cells */
2933       *numFaceVertices = 9; /* Face has 9 vertices */
2934       break;
2935     default:
2936       SETERRQ2(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid number of face corners %D for dimension %D", numCorners, cellDim);
2937     }
2938     break;
2939   default:
2940     SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid cell dimension %D", cellDim);
2941   }
2942   PetscFunctionReturn(0);
2943 }
2944 
2945 /*@
2946   DMPlexGetDepthLabel - Get the DMLabel recording the depth of each point
2947 
2948   Not Collective
2949 
2950   Input Parameter:
2951 . dm    - The DMPlex object
2952 
2953   Output Parameter:
2954 . depthLabel - The DMLabel recording point depth
2955 
2956   Level: developer
2957 
2958 .keywords: mesh, points
2959 .seealso: DMPlexGetDepth(), DMPlexGetHeightStratum(), DMPlexGetDepthStratum()
2960 @*/
2961 PetscErrorCode DMPlexGetDepthLabel(DM dm, DMLabel *depthLabel)
2962 {
2963   PetscErrorCode ierr;
2964 
2965   PetscFunctionBegin;
2966   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2967   PetscValidPointer(depthLabel, 2);
2968   if (!dm->depthLabel) {ierr = DMGetLabel(dm, "depth", &dm->depthLabel);CHKERRQ(ierr);}
2969   *depthLabel = dm->depthLabel;
2970   PetscFunctionReturn(0);
2971 }
2972 
2973 /*@
2974   DMPlexGetDepth - Get the depth of the DAG representing this mesh
2975 
2976   Not Collective
2977 
2978   Input Parameter:
2979 . dm    - The DMPlex object
2980 
2981   Output Parameter:
2982 . depth - The number of strata (breadth first levels) in the DAG
2983 
2984   Level: developer
2985 
2986 .keywords: mesh, points
2987 .seealso: DMPlexGetDepthLabel(), DMPlexGetHeightStratum(), DMPlexGetDepthStratum()
2988 @*/
2989 PetscErrorCode DMPlexGetDepth(DM dm, PetscInt *depth)
2990 {
2991   DMLabel        label;
2992   PetscInt       d = 0;
2993   PetscErrorCode ierr;
2994 
2995   PetscFunctionBegin;
2996   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2997   PetscValidPointer(depth, 2);
2998   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
2999   if (label) {ierr = DMLabelGetNumValues(label, &d);CHKERRQ(ierr);}
3000   *depth = d-1;
3001   PetscFunctionReturn(0);
3002 }
3003 
3004 /*@
3005   DMPlexGetDepthStratum - Get the bounds [start, end) for all points at a certain depth.
3006 
3007   Not Collective
3008 
3009   Input Parameters:
3010 + dm           - The DMPlex object
3011 - stratumValue - The requested depth
3012 
3013   Output Parameters:
3014 + start - The first point at this depth
3015 - end   - One beyond the last point at this depth
3016 
3017   Level: developer
3018 
3019 .keywords: mesh, points
3020 .seealso: DMPlexGetHeightStratum(), DMPlexGetDepth()
3021 @*/
3022 PetscErrorCode DMPlexGetDepthStratum(DM dm, PetscInt stratumValue, PetscInt *start, PetscInt *end)
3023 {
3024   DMLabel        label;
3025   PetscInt       pStart, pEnd;
3026   PetscErrorCode ierr;
3027 
3028   PetscFunctionBegin;
3029   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3030   if (start) {PetscValidPointer(start, 3); *start = 0;}
3031   if (end)   {PetscValidPointer(end,   4); *end   = 0;}
3032   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
3033   if (pStart == pEnd) PetscFunctionReturn(0);
3034   if (stratumValue < 0) {
3035     if (start) *start = pStart;
3036     if (end)   *end   = pEnd;
3037     PetscFunctionReturn(0);
3038   }
3039   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
3040   if (!label) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "No label named depth was found");
3041   ierr = DMLabelGetStratumBounds(label, stratumValue, start, end);CHKERRQ(ierr);
3042   PetscFunctionReturn(0);
3043 }
3044 
3045 /*@
3046   DMPlexGetHeightStratum - Get the bounds [start, end) for all points at a certain height.
3047 
3048   Not Collective
3049 
3050   Input Parameters:
3051 + dm           - The DMPlex object
3052 - stratumValue - The requested height
3053 
3054   Output Parameters:
3055 + start - The first point at this height
3056 - end   - One beyond the last point at this height
3057 
3058   Level: developer
3059 
3060 .keywords: mesh, points
3061 .seealso: DMPlexGetDepthStratum(), DMPlexGetDepth()
3062 @*/
3063 PetscErrorCode DMPlexGetHeightStratum(DM dm, PetscInt stratumValue, PetscInt *start, PetscInt *end)
3064 {
3065   DMLabel        label;
3066   PetscInt       depth, pStart, pEnd;
3067   PetscErrorCode ierr;
3068 
3069   PetscFunctionBegin;
3070   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3071   if (start) {PetscValidPointer(start, 3); *start = 0;}
3072   if (end)   {PetscValidPointer(end,   4); *end   = 0;}
3073   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
3074   if (pStart == pEnd) PetscFunctionReturn(0);
3075   if (stratumValue < 0) {
3076     if (start) *start = pStart;
3077     if (end)   *end   = pEnd;
3078     PetscFunctionReturn(0);
3079   }
3080   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
3081   if (!label) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "No label named depth was found");
3082   ierr = DMLabelGetNumValues(label, &depth);CHKERRQ(ierr);
3083   ierr = DMLabelGetStratumBounds(label, depth-1-stratumValue, start, end);CHKERRQ(ierr);
3084   PetscFunctionReturn(0);
3085 }
3086 
3087 /* Set the number of dof on each point and separate by fields */
3088 static PetscErrorCode DMPlexCreateSectionInitial(DM dm, PetscInt dim, PetscInt numFields,const PetscInt numComp[],const PetscInt numDof[], PetscSection *section)
3089 {
3090   PetscInt      *pMax;
3091   PetscInt       depth, cellHeight, pStart = 0, pEnd = 0;
3092   PetscInt       Nf, p, d, dep, f;
3093   PetscBool     *isFE;
3094   PetscErrorCode ierr;
3095 
3096   PetscFunctionBegin;
3097   ierr = PetscMalloc1(numFields, &isFE);CHKERRQ(ierr);
3098   ierr = DMGetNumFields(dm, &Nf);CHKERRQ(ierr);
3099   for (f = 0; f < numFields; ++f) {
3100     PetscObject  obj;
3101     PetscClassId id;
3102 
3103     isFE[f] = PETSC_FALSE;
3104     if (f >= Nf) continue;
3105     ierr = DMGetField(dm, f, &obj);CHKERRQ(ierr);
3106     ierr = PetscObjectGetClassId(obj, &id);CHKERRQ(ierr);
3107     if (id == PETSCFE_CLASSID)      {isFE[f] = PETSC_TRUE;}
3108     else if (id == PETSCFV_CLASSID) {isFE[f] = PETSC_FALSE;}
3109   }
3110   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), section);CHKERRQ(ierr);
3111   if (numFields > 0) {
3112     ierr = PetscSectionSetNumFields(*section, numFields);CHKERRQ(ierr);
3113     if (numComp) {
3114       for (f = 0; f < numFields; ++f) {
3115         ierr = PetscSectionSetFieldComponents(*section, f, numComp[f]);CHKERRQ(ierr);
3116         if (isFE[f]) {
3117           PetscFE           fe;
3118           PetscDualSpace    dspace;
3119           const PetscInt    ***perms;
3120           const PetscScalar ***flips;
3121           const PetscInt    *numDof;
3122 
3123           ierr = DMGetField(dm,f,(PetscObject *) &fe);CHKERRQ(ierr);
3124           ierr = PetscFEGetDualSpace(fe,&dspace);CHKERRQ(ierr);
3125           ierr = PetscDualSpaceGetSymmetries(dspace,&perms,&flips);CHKERRQ(ierr);
3126           ierr = PetscDualSpaceGetNumDof(dspace,&numDof);CHKERRQ(ierr);
3127           if (perms || flips) {
3128             DM               K;
3129             DMLabel          depthLabel;
3130             PetscInt         depth, h;
3131             PetscSectionSym  sym;
3132 
3133             ierr = PetscDualSpaceGetDM(dspace,&K);CHKERRQ(ierr);
3134             ierr = DMPlexGetDepthLabel(dm,&depthLabel);CHKERRQ(ierr);
3135             ierr = DMPlexGetDepth(dm,&depth);CHKERRQ(ierr);
3136             ierr = PetscSectionSymCreateLabel(PetscObjectComm((PetscObject)*section),depthLabel,&sym);CHKERRQ(ierr);
3137             for (h = 0; h <= depth; h++) {
3138               PetscDualSpace    hspace;
3139               PetscInt          kStart, kEnd;
3140               PetscInt          kConeSize;
3141               const PetscInt    **perms0 = NULL;
3142               const PetscScalar **flips0 = NULL;
3143 
3144               ierr = PetscDualSpaceGetHeightSubspace(dspace,h,&hspace);CHKERRQ(ierr);
3145               ierr = DMPlexGetHeightStratum(K,h,&kStart,&kEnd);CHKERRQ(ierr);
3146               if (!hspace) continue;
3147               ierr = PetscDualSpaceGetSymmetries(hspace,&perms,&flips);CHKERRQ(ierr);
3148               if (perms) perms0 = perms[0];
3149               if (flips) flips0 = flips[0];
3150               if (!(perms0 || flips0)) continue;
3151               ierr = DMPlexGetConeSize(K,kStart,&kConeSize);CHKERRQ(ierr);
3152               ierr = PetscSectionSymLabelSetStratum(sym,depth - h,numDof[depth - h],-kConeSize,kConeSize,PETSC_USE_POINTER,perms0 ? &perms0[-kConeSize] : NULL,flips0 ? &flips0[-kConeSize] : NULL);CHKERRQ(ierr);
3153             }
3154             ierr = PetscSectionSetFieldSym(*section,f,sym);CHKERRQ(ierr);
3155             ierr = PetscSectionSymDestroy(&sym);CHKERRQ(ierr);
3156           }
3157         }
3158       }
3159     }
3160   }
3161   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
3162   ierr = PetscSectionSetChart(*section, pStart, pEnd);CHKERRQ(ierr);
3163   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3164   ierr = PetscMalloc1(depth+1,&pMax);CHKERRQ(ierr);
3165   ierr = DMPlexGetHybridBounds(dm, depth >= 0 ? &pMax[depth] : NULL, depth>1 ? &pMax[depth-1] : NULL, depth>2 ? &pMax[1] : NULL, &pMax[0]);CHKERRQ(ierr);
3166   ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
3167   for (dep = 0; dep <= depth - cellHeight; ++dep) {
3168     d    = dim == depth ? dep : (!dep ? 0 : dim);
3169     ierr = DMPlexGetDepthStratum(dm, dep, &pStart, &pEnd);CHKERRQ(ierr);
3170     pMax[dep] = pMax[dep] < 0 ? pEnd : pMax[dep];
3171     for (p = pStart; p < pEnd; ++p) {
3172       PetscInt tot = 0;
3173 
3174       for (f = 0; f < numFields; ++f) {
3175         if (isFE[f] && p >= pMax[dep]) continue;
3176         ierr = PetscSectionSetFieldDof(*section, p, f, numDof[f*(dim+1)+d]);CHKERRQ(ierr);
3177         tot += numDof[f*(dim+1)+d];
3178       }
3179       ierr = PetscSectionSetDof(*section, p, tot);CHKERRQ(ierr);
3180     }
3181   }
3182   ierr = PetscFree(pMax);CHKERRQ(ierr);
3183   ierr = PetscFree(isFE);CHKERRQ(ierr);
3184   PetscFunctionReturn(0);
3185 }
3186 
3187 /* Set the number of dof on each point and separate by fields
3188    If bcComps is NULL or the IS is NULL, constrain every dof on the point
3189 */
3190 static PetscErrorCode DMPlexCreateSectionBCDof(DM dm, PetscInt numBC, const PetscInt bcField[], const IS bcComps[], const IS bcPoints[], PetscSection section)
3191 {
3192   PetscInt       numFields;
3193   PetscInt       bc;
3194   PetscSection   aSec;
3195   PetscErrorCode ierr;
3196 
3197   PetscFunctionBegin;
3198   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
3199   for (bc = 0; bc < numBC; ++bc) {
3200     PetscInt        field = 0;
3201     const PetscInt *comp;
3202     const PetscInt *idx;
3203     PetscInt        Nc = -1, n, i;
3204 
3205     if (numFields) field = bcField[bc];
3206     if (bcComps && bcComps[bc]) {ierr = ISGetLocalSize(bcComps[bc], &Nc);CHKERRQ(ierr);}
3207     if (bcComps && bcComps[bc]) {ierr = ISGetIndices(bcComps[bc], &comp);CHKERRQ(ierr);}
3208     ierr = ISGetLocalSize(bcPoints[bc], &n);CHKERRQ(ierr);
3209     ierr = ISGetIndices(bcPoints[bc], &idx);CHKERRQ(ierr);
3210     for (i = 0; i < n; ++i) {
3211       const PetscInt p = idx[i];
3212       PetscInt       numConst;
3213 
3214       if (numFields) {
3215         ierr = PetscSectionGetFieldDof(section, p, field, &numConst);CHKERRQ(ierr);
3216       } else {
3217         ierr = PetscSectionGetDof(section, p, &numConst);CHKERRQ(ierr);
3218       }
3219       /* If Nc < 0, constrain every dof on the point */
3220       if (Nc > 0) numConst = PetscMin(numConst, Nc);
3221       if (numFields) {ierr = PetscSectionAddFieldConstraintDof(section, p, field, numConst);CHKERRQ(ierr);}
3222       ierr = PetscSectionAddConstraintDof(section, p, numConst);CHKERRQ(ierr);
3223     }
3224     ierr = ISRestoreIndices(bcPoints[bc], &idx);CHKERRQ(ierr);
3225     if (bcComps && bcComps[bc]) {ierr = ISRestoreIndices(bcComps[bc], &comp);CHKERRQ(ierr);}
3226   }
3227   ierr = DMPlexGetAnchors(dm, &aSec, NULL);CHKERRQ(ierr);
3228   if (aSec) {
3229     PetscInt aStart, aEnd, a;
3230 
3231     ierr = PetscSectionGetChart(aSec, &aStart, &aEnd);CHKERRQ(ierr);
3232     for (a = aStart; a < aEnd; a++) {
3233       PetscInt dof, f;
3234 
3235       ierr = PetscSectionGetDof(aSec, a, &dof);CHKERRQ(ierr);
3236       if (dof) {
3237         /* if there are point-to-point constraints, then all dofs are constrained */
3238         ierr = PetscSectionGetDof(section, a, &dof);CHKERRQ(ierr);
3239         ierr = PetscSectionSetConstraintDof(section, a, dof);CHKERRQ(ierr);
3240         for (f = 0; f < numFields; f++) {
3241           ierr = PetscSectionGetFieldDof(section, a, f, &dof);CHKERRQ(ierr);
3242           ierr = PetscSectionSetFieldConstraintDof(section, a, f, dof);CHKERRQ(ierr);
3243         }
3244       }
3245     }
3246   }
3247   PetscFunctionReturn(0);
3248 }
3249 
3250 /* Set the constrained field indices on each point
3251    If bcComps is NULL or the IS is NULL, constrain every dof on the point
3252 */
3253 static PetscErrorCode DMPlexCreateSectionBCIndicesField(DM dm, PetscInt numBC,const PetscInt bcField[], const IS bcComps[], const IS bcPoints[], PetscSection section)
3254 {
3255   PetscSection   aSec;
3256   PetscInt      *indices;
3257   PetscInt       numFields, cdof, maxDof = 0, pStart, pEnd, p, bc, f, d;
3258   PetscErrorCode ierr;
3259 
3260   PetscFunctionBegin;
3261   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
3262   if (!numFields) PetscFunctionReturn(0);
3263   /* Initialize all field indices to -1 */
3264   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
3265   for (p = pStart; p < pEnd; ++p) {ierr = PetscSectionGetConstraintDof(section, p, &cdof);CHKERRQ(ierr); maxDof = PetscMax(maxDof, cdof);}
3266   ierr = PetscMalloc1(maxDof, &indices);CHKERRQ(ierr);
3267   for (d = 0; d < maxDof; ++d) indices[d] = -1;
3268   for (p = pStart; p < pEnd; ++p) for (f = 0; f < numFields; ++f) {ierr = PetscSectionSetFieldConstraintIndices(section, p, f, indices);CHKERRQ(ierr);}
3269   /* Handle BC constraints */
3270   for (bc = 0; bc < numBC; ++bc) {
3271     const PetscInt  field = bcField[bc];
3272     const PetscInt *comp, *idx;
3273     PetscInt        Nc = -1, n, i;
3274 
3275     if (bcComps && bcComps[bc]) {ierr = ISGetLocalSize(bcComps[bc], &Nc);CHKERRQ(ierr);}
3276     if (bcComps && bcComps[bc]) {ierr = ISGetIndices(bcComps[bc], &comp);CHKERRQ(ierr);}
3277     ierr = ISGetLocalSize(bcPoints[bc], &n);CHKERRQ(ierr);
3278     ierr = ISGetIndices(bcPoints[bc], &idx);CHKERRQ(ierr);
3279     for (i = 0; i < n; ++i) {
3280       const PetscInt  p = idx[i];
3281       const PetscInt *find;
3282       PetscInt        fdof, fcdof, c;
3283 
3284       ierr = PetscSectionGetFieldDof(section, p, field, &fdof);CHKERRQ(ierr);
3285       if (!fdof) continue;
3286       if (Nc < 0) {
3287         for (d = 0; d < fdof; ++d) indices[d] = d;
3288         fcdof = fdof;
3289       } else {
3290         ierr = PetscSectionGetFieldConstraintDof(section, p, field, &fcdof);CHKERRQ(ierr);
3291         ierr = PetscSectionGetFieldConstraintIndices(section, p, field, &find);CHKERRQ(ierr);
3292         for (d = 0; d < fcdof; ++d) {if (find[d] < 0) break; indices[d] = find[d];}
3293         for (c = 0; c < Nc; ++c) indices[d++] = comp[c];
3294         ierr = PetscSortRemoveDupsInt(&d, indices);CHKERRQ(ierr);
3295         for (c = d; c < fcdof; ++c) indices[c] = -1;
3296         fcdof = d;
3297       }
3298       ierr = PetscSectionSetFieldConstraintDof(section, p, field, fcdof);CHKERRQ(ierr);
3299       ierr = PetscSectionSetFieldConstraintIndices(section, p, field, indices);CHKERRQ(ierr);
3300     }
3301     if (bcComps && bcComps[bc]) {ierr = ISRestoreIndices(bcComps[bc], &comp);CHKERRQ(ierr);}
3302     ierr = ISRestoreIndices(bcPoints[bc], &idx);CHKERRQ(ierr);
3303   }
3304   /* Handle anchors */
3305   ierr = DMPlexGetAnchors(dm, &aSec, NULL);CHKERRQ(ierr);
3306   if (aSec) {
3307     PetscInt aStart, aEnd, a;
3308 
3309     for (d = 0; d < maxDof; ++d) indices[d] = d;
3310     ierr = PetscSectionGetChart(aSec, &aStart, &aEnd);CHKERRQ(ierr);
3311     for (a = aStart; a < aEnd; a++) {
3312       PetscInt dof, f;
3313 
3314       ierr = PetscSectionGetDof(aSec, a, &dof);CHKERRQ(ierr);
3315       if (dof) {
3316         /* if there are point-to-point constraints, then all dofs are constrained */
3317         for (f = 0; f < numFields; f++) {
3318           ierr = PetscSectionSetFieldConstraintIndices(section, a, f, indices);CHKERRQ(ierr);
3319         }
3320       }
3321     }
3322   }
3323   ierr = PetscFree(indices);CHKERRQ(ierr);
3324   PetscFunctionReturn(0);
3325 }
3326 
3327 /* Set the constrained indices on each point */
3328 static PetscErrorCode DMPlexCreateSectionBCIndices(DM dm, PetscSection section)
3329 {
3330   PetscInt      *indices;
3331   PetscInt       numFields, maxDof, pStart, pEnd, p, f, d;
3332   PetscErrorCode ierr;
3333 
3334   PetscFunctionBegin;
3335   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
3336   ierr = PetscSectionGetMaxDof(section, &maxDof);CHKERRQ(ierr);
3337   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
3338   ierr = PetscMalloc1(maxDof, &indices);CHKERRQ(ierr);
3339   for (d = 0; d < maxDof; ++d) indices[d] = -1;
3340   for (p = pStart; p < pEnd; ++p) {
3341     PetscInt cdof, d;
3342 
3343     ierr = PetscSectionGetConstraintDof(section, p, &cdof);CHKERRQ(ierr);
3344     if (cdof) {
3345       if (numFields) {
3346         PetscInt numConst = 0, foff = 0;
3347 
3348         for (f = 0; f < numFields; ++f) {
3349           const PetscInt *find;
3350           PetscInt        fcdof, fdof;
3351 
3352           ierr = PetscSectionGetFieldDof(section, p, f, &fdof);CHKERRQ(ierr);
3353           ierr = PetscSectionGetFieldConstraintDof(section, p, f, &fcdof);CHKERRQ(ierr);
3354           /* Change constraint numbering from field component to local dof number */
3355           ierr = PetscSectionGetFieldConstraintIndices(section, p, f, &find);CHKERRQ(ierr);
3356           for (d = 0; d < fcdof; ++d) indices[numConst+d] = find[d] + foff;
3357           numConst += fcdof;
3358           foff     += fdof;
3359         }
3360         if (cdof != numConst) {ierr = PetscSectionSetConstraintDof(section, p, numConst);CHKERRQ(ierr);}
3361       } else {
3362         for (d = 0; d < cdof; ++d) indices[d] = d;
3363       }
3364       ierr = PetscSectionSetConstraintIndices(section, p, indices);CHKERRQ(ierr);
3365     }
3366   }
3367   ierr = PetscFree(indices);CHKERRQ(ierr);
3368   PetscFunctionReturn(0);
3369 }
3370 
3371 /*@C
3372   DMPlexCreateSection - Create a PetscSection based upon the dof layout specification provided.
3373 
3374   Not Collective
3375 
3376   Input Parameters:
3377 + dm        - The DMPlex object
3378 . dim       - The spatial dimension of the problem
3379 . numFields - The number of fields in the problem
3380 . numComp   - An array of size numFields that holds the number of components for each field
3381 . numDof    - An array of size numFields*(dim+1) which holds the number of dof for each field on a mesh piece of dimension d
3382 . numBC     - The number of boundary conditions
3383 . bcField   - An array of size numBC giving the field number for each boundry condition
3384 . bcComps   - [Optional] An array of size numBC giving an IS holding the field components to which each boundary condition applies
3385 . bcPoints  - An array of size numBC giving an IS holding the Plex points to which each boundary condition applies
3386 - perm      - Optional permutation of the chart, or NULL
3387 
3388   Output Parameter:
3389 . section - The PetscSection object
3390 
3391   Notes: numDof[f*(dim+1)+d] gives the number of dof for field f on points of dimension d. For instance, numDof[1] is the
3392   number of dof for field 0 on each edge.
3393 
3394   The chart permutation is the same one set using PetscSectionSetPermutation()
3395 
3396   Level: developer
3397 
3398   Fortran Notes:
3399   A Fortran 90 version is available as DMPlexCreateSectionF90()
3400 
3401 .keywords: mesh, elements
3402 .seealso: DMPlexCreate(), PetscSectionCreate(), PetscSectionSetPermutation()
3403 @*/
3404 PetscErrorCode DMPlexCreateSection(DM dm, PetscInt dim, PetscInt numFields,const PetscInt numComp[],const PetscInt numDof[], PetscInt numBC,const PetscInt bcField[], const IS bcComps[], const IS bcPoints[], IS perm, PetscSection *section)
3405 {
3406   PetscSection   aSec;
3407   PetscErrorCode ierr;
3408 
3409   PetscFunctionBegin;
3410   ierr = DMPlexCreateSectionInitial(dm, dim, numFields, numComp, numDof, section);CHKERRQ(ierr);
3411   ierr = DMPlexCreateSectionBCDof(dm, numBC, bcField, bcComps, bcPoints, *section);CHKERRQ(ierr);
3412   if (perm) {ierr = PetscSectionSetPermutation(*section, perm);CHKERRQ(ierr);}
3413   ierr = PetscSectionSetUp(*section);CHKERRQ(ierr);
3414   ierr = DMPlexGetAnchors(dm,&aSec,NULL);CHKERRQ(ierr);
3415   if (numBC || aSec) {
3416     ierr = DMPlexCreateSectionBCIndicesField(dm, numBC, bcField, bcComps, bcPoints, *section);CHKERRQ(ierr);
3417     ierr = DMPlexCreateSectionBCIndices(dm, *section);CHKERRQ(ierr);
3418   }
3419   ierr = PetscSectionViewFromOptions(*section,NULL,"-section_view");CHKERRQ(ierr);
3420   PetscFunctionReturn(0);
3421 }
3422 
3423 PetscErrorCode DMCreateCoordinateDM_Plex(DM dm, DM *cdm)
3424 {
3425   PetscSection   section, s;
3426   Mat            m;
3427   PetscInt       maxHeight;
3428   PetscErrorCode ierr;
3429 
3430   PetscFunctionBegin;
3431   ierr = DMClone(dm, cdm);CHKERRQ(ierr);
3432   ierr = DMPlexGetMaxProjectionHeight(dm, &maxHeight);CHKERRQ(ierr);
3433   ierr = DMPlexSetMaxProjectionHeight(*cdm, maxHeight);CHKERRQ(ierr);
3434   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &section);CHKERRQ(ierr);
3435   ierr = DMSetDefaultSection(*cdm, section);CHKERRQ(ierr);
3436   ierr = PetscSectionDestroy(&section);CHKERRQ(ierr);
3437   ierr = PetscSectionCreate(PETSC_COMM_SELF, &s);CHKERRQ(ierr);
3438   ierr = MatCreate(PETSC_COMM_SELF, &m);CHKERRQ(ierr);
3439   ierr = DMSetDefaultConstraints(*cdm, s, m);CHKERRQ(ierr);
3440   ierr = PetscSectionDestroy(&s);CHKERRQ(ierr);
3441   ierr = MatDestroy(&m);CHKERRQ(ierr);
3442   PetscFunctionReturn(0);
3443 }
3444 
3445 /*@C
3446   DMPlexGetConeSection - Return a section which describes the layout of cone data
3447 
3448   Not Collective
3449 
3450   Input Parameters:
3451 . dm        - The DMPlex object
3452 
3453   Output Parameter:
3454 . section - The PetscSection object
3455 
3456   Level: developer
3457 
3458 .seealso: DMPlexGetSupportSection(), DMPlexGetCones(), DMPlexGetConeOrientations()
3459 @*/
3460 PetscErrorCode DMPlexGetConeSection(DM dm, PetscSection *section)
3461 {
3462   DM_Plex *mesh = (DM_Plex*) dm->data;
3463 
3464   PetscFunctionBegin;
3465   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3466   if (section) *section = mesh->coneSection;
3467   PetscFunctionReturn(0);
3468 }
3469 
3470 /*@C
3471   DMPlexGetSupportSection - Return a section which describes the layout of support data
3472 
3473   Not Collective
3474 
3475   Input Parameters:
3476 . dm        - The DMPlex object
3477 
3478   Output Parameter:
3479 . section - The PetscSection object
3480 
3481   Level: developer
3482 
3483 .seealso: DMPlexGetConeSection()
3484 @*/
3485 PetscErrorCode DMPlexGetSupportSection(DM dm, PetscSection *section)
3486 {
3487   DM_Plex *mesh = (DM_Plex*) dm->data;
3488 
3489   PetscFunctionBegin;
3490   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3491   if (section) *section = mesh->supportSection;
3492   PetscFunctionReturn(0);
3493 }
3494 
3495 /*@C
3496   DMPlexGetCones - Return cone data
3497 
3498   Not Collective
3499 
3500   Input Parameters:
3501 . dm        - The DMPlex object
3502 
3503   Output Parameter:
3504 . cones - The cone for each point
3505 
3506   Level: developer
3507 
3508 .seealso: DMPlexGetConeSection()
3509 @*/
3510 PetscErrorCode DMPlexGetCones(DM dm, PetscInt *cones[])
3511 {
3512   DM_Plex *mesh = (DM_Plex*) dm->data;
3513 
3514   PetscFunctionBegin;
3515   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3516   if (cones) *cones = mesh->cones;
3517   PetscFunctionReturn(0);
3518 }
3519 
3520 /*@C
3521   DMPlexGetConeOrientations - Return cone orientation data
3522 
3523   Not Collective
3524 
3525   Input Parameters:
3526 . dm        - The DMPlex object
3527 
3528   Output Parameter:
3529 . coneOrientations - The cone orientation for each point
3530 
3531   Level: developer
3532 
3533 .seealso: DMPlexGetConeSection()
3534 @*/
3535 PetscErrorCode DMPlexGetConeOrientations(DM dm, PetscInt *coneOrientations[])
3536 {
3537   DM_Plex *mesh = (DM_Plex*) dm->data;
3538 
3539   PetscFunctionBegin;
3540   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3541   if (coneOrientations) *coneOrientations = mesh->coneOrientations;
3542   PetscFunctionReturn(0);
3543 }
3544 
3545 /******************************** FEM Support **********************************/
3546 
3547 PetscErrorCode DMPlexCreateSpectralClosurePermutation(DM dm, PetscInt point, PetscSection section)
3548 {
3549   DMLabel        label;
3550   PetscInt      *perm;
3551   PetscInt       dim, depth, eStart, k, Nf, f, Nc, c, i, j, size = 0, offset = 0, foffset = 0;
3552   PetscErrorCode ierr;
3553 
3554   PetscFunctionBegin;
3555   if (point < 0) {ierr = DMPlexGetDepthStratum(dm, 1, &point, NULL);CHKERRQ(ierr);}
3556   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
3557   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
3558   ierr = DMLabelGetValue(label, point, &depth);CHKERRQ(ierr);
3559   if (depth == 1) {eStart = point;}
3560   else if  (depth == dim) {
3561     const PetscInt *cone;
3562 
3563     ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
3564     eStart = cone[0];
3565   } else SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Point %D of depth %D cannot be used to bootstrap spectral ordering", point, depth);
3566   if (!section) {ierr = DMGetDefaultSection(dm, &section);CHKERRQ(ierr);}
3567   ierr = PetscSectionGetNumFields(section, &Nf);CHKERRQ(ierr);
3568   if (dim <= 1) PetscFunctionReturn(0);
3569   for (f = 0; f < Nf; ++f) {
3570     /* An order k SEM disc has k-1 dofs on an edge */
3571     ierr = PetscSectionGetFieldDof(section, eStart, f, &k);CHKERRQ(ierr);
3572     ierr = PetscSectionGetFieldComponents(section, f, &Nc);CHKERRQ(ierr);
3573     k = k/Nc + 1;
3574     size += PetscPowInt(k+1, dim)*Nc;
3575   }
3576   ierr = PetscMalloc1(size, &perm);CHKERRQ(ierr);
3577   for (f = 0; f < Nf; ++f) {
3578     switch (dim) {
3579     case 2:
3580       /* The original quad closure is oriented clockwise, {f, e_b, e_r, e_t, e_l, v_lb, v_rb, v_tr, v_tl} */
3581       ierr = PetscSectionGetFieldDof(section, eStart, f, &k);CHKERRQ(ierr);
3582       ierr = PetscSectionGetFieldComponents(section, f, &Nc);CHKERRQ(ierr);
3583       k = k/Nc + 1;
3584       /* The SEM order is
3585 
3586          v_lb, {e_b}, v_rb,
3587          e^{(k-1)-i}_l, {f^{i*(k-1)}}, e^i_r,
3588          v_lt, reverse {e_t}, v_rt
3589       */
3590       {
3591         const PetscInt of   = 0;
3592         const PetscInt oeb  = of   + PetscSqr(k-1);
3593         const PetscInt oer  = oeb  + (k-1);
3594         const PetscInt oet  = oer  + (k-1);
3595         const PetscInt oel  = oet  + (k-1);
3596         const PetscInt ovlb = oel  + (k-1);
3597         const PetscInt ovrb = ovlb + 1;
3598         const PetscInt ovrt = ovrb + 1;
3599         const PetscInt ovlt = ovrt + 1;
3600         PetscInt       o;
3601 
3602         /* bottom */
3603         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovlb*Nc + c + foffset;
3604         for (o = oeb; o < oer; ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
3605         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovrb*Nc + c + foffset;
3606         /* middle */
3607         for (i = 0; i < k-1; ++i) {
3608           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oel+(k-2)-i)*Nc + c + foffset;
3609           for (o = of+(k-1)*i; o < of+(k-1)*(i+1); ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
3610           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oer+i)*Nc + c + foffset;
3611         }
3612         /* top */
3613         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovlt*Nc + c + foffset;
3614         for (o = oel-1; o >= oet; --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
3615         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovrt*Nc + c + foffset;
3616         foffset = offset;
3617       }
3618       break;
3619     case 3:
3620       /* The original hex closure is
3621 
3622          {c,
3623           f_b, f_t, f_f, f_b, f_r, f_l,
3624           e_bl, e_bb, e_br, e_bf,  e_tf, e_tr, e_tb, e_tl,  e_rf, e_lf, e_lb, e_rb,
3625           v_blf, v_blb, v_brb, v_brf, v_tlf, v_trf, v_trb, v_tlb}
3626       */
3627       ierr = PetscSectionGetFieldDof(section, eStart, f, &k);CHKERRQ(ierr);
3628       ierr = PetscSectionGetFieldComponents(section, f, &Nc);CHKERRQ(ierr);
3629       k = k/Nc + 1;
3630       /* The SEM order is
3631          Bottom Slice
3632          v_blf, {e^{(k-1)-n}_bf}, v_brf,
3633          e^{i}_bl, f^{n*(k-1)+(k-1)-i}_b, e^{(k-1)-i}_br,
3634          v_blb, {e_bb}, v_brb,
3635 
3636          Middle Slice (j)
3637          {e^{(k-1)-j}_lf}, {f^{j*(k-1)+n}_f}, e^j_rf,
3638          f^{i*(k-1)+j}_l, {c^{(j*(k-1) + i)*(k-1)+n}_t}, f^{j*(k-1)+i}_r,
3639          e^j_lb, {f^{j*(k-1)+(k-1)-n}_b}, e^{(k-1)-j}_rb,
3640 
3641          Top Slice
3642          v_tlf, {e_tf}, v_trf,
3643          e^{(k-1)-i}_tl, {f^{i*(k-1)}_t}, e^{i}_tr,
3644          v_tlb, {e^{(k-1)-n}_tb}, v_trb,
3645       */
3646       {
3647         const PetscInt oc    = 0;
3648         const PetscInt ofb   = oc    + PetscSqr(k-1)*(k-1);
3649         const PetscInt oft   = ofb   + PetscSqr(k-1);
3650         const PetscInt off   = oft   + PetscSqr(k-1);
3651         const PetscInt ofk   = off   + PetscSqr(k-1);
3652         const PetscInt ofr   = ofk   + PetscSqr(k-1);
3653         const PetscInt ofl   = ofr   + PetscSqr(k-1);
3654         const PetscInt oebl  = ofl   + PetscSqr(k-1);
3655         const PetscInt oebb  = oebl  + (k-1);
3656         const PetscInt oebr  = oebb  + (k-1);
3657         const PetscInt oebf  = oebr  + (k-1);
3658         const PetscInt oetf  = oebf  + (k-1);
3659         const PetscInt oetr  = oetf  + (k-1);
3660         const PetscInt oetb  = oetr  + (k-1);
3661         const PetscInt oetl  = oetb  + (k-1);
3662         const PetscInt oerf  = oetl  + (k-1);
3663         const PetscInt oelf  = oerf  + (k-1);
3664         const PetscInt oelb  = oelf  + (k-1);
3665         const PetscInt oerb  = oelb  + (k-1);
3666         const PetscInt ovblf = oerb  + (k-1);
3667         const PetscInt ovblb = ovblf + 1;
3668         const PetscInt ovbrb = ovblb + 1;
3669         const PetscInt ovbrf = ovbrb + 1;
3670         const PetscInt ovtlf = ovbrf + 1;
3671         const PetscInt ovtrf = ovtlf + 1;
3672         const PetscInt ovtrb = ovtrf + 1;
3673         const PetscInt ovtlb = ovtrb + 1;
3674         PetscInt       o, n;
3675 
3676         /* Bottom Slice */
3677         /*   bottom */
3678         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovblf*Nc + c + foffset;
3679         for (o = oetf-1; o >= oebf; --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
3680         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovbrf*Nc + c + foffset;
3681         /*   middle */
3682         for (i = 0; i < k-1; ++i) {
3683           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oebl+i)*Nc + c + foffset;
3684           for (n = 0; n < k-1; ++n) {o = ofb+n*(k-1)+i; for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;}
3685           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oebr+(k-2)-i)*Nc + c + foffset;
3686         }
3687         /*   top */
3688         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovblb*Nc + c + foffset;
3689         for (o = oebb; o < oebr; ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
3690         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovbrb*Nc + c + foffset;
3691 
3692         /* Middle Slice */
3693         for (j = 0; j < k-1; ++j) {
3694           /*   bottom */
3695           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oelf+(k-2)-j)*Nc + c + foffset;
3696           for (o = off+j*(k-1); o < off+(j+1)*(k-1); ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
3697           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oerf+j)*Nc + c + foffset;
3698           /*   middle */
3699           for (i = 0; i < k-1; ++i) {
3700             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (ofl+i*(k-1)+j)*Nc + c + foffset;
3701             for (n = 0; n < k-1; ++n) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oc+(j*(k-1)+i)*(k-1)+n)*Nc + c + foffset;
3702             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (ofr+j*(k-1)+i)*Nc + c + foffset;
3703           }
3704           /*   top */
3705           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oelb+j)*Nc + c + foffset;
3706           for (o = ofk+j*(k-1)+(k-2); o >= ofk+j*(k-1); --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
3707           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oerb+(k-2)-j)*Nc + c + foffset;
3708         }
3709 
3710         /* Top Slice */
3711         /*   bottom */
3712         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtlf*Nc + c + foffset;
3713         for (o = oetf; o < oetr; ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
3714         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtrf*Nc + c + foffset;
3715         /*   middle */
3716         for (i = 0; i < k-1; ++i) {
3717           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oetl+(k-2)-i)*Nc + c + foffset;
3718           for (n = 0; n < k-1; ++n) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oft+i*(k-1)+n)*Nc + c + foffset;
3719           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oetr+i)*Nc + c + foffset;
3720         }
3721         /*   top */
3722         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtlb*Nc + c + foffset;
3723         for (o = oetl-1; o >= oetb; --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
3724         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtrb*Nc + c + foffset;
3725 
3726         foffset = offset;
3727       }
3728       break;
3729     default: SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_OUTOFRANGE, "No spectral ordering for dimension %D", dim);
3730     }
3731   }
3732   if (offset != size) SETERRQ2(PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Number of permutation entries %D != %D", offset, size);
3733   /* Check permutation */
3734   {
3735     PetscInt *check;
3736 
3737     ierr = PetscMalloc1(size, &check);CHKERRQ(ierr);
3738     for (i = 0; i < size; ++i) {check[i] = -1; if (perm[i] < 0 || perm[i] >= size) SETERRQ2(PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Invalid permutation index p[%D] = %D", i, perm[i]);}
3739     for (i = 0; i < size; ++i) check[perm[i]] = i;
3740     for (i = 0; i < size; ++i) {if (check[i] < 0) SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Missing permutation index %D", i);}
3741     ierr = PetscFree(check);CHKERRQ(ierr);
3742   }
3743   ierr = PetscSectionSetClosurePermutation_Internal(section, (PetscObject) dm, size, PETSC_OWN_POINTER, perm);CHKERRQ(ierr);
3744   PetscFunctionReturn(0);
3745 }
3746 
3747 PetscErrorCode DMPlexGetPointDualSpaceFEM(DM dm, PetscInt point, PetscInt field, PetscDualSpace *dspace)
3748 {
3749   PetscDS        prob;
3750   PetscInt       depth, Nf, h;
3751   DMLabel        label;
3752   PetscErrorCode ierr;
3753 
3754   PetscFunctionBeginHot;
3755   prob    = dm->prob;
3756   Nf      = prob->Nf;
3757   label   = dm->depthLabel;
3758   *dspace = NULL;
3759   if (field < Nf) {
3760     PetscObject disc = prob->disc[field];
3761 
3762     if (disc->classid == PETSCFE_CLASSID) {
3763       PetscDualSpace dsp;
3764 
3765       ierr = PetscFEGetDualSpace((PetscFE)disc,&dsp);CHKERRQ(ierr);
3766       ierr = DMLabelGetNumValues(label,&depth);CHKERRQ(ierr);
3767       ierr = DMLabelGetValue(label,point,&h);CHKERRQ(ierr);
3768       h    = depth - 1 - h;
3769       if (h) {
3770         ierr = PetscDualSpaceGetHeightSubspace(dsp,h,dspace);CHKERRQ(ierr);
3771       } else {
3772         *dspace = dsp;
3773       }
3774     }
3775   }
3776   PetscFunctionReturn(0);
3777 }
3778 
3779 
3780 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecGetClosure_Depth1_Static(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
3781 {
3782   PetscScalar    *array, *vArray;
3783   const PetscInt *cone, *coneO;
3784   PetscInt        pStart, pEnd, p, numPoints, size = 0, offset = 0;
3785   PetscErrorCode  ierr;
3786 
3787   PetscFunctionBeginHot;
3788   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
3789   ierr = DMPlexGetConeSize(dm, point, &numPoints);CHKERRQ(ierr);
3790   ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
3791   ierr = DMPlexGetConeOrientation(dm, point, &coneO);CHKERRQ(ierr);
3792   if (!values || !*values) {
3793     if ((point >= pStart) && (point < pEnd)) {
3794       PetscInt dof;
3795 
3796       ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
3797       size += dof;
3798     }
3799     for (p = 0; p < numPoints; ++p) {
3800       const PetscInt cp = cone[p];
3801       PetscInt       dof;
3802 
3803       if ((cp < pStart) || (cp >= pEnd)) continue;
3804       ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
3805       size += dof;
3806     }
3807     if (!values) {
3808       if (csize) *csize = size;
3809       PetscFunctionReturn(0);
3810     }
3811     ierr = DMGetWorkArray(dm, size, MPIU_SCALAR, &array);CHKERRQ(ierr);
3812   } else {
3813     array = *values;
3814   }
3815   size = 0;
3816   ierr = VecGetArray(v, &vArray);CHKERRQ(ierr);
3817   if ((point >= pStart) && (point < pEnd)) {
3818     PetscInt     dof, off, d;
3819     PetscScalar *varr;
3820 
3821     ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
3822     ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
3823     varr = &vArray[off];
3824     for (d = 0; d < dof; ++d, ++offset) {
3825       array[offset] = varr[d];
3826     }
3827     size += dof;
3828   }
3829   for (p = 0; p < numPoints; ++p) {
3830     const PetscInt cp = cone[p];
3831     PetscInt       o  = coneO[p];
3832     PetscInt       dof, off, d;
3833     PetscScalar   *varr;
3834 
3835     if ((cp < pStart) || (cp >= pEnd)) continue;
3836     ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
3837     ierr = PetscSectionGetOffset(section, cp, &off);CHKERRQ(ierr);
3838     varr = &vArray[off];
3839     if (o >= 0) {
3840       for (d = 0; d < dof; ++d, ++offset) {
3841         array[offset] = varr[d];
3842       }
3843     } else {
3844       for (d = dof-1; d >= 0; --d, ++offset) {
3845         array[offset] = varr[d];
3846       }
3847     }
3848     size += dof;
3849   }
3850   ierr = VecRestoreArray(v, &vArray);CHKERRQ(ierr);
3851   if (!*values) {
3852     if (csize) *csize = size;
3853     *values = array;
3854   } else {
3855     if (size > *csize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Size of input array %D < actual size %D", *csize, size);
3856     *csize = size;
3857   }
3858   PetscFunctionReturn(0);
3859 }
3860 
3861 static PetscErrorCode DMPlexGetCompressedClosure(DM dm, PetscSection section, PetscInt point, PetscInt *numPoints, PetscInt **points, PetscSection *clSec, IS *clPoints, const PetscInt **clp)
3862 {
3863   const PetscInt *cla;
3864   PetscInt       np, *pts = NULL;
3865   PetscErrorCode ierr;
3866 
3867   PetscFunctionBeginHot;
3868   ierr = PetscSectionGetClosureIndex(section, (PetscObject) dm, clSec, clPoints);CHKERRQ(ierr);
3869   if (!*clPoints) {
3870     PetscInt pStart, pEnd, p, q;
3871 
3872     ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
3873     ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &np, &pts);CHKERRQ(ierr);
3874     /* Compress out points not in the section */
3875     for (p = 0, q = 0; p < np; p++) {
3876       PetscInt r = pts[2*p];
3877       if ((r >= pStart) && (r < pEnd)) {
3878         pts[q*2]   = r;
3879         pts[q*2+1] = pts[2*p+1];
3880         ++q;
3881       }
3882     }
3883     np = q;
3884     cla = NULL;
3885   } else {
3886     PetscInt dof, off;
3887 
3888     ierr = PetscSectionGetDof(*clSec, point, &dof);CHKERRQ(ierr);
3889     ierr = PetscSectionGetOffset(*clSec, point, &off);CHKERRQ(ierr);
3890     ierr = ISGetIndices(*clPoints, &cla);CHKERRQ(ierr);
3891     np   = dof/2;
3892     pts  = (PetscInt *) &cla[off];
3893   }
3894   *numPoints = np;
3895   *points    = pts;
3896   *clp       = cla;
3897 
3898   PetscFunctionReturn(0);
3899 }
3900 
3901 static PetscErrorCode DMPlexRestoreCompressedClosure(DM dm, PetscSection section, PetscInt point, PetscInt *numPoints, PetscInt **points, PetscSection *clSec, IS *clPoints, const PetscInt **clp)
3902 {
3903   PetscErrorCode ierr;
3904 
3905   PetscFunctionBeginHot;
3906   if (!*clPoints) {
3907     ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, numPoints, points);CHKERRQ(ierr);
3908   } else {
3909     ierr = ISRestoreIndices(*clPoints, clp);CHKERRQ(ierr);
3910   }
3911   *numPoints = 0;
3912   *points    = NULL;
3913   *clSec     = NULL;
3914   *clPoints  = NULL;
3915   *clp       = NULL;
3916   PetscFunctionReturn(0);
3917 }
3918 
3919 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecGetClosure_Static(DM dm, PetscSection section, PetscInt numPoints, const PetscInt points[], const PetscInt clperm[], const PetscScalar vArray[], PetscInt *size, PetscScalar array[])
3920 {
3921   PetscInt          offset = 0, p;
3922   const PetscInt    **perms = NULL;
3923   const PetscScalar **flips = NULL;
3924   PetscErrorCode    ierr;
3925 
3926   PetscFunctionBeginHot;
3927   *size = 0;
3928   ierr = PetscSectionGetPointSyms(section,numPoints,points,&perms,&flips);CHKERRQ(ierr);
3929   for (p = 0; p < numPoints; p++) {
3930     const PetscInt    point = points[2*p];
3931     const PetscInt    *perm = perms ? perms[p] : NULL;
3932     const PetscScalar *flip = flips ? flips[p] : NULL;
3933     PetscInt          dof, off, d;
3934     const PetscScalar *varr;
3935 
3936     ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
3937     ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
3938     varr = &vArray[off];
3939     if (clperm) {
3940       if (perm) {
3941         for (d = 0; d < dof; d++) array[clperm[offset + perm[d]]]  = varr[d];
3942       } else {
3943         for (d = 0; d < dof; d++) array[clperm[offset +      d ]]  = varr[d];
3944       }
3945       if (flip) {
3946         for (d = 0; d < dof; d++) array[clperm[offset +      d ]] *= flip[d];
3947       }
3948     } else {
3949       if (perm) {
3950         for (d = 0; d < dof; d++) array[offset + perm[d]]  = varr[d];
3951       } else {
3952         for (d = 0; d < dof; d++) array[offset +      d ]  = varr[d];
3953       }
3954       if (flip) {
3955         for (d = 0; d < dof; d++) array[offset +      d ] *= flip[d];
3956       }
3957     }
3958     offset += dof;
3959   }
3960   ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms,&flips);CHKERRQ(ierr);
3961   *size = offset;
3962   PetscFunctionReturn(0);
3963 }
3964 
3965 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecGetClosure_Fields_Static(DM dm, PetscSection section, PetscInt numPoints, const PetscInt points[], PetscInt numFields, const PetscInt clperm[], const PetscScalar vArray[], PetscInt *size, PetscScalar array[])
3966 {
3967   PetscInt          offset = 0, f;
3968   PetscErrorCode    ierr;
3969 
3970   PetscFunctionBeginHot;
3971   *size = 0;
3972   for (f = 0; f < numFields; ++f) {
3973     PetscInt          p;
3974     const PetscInt    **perms = NULL;
3975     const PetscScalar **flips = NULL;
3976 
3977     ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
3978     for (p = 0; p < numPoints; p++) {
3979       const PetscInt    point = points[2*p];
3980       PetscInt          fdof, foff, b;
3981       const PetscScalar *varr;
3982       const PetscInt    *perm = perms ? perms[p] : NULL;
3983       const PetscScalar *flip = flips ? flips[p] : NULL;
3984 
3985       ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
3986       ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
3987       varr = &vArray[foff];
3988       if (clperm) {
3989         if (perm) {for (b = 0; b < fdof; b++) {array[clperm[offset + perm[b]]]  = varr[b];}}
3990         else      {for (b = 0; b < fdof; b++) {array[clperm[offset +      b ]]  = varr[b];}}
3991         if (flip) {for (b = 0; b < fdof; b++) {array[clperm[offset +      b ]] *= flip[b];}}
3992       } else {
3993         if (perm) {for (b = 0; b < fdof; b++) {array[offset + perm[b]]  = varr[b];}}
3994         else      {for (b = 0; b < fdof; b++) {array[offset +      b ]  = varr[b];}}
3995         if (flip) {for (b = 0; b < fdof; b++) {array[offset +      b ] *= flip[b];}}
3996       }
3997       offset += fdof;
3998     }
3999     ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4000   }
4001   *size = offset;
4002   PetscFunctionReturn(0);
4003 }
4004 
4005 /*@C
4006   DMPlexVecGetClosure - Get an array of the values on the closure of 'point'
4007 
4008   Not collective
4009 
4010   Input Parameters:
4011 + dm - The DM
4012 . section - The section describing the layout in v, or NULL to use the default section
4013 . v - The local vector
4014 . point - The point in the DM
4015 . csize - The size of the input values array, or NULL
4016 - values - An array to use for the values, or NULL to have it allocated automatically
4017 
4018   Output Parameters:
4019 + csize - The number of values in the closure
4020 - values - The array of values. If the user provided NULL, it is a borrowed array and should not be freed
4021 
4022 $ Note that DMPlexVecGetClosure/DMPlexVecRestoreClosure only allocates the values array if it set to NULL in the
4023 $ calling function. This is because DMPlexVecGetClosure() is typically called in the inner loop of a Vec or Mat
4024 $ assembly function, and a user may already have allocated storage for this operation.
4025 $
4026 $ A typical use could be
4027 $
4028 $  values = NULL;
4029 $  ierr = DMPlexVecGetClosure(dm, NULL, v, p, &clSize, &values);CHKERRQ(ierr);
4030 $  for (cl = 0; cl < clSize; ++cl) {
4031 $    <Compute on closure>
4032 $  }
4033 $  ierr = DMPlexVecRestoreClosure(dm, NULL, v, p, &clSize, &values);CHKERRQ(ierr);
4034 $
4035 $ or
4036 $
4037 $  PetscMalloc1(clMaxSize, &values);
4038 $  for (p = pStart; p < pEnd; ++p) {
4039 $    clSize = clMaxSize;
4040 $    ierr = DMPlexVecGetClosure(dm, NULL, v, p, &clSize, &values);CHKERRQ(ierr);
4041 $    for (cl = 0; cl < clSize; ++cl) {
4042 $      <Compute on closure>
4043 $    }
4044 $  }
4045 $  PetscFree(values);
4046 
4047   Fortran Notes:
4048   Since it returns an array, this routine is only available in Fortran 90, and you must
4049   include petsc.h90 in your code.
4050 
4051   The csize argument is not present in the Fortran 90 binding since it is internal to the array.
4052 
4053   Level: intermediate
4054 
4055 .seealso DMPlexVecRestoreClosure(), DMPlexVecSetClosure(), DMPlexMatSetClosure()
4056 @*/
4057 PetscErrorCode DMPlexVecGetClosure(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
4058 {
4059   PetscSection       clSection;
4060   IS                 clPoints;
4061   PetscScalar       *array;
4062   const PetscScalar *vArray;
4063   PetscInt          *points = NULL;
4064   const PetscInt    *clp, *perm;
4065   PetscInt           depth, numFields, numPoints, size;
4066   PetscErrorCode     ierr;
4067 
4068   PetscFunctionBeginHot;
4069   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4070   if (!section) {ierr = DMGetDefaultSection(dm, &section);CHKERRQ(ierr);}
4071   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
4072   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
4073   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
4074   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
4075   if (depth == 1 && numFields < 2) {
4076     ierr = DMPlexVecGetClosure_Depth1_Static(dm, section, v, point, csize, values);CHKERRQ(ierr);
4077     PetscFunctionReturn(0);
4078   }
4079   /* Get points */
4080   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
4081   ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, NULL, &perm);CHKERRQ(ierr);
4082   /* Get array */
4083   if (!values || !*values) {
4084     PetscInt asize = 0, dof, p;
4085 
4086     for (p = 0; p < numPoints*2; p += 2) {
4087       ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
4088       asize += dof;
4089     }
4090     if (!values) {
4091       ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
4092       if (csize) *csize = asize;
4093       PetscFunctionReturn(0);
4094     }
4095     ierr = DMGetWorkArray(dm, asize, MPIU_SCALAR, &array);CHKERRQ(ierr);
4096   } else {
4097     array = *values;
4098   }
4099   ierr = VecGetArrayRead(v, &vArray);CHKERRQ(ierr);
4100   /* Get values */
4101   if (numFields > 0) {ierr = DMPlexVecGetClosure_Fields_Static(dm, section, numPoints, points, numFields, perm, vArray, &size, array);CHKERRQ(ierr);}
4102   else               {ierr = DMPlexVecGetClosure_Static(dm, section, numPoints, points, perm, vArray, &size, array);CHKERRQ(ierr);}
4103   /* Cleanup points */
4104   ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
4105   /* Cleanup array */
4106   ierr = VecRestoreArrayRead(v, &vArray);CHKERRQ(ierr);
4107   if (!*values) {
4108     if (csize) *csize = size;
4109     *values = array;
4110   } else {
4111     if (size > *csize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Size of input array %D < actual size %D", *csize, size);
4112     *csize = size;
4113   }
4114   PetscFunctionReturn(0);
4115 }
4116 
4117 /*@C
4118   DMPlexVecRestoreClosure - Restore the array of the values on the closure of 'point'
4119 
4120   Not collective
4121 
4122   Input Parameters:
4123 + dm - The DM
4124 . section - The section describing the layout in v, or NULL to use the default section
4125 . v - The local vector
4126 . point - The point in the DM
4127 . csize - The number of values in the closure, or NULL
4128 - values - The array of values, which is a borrowed array and should not be freed
4129 
4130   Note that the array values are discarded and not copied back into v. In order to copy values back to v, use DMPlexVecSetClosure()
4131 
4132   Fortran Notes:
4133   Since it returns an array, this routine is only available in Fortran 90, and you must
4134   include petsc.h90 in your code.
4135 
4136   The csize argument is not present in the Fortran 90 binding since it is internal to the array.
4137 
4138   Level: intermediate
4139 
4140 .seealso DMPlexVecGetClosure(), DMPlexVecSetClosure(), DMPlexMatSetClosure()
4141 @*/
4142 PetscErrorCode DMPlexVecRestoreClosure(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
4143 {
4144   PetscInt       size = 0;
4145   PetscErrorCode ierr;
4146 
4147   PetscFunctionBegin;
4148   /* Should work without recalculating size */
4149   ierr = DMRestoreWorkArray(dm, size, MPIU_SCALAR, (void*) values);CHKERRQ(ierr);
4150   *values = NULL;
4151   PetscFunctionReturn(0);
4152 }
4153 
4154 PETSC_STATIC_INLINE void add   (PetscScalar *x, PetscScalar y) {*x += y;}
4155 PETSC_STATIC_INLINE void insert(PetscScalar *x, PetscScalar y) {*x  = y;}
4156 
4157 PETSC_STATIC_INLINE PetscErrorCode updatePoint_private(PetscSection section, PetscInt point, PetscInt dof, void (*fuse)(PetscScalar*, PetscScalar), PetscBool setBC, const PetscInt perm[], const PetscScalar flip[], const PetscInt clperm[], const PetscScalar values[], PetscInt offset, PetscScalar array[])
4158 {
4159   PetscInt        cdof;   /* The number of constraints on this point */
4160   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
4161   PetscScalar    *a;
4162   PetscInt        off, cind = 0, k;
4163   PetscErrorCode  ierr;
4164 
4165   PetscFunctionBegin;
4166   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
4167   ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
4168   a    = &array[off];
4169   if (!cdof || setBC) {
4170     if (clperm) {
4171       if (perm) {for (k = 0; k < dof; ++k) {fuse(&a[k], values[clperm[offset+perm[k]]] * (flip ? flip[perm[k]] : 1.));}}
4172       else      {for (k = 0; k < dof; ++k) {fuse(&a[k], values[clperm[offset+     k ]] * (flip ? flip[     k ] : 1.));}}
4173     } else {
4174       if (perm) {for (k = 0; k < dof; ++k) {fuse(&a[k], values[offset+perm[k]] * (flip ? flip[perm[k]] : 1.));}}
4175       else      {for (k = 0; k < dof; ++k) {fuse(&a[k], values[offset+     k ] * (flip ? flip[     k ] : 1.));}}
4176     }
4177   } else {
4178     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
4179     if (clperm) {
4180       if (perm) {for (k = 0; k < dof; ++k) {
4181           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
4182           fuse(&a[k], values[clperm[offset+perm[k]]] * (flip ? flip[perm[k]] : 1.));
4183         }
4184       } else {
4185         for (k = 0; k < dof; ++k) {
4186           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
4187           fuse(&a[k], values[clperm[offset+     k ]] * (flip ? flip[     k ] : 1.));
4188         }
4189       }
4190     } else {
4191       if (perm) {
4192         for (k = 0; k < dof; ++k) {
4193           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
4194           fuse(&a[k], values[offset+perm[k]] * (flip ? flip[perm[k]] : 1.));
4195         }
4196       } else {
4197         for (k = 0; k < dof; ++k) {
4198           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
4199           fuse(&a[k], values[offset+     k ] * (flip ? flip[     k ] : 1.));
4200         }
4201       }
4202     }
4203   }
4204   PetscFunctionReturn(0);
4205 }
4206 
4207 PETSC_STATIC_INLINE PetscErrorCode updatePointBC_private(PetscSection section, PetscInt point, PetscInt dof, void (*fuse)(PetscScalar*, PetscScalar), const PetscInt perm[], const PetscScalar flip[], const PetscInt clperm[], const PetscScalar values[], PetscInt offset, PetscScalar array[])
4208 {
4209   PetscInt        cdof;   /* The number of constraints on this point */
4210   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
4211   PetscScalar    *a;
4212   PetscInt        off, cind = 0, k;
4213   PetscErrorCode  ierr;
4214 
4215   PetscFunctionBegin;
4216   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
4217   ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
4218   a    = &array[off];
4219   if (cdof) {
4220     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
4221     if (clperm) {
4222       if (perm) {
4223         for (k = 0; k < dof; ++k) {
4224           if ((cind < cdof) && (k == cdofs[cind])) {
4225             fuse(&a[k], values[clperm[offset+perm[k]]] * (flip ? flip[perm[k]] : 1.));
4226             cind++;
4227           }
4228         }
4229       } else {
4230         for (k = 0; k < dof; ++k) {
4231           if ((cind < cdof) && (k == cdofs[cind])) {
4232             fuse(&a[k], values[clperm[offset+     k ]] * (flip ? flip[     k ] : 1.));
4233             cind++;
4234           }
4235         }
4236       }
4237     } else {
4238       if (perm) {
4239         for (k = 0; k < dof; ++k) {
4240           if ((cind < cdof) && (k == cdofs[cind])) {
4241             fuse(&a[k], values[offset+perm[k]] * (flip ? flip[perm[k]] : 1.));
4242             cind++;
4243           }
4244         }
4245       } else {
4246         for (k = 0; k < dof; ++k) {
4247           if ((cind < cdof) && (k == cdofs[cind])) {
4248             fuse(&a[k], values[offset+     k ] * (flip ? flip[     k ] : 1.));
4249             cind++;
4250           }
4251         }
4252       }
4253     }
4254   }
4255   PetscFunctionReturn(0);
4256 }
4257 
4258 PETSC_STATIC_INLINE PetscErrorCode updatePointFields_private(PetscSection section, PetscInt point, const PetscInt *perm, const PetscScalar *flip, PetscInt f, void (*fuse)(PetscScalar*, PetscScalar), PetscBool setBC, const PetscInt clperm[], const PetscScalar values[], PetscInt *offset, PetscScalar array[])
4259 {
4260   PetscScalar    *a;
4261   PetscInt        fdof, foff, fcdof, foffset = *offset;
4262   const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
4263   PetscInt        cind = 0, b;
4264   PetscErrorCode  ierr;
4265 
4266   PetscFunctionBegin;
4267   ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
4268   ierr = PetscSectionGetFieldConstraintDof(section, point, f, &fcdof);CHKERRQ(ierr);
4269   ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
4270   a    = &array[foff];
4271   if (!fcdof || setBC) {
4272     if (clperm) {
4273       if (perm) {for (b = 0; b < fdof; b++) {fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));}}
4274       else      {for (b = 0; b < fdof; b++) {fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));}}
4275     } else {
4276       if (perm) {for (b = 0; b < fdof; b++) {fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));}}
4277       else      {for (b = 0; b < fdof; b++) {fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));}}
4278     }
4279   } else {
4280     ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
4281     if (clperm) {
4282       if (perm) {
4283         for (b = 0; b < fdof; b++) {
4284           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
4285           fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));
4286         }
4287       } else {
4288         for (b = 0; b < fdof; b++) {
4289           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
4290           fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));
4291         }
4292       }
4293     } else {
4294       if (perm) {
4295         for (b = 0; b < fdof; b++) {
4296           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
4297           fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));
4298         }
4299       } else {
4300         for (b = 0; b < fdof; b++) {
4301           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
4302           fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));
4303         }
4304       }
4305     }
4306   }
4307   *offset += fdof;
4308   PetscFunctionReturn(0);
4309 }
4310 
4311 PETSC_STATIC_INLINE PetscErrorCode updatePointFieldsBC_private(PetscSection section, PetscInt point, const PetscInt perm[], const PetscScalar flip[], PetscInt f, PetscInt Ncc, const PetscInt comps[], void (*fuse)(PetscScalar*, PetscScalar), const PetscInt clperm[], const PetscScalar values[], PetscInt *offset, PetscScalar array[])
4312 {
4313   PetscScalar    *a;
4314   PetscInt        fdof, foff, fcdof, foffset = *offset;
4315   const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
4316   PetscInt        cind = 0, ncind = 0, b;
4317   PetscBool       ncSet, fcSet;
4318   PetscErrorCode  ierr;
4319 
4320   PetscFunctionBegin;
4321   ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
4322   ierr = PetscSectionGetFieldConstraintDof(section, point, f, &fcdof);CHKERRQ(ierr);
4323   ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
4324   a    = &array[foff];
4325   if (fcdof) {
4326     /* We just override fcdof and fcdofs with Ncc and comps */
4327     ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
4328     if (clperm) {
4329       if (perm) {
4330         if (comps) {
4331           for (b = 0; b < fdof; b++) {
4332             ncSet = fcSet = PETSC_FALSE;
4333             if ((ncind < Ncc)  && (b == comps[ncind])) {++ncind; ncSet = PETSC_TRUE;}
4334             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
4335             if (ncSet && fcSet) {fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));}
4336           }
4337         } else {
4338           for (b = 0; b < fdof; b++) {
4339             if ((cind < fcdof) && (b == fcdofs[cind])) {
4340               fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));
4341               ++cind;
4342             }
4343           }
4344         }
4345       } else {
4346         if (comps) {
4347           for (b = 0; b < fdof; b++) {
4348             ncSet = fcSet = PETSC_FALSE;
4349             if ((ncind < Ncc)  && (b == comps[ncind])) {++ncind; ncSet = PETSC_TRUE;}
4350             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
4351             if (ncSet && fcSet) {fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));}
4352           }
4353         } else {
4354           for (b = 0; b < fdof; b++) {
4355             if ((cind < fcdof) && (b == fcdofs[cind])) {
4356               fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));
4357               ++cind;
4358             }
4359           }
4360         }
4361       }
4362     } else {
4363       if (perm) {
4364         if (comps) {
4365           for (b = 0; b < fdof; b++) {
4366             ncSet = fcSet = PETSC_FALSE;
4367             if ((ncind < Ncc)  && (b == comps[ncind])) {++ncind; ncSet = PETSC_TRUE;}
4368             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
4369             if (ncSet && fcSet) {fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));}
4370           }
4371         } else {
4372           for (b = 0; b < fdof; b++) {
4373             if ((cind < fcdof) && (b == fcdofs[cind])) {
4374               fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));
4375               ++cind;
4376             }
4377           }
4378         }
4379       } else {
4380         if (comps) {
4381           for (b = 0; b < fdof; b++) {
4382             ncSet = fcSet = PETSC_FALSE;
4383             if ((ncind < Ncc)  && (b == comps[ncind])) {++ncind; ncSet = PETSC_TRUE;}
4384             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
4385             if (ncSet && fcSet) {fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));}
4386           }
4387         } else {
4388           for (b = 0; b < fdof; b++) {
4389             if ((cind < fcdof) && (b == fcdofs[cind])) {
4390               fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));
4391               ++cind;
4392             }
4393           }
4394         }
4395       }
4396     }
4397   }
4398   *offset += fdof;
4399   PetscFunctionReturn(0);
4400 }
4401 
4402 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecSetClosure_Depth1_Static(DM dm, PetscSection section, Vec v, PetscInt point, const PetscScalar values[], InsertMode mode)
4403 {
4404   PetscScalar    *array;
4405   const PetscInt *cone, *coneO;
4406   PetscInt        pStart, pEnd, p, numPoints, off, dof;
4407   PetscErrorCode  ierr;
4408 
4409   PetscFunctionBeginHot;
4410   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
4411   ierr = DMPlexGetConeSize(dm, point, &numPoints);CHKERRQ(ierr);
4412   ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
4413   ierr = DMPlexGetConeOrientation(dm, point, &coneO);CHKERRQ(ierr);
4414   ierr = VecGetArray(v, &array);CHKERRQ(ierr);
4415   for (p = 0, off = 0; p <= numPoints; ++p, off += dof) {
4416     const PetscInt cp = !p ? point : cone[p-1];
4417     const PetscInt o  = !p ? 0     : coneO[p-1];
4418 
4419     if ((cp < pStart) || (cp >= pEnd)) {dof = 0; continue;}
4420     ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
4421     /* ADD_VALUES */
4422     {
4423       const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
4424       PetscScalar    *a;
4425       PetscInt        cdof, coff, cind = 0, k;
4426 
4427       ierr = PetscSectionGetConstraintDof(section, cp, &cdof);CHKERRQ(ierr);
4428       ierr = PetscSectionGetOffset(section, cp, &coff);CHKERRQ(ierr);
4429       a    = &array[coff];
4430       if (!cdof) {
4431         if (o >= 0) {
4432           for (k = 0; k < dof; ++k) {
4433             a[k] += values[off+k];
4434           }
4435         } else {
4436           for (k = 0; k < dof; ++k) {
4437             a[k] += values[off+dof-k-1];
4438           }
4439         }
4440       } else {
4441         ierr = PetscSectionGetConstraintIndices(section, cp, &cdofs);CHKERRQ(ierr);
4442         if (o >= 0) {
4443           for (k = 0; k < dof; ++k) {
4444             if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
4445             a[k] += values[off+k];
4446           }
4447         } else {
4448           for (k = 0; k < dof; ++k) {
4449             if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
4450             a[k] += values[off+dof-k-1];
4451           }
4452         }
4453       }
4454     }
4455   }
4456   ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
4457   PetscFunctionReturn(0);
4458 }
4459 
4460 /*@C
4461   DMPlexVecSetClosure - Set an array of the values on the closure of 'point'
4462 
4463   Not collective
4464 
4465   Input Parameters:
4466 + dm - The DM
4467 . section - The section describing the layout in v, or NULL to use the default section
4468 . v - The local vector
4469 . point - The point in the DM
4470 . values - The array of values
4471 - mode - The insert mode. One of INSERT_ALL_VALUES, ADD_ALL_VALUES, INSERT_VALUES, ADD_VALUES, INSERT_BC_VALUES, and ADD_BC_VALUES,
4472          where INSERT_ALL_VALUES and ADD_ALL_VALUES also overwrite boundary conditions.
4473 
4474   Fortran Notes:
4475   This routine is only available in Fortran 90, and you must include petsc.h90 in your code.
4476 
4477   Level: intermediate
4478 
4479 .seealso DMPlexVecGetClosure(), DMPlexMatSetClosure()
4480 @*/
4481 PetscErrorCode DMPlexVecSetClosure(DM dm, PetscSection section, Vec v, PetscInt point, const PetscScalar values[], InsertMode mode)
4482 {
4483   PetscSection    clSection;
4484   IS              clPoints;
4485   PetscScalar    *array;
4486   PetscInt       *points = NULL;
4487   const PetscInt *clp, *clperm;
4488   PetscInt        depth, numFields, numPoints, p;
4489   PetscErrorCode  ierr;
4490 
4491   PetscFunctionBeginHot;
4492   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4493   if (!section) {ierr = DMGetDefaultSection(dm, &section);CHKERRQ(ierr);}
4494   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
4495   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
4496   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
4497   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
4498   if (depth == 1 && numFields < 2 && mode == ADD_VALUES) {
4499     ierr = DMPlexVecSetClosure_Depth1_Static(dm, section, v, point, values, mode);CHKERRQ(ierr);
4500     PetscFunctionReturn(0);
4501   }
4502   /* Get points */
4503   ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, NULL, &clperm);CHKERRQ(ierr);
4504   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
4505   /* Get array */
4506   ierr = VecGetArray(v, &array);CHKERRQ(ierr);
4507   /* Get values */
4508   if (numFields > 0) {
4509     PetscInt offset = 0, f;
4510     for (f = 0; f < numFields; ++f) {
4511       const PetscInt    **perms = NULL;
4512       const PetscScalar **flips = NULL;
4513 
4514       ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4515       switch (mode) {
4516       case INSERT_VALUES:
4517         for (p = 0; p < numPoints; p++) {
4518           const PetscInt    point = points[2*p];
4519           const PetscInt    *perm = perms ? perms[p] : NULL;
4520           const PetscScalar *flip = flips ? flips[p] : NULL;
4521           updatePointFields_private(section, point, perm, flip, f, insert, PETSC_FALSE, clperm, values, &offset, array);
4522         } break;
4523       case INSERT_ALL_VALUES:
4524         for (p = 0; p < numPoints; p++) {
4525           const PetscInt    point = points[2*p];
4526           const PetscInt    *perm = perms ? perms[p] : NULL;
4527           const PetscScalar *flip = flips ? flips[p] : NULL;
4528           updatePointFields_private(section, point, perm, flip, f, insert, PETSC_TRUE, clperm, values, &offset, array);
4529         } break;
4530       case INSERT_BC_VALUES:
4531         for (p = 0; p < numPoints; p++) {
4532           const PetscInt    point = points[2*p];
4533           const PetscInt    *perm = perms ? perms[p] : NULL;
4534           const PetscScalar *flip = flips ? flips[p] : NULL;
4535           updatePointFieldsBC_private(section, point, perm, flip, f, -1, NULL, insert, clperm, values, &offset, array);
4536         } break;
4537       case ADD_VALUES:
4538         for (p = 0; p < numPoints; p++) {
4539           const PetscInt    point = points[2*p];
4540           const PetscInt    *perm = perms ? perms[p] : NULL;
4541           const PetscScalar *flip = flips ? flips[p] : NULL;
4542           updatePointFields_private(section, point, perm, flip, f, add, PETSC_FALSE, clperm, values, &offset, array);
4543         } break;
4544       case ADD_ALL_VALUES:
4545         for (p = 0; p < numPoints; p++) {
4546           const PetscInt    point = points[2*p];
4547           const PetscInt    *perm = perms ? perms[p] : NULL;
4548           const PetscScalar *flip = flips ? flips[p] : NULL;
4549           updatePointFields_private(section, point, perm, flip, f, add, PETSC_TRUE, clperm, values, &offset, array);
4550         } break;
4551       case ADD_BC_VALUES:
4552         for (p = 0; p < numPoints; p++) {
4553           const PetscInt    point = points[2*p];
4554           const PetscInt    *perm = perms ? perms[p] : NULL;
4555           const PetscScalar *flip = flips ? flips[p] : NULL;
4556           updatePointFieldsBC_private(section, point, perm, flip, f, -1, NULL, add, clperm, values, &offset, array);
4557         } break;
4558       default:
4559         SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %d", mode);
4560       }
4561       ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4562     }
4563   } else {
4564     PetscInt dof, off;
4565     const PetscInt    **perms = NULL;
4566     const PetscScalar **flips = NULL;
4567 
4568     ierr = PetscSectionGetPointSyms(section,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4569     switch (mode) {
4570     case INSERT_VALUES:
4571       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
4572         const PetscInt    point = points[2*p];
4573         const PetscInt    *perm = perms ? perms[p] : NULL;
4574         const PetscScalar *flip = flips ? flips[p] : NULL;
4575         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4576         updatePoint_private(section, point, dof, insert, PETSC_FALSE, perm, flip, clperm, values, off, array);
4577       } break;
4578     case INSERT_ALL_VALUES:
4579       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
4580         const PetscInt    point = points[2*p];
4581         const PetscInt    *perm = perms ? perms[p] : NULL;
4582         const PetscScalar *flip = flips ? flips[p] : NULL;
4583         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4584         updatePoint_private(section, point, dof, insert, PETSC_TRUE,  perm, flip, clperm, values, off, array);
4585       } break;
4586     case INSERT_BC_VALUES:
4587       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
4588         const PetscInt    point = points[2*p];
4589         const PetscInt    *perm = perms ? perms[p] : NULL;
4590         const PetscScalar *flip = flips ? flips[p] : NULL;
4591         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4592         updatePointBC_private(section, point, dof, insert,  perm, flip, clperm, values, off, array);
4593       } break;
4594     case ADD_VALUES:
4595       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
4596         const PetscInt    point = points[2*p];
4597         const PetscInt    *perm = perms ? perms[p] : NULL;
4598         const PetscScalar *flip = flips ? flips[p] : NULL;
4599         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4600         updatePoint_private(section, point, dof, add,    PETSC_FALSE, perm, flip, clperm, values, off, array);
4601       } break;
4602     case ADD_ALL_VALUES:
4603       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
4604         const PetscInt    point = points[2*p];
4605         const PetscInt    *perm = perms ? perms[p] : NULL;
4606         const PetscScalar *flip = flips ? flips[p] : NULL;
4607         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4608         updatePoint_private(section, point, dof, add,    PETSC_TRUE,  perm, flip, clperm, values, off, array);
4609       } break;
4610     case ADD_BC_VALUES:
4611       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
4612         const PetscInt    point = points[2*p];
4613         const PetscInt    *perm = perms ? perms[p] : NULL;
4614         const PetscScalar *flip = flips ? flips[p] : NULL;
4615         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4616         updatePointBC_private(section, point, dof, add,  perm, flip, clperm, values, off, array);
4617       } break;
4618     default:
4619       SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %d", mode);
4620     }
4621     ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4622   }
4623   /* Cleanup points */
4624   ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
4625   /* Cleanup array */
4626   ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
4627   PetscFunctionReturn(0);
4628 }
4629 
4630 PetscErrorCode DMPlexVecSetFieldClosure_Internal(DM dm, PetscSection section, Vec v, PetscBool fieldActive[], PetscInt point, PetscInt Ncc, const PetscInt comps[], const PetscScalar values[], InsertMode mode)
4631 {
4632   PetscSection      clSection;
4633   IS                clPoints;
4634   PetscScalar       *array;
4635   PetscInt          *points = NULL;
4636   const PetscInt    *clp, *clperm;
4637   PetscInt          numFields, numPoints, p;
4638   PetscInt          offset = 0, f;
4639   PetscErrorCode    ierr;
4640 
4641   PetscFunctionBeginHot;
4642   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4643   if (!section) {ierr = DMGetDefaultSection(dm, &section);CHKERRQ(ierr);}
4644   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
4645   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
4646   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
4647   /* Get points */
4648   ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, NULL, &clperm);CHKERRQ(ierr);
4649   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
4650   /* Get array */
4651   ierr = VecGetArray(v, &array);CHKERRQ(ierr);
4652   /* Get values */
4653   for (f = 0; f < numFields; ++f) {
4654     const PetscInt    **perms = NULL;
4655     const PetscScalar **flips = NULL;
4656 
4657     if (!fieldActive[f]) {
4658       for (p = 0; p < numPoints*2; p += 2) {
4659         PetscInt fdof;
4660         ierr = PetscSectionGetFieldDof(section, points[p], f, &fdof);CHKERRQ(ierr);
4661         offset += fdof;
4662       }
4663       continue;
4664     }
4665     ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4666     switch (mode) {
4667     case INSERT_VALUES:
4668       for (p = 0; p < numPoints; p++) {
4669         const PetscInt    point = points[2*p];
4670         const PetscInt    *perm = perms ? perms[p] : NULL;
4671         const PetscScalar *flip = flips ? flips[p] : NULL;
4672         updatePointFields_private(section, point, perm, flip, f, insert, PETSC_FALSE, clperm, values, &offset, array);
4673       } break;
4674     case INSERT_ALL_VALUES:
4675       for (p = 0; p < numPoints; p++) {
4676         const PetscInt    point = points[2*p];
4677         const PetscInt    *perm = perms ? perms[p] : NULL;
4678         const PetscScalar *flip = flips ? flips[p] : NULL;
4679         updatePointFields_private(section, point, perm, flip, f, insert, PETSC_TRUE, clperm, values, &offset, array);
4680         } break;
4681     case INSERT_BC_VALUES:
4682       for (p = 0; p < numPoints; p++) {
4683         const PetscInt    point = points[2*p];
4684         const PetscInt    *perm = perms ? perms[p] : NULL;
4685         const PetscScalar *flip = flips ? flips[p] : NULL;
4686         updatePointFieldsBC_private(section, point, perm, flip, f, Ncc, comps, insert, clperm, values, &offset, array);
4687       } break;
4688     case ADD_VALUES:
4689       for (p = 0; p < numPoints; p++) {
4690         const PetscInt    point = points[2*p];
4691         const PetscInt    *perm = perms ? perms[p] : NULL;
4692         const PetscScalar *flip = flips ? flips[p] : NULL;
4693         updatePointFields_private(section, point, perm, flip, f, add, PETSC_FALSE, clperm, values, &offset, array);
4694       } break;
4695     case ADD_ALL_VALUES:
4696       for (p = 0; p < numPoints; p++) {
4697         const PetscInt    point = points[2*p];
4698         const PetscInt    *perm = perms ? perms[p] : NULL;
4699         const PetscScalar *flip = flips ? flips[p] : NULL;
4700         updatePointFields_private(section, point, perm, flip, f, add, PETSC_TRUE, clperm, values, &offset, array);
4701       } break;
4702     default:
4703       SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %d", mode);
4704     }
4705     ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4706   }
4707   /* Cleanup points */
4708   ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
4709   /* Cleanup array */
4710   ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
4711   PetscFunctionReturn(0);
4712 }
4713 
4714 static PetscErrorCode DMPlexPrintMatSetValues(PetscViewer viewer, Mat A, PetscInt point, PetscInt numRIndices, const PetscInt rindices[], PetscInt numCIndices, const PetscInt cindices[], const PetscScalar values[])
4715 {
4716   PetscMPIInt    rank;
4717   PetscInt       i, j;
4718   PetscErrorCode ierr;
4719 
4720   PetscFunctionBegin;
4721   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRQ(ierr);
4722   ierr = PetscViewerASCIIPrintf(viewer, "[%d]mat for point %D\n", rank, point);CHKERRQ(ierr);
4723   for (i = 0; i < numRIndices; i++) {ierr = PetscViewerASCIIPrintf(viewer, "[%d]mat row indices[%D] = %D\n", rank, i, rindices[i]);CHKERRQ(ierr);}
4724   for (i = 0; i < numCIndices; i++) {ierr = PetscViewerASCIIPrintf(viewer, "[%d]mat col indices[%D] = %D\n", rank, i, cindices[i]);CHKERRQ(ierr);}
4725   numCIndices = numCIndices ? numCIndices : numRIndices;
4726   for (i = 0; i < numRIndices; i++) {
4727     ierr = PetscViewerASCIIPrintf(viewer, "[%d]", rank);CHKERRQ(ierr);
4728     for (j = 0; j < numCIndices; j++) {
4729 #if defined(PETSC_USE_COMPLEX)
4730       ierr = PetscViewerASCIIPrintf(viewer, " (%g,%g)", (double)PetscRealPart(values[i*numCIndices+j]), (double)PetscImaginaryPart(values[i*numCIndices+j]));CHKERRQ(ierr);
4731 #else
4732       ierr = PetscViewerASCIIPrintf(viewer, " %g", (double)values[i*numCIndices+j]);CHKERRQ(ierr);
4733 #endif
4734     }
4735     ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
4736   }
4737   PetscFunctionReturn(0);
4738 }
4739 
4740 /* . off - The global offset of this point */
4741 PetscErrorCode DMPlexGetIndicesPoint_Internal(PetscSection section, PetscInt point, PetscInt off, PetscInt *loff, PetscBool setBC, const PetscInt perm[], PetscInt indices[])
4742 {
4743   PetscInt        dof;    /* The number of unknowns on this point */
4744   PetscInt        cdof;   /* The number of constraints on this point */
4745   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
4746   PetscInt        cind = 0, k;
4747   PetscErrorCode  ierr;
4748 
4749   PetscFunctionBegin;
4750   ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4751   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
4752   if (!cdof || setBC) {
4753     if (perm) {
4754       for (k = 0; k < dof; k++) indices[*loff+perm[k]] = off + k;
4755     } else {
4756       for (k = 0; k < dof; k++) indices[*loff+k] = off + k;
4757     }
4758   } else {
4759     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
4760     if (perm) {
4761       for (k = 0; k < dof; ++k) {
4762         if ((cind < cdof) && (k == cdofs[cind])) {
4763           /* Insert check for returning constrained indices */
4764           indices[*loff+perm[k]] = -(off+k+1);
4765           ++cind;
4766         } else {
4767           indices[*loff+perm[k]] = off+k-cind;
4768         }
4769       }
4770     } else {
4771       for (k = 0; k < dof; ++k) {
4772         if ((cind < cdof) && (k == cdofs[cind])) {
4773           /* Insert check for returning constrained indices */
4774           indices[*loff+k] = -(off+k+1);
4775           ++cind;
4776         } else {
4777           indices[*loff+k] = off+k-cind;
4778         }
4779       }
4780     }
4781   }
4782   *loff += dof;
4783   PetscFunctionReturn(0);
4784 }
4785 
4786 /* . off - The global offset of this point */
4787 PetscErrorCode DMPlexGetIndicesPointFields_Internal(PetscSection section, PetscInt point, PetscInt off, PetscInt foffs[], PetscBool setBC, const PetscInt ***perms, PetscInt permsoff, PetscInt indices[])
4788 {
4789   PetscInt       numFields, foff, f;
4790   PetscErrorCode ierr;
4791 
4792   PetscFunctionBegin;
4793   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
4794   for (f = 0, foff = 0; f < numFields; ++f) {
4795     PetscInt        fdof, cfdof;
4796     const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
4797     PetscInt        cind = 0, b;
4798     const PetscInt  *perm = (perms && perms[f]) ? perms[f][permsoff] : NULL;
4799 
4800     ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
4801     ierr = PetscSectionGetFieldConstraintDof(section, point, f, &cfdof);CHKERRQ(ierr);
4802     if (!cfdof || setBC) {
4803       if (perm) {for (b = 0; b < fdof; b++) {indices[foffs[f]+perm[b]] = off+foff+b;}}
4804       else      {for (b = 0; b < fdof; b++) {indices[foffs[f]+     b ] = off+foff+b;}}
4805     } else {
4806       ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
4807       if (perm) {
4808         for (b = 0; b < fdof; b++) {
4809           if ((cind < cfdof) && (b == fcdofs[cind])) {
4810             indices[foffs[f]+perm[b]] = -(off+foff+b+1);
4811             ++cind;
4812           } else {
4813             indices[foffs[f]+perm[b]] = off+foff+b-cind;
4814           }
4815         }
4816       } else {
4817         for (b = 0; b < fdof; b++) {
4818           if ((cind < cfdof) && (b == fcdofs[cind])) {
4819             indices[foffs[f]+b] = -(off+foff+b+1);
4820             ++cind;
4821           } else {
4822             indices[foffs[f]+b] = off+foff+b-cind;
4823           }
4824         }
4825       }
4826     }
4827     foff     += (setBC ? fdof : (fdof - cfdof));
4828     foffs[f] += fdof;
4829   }
4830   PetscFunctionReturn(0);
4831 }
4832 
4833 PetscErrorCode DMPlexAnchorsModifyMat(DM dm, PetscSection section, PetscInt numPoints, PetscInt numIndices, const PetscInt points[], const PetscInt ***perms, const PetscScalar values[], PetscInt *outNumPoints, PetscInt *outNumIndices, PetscInt *outPoints[], PetscScalar *outValues[], PetscInt offsets[], PetscBool multiplyLeft)
4834 {
4835   Mat             cMat;
4836   PetscSection    aSec, cSec;
4837   IS              aIS;
4838   PetscInt        aStart = -1, aEnd = -1;
4839   const PetscInt  *anchors;
4840   PetscInt        numFields, f, p, q, newP = 0;
4841   PetscInt        newNumPoints = 0, newNumIndices = 0;
4842   PetscInt        *newPoints, *indices, *newIndices;
4843   PetscInt        maxAnchor, maxDof;
4844   PetscInt        newOffsets[32];
4845   PetscInt        *pointMatOffsets[32];
4846   PetscInt        *newPointOffsets[32];
4847   PetscScalar     *pointMat[32];
4848   PetscScalar     *newValues=NULL,*tmpValues;
4849   PetscBool       anyConstrained = PETSC_FALSE;
4850   PetscErrorCode  ierr;
4851 
4852   PetscFunctionBegin;
4853   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4854   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
4855   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
4856 
4857   ierr = DMPlexGetAnchors(dm,&aSec,&aIS);CHKERRQ(ierr);
4858   /* if there are point-to-point constraints */
4859   if (aSec) {
4860     ierr = PetscMemzero(newOffsets, 32 * sizeof(PetscInt));CHKERRQ(ierr);
4861     ierr = ISGetIndices(aIS,&anchors);CHKERRQ(ierr);
4862     ierr = PetscSectionGetChart(aSec,&aStart,&aEnd);CHKERRQ(ierr);
4863     /* figure out how many points are going to be in the new element matrix
4864      * (we allow double counting, because it's all just going to be summed
4865      * into the global matrix anyway) */
4866     for (p = 0; p < 2*numPoints; p+=2) {
4867       PetscInt b    = points[p];
4868       PetscInt bDof = 0, bSecDof;
4869 
4870       ierr = PetscSectionGetDof(section,b,&bSecDof);CHKERRQ(ierr);
4871       if (!bSecDof) {
4872         continue;
4873       }
4874       if (b >= aStart && b < aEnd) {
4875         ierr = PetscSectionGetDof(aSec,b,&bDof);CHKERRQ(ierr);
4876       }
4877       if (bDof) {
4878         /* this point is constrained */
4879         /* it is going to be replaced by its anchors */
4880         PetscInt bOff, q;
4881 
4882         anyConstrained = PETSC_TRUE;
4883         newNumPoints  += bDof;
4884         ierr = PetscSectionGetOffset(aSec,b,&bOff);CHKERRQ(ierr);
4885         for (q = 0; q < bDof; q++) {
4886           PetscInt a = anchors[bOff + q];
4887           PetscInt aDof;
4888 
4889           ierr           = PetscSectionGetDof(section,a,&aDof);CHKERRQ(ierr);
4890           newNumIndices += aDof;
4891           for (f = 0; f < numFields; ++f) {
4892             PetscInt fDof;
4893 
4894             ierr             = PetscSectionGetFieldDof(section, a, f, &fDof);CHKERRQ(ierr);
4895             newOffsets[f+1] += fDof;
4896           }
4897         }
4898       }
4899       else {
4900         /* this point is not constrained */
4901         newNumPoints++;
4902         newNumIndices += bSecDof;
4903         for (f = 0; f < numFields; ++f) {
4904           PetscInt fDof;
4905 
4906           ierr = PetscSectionGetFieldDof(section, b, f, &fDof);CHKERRQ(ierr);
4907           newOffsets[f+1] += fDof;
4908         }
4909       }
4910     }
4911   }
4912   if (!anyConstrained) {
4913     if (outNumPoints)  *outNumPoints  = 0;
4914     if (outNumIndices) *outNumIndices = 0;
4915     if (outPoints)     *outPoints     = NULL;
4916     if (outValues)     *outValues     = NULL;
4917     if (aSec) {ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);}
4918     PetscFunctionReturn(0);
4919   }
4920 
4921   if (outNumPoints)  *outNumPoints  = newNumPoints;
4922   if (outNumIndices) *outNumIndices = newNumIndices;
4923 
4924   for (f = 0; f < numFields; ++f) newOffsets[f+1] += newOffsets[f];
4925 
4926   if (!outPoints && !outValues) {
4927     if (offsets) {
4928       for (f = 0; f <= numFields; f++) {
4929         offsets[f] = newOffsets[f];
4930       }
4931     }
4932     if (aSec) {ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);}
4933     PetscFunctionReturn(0);
4934   }
4935 
4936   if (numFields && newOffsets[numFields] != newNumIndices) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", newOffsets[numFields], newNumIndices);
4937 
4938   ierr = DMGetDefaultConstraints(dm, &cSec, &cMat);CHKERRQ(ierr);
4939 
4940   /* workspaces */
4941   if (numFields) {
4942     for (f = 0; f < numFields; f++) {
4943       ierr = DMGetWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[f]);CHKERRQ(ierr);
4944       ierr = DMGetWorkArray(dm,numPoints+1,MPIU_INT,&newPointOffsets[f]);CHKERRQ(ierr);
4945     }
4946   }
4947   else {
4948     ierr = DMGetWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[0]);CHKERRQ(ierr);
4949     ierr = DMGetWorkArray(dm,numPoints,MPIU_INT,&newPointOffsets[0]);CHKERRQ(ierr);
4950   }
4951 
4952   /* get workspaces for the point-to-point matrices */
4953   if (numFields) {
4954     PetscInt totalOffset, totalMatOffset;
4955 
4956     for (p = 0; p < numPoints; p++) {
4957       PetscInt b    = points[2*p];
4958       PetscInt bDof = 0, bSecDof;
4959 
4960       ierr = PetscSectionGetDof(section,b,&bSecDof);CHKERRQ(ierr);
4961       if (!bSecDof) {
4962         for (f = 0; f < numFields; f++) {
4963           newPointOffsets[f][p + 1] = 0;
4964           pointMatOffsets[f][p + 1] = 0;
4965         }
4966         continue;
4967       }
4968       if (b >= aStart && b < aEnd) {
4969         ierr = PetscSectionGetDof(aSec, b, &bDof);CHKERRQ(ierr);
4970       }
4971       if (bDof) {
4972         for (f = 0; f < numFields; f++) {
4973           PetscInt fDof, q, bOff, allFDof = 0;
4974 
4975           ierr = PetscSectionGetFieldDof(section, b, f, &fDof);CHKERRQ(ierr);
4976           ierr = PetscSectionGetOffset(aSec, b, &bOff);CHKERRQ(ierr);
4977           for (q = 0; q < bDof; q++) {
4978             PetscInt a = anchors[bOff + q];
4979             PetscInt aFDof;
4980 
4981             ierr     = PetscSectionGetFieldDof(section, a, f, &aFDof);CHKERRQ(ierr);
4982             allFDof += aFDof;
4983           }
4984           newPointOffsets[f][p+1] = allFDof;
4985           pointMatOffsets[f][p+1] = fDof * allFDof;
4986         }
4987       }
4988       else {
4989         for (f = 0; f < numFields; f++) {
4990           PetscInt fDof;
4991 
4992           ierr = PetscSectionGetFieldDof(section, b, f, &fDof);CHKERRQ(ierr);
4993           newPointOffsets[f][p+1] = fDof;
4994           pointMatOffsets[f][p+1] = 0;
4995         }
4996       }
4997     }
4998     for (f = 0, totalOffset = 0, totalMatOffset = 0; f < numFields; f++) {
4999       newPointOffsets[f][0] = totalOffset;
5000       pointMatOffsets[f][0] = totalMatOffset;
5001       for (p = 0; p < numPoints; p++) {
5002         newPointOffsets[f][p+1] += newPointOffsets[f][p];
5003         pointMatOffsets[f][p+1] += pointMatOffsets[f][p];
5004       }
5005       totalOffset    = newPointOffsets[f][numPoints];
5006       totalMatOffset = pointMatOffsets[f][numPoints];
5007       ierr = DMGetWorkArray(dm,pointMatOffsets[f][numPoints],MPIU_SCALAR,&pointMat[f]);CHKERRQ(ierr);
5008     }
5009   }
5010   else {
5011     for (p = 0; p < numPoints; p++) {
5012       PetscInt b    = points[2*p];
5013       PetscInt bDof = 0, bSecDof;
5014 
5015       ierr = PetscSectionGetDof(section,b,&bSecDof);CHKERRQ(ierr);
5016       if (!bSecDof) {
5017         newPointOffsets[0][p + 1] = 0;
5018         pointMatOffsets[0][p + 1] = 0;
5019         continue;
5020       }
5021       if (b >= aStart && b < aEnd) {
5022         ierr = PetscSectionGetDof(aSec, b, &bDof);CHKERRQ(ierr);
5023       }
5024       if (bDof) {
5025         PetscInt bOff, q, allDof = 0;
5026 
5027         ierr = PetscSectionGetOffset(aSec, b, &bOff);CHKERRQ(ierr);
5028         for (q = 0; q < bDof; q++) {
5029           PetscInt a = anchors[bOff + q], aDof;
5030 
5031           ierr    = PetscSectionGetDof(section, a, &aDof);CHKERRQ(ierr);
5032           allDof += aDof;
5033         }
5034         newPointOffsets[0][p+1] = allDof;
5035         pointMatOffsets[0][p+1] = bSecDof * allDof;
5036       }
5037       else {
5038         newPointOffsets[0][p+1] = bSecDof;
5039         pointMatOffsets[0][p+1] = 0;
5040       }
5041     }
5042     newPointOffsets[0][0] = 0;
5043     pointMatOffsets[0][0] = 0;
5044     for (p = 0; p < numPoints; p++) {
5045       newPointOffsets[0][p+1] += newPointOffsets[0][p];
5046       pointMatOffsets[0][p+1] += pointMatOffsets[0][p];
5047     }
5048     ierr = DMGetWorkArray(dm,pointMatOffsets[0][numPoints],MPIU_SCALAR,&pointMat[0]);CHKERRQ(ierr);
5049   }
5050 
5051   /* output arrays */
5052   ierr = DMGetWorkArray(dm,2*newNumPoints,MPIU_INT,&newPoints);CHKERRQ(ierr);
5053 
5054   /* get the point-to-point matrices; construct newPoints */
5055   ierr = PetscSectionGetMaxDof(aSec, &maxAnchor);CHKERRQ(ierr);
5056   ierr = PetscSectionGetMaxDof(section, &maxDof);CHKERRQ(ierr);
5057   ierr = DMGetWorkArray(dm,maxDof,MPIU_INT,&indices);CHKERRQ(ierr);
5058   ierr = DMGetWorkArray(dm,maxAnchor*maxDof,MPIU_INT,&newIndices);CHKERRQ(ierr);
5059   if (numFields) {
5060     for (p = 0, newP = 0; p < numPoints; p++) {
5061       PetscInt b    = points[2*p];
5062       PetscInt o    = points[2*p+1];
5063       PetscInt bDof = 0, bSecDof;
5064 
5065       ierr = PetscSectionGetDof(section, b, &bSecDof);CHKERRQ(ierr);
5066       if (!bSecDof) {
5067         continue;
5068       }
5069       if (b >= aStart && b < aEnd) {
5070         ierr = PetscSectionGetDof(aSec, b, &bDof);CHKERRQ(ierr);
5071       }
5072       if (bDof) {
5073         PetscInt fStart[32], fEnd[32], fAnchorStart[32], fAnchorEnd[32], bOff, q;
5074 
5075         fStart[0] = 0;
5076         fEnd[0]   = 0;
5077         for (f = 0; f < numFields; f++) {
5078           PetscInt fDof;
5079 
5080           ierr        = PetscSectionGetFieldDof(cSec, b, f, &fDof);CHKERRQ(ierr);
5081           fStart[f+1] = fStart[f] + fDof;
5082           fEnd[f+1]   = fStart[f+1];
5083         }
5084         ierr = PetscSectionGetOffset(cSec, b, &bOff);CHKERRQ(ierr);
5085         ierr = DMPlexGetIndicesPointFields_Internal(cSec, b, bOff, fEnd, PETSC_TRUE, perms, p, indices);CHKERRQ(ierr);
5086 
5087         fAnchorStart[0] = 0;
5088         fAnchorEnd[0]   = 0;
5089         for (f = 0; f < numFields; f++) {
5090           PetscInt fDof = newPointOffsets[f][p + 1] - newPointOffsets[f][p];
5091 
5092           fAnchorStart[f+1] = fAnchorStart[f] + fDof;
5093           fAnchorEnd[f+1]   = fAnchorStart[f + 1];
5094         }
5095         ierr = PetscSectionGetOffset(aSec, b, &bOff);CHKERRQ(ierr);
5096         for (q = 0; q < bDof; q++) {
5097           PetscInt a = anchors[bOff + q], aOff;
5098 
5099           /* we take the orientation of ap into account in the order that we constructed the indices above: the newly added points have no orientation */
5100           newPoints[2*(newP + q)]     = a;
5101           newPoints[2*(newP + q) + 1] = 0;
5102           ierr = PetscSectionGetOffset(section, a, &aOff);CHKERRQ(ierr);
5103           ierr = DMPlexGetIndicesPointFields_Internal(section, a, aOff, fAnchorEnd, PETSC_TRUE, NULL, -1, newIndices);CHKERRQ(ierr);
5104         }
5105         newP += bDof;
5106 
5107         if (outValues) {
5108           /* get the point-to-point submatrix */
5109           for (f = 0; f < numFields; f++) {
5110             ierr = MatGetValues(cMat,fEnd[f]-fStart[f],indices + fStart[f],fAnchorEnd[f] - fAnchorStart[f],newIndices + fAnchorStart[f],pointMat[f] + pointMatOffsets[f][p]);CHKERRQ(ierr);
5111           }
5112         }
5113       }
5114       else {
5115         newPoints[2 * newP]     = b;
5116         newPoints[2 * newP + 1] = o;
5117         newP++;
5118       }
5119     }
5120   } else {
5121     for (p = 0; p < numPoints; p++) {
5122       PetscInt b    = points[2*p];
5123       PetscInt o    = points[2*p+1];
5124       PetscInt bDof = 0, bSecDof;
5125 
5126       ierr = PetscSectionGetDof(section, b, &bSecDof);CHKERRQ(ierr);
5127       if (!bSecDof) {
5128         continue;
5129       }
5130       if (b >= aStart && b < aEnd) {
5131         ierr = PetscSectionGetDof(aSec, b, &bDof);CHKERRQ(ierr);
5132       }
5133       if (bDof) {
5134         PetscInt bEnd = 0, bAnchorEnd = 0, bOff;
5135 
5136         ierr = PetscSectionGetOffset(cSec, b, &bOff);CHKERRQ(ierr);
5137         ierr = DMPlexGetIndicesPoint_Internal(cSec, b, bOff, &bEnd, PETSC_TRUE, (perms && perms[0]) ? perms[0][p] : NULL, indices);CHKERRQ(ierr);
5138 
5139         ierr = PetscSectionGetOffset (aSec, b, &bOff);CHKERRQ(ierr);
5140         for (q = 0; q < bDof; q++) {
5141           PetscInt a = anchors[bOff + q], aOff;
5142 
5143           /* we take the orientation of ap into account in the order that we constructed the indices above: the newly added points have no orientation */
5144 
5145           newPoints[2*(newP + q)]     = a;
5146           newPoints[2*(newP + q) + 1] = 0;
5147           ierr = PetscSectionGetOffset(section, a, &aOff);CHKERRQ(ierr);
5148           ierr = DMPlexGetIndicesPoint_Internal(section, a, aOff, &bAnchorEnd, PETSC_TRUE, NULL, newIndices);CHKERRQ(ierr);
5149         }
5150         newP += bDof;
5151 
5152         /* get the point-to-point submatrix */
5153         if (outValues) {
5154           ierr = MatGetValues(cMat,bEnd,indices,bAnchorEnd,newIndices,pointMat[0] + pointMatOffsets[0][p]);CHKERRQ(ierr);
5155         }
5156       }
5157       else {
5158         newPoints[2 * newP]     = b;
5159         newPoints[2 * newP + 1] = o;
5160         newP++;
5161       }
5162     }
5163   }
5164 
5165   if (outValues) {
5166     ierr = DMGetWorkArray(dm,newNumIndices*numIndices,MPIU_SCALAR,&tmpValues);CHKERRQ(ierr);
5167     ierr = PetscMemzero(tmpValues,newNumIndices*numIndices*sizeof(*tmpValues));CHKERRQ(ierr);
5168     /* multiply constraints on the right */
5169     if (numFields) {
5170       for (f = 0; f < numFields; f++) {
5171         PetscInt oldOff = offsets[f];
5172 
5173         for (p = 0; p < numPoints; p++) {
5174           PetscInt cStart = newPointOffsets[f][p];
5175           PetscInt b      = points[2 * p];
5176           PetscInt c, r, k;
5177           PetscInt dof;
5178 
5179           ierr = PetscSectionGetFieldDof(section,b,f,&dof);CHKERRQ(ierr);
5180           if (!dof) {
5181             continue;
5182           }
5183           if (pointMatOffsets[f][p] < pointMatOffsets[f][p + 1]) {
5184             PetscInt nCols         = newPointOffsets[f][p+1]-cStart;
5185             const PetscScalar *mat = pointMat[f] + pointMatOffsets[f][p];
5186 
5187             for (r = 0; r < numIndices; r++) {
5188               for (c = 0; c < nCols; c++) {
5189                 for (k = 0; k < dof; k++) {
5190                   tmpValues[r * newNumIndices + cStart + c] += values[r * numIndices + oldOff + k] * mat[k * nCols + c];
5191                 }
5192               }
5193             }
5194           }
5195           else {
5196             /* copy this column as is */
5197             for (r = 0; r < numIndices; r++) {
5198               for (c = 0; c < dof; c++) {
5199                 tmpValues[r * newNumIndices + cStart + c] = values[r * numIndices + oldOff + c];
5200               }
5201             }
5202           }
5203           oldOff += dof;
5204         }
5205       }
5206     }
5207     else {
5208       PetscInt oldOff = 0;
5209       for (p = 0; p < numPoints; p++) {
5210         PetscInt cStart = newPointOffsets[0][p];
5211         PetscInt b      = points[2 * p];
5212         PetscInt c, r, k;
5213         PetscInt dof;
5214 
5215         ierr = PetscSectionGetDof(section,b,&dof);CHKERRQ(ierr);
5216         if (!dof) {
5217           continue;
5218         }
5219         if (pointMatOffsets[0][p] < pointMatOffsets[0][p + 1]) {
5220           PetscInt nCols         = newPointOffsets[0][p+1]-cStart;
5221           const PetscScalar *mat = pointMat[0] + pointMatOffsets[0][p];
5222 
5223           for (r = 0; r < numIndices; r++) {
5224             for (c = 0; c < nCols; c++) {
5225               for (k = 0; k < dof; k++) {
5226                 tmpValues[r * newNumIndices + cStart + c] += mat[k * nCols + c] * values[r * numIndices + oldOff + k];
5227               }
5228             }
5229           }
5230         }
5231         else {
5232           /* copy this column as is */
5233           for (r = 0; r < numIndices; r++) {
5234             for (c = 0; c < dof; c++) {
5235               tmpValues[r * newNumIndices + cStart + c] = values[r * numIndices + oldOff + c];
5236             }
5237           }
5238         }
5239         oldOff += dof;
5240       }
5241     }
5242 
5243     if (multiplyLeft) {
5244       ierr = DMGetWorkArray(dm,newNumIndices*newNumIndices,MPIU_SCALAR,&newValues);CHKERRQ(ierr);
5245       ierr = PetscMemzero(newValues,newNumIndices*newNumIndices*sizeof(*newValues));CHKERRQ(ierr);
5246       /* multiply constraints transpose on the left */
5247       if (numFields) {
5248         for (f = 0; f < numFields; f++) {
5249           PetscInt oldOff = offsets[f];
5250 
5251           for (p = 0; p < numPoints; p++) {
5252             PetscInt rStart = newPointOffsets[f][p];
5253             PetscInt b      = points[2 * p];
5254             PetscInt c, r, k;
5255             PetscInt dof;
5256 
5257             ierr = PetscSectionGetFieldDof(section,b,f,&dof);CHKERRQ(ierr);
5258             if (pointMatOffsets[f][p] < pointMatOffsets[f][p + 1]) {
5259               PetscInt nRows                        = newPointOffsets[f][p+1]-rStart;
5260               const PetscScalar *PETSC_RESTRICT mat = pointMat[f] + pointMatOffsets[f][p];
5261 
5262               for (r = 0; r < nRows; r++) {
5263                 for (c = 0; c < newNumIndices; c++) {
5264                   for (k = 0; k < dof; k++) {
5265                     newValues[(rStart + r) * newNumIndices + c] += mat[k * nRows + r] * tmpValues[(oldOff + k) * newNumIndices + c];
5266                   }
5267                 }
5268               }
5269             }
5270             else {
5271               /* copy this row as is */
5272               for (r = 0; r < dof; r++) {
5273                 for (c = 0; c < newNumIndices; c++) {
5274                   newValues[(rStart + r) * newNumIndices + c] = tmpValues[(oldOff + r) * newNumIndices + c];
5275                 }
5276               }
5277             }
5278             oldOff += dof;
5279           }
5280         }
5281       }
5282       else {
5283         PetscInt oldOff = 0;
5284 
5285         for (p = 0; p < numPoints; p++) {
5286           PetscInt rStart = newPointOffsets[0][p];
5287           PetscInt b      = points[2 * p];
5288           PetscInt c, r, k;
5289           PetscInt dof;
5290 
5291           ierr = PetscSectionGetDof(section,b,&dof);CHKERRQ(ierr);
5292           if (pointMatOffsets[0][p] < pointMatOffsets[0][p + 1]) {
5293             PetscInt nRows                        = newPointOffsets[0][p+1]-rStart;
5294             const PetscScalar *PETSC_RESTRICT mat = pointMat[0] + pointMatOffsets[0][p];
5295 
5296             for (r = 0; r < nRows; r++) {
5297               for (c = 0; c < newNumIndices; c++) {
5298                 for (k = 0; k < dof; k++) {
5299                   newValues[(rStart + r) * newNumIndices + c] += mat[k * nRows + r] * tmpValues[(oldOff + k) * newNumIndices + c];
5300                 }
5301               }
5302             }
5303           }
5304           else {
5305             /* copy this row as is */
5306             for (r = 0; r < dof; r++) {
5307               for (c = 0; c < newNumIndices; c++) {
5308                 newValues[(rStart + r) * newNumIndices + c] = tmpValues[(oldOff + r) * newNumIndices + c];
5309               }
5310             }
5311           }
5312           oldOff += dof;
5313         }
5314       }
5315 
5316       ierr = DMRestoreWorkArray(dm,newNumIndices*numIndices,MPIU_SCALAR,&tmpValues);CHKERRQ(ierr);
5317     }
5318     else {
5319       newValues = tmpValues;
5320     }
5321   }
5322 
5323   /* clean up */
5324   ierr = DMRestoreWorkArray(dm,maxDof,MPIU_INT,&indices);CHKERRQ(ierr);
5325   ierr = DMRestoreWorkArray(dm,maxAnchor*maxDof,MPIU_INT,&newIndices);CHKERRQ(ierr);
5326 
5327   if (numFields) {
5328     for (f = 0; f < numFields; f++) {
5329       ierr = DMRestoreWorkArray(dm,pointMatOffsets[f][numPoints],MPIU_SCALAR,&pointMat[f]);CHKERRQ(ierr);
5330       ierr = DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[f]);CHKERRQ(ierr);
5331       ierr = DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&newPointOffsets[f]);CHKERRQ(ierr);
5332     }
5333   }
5334   else {
5335     ierr = DMRestoreWorkArray(dm,pointMatOffsets[0][numPoints],MPIU_SCALAR,&pointMat[0]);CHKERRQ(ierr);
5336     ierr = DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[0]);CHKERRQ(ierr);
5337     ierr = DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&newPointOffsets[0]);CHKERRQ(ierr);
5338   }
5339   ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);
5340 
5341   /* output */
5342   if (outPoints) {
5343     *outPoints = newPoints;
5344   }
5345   else {
5346     ierr = DMRestoreWorkArray(dm,2*newNumPoints,MPIU_INT,&newPoints);CHKERRQ(ierr);
5347   }
5348   if (outValues) {
5349     *outValues = newValues;
5350   }
5351   for (f = 0; f <= numFields; f++) {
5352     offsets[f] = newOffsets[f];
5353   }
5354   PetscFunctionReturn(0);
5355 }
5356 
5357 /*@C
5358   DMPlexGetClosureIndices - Get the global indices in a vector v for all points in the closure of the given point
5359 
5360   Not collective
5361 
5362   Input Parameters:
5363 + dm - The DM
5364 . section - The section describing the layout in v, or NULL to use the default section
5365 . globalSection - The section describing the parallel layout in v, or NULL to use the default section
5366 - point - The mesh point
5367 
5368   Output parameters:
5369 + numIndices - The number of indices
5370 . indices - The indices
5371 - outOffsets - Field offset if not NULL
5372 
5373   Note: Must call DMPlexRestoreClosureIndices() to free allocated memory
5374 
5375   Level: advanced
5376 
5377 .seealso DMPlexRestoreClosureIndices(), DMPlexVecGetClosure(), DMPlexMatSetClosure()
5378 @*/
5379 PetscErrorCode DMPlexGetClosureIndices(DM dm, PetscSection section, PetscSection globalSection, PetscInt point, PetscInt *numIndices, PetscInt **indices, PetscInt *outOffsets)
5380 {
5381   PetscSection    clSection;
5382   IS              clPoints;
5383   const PetscInt *clp;
5384   const PetscInt  **perms[32] = {NULL};
5385   PetscInt       *points = NULL, *pointsNew;
5386   PetscInt        numPoints, numPointsNew;
5387   PetscInt        offsets[32];
5388   PetscInt        Nf, Nind, NindNew, off, globalOff, f, p;
5389   PetscErrorCode  ierr;
5390 
5391   PetscFunctionBegin;
5392   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5393   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
5394   PetscValidHeaderSpecific(globalSection, PETSC_SECTION_CLASSID, 3);
5395   if (numIndices) PetscValidPointer(numIndices, 4);
5396   PetscValidPointer(indices, 5);
5397   ierr = PetscSectionGetNumFields(section, &Nf);CHKERRQ(ierr);
5398   if (Nf > 31) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", Nf);
5399   ierr = PetscMemzero(offsets, 32 * sizeof(PetscInt));CHKERRQ(ierr);
5400   /* Get points in closure */
5401   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5402   /* Get number of indices and indices per field */
5403   for (p = 0, Nind = 0; p < numPoints*2; p += 2) {
5404     PetscInt dof, fdof;
5405 
5406     ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5407     for (f = 0; f < Nf; ++f) {
5408       ierr = PetscSectionGetFieldDof(section, points[p], f, &fdof);CHKERRQ(ierr);
5409       offsets[f+1] += fdof;
5410     }
5411     Nind += dof;
5412   }
5413   for (f = 1; f < Nf; ++f) offsets[f+1] += offsets[f];
5414   if (Nf && offsets[Nf] != Nind) SETERRQ2(PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", offsets[Nf], Nind);
5415   if (!Nf) offsets[1] = Nind;
5416   /* Get dual space symmetries */
5417   for (f = 0; f < PetscMax(1,Nf); f++) {
5418     if (Nf) {ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms[f],NULL);CHKERRQ(ierr);}
5419     else    {ierr = PetscSectionGetPointSyms(section,numPoints,points,&perms[f],NULL);CHKERRQ(ierr);}
5420   }
5421   /* Correct for hanging node constraints */
5422   {
5423     ierr = DMPlexAnchorsModifyMat(dm, section, numPoints, Nind, points, perms, NULL, &numPointsNew, &NindNew, &pointsNew, NULL, offsets, PETSC_TRUE);CHKERRQ(ierr);
5424     if (numPointsNew) {
5425       for (f = 0; f < PetscMax(1,Nf); f++) {
5426         if (Nf) {ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms[f],NULL);CHKERRQ(ierr);}
5427         else    {ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms[f],NULL);CHKERRQ(ierr);}
5428       }
5429       for (f = 0; f < PetscMax(1,Nf); f++) {
5430         if (Nf) {ierr = PetscSectionGetFieldPointSyms(section,f,numPointsNew,pointsNew,&perms[f],NULL);CHKERRQ(ierr);}
5431         else    {ierr = PetscSectionGetPointSyms(section,numPointsNew,pointsNew,&perms[f],NULL);CHKERRQ(ierr);}
5432       }
5433       ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5434       numPoints = numPointsNew;
5435       Nind      = NindNew;
5436       points    = pointsNew;
5437     }
5438   }
5439   /* Calculate indices */
5440   ierr = DMGetWorkArray(dm, Nind, MPIU_INT, indices);CHKERRQ(ierr);
5441   if (Nf) {
5442     if (outOffsets) {
5443       PetscInt f;
5444 
5445       for (f = 0; f <= Nf; f++) {
5446         outOffsets[f] = offsets[f];
5447       }
5448     }
5449     for (p = 0; p < numPoints; p++) {
5450       ierr = PetscSectionGetOffset(globalSection, points[2*p], &globalOff);CHKERRQ(ierr);
5451       DMPlexGetIndicesPointFields_Internal(section, points[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, offsets, PETSC_FALSE, perms, p, *indices);
5452     }
5453   } else {
5454     for (p = 0, off = 0; p < numPoints; p++) {
5455       const PetscInt *perm = perms[0] ? perms[0][p] : NULL;
5456 
5457       ierr = PetscSectionGetOffset(globalSection, points[2*p], &globalOff);CHKERRQ(ierr);
5458       DMPlexGetIndicesPoint_Internal(section, points[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, *indices);
5459     }
5460   }
5461   /* Cleanup points */
5462   for (f = 0; f < PetscMax(1,Nf); f++) {
5463     if (Nf) {ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms[f],NULL);CHKERRQ(ierr);}
5464     else    {ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms[f],NULL);CHKERRQ(ierr);}
5465   }
5466   if (numPointsNew) {
5467     ierr = DMRestoreWorkArray(dm, 2*numPointsNew, MPIU_INT, &pointsNew);CHKERRQ(ierr);
5468   } else {
5469     ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5470   }
5471   if (numIndices) *numIndices = Nind;
5472   PetscFunctionReturn(0);
5473 }
5474 
5475 /*@C
5476   DMPlexRestoreClosureIndices - Restore the indices in a vector v for all points in the closure of the given point
5477 
5478   Not collective
5479 
5480   Input Parameters:
5481 + dm - The DM
5482 . section - The section describing the layout in v, or NULL to use the default section
5483 . globalSection - The section describing the parallel layout in v, or NULL to use the default section
5484 . point - The mesh point
5485 . numIndices - The number of indices
5486 . indices - The indices
5487 - outOffsets - Field offset if not NULL
5488 
5489   Level: advanced
5490 
5491 .seealso DMPlexGetClosureIndices(), DMPlexVecGetClosure(), DMPlexMatSetClosure()
5492 @*/
5493 PetscErrorCode DMPlexRestoreClosureIndices(DM dm, PetscSection section, PetscSection globalSection, PetscInt point, PetscInt *numIndices, PetscInt **indices,PetscInt *outOffsets)
5494 {
5495   PetscErrorCode ierr;
5496 
5497   PetscFunctionBegin;
5498   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5499   PetscValidPointer(indices, 5);
5500   ierr = DMRestoreWorkArray(dm, 0, MPIU_INT, indices);CHKERRQ(ierr);
5501   PetscFunctionReturn(0);
5502 }
5503 
5504 /*@C
5505   DMPlexMatSetClosure - Set an array of the values on the closure of 'point'
5506 
5507   Not collective
5508 
5509   Input Parameters:
5510 + dm - The DM
5511 . section - The section describing the layout in v, or NULL to use the default section
5512 . globalSection - The section describing the layout in v, or NULL to use the default global section
5513 . A - The matrix
5514 . point - The point in the DM
5515 . values - The array of values
5516 - mode - The insert mode, where INSERT_ALL_VALUES and ADD_ALL_VALUES also overwrite boundary conditions
5517 
5518   Fortran Notes:
5519   This routine is only available in Fortran 90, and you must include petsc.h90 in your code.
5520 
5521   Level: intermediate
5522 
5523 .seealso DMPlexVecGetClosure(), DMPlexVecSetClosure()
5524 @*/
5525 PetscErrorCode DMPlexMatSetClosure(DM dm, PetscSection section, PetscSection globalSection, Mat A, PetscInt point, const PetscScalar values[], InsertMode mode)
5526 {
5527   DM_Plex            *mesh   = (DM_Plex*) dm->data;
5528   PetscSection        clSection;
5529   IS                  clPoints;
5530   PetscInt           *points = NULL, *newPoints;
5531   const PetscInt     *clp;
5532   PetscInt           *indices;
5533   PetscInt            offsets[32];
5534   const PetscInt    **perms[32] = {NULL};
5535   const PetscScalar **flips[32] = {NULL};
5536   PetscInt            numFields, numPoints, newNumPoints, numIndices, newNumIndices, dof, off, globalOff, p, f;
5537   PetscScalar        *valCopy = NULL;
5538   PetscScalar        *newValues;
5539   PetscErrorCode      ierr;
5540 
5541   PetscFunctionBegin;
5542   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5543   if (!section) {ierr = DMGetDefaultSection(dm, &section);CHKERRQ(ierr);}
5544   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
5545   if (!globalSection) {ierr = DMGetDefaultGlobalSection(dm, &globalSection);CHKERRQ(ierr);}
5546   PetscValidHeaderSpecific(globalSection, PETSC_SECTION_CLASSID, 3);
5547   PetscValidHeaderSpecific(A, MAT_CLASSID, 4);
5548   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5549   if (numFields > 31) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", numFields);
5550   ierr = PetscMemzero(offsets, 32 * sizeof(PetscInt));CHKERRQ(ierr);
5551   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5552   for (p = 0, numIndices = 0; p < numPoints*2; p += 2) {
5553     PetscInt fdof;
5554 
5555     ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5556     for (f = 0; f < numFields; ++f) {
5557       ierr          = PetscSectionGetFieldDof(section, points[p], f, &fdof);CHKERRQ(ierr);
5558       offsets[f+1] += fdof;
5559     }
5560     numIndices += dof;
5561   }
5562   for (f = 1; f < numFields; ++f) offsets[f+1] += offsets[f];
5563 
5564   if (numFields && offsets[numFields] != numIndices) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", offsets[numFields], numIndices);
5565   /* Get symmetries */
5566   for (f = 0; f < PetscMax(1,numFields); f++) {
5567     if (numFields) {ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms[f],&flips[f]);CHKERRQ(ierr);}
5568     else           {ierr = PetscSectionGetPointSyms(section,numPoints,points,&perms[f],&flips[f]);CHKERRQ(ierr);}
5569     if (values && flips[f]) { /* may need to apply sign changes to the element matrix */
5570       PetscInt foffset = offsets[f];
5571 
5572       for (p = 0; p < numPoints; p++) {
5573         PetscInt point          = points[2*p], fdof;
5574         const PetscScalar *flip = flips[f] ? flips[f][p] : NULL;
5575 
5576         if (!numFields) {
5577           ierr = PetscSectionGetDof(section,point,&fdof);CHKERRQ(ierr);
5578         } else {
5579           ierr = PetscSectionGetFieldDof(section,point,f,&fdof);CHKERRQ(ierr);
5580         }
5581         if (flip) {
5582           PetscInt i, j, k;
5583 
5584           if (!valCopy) {
5585             ierr = DMGetWorkArray(dm,numIndices*numIndices,MPIU_SCALAR,&valCopy);CHKERRQ(ierr);
5586             for (j = 0; j < numIndices * numIndices; j++) valCopy[j] = values[j];
5587             values = valCopy;
5588           }
5589           for (i = 0; i < fdof; i++) {
5590             PetscScalar fval = flip[i];
5591 
5592             for (k = 0; k < numIndices; k++) {
5593               valCopy[numIndices * (foffset + i) + k] *= fval;
5594               valCopy[numIndices * k + (foffset + i)] *= fval;
5595             }
5596           }
5597         }
5598         foffset += fdof;
5599       }
5600     }
5601   }
5602   ierr = DMPlexAnchorsModifyMat(dm,section,numPoints,numIndices,points,perms,values,&newNumPoints,&newNumIndices,&newPoints,&newValues,offsets,PETSC_TRUE);CHKERRQ(ierr);
5603   if (newNumPoints) {
5604     if (valCopy) {
5605       ierr = DMRestoreWorkArray(dm,numIndices*numIndices,MPIU_SCALAR,&valCopy);CHKERRQ(ierr);
5606     }
5607     for (f = 0; f < PetscMax(1,numFields); f++) {
5608       if (numFields) {ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms[f],&flips[f]);CHKERRQ(ierr);}
5609       else           {ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms[f],&flips[f]);CHKERRQ(ierr);}
5610     }
5611     for (f = 0; f < PetscMax(1,numFields); f++) {
5612       if (numFields) {ierr = PetscSectionGetFieldPointSyms(section,f,newNumPoints,newPoints,&perms[f],&flips[f]);CHKERRQ(ierr);}
5613       else           {ierr = PetscSectionGetPointSyms(section,newNumPoints,newPoints,&perms[f],&flips[f]);CHKERRQ(ierr);}
5614     }
5615     ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5616     numPoints  = newNumPoints;
5617     numIndices = newNumIndices;
5618     points     = newPoints;
5619     values     = newValues;
5620   }
5621   ierr = DMGetWorkArray(dm, numIndices, MPIU_INT, &indices);CHKERRQ(ierr);
5622   if (numFields) {
5623     for (p = 0; p < numPoints; p++) {
5624       ierr = PetscSectionGetOffset(globalSection, points[2*p], &globalOff);CHKERRQ(ierr);
5625       DMPlexGetIndicesPointFields_Internal(section, points[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, offsets, PETSC_FALSE, perms, p, indices);
5626     }
5627   } else {
5628     for (p = 0, off = 0; p < numPoints; p++) {
5629       const PetscInt *perm = perms[0] ? perms[0][p] : NULL;
5630       ierr = PetscSectionGetOffset(globalSection, points[2*p], &globalOff);CHKERRQ(ierr);
5631       DMPlexGetIndicesPoint_Internal(section, points[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, indices);
5632     }
5633   }
5634   if (mesh->printSetValues) {ierr = DMPlexPrintMatSetValues(PETSC_VIEWER_STDOUT_SELF, A, point, numIndices, indices, 0, NULL, values);CHKERRQ(ierr);}
5635   ierr = MatSetValues(A, numIndices, indices, numIndices, indices, values, mode);
5636   if (mesh->printFEM > 1) {
5637     PetscInt i;
5638     ierr = PetscPrintf(PETSC_COMM_SELF, "  Indices:");CHKERRQ(ierr);
5639     for (i = 0; i < numIndices; ++i) {ierr = PetscPrintf(PETSC_COMM_SELF, " %D", indices[i]);CHKERRQ(ierr);}
5640     ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
5641   }
5642   if (ierr) {
5643     PetscMPIInt    rank;
5644     PetscErrorCode ierr2;
5645 
5646     ierr2 = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRQ(ierr2);
5647     ierr2 = (*PetscErrorPrintf)("[%d]ERROR in DMPlexMatSetClosure\n", rank);CHKERRQ(ierr2);
5648     ierr2 = DMPlexPrintMatSetValues(PETSC_VIEWER_STDERR_SELF, A, point, numIndices, indices, 0, NULL, values);CHKERRQ(ierr2);
5649     ierr2 = DMRestoreWorkArray(dm, numIndices, MPIU_INT, &indices);CHKERRQ(ierr2);
5650     CHKERRQ(ierr);
5651   }
5652   for (f = 0; f < PetscMax(1,numFields); f++) {
5653     if (numFields) {ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms[f],&flips[f]);CHKERRQ(ierr);}
5654     else           {ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms[f],&flips[f]);CHKERRQ(ierr);}
5655   }
5656   if (newNumPoints) {
5657     ierr = DMRestoreWorkArray(dm,newNumIndices*newNumIndices,MPIU_SCALAR,&newValues);CHKERRQ(ierr);
5658     ierr = DMRestoreWorkArray(dm,2*newNumPoints,MPIU_INT,&newPoints);CHKERRQ(ierr);
5659   }
5660   else {
5661     if (valCopy) {
5662       ierr = DMRestoreWorkArray(dm,numIndices*numIndices,MPIU_SCALAR,&valCopy);CHKERRQ(ierr);
5663     }
5664     ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5665   }
5666   ierr = DMRestoreWorkArray(dm, numIndices, MPIU_INT, &indices);CHKERRQ(ierr);
5667   PetscFunctionReturn(0);
5668 }
5669 
5670 PetscErrorCode DMPlexMatSetClosureRefined(DM dmf, PetscSection fsection, PetscSection globalFSection, DM dmc, PetscSection csection, PetscSection globalCSection, Mat A, PetscInt point, const PetscScalar values[], InsertMode mode)
5671 {
5672   DM_Plex        *mesh   = (DM_Plex*) dmf->data;
5673   PetscInt       *fpoints = NULL, *ftotpoints = NULL;
5674   PetscInt       *cpoints = NULL;
5675   PetscInt       *findices, *cindices;
5676   PetscInt        foffsets[32], coffsets[32];
5677   CellRefiner     cellRefiner;
5678   PetscInt        numFields, numSubcells, maxFPoints, numFPoints, numCPoints, numFIndices, numCIndices, dof, off, globalOff, pStart, pEnd, p, q, r, s, f;
5679   PetscErrorCode  ierr;
5680 
5681   PetscFunctionBegin;
5682   PetscValidHeaderSpecific(dmf, DM_CLASSID, 1);
5683   PetscValidHeaderSpecific(dmc, DM_CLASSID, 4);
5684   if (!fsection) {ierr = DMGetDefaultSection(dmf, &fsection);CHKERRQ(ierr);}
5685   PetscValidHeaderSpecific(fsection, PETSC_SECTION_CLASSID, 2);
5686   if (!csection) {ierr = DMGetDefaultSection(dmc, &csection);CHKERRQ(ierr);}
5687   PetscValidHeaderSpecific(csection, PETSC_SECTION_CLASSID, 5);
5688   if (!globalFSection) {ierr = DMGetDefaultGlobalSection(dmf, &globalFSection);CHKERRQ(ierr);}
5689   PetscValidHeaderSpecific(globalFSection, PETSC_SECTION_CLASSID, 3);
5690   if (!globalCSection) {ierr = DMGetDefaultGlobalSection(dmc, &globalCSection);CHKERRQ(ierr);}
5691   PetscValidHeaderSpecific(globalCSection, PETSC_SECTION_CLASSID, 6);
5692   PetscValidHeaderSpecific(A, MAT_CLASSID, 7);
5693   ierr = PetscSectionGetNumFields(fsection, &numFields);CHKERRQ(ierr);
5694   if (numFields > 31) SETERRQ1(PetscObjectComm((PetscObject)dmf), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", numFields);
5695   ierr = PetscMemzero(foffsets, 32 * sizeof(PetscInt));CHKERRQ(ierr);
5696   ierr = PetscMemzero(coffsets, 32 * sizeof(PetscInt));CHKERRQ(ierr);
5697   /* Column indices */
5698   ierr = DMPlexGetTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
5699   maxFPoints = numCPoints;
5700   /* Compress out points not in the section */
5701   /*   TODO: Squeeze out points with 0 dof as well */
5702   ierr = PetscSectionGetChart(csection, &pStart, &pEnd);CHKERRQ(ierr);
5703   for (p = 0, q = 0; p < numCPoints*2; p += 2) {
5704     if ((cpoints[p] >= pStart) && (cpoints[p] < pEnd)) {
5705       cpoints[q*2]   = cpoints[p];
5706       cpoints[q*2+1] = cpoints[p+1];
5707       ++q;
5708     }
5709   }
5710   numCPoints = q;
5711   for (p = 0, numCIndices = 0; p < numCPoints*2; p += 2) {
5712     PetscInt fdof;
5713 
5714     ierr = PetscSectionGetDof(csection, cpoints[p], &dof);CHKERRQ(ierr);
5715     if (!dof) continue;
5716     for (f = 0; f < numFields; ++f) {
5717       ierr           = PetscSectionGetFieldDof(csection, cpoints[p], f, &fdof);CHKERRQ(ierr);
5718       coffsets[f+1] += fdof;
5719     }
5720     numCIndices += dof;
5721   }
5722   for (f = 1; f < numFields; ++f) coffsets[f+1] += coffsets[f];
5723   /* Row indices */
5724   ierr = DMPlexGetCellRefiner_Internal(dmc, &cellRefiner);CHKERRQ(ierr);
5725   ierr = CellRefinerGetAffineTransforms_Internal(cellRefiner, &numSubcells, NULL, NULL, NULL);CHKERRQ(ierr);
5726   ierr = DMGetWorkArray(dmf, maxFPoints*2*numSubcells, MPIU_INT, &ftotpoints);CHKERRQ(ierr);
5727   for (r = 0, q = 0; r < numSubcells; ++r) {
5728     /* TODO Map from coarse to fine cells */
5729     ierr = DMPlexGetTransitiveClosure(dmf, point*numSubcells + r, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
5730     /* Compress out points not in the section */
5731     ierr = PetscSectionGetChart(fsection, &pStart, &pEnd);CHKERRQ(ierr);
5732     for (p = 0; p < numFPoints*2; p += 2) {
5733       if ((fpoints[p] >= pStart) && (fpoints[p] < pEnd)) {
5734         ierr = PetscSectionGetDof(fsection, fpoints[p], &dof);CHKERRQ(ierr);
5735         if (!dof) continue;
5736         for (s = 0; s < q; ++s) if (fpoints[p] == ftotpoints[s*2]) break;
5737         if (s < q) continue;
5738         ftotpoints[q*2]   = fpoints[p];
5739         ftotpoints[q*2+1] = fpoints[p+1];
5740         ++q;
5741       }
5742     }
5743     ierr = DMPlexRestoreTransitiveClosure(dmf, point, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
5744   }
5745   numFPoints = q;
5746   for (p = 0, numFIndices = 0; p < numFPoints*2; p += 2) {
5747     PetscInt fdof;
5748 
5749     ierr = PetscSectionGetDof(fsection, ftotpoints[p], &dof);CHKERRQ(ierr);
5750     if (!dof) continue;
5751     for (f = 0; f < numFields; ++f) {
5752       ierr           = PetscSectionGetFieldDof(fsection, ftotpoints[p], f, &fdof);CHKERRQ(ierr);
5753       foffsets[f+1] += fdof;
5754     }
5755     numFIndices += dof;
5756   }
5757   for (f = 1; f < numFields; ++f) foffsets[f+1] += foffsets[f];
5758 
5759   if (numFields && foffsets[numFields] != numFIndices) SETERRQ2(PetscObjectComm((PetscObject)dmf), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", foffsets[numFields], numFIndices);
5760   if (numFields && coffsets[numFields] != numCIndices) SETERRQ2(PetscObjectComm((PetscObject)dmc), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", coffsets[numFields], numCIndices);
5761   ierr = DMGetWorkArray(dmf, numFIndices, MPIU_INT, &findices);CHKERRQ(ierr);
5762   ierr = DMGetWorkArray(dmc, numCIndices, MPIU_INT, &cindices);CHKERRQ(ierr);
5763   if (numFields) {
5764     const PetscInt **permsF[32] = {NULL};
5765     const PetscInt **permsC[32] = {NULL};
5766 
5767     for (f = 0; f < numFields; f++) {
5768       ierr = PetscSectionGetFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL);CHKERRQ(ierr);
5769       ierr = PetscSectionGetFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL);CHKERRQ(ierr);
5770     }
5771     for (p = 0; p < numFPoints; p++) {
5772       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff);CHKERRQ(ierr);
5773       DMPlexGetIndicesPointFields_Internal(fsection, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, foffsets, PETSC_FALSE, permsF, p, findices);
5774     }
5775     for (p = 0; p < numCPoints; p++) {
5776       ierr = PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff);CHKERRQ(ierr);
5777       DMPlexGetIndicesPointFields_Internal(csection, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, coffsets, PETSC_FALSE, permsC, p, cindices);
5778     }
5779     for (f = 0; f < numFields; f++) {
5780       ierr = PetscSectionRestoreFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL);CHKERRQ(ierr);
5781       ierr = PetscSectionRestoreFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL);CHKERRQ(ierr);
5782     }
5783   } else {
5784     const PetscInt **permsF = NULL;
5785     const PetscInt **permsC = NULL;
5786 
5787     ierr = PetscSectionGetPointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL);CHKERRQ(ierr);
5788     ierr = PetscSectionGetPointSyms(csection,numCPoints,cpoints,&permsC,NULL);CHKERRQ(ierr);
5789     for (p = 0, off = 0; p < numFPoints; p++) {
5790       const PetscInt *perm = permsF ? permsF[p] : NULL;
5791 
5792       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff);CHKERRQ(ierr);
5793       ierr = DMPlexGetIndicesPoint_Internal(fsection, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, findices);CHKERRQ(ierr);
5794     }
5795     for (p = 0, off = 0; p < numCPoints; p++) {
5796       const PetscInt *perm = permsC ? permsC[p] : NULL;
5797 
5798       ierr = PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff);CHKERRQ(ierr);
5799       ierr = DMPlexGetIndicesPoint_Internal(csection, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, cindices);CHKERRQ(ierr);
5800     }
5801     ierr = PetscSectionRestorePointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL);CHKERRQ(ierr);
5802     ierr = PetscSectionRestorePointSyms(csection,numCPoints,cpoints,&permsC,NULL);CHKERRQ(ierr);
5803   }
5804   if (mesh->printSetValues) {ierr = DMPlexPrintMatSetValues(PETSC_VIEWER_STDOUT_SELF, A, point, numFIndices, findices, numCIndices, cindices, values);CHKERRQ(ierr);}
5805   /* TODO: flips */
5806   ierr = MatSetValues(A, numFIndices, findices, numCIndices, cindices, values, mode);
5807   if (ierr) {
5808     PetscMPIInt    rank;
5809     PetscErrorCode ierr2;
5810 
5811     ierr2 = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRQ(ierr2);
5812     ierr2 = (*PetscErrorPrintf)("[%d]ERROR in DMPlexMatSetClosure\n", rank);CHKERRQ(ierr2);
5813     ierr2 = DMPlexPrintMatSetValues(PETSC_VIEWER_STDERR_SELF, A, point, numFIndices, findices, numCIndices, cindices, values);CHKERRQ(ierr2);
5814     ierr2 = DMRestoreWorkArray(dmf, numFIndices, MPIU_INT, &findices);CHKERRQ(ierr2);
5815     ierr2 = DMRestoreWorkArray(dmc, numCIndices, MPIU_INT, &cindices);CHKERRQ(ierr2);
5816     CHKERRQ(ierr);
5817   }
5818   ierr = DMRestoreWorkArray(dmf, numCPoints*2*4, MPIU_INT, &ftotpoints);CHKERRQ(ierr);
5819   ierr = DMPlexRestoreTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
5820   ierr = DMRestoreWorkArray(dmf, numFIndices, MPIU_INT, &findices);CHKERRQ(ierr);
5821   ierr = DMRestoreWorkArray(dmc, numCIndices, MPIU_INT, &cindices);CHKERRQ(ierr);
5822   PetscFunctionReturn(0);
5823 }
5824 
5825 PetscErrorCode DMPlexMatGetClosureIndicesRefined(DM dmf, PetscSection fsection, PetscSection globalFSection, DM dmc, PetscSection csection, PetscSection globalCSection, PetscInt point, PetscInt cindices[], PetscInt findices[])
5826 {
5827   PetscInt      *fpoints = NULL, *ftotpoints = NULL;
5828   PetscInt      *cpoints = NULL;
5829   PetscInt       foffsets[32], coffsets[32];
5830   CellRefiner    cellRefiner;
5831   PetscInt       numFields, numSubcells, maxFPoints, numFPoints, numCPoints, numFIndices, numCIndices, dof, off, globalOff, pStart, pEnd, p, q, r, s, f;
5832   PetscErrorCode ierr;
5833 
5834   PetscFunctionBegin;
5835   PetscValidHeaderSpecific(dmf, DM_CLASSID, 1);
5836   PetscValidHeaderSpecific(dmc, DM_CLASSID, 4);
5837   if (!fsection) {ierr = DMGetDefaultSection(dmf, &fsection);CHKERRQ(ierr);}
5838   PetscValidHeaderSpecific(fsection, PETSC_SECTION_CLASSID, 2);
5839   if (!csection) {ierr = DMGetDefaultSection(dmc, &csection);CHKERRQ(ierr);}
5840   PetscValidHeaderSpecific(csection, PETSC_SECTION_CLASSID, 5);
5841   if (!globalFSection) {ierr = DMGetDefaultGlobalSection(dmf, &globalFSection);CHKERRQ(ierr);}
5842   PetscValidHeaderSpecific(globalFSection, PETSC_SECTION_CLASSID, 3);
5843   if (!globalCSection) {ierr = DMGetDefaultGlobalSection(dmc, &globalCSection);CHKERRQ(ierr);}
5844   PetscValidHeaderSpecific(globalCSection, PETSC_SECTION_CLASSID, 6);
5845   ierr = PetscSectionGetNumFields(fsection, &numFields);CHKERRQ(ierr);
5846   if (numFields > 31) SETERRQ1(PetscObjectComm((PetscObject)dmf), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", numFields);
5847   ierr = PetscMemzero(foffsets, 32 * sizeof(PetscInt));CHKERRQ(ierr);
5848   ierr = PetscMemzero(coffsets, 32 * sizeof(PetscInt));CHKERRQ(ierr);
5849   /* Column indices */
5850   ierr = DMPlexGetTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
5851   maxFPoints = numCPoints;
5852   /* Compress out points not in the section */
5853   /*   TODO: Squeeze out points with 0 dof as well */
5854   ierr = PetscSectionGetChart(csection, &pStart, &pEnd);CHKERRQ(ierr);
5855   for (p = 0, q = 0; p < numCPoints*2; p += 2) {
5856     if ((cpoints[p] >= pStart) && (cpoints[p] < pEnd)) {
5857       cpoints[q*2]   = cpoints[p];
5858       cpoints[q*2+1] = cpoints[p+1];
5859       ++q;
5860     }
5861   }
5862   numCPoints = q;
5863   for (p = 0, numCIndices = 0; p < numCPoints*2; p += 2) {
5864     PetscInt fdof;
5865 
5866     ierr = PetscSectionGetDof(csection, cpoints[p], &dof);CHKERRQ(ierr);
5867     if (!dof) continue;
5868     for (f = 0; f < numFields; ++f) {
5869       ierr           = PetscSectionGetFieldDof(csection, cpoints[p], f, &fdof);CHKERRQ(ierr);
5870       coffsets[f+1] += fdof;
5871     }
5872     numCIndices += dof;
5873   }
5874   for (f = 1; f < numFields; ++f) coffsets[f+1] += coffsets[f];
5875   /* Row indices */
5876   ierr = DMPlexGetCellRefiner_Internal(dmc, &cellRefiner);CHKERRQ(ierr);
5877   ierr = CellRefinerGetAffineTransforms_Internal(cellRefiner, &numSubcells, NULL, NULL, NULL);CHKERRQ(ierr);
5878   ierr = DMGetWorkArray(dmf, maxFPoints*2*numSubcells, MPIU_INT, &ftotpoints);CHKERRQ(ierr);
5879   for (r = 0, q = 0; r < numSubcells; ++r) {
5880     /* TODO Map from coarse to fine cells */
5881     ierr = DMPlexGetTransitiveClosure(dmf, point*numSubcells + r, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
5882     /* Compress out points not in the section */
5883     ierr = PetscSectionGetChart(fsection, &pStart, &pEnd);CHKERRQ(ierr);
5884     for (p = 0; p < numFPoints*2; p += 2) {
5885       if ((fpoints[p] >= pStart) && (fpoints[p] < pEnd)) {
5886         ierr = PetscSectionGetDof(fsection, fpoints[p], &dof);CHKERRQ(ierr);
5887         if (!dof) continue;
5888         for (s = 0; s < q; ++s) if (fpoints[p] == ftotpoints[s*2]) break;
5889         if (s < q) continue;
5890         ftotpoints[q*2]   = fpoints[p];
5891         ftotpoints[q*2+1] = fpoints[p+1];
5892         ++q;
5893       }
5894     }
5895     ierr = DMPlexRestoreTransitiveClosure(dmf, point, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
5896   }
5897   numFPoints = q;
5898   for (p = 0, numFIndices = 0; p < numFPoints*2; p += 2) {
5899     PetscInt fdof;
5900 
5901     ierr = PetscSectionGetDof(fsection, ftotpoints[p], &dof);CHKERRQ(ierr);
5902     if (!dof) continue;
5903     for (f = 0; f < numFields; ++f) {
5904       ierr           = PetscSectionGetFieldDof(fsection, ftotpoints[p], f, &fdof);CHKERRQ(ierr);
5905       foffsets[f+1] += fdof;
5906     }
5907     numFIndices += dof;
5908   }
5909   for (f = 1; f < numFields; ++f) foffsets[f+1] += foffsets[f];
5910 
5911   if (numFields && foffsets[numFields] != numFIndices) SETERRQ2(PetscObjectComm((PetscObject)dmf), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", foffsets[numFields], numFIndices);
5912   if (numFields && coffsets[numFields] != numCIndices) SETERRQ2(PetscObjectComm((PetscObject)dmc), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", coffsets[numFields], numCIndices);
5913   if (numFields) {
5914     const PetscInt **permsF[32] = {NULL};
5915     const PetscInt **permsC[32] = {NULL};
5916 
5917     for (f = 0; f < numFields; f++) {
5918       ierr = PetscSectionGetFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL);CHKERRQ(ierr);
5919       ierr = PetscSectionGetFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL);CHKERRQ(ierr);
5920     }
5921     for (p = 0; p < numFPoints; p++) {
5922       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff);CHKERRQ(ierr);
5923       DMPlexGetIndicesPointFields_Internal(fsection, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, foffsets, PETSC_FALSE, permsF, p, findices);
5924     }
5925     for (p = 0; p < numCPoints; p++) {
5926       ierr = PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff);CHKERRQ(ierr);
5927       DMPlexGetIndicesPointFields_Internal(csection, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, coffsets, PETSC_FALSE, permsC, p, cindices);
5928     }
5929     for (f = 0; f < numFields; f++) {
5930       ierr = PetscSectionRestoreFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL);CHKERRQ(ierr);
5931       ierr = PetscSectionRestoreFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL);CHKERRQ(ierr);
5932     }
5933   } else {
5934     const PetscInt **permsF = NULL;
5935     const PetscInt **permsC = NULL;
5936 
5937     ierr = PetscSectionGetPointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL);CHKERRQ(ierr);
5938     ierr = PetscSectionGetPointSyms(csection,numCPoints,cpoints,&permsC,NULL);CHKERRQ(ierr);
5939     for (p = 0, off = 0; p < numFPoints; p++) {
5940       const PetscInt *perm = permsF ? permsF[p] : NULL;
5941 
5942       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff);CHKERRQ(ierr);
5943       DMPlexGetIndicesPoint_Internal(fsection, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, findices);
5944     }
5945     for (p = 0, off = 0; p < numCPoints; p++) {
5946       const PetscInt *perm = permsC ? permsC[p] : NULL;
5947 
5948       ierr = PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff);CHKERRQ(ierr);
5949       DMPlexGetIndicesPoint_Internal(csection, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, cindices);
5950     }
5951     ierr = PetscSectionRestorePointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL);CHKERRQ(ierr);
5952     ierr = PetscSectionRestorePointSyms(csection,numCPoints,cpoints,&permsC,NULL);CHKERRQ(ierr);
5953   }
5954   ierr = DMRestoreWorkArray(dmf, numCPoints*2*4, MPIU_INT, &ftotpoints);CHKERRQ(ierr);
5955   ierr = DMPlexRestoreTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
5956   PetscFunctionReturn(0);
5957 }
5958 
5959 /*@
5960   DMPlexGetHybridBounds - Get the first mesh point of each dimension which is a hybrid
5961 
5962   Input Parameter:
5963 . dm - The DMPlex object
5964 
5965   Output Parameters:
5966 + cMax - The first hybrid cell
5967 . fMax - The first hybrid face
5968 . eMax - The first hybrid edge
5969 - vMax - The first hybrid vertex
5970 
5971   Level: developer
5972 
5973 .seealso DMPlexCreateHybridMesh(), DMPlexSetHybridBounds()
5974 @*/
5975 PetscErrorCode DMPlexGetHybridBounds(DM dm, PetscInt *cMax, PetscInt *fMax, PetscInt *eMax, PetscInt *vMax)
5976 {
5977   DM_Plex       *mesh = (DM_Plex*) dm->data;
5978   PetscInt       dim;
5979   PetscErrorCode ierr;
5980 
5981   PetscFunctionBegin;
5982   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5983   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
5984   if (cMax) *cMax = mesh->hybridPointMax[dim];
5985   if (fMax) *fMax = mesh->hybridPointMax[dim-1];
5986   if (eMax) *eMax = mesh->hybridPointMax[1];
5987   if (vMax) *vMax = mesh->hybridPointMax[0];
5988   PetscFunctionReturn(0);
5989 }
5990 
5991 /*@
5992   DMPlexSetHybridBounds - Set the first mesh point of each dimension which is a hybrid
5993 
5994   Input Parameters:
5995 . dm   - The DMPlex object
5996 . cMax - The first hybrid cell
5997 . fMax - The first hybrid face
5998 . eMax - The first hybrid edge
5999 - vMax - The first hybrid vertex
6000 
6001   Level: developer
6002 
6003 .seealso DMPlexCreateHybridMesh(), DMPlexGetHybridBounds()
6004 @*/
6005 PetscErrorCode DMPlexSetHybridBounds(DM dm, PetscInt cMax, PetscInt fMax, PetscInt eMax, PetscInt vMax)
6006 {
6007   DM_Plex       *mesh = (DM_Plex*) dm->data;
6008   PetscInt       dim;
6009   PetscErrorCode ierr;
6010 
6011   PetscFunctionBegin;
6012   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6013   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
6014   if (cMax >= 0) mesh->hybridPointMax[dim]   = cMax;
6015   if (fMax >= 0) mesh->hybridPointMax[dim-1] = fMax;
6016   if (eMax >= 0) mesh->hybridPointMax[1]     = eMax;
6017   if (vMax >= 0) mesh->hybridPointMax[0]     = vMax;
6018   PetscFunctionReturn(0);
6019 }
6020 
6021 /*@C
6022   DMPlexGetVTKCellHeight - Returns the height in the DAG used to determine which points are cells (normally 0)
6023 
6024   Input Parameter:
6025 . dm   - The DMPlex object
6026 
6027   Output Parameter:
6028 . cellHeight - The height of a cell
6029 
6030   Level: developer
6031 
6032 .seealso DMPlexSetVTKCellHeight()
6033 @*/
6034 PetscErrorCode DMPlexGetVTKCellHeight(DM dm, PetscInt *cellHeight)
6035 {
6036   DM_Plex *mesh = (DM_Plex*) dm->data;
6037 
6038   PetscFunctionBegin;
6039   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6040   PetscValidPointer(cellHeight, 2);
6041   *cellHeight = mesh->vtkCellHeight;
6042   PetscFunctionReturn(0);
6043 }
6044 
6045 /*@C
6046   DMPlexSetVTKCellHeight - Sets the height in the DAG used to determine which points are cells (normally 0)
6047 
6048   Input Parameters:
6049 + dm   - The DMPlex object
6050 - cellHeight - The height of a cell
6051 
6052   Level: developer
6053 
6054 .seealso DMPlexGetVTKCellHeight()
6055 @*/
6056 PetscErrorCode DMPlexSetVTKCellHeight(DM dm, PetscInt cellHeight)
6057 {
6058   DM_Plex *mesh = (DM_Plex*) dm->data;
6059 
6060   PetscFunctionBegin;
6061   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6062   mesh->vtkCellHeight = cellHeight;
6063   PetscFunctionReturn(0);
6064 }
6065 
6066 /* We can easily have a form that takes an IS instead */
6067 static PetscErrorCode DMPlexCreateNumbering_Private(DM dm, PetscInt pStart, PetscInt pEnd, PetscInt shift, PetscInt *globalSize, PetscSF sf, IS *numbering)
6068 {
6069   PetscSection   section, globalSection;
6070   PetscInt      *numbers, p;
6071   PetscErrorCode ierr;
6072 
6073   PetscFunctionBegin;
6074   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &section);CHKERRQ(ierr);
6075   ierr = PetscSectionSetChart(section, pStart, pEnd);CHKERRQ(ierr);
6076   for (p = pStart; p < pEnd; ++p) {
6077     ierr = PetscSectionSetDof(section, p, 1);CHKERRQ(ierr);
6078   }
6079   ierr = PetscSectionSetUp(section);CHKERRQ(ierr);
6080   ierr = PetscSectionCreateGlobalSection(section, sf, PETSC_FALSE, PETSC_FALSE, &globalSection);CHKERRQ(ierr);
6081   ierr = PetscMalloc1(pEnd - pStart, &numbers);CHKERRQ(ierr);
6082   for (p = pStart; p < pEnd; ++p) {
6083     ierr = PetscSectionGetOffset(globalSection, p, &numbers[p-pStart]);CHKERRQ(ierr);
6084     if (numbers[p-pStart] < 0) numbers[p-pStart] -= shift;
6085     else                       numbers[p-pStart] += shift;
6086   }
6087   ierr = ISCreateGeneral(PetscObjectComm((PetscObject) dm), pEnd - pStart, numbers, PETSC_OWN_POINTER, numbering);CHKERRQ(ierr);
6088   if (globalSize) {
6089     PetscLayout layout;
6090     ierr = PetscSectionGetPointLayout(PetscObjectComm((PetscObject) dm), globalSection, &layout);CHKERRQ(ierr);
6091     ierr = PetscLayoutGetSize(layout, globalSize);CHKERRQ(ierr);
6092     ierr = PetscLayoutDestroy(&layout);CHKERRQ(ierr);
6093   }
6094   ierr = PetscSectionDestroy(&section);CHKERRQ(ierr);
6095   ierr = PetscSectionDestroy(&globalSection);CHKERRQ(ierr);
6096   PetscFunctionReturn(0);
6097 }
6098 
6099 PetscErrorCode DMPlexCreateCellNumbering_Internal(DM dm, PetscBool includeHybrid, IS *globalCellNumbers)
6100 {
6101   PetscInt       cellHeight, cStart, cEnd, cMax;
6102   PetscErrorCode ierr;
6103 
6104   PetscFunctionBegin;
6105   ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
6106   ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
6107   ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
6108   if (cMax >= 0 && !includeHybrid) cEnd = PetscMin(cEnd, cMax);
6109   ierr = DMPlexCreateNumbering_Private(dm, cStart, cEnd, 0, NULL, dm->sf, globalCellNumbers);CHKERRQ(ierr);
6110   PetscFunctionReturn(0);
6111 }
6112 
6113 /*@C
6114   DMPlexGetCellNumbering - Get a global cell numbering for all cells on this process
6115 
6116   Input Parameter:
6117 . dm   - The DMPlex object
6118 
6119   Output Parameter:
6120 . globalCellNumbers - Global cell numbers for all cells on this process
6121 
6122   Level: developer
6123 
6124 .seealso DMPlexGetVertexNumbering()
6125 @*/
6126 PetscErrorCode DMPlexGetCellNumbering(DM dm, IS *globalCellNumbers)
6127 {
6128   DM_Plex       *mesh = (DM_Plex*) dm->data;
6129   PetscErrorCode ierr;
6130 
6131   PetscFunctionBegin;
6132   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6133   if (!mesh->globalCellNumbers) {ierr = DMPlexCreateCellNumbering_Internal(dm, PETSC_FALSE, &mesh->globalCellNumbers);CHKERRQ(ierr);}
6134   *globalCellNumbers = mesh->globalCellNumbers;
6135   PetscFunctionReturn(0);
6136 }
6137 
6138 PetscErrorCode DMPlexCreateVertexNumbering_Internal(DM dm, PetscBool includeHybrid, IS *globalVertexNumbers)
6139 {
6140   PetscInt       vStart, vEnd, vMax;
6141   PetscErrorCode ierr;
6142 
6143   PetscFunctionBegin;
6144   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6145   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
6146   ierr = DMPlexGetHybridBounds(dm, NULL, NULL, NULL, &vMax);CHKERRQ(ierr);
6147   if (vMax >= 0 && !includeHybrid) vEnd = PetscMin(vEnd, vMax);
6148   ierr = DMPlexCreateNumbering_Private(dm, vStart, vEnd, 0, NULL, dm->sf, globalVertexNumbers);CHKERRQ(ierr);
6149   PetscFunctionReturn(0);
6150 }
6151 
6152 /*@C
6153   DMPlexGetVertexNumbering - Get a global certex numbering for all vertices on this process
6154 
6155   Input Parameter:
6156 . dm   - The DMPlex object
6157 
6158   Output Parameter:
6159 . globalVertexNumbers - Global vertex numbers for all vertices on this process
6160 
6161   Level: developer
6162 
6163 .seealso DMPlexGetCellNumbering()
6164 @*/
6165 PetscErrorCode DMPlexGetVertexNumbering(DM dm, IS *globalVertexNumbers)
6166 {
6167   DM_Plex       *mesh = (DM_Plex*) dm->data;
6168   PetscErrorCode ierr;
6169 
6170   PetscFunctionBegin;
6171   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6172   if (!mesh->globalVertexNumbers) {ierr = DMPlexCreateVertexNumbering_Internal(dm, PETSC_FALSE, &mesh->globalVertexNumbers);CHKERRQ(ierr);}
6173   *globalVertexNumbers = mesh->globalVertexNumbers;
6174   PetscFunctionReturn(0);
6175 }
6176 
6177 /*@C
6178   DMPlexCreatePointNumbering - Create a global numbering for all points on this process
6179 
6180   Input Parameter:
6181 . dm   - The DMPlex object
6182 
6183   Output Parameter:
6184 . globalPointNumbers - Global numbers for all points on this process
6185 
6186   Level: developer
6187 
6188 .seealso DMPlexGetCellNumbering()
6189 @*/
6190 PetscErrorCode DMPlexCreatePointNumbering(DM dm, IS *globalPointNumbers)
6191 {
6192   IS             nums[4];
6193   PetscInt       depths[4];
6194   PetscInt       depth, d, shift = 0;
6195   PetscErrorCode ierr;
6196 
6197   PetscFunctionBegin;
6198   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6199   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
6200   /* For unstratified meshes use dim instead of depth */
6201   if (depth < 0) {ierr = DMGetDimension(dm, &depth);CHKERRQ(ierr);}
6202   depths[0] = depth; depths[1] = 0;
6203   for (d = 2; d <= depth; ++d) depths[d] = depth-d+1;
6204   for (d = 0; d <= depth; ++d) {
6205     PetscInt pStart, pEnd, gsize;
6206 
6207     ierr = DMPlexGetDepthStratum(dm, depths[d], &pStart, &pEnd);CHKERRQ(ierr);
6208     ierr = DMPlexCreateNumbering_Private(dm, pStart, pEnd, shift, &gsize, dm->sf, &nums[d]);CHKERRQ(ierr);
6209     shift += gsize;
6210   }
6211   ierr = ISConcatenate(PetscObjectComm((PetscObject) dm), depth+1, nums, globalPointNumbers);CHKERRQ(ierr);
6212   for (d = 0; d <= depth; ++d) {ierr = ISDestroy(&nums[d]);CHKERRQ(ierr);}
6213   PetscFunctionReturn(0);
6214 }
6215 
6216 
6217 /*@
6218   DMPlexCreateRankField - Create a cell field whose value is the rank of the owner
6219 
6220   Input Parameter:
6221 . dm - The DMPlex object
6222 
6223   Output Parameter:
6224 . ranks - The rank field
6225 
6226   Options Database Keys:
6227 . -dm_partition_view - Adds the rank field into the DM output from -dm_view using the same viewer
6228 
6229   Level: intermediate
6230 
6231 .seealso: DMView()
6232 @*/
6233 PetscErrorCode DMPlexCreateRankField(DM dm, Vec *ranks)
6234 {
6235   DM             rdm;
6236   PetscDS        prob;
6237   PetscFE        fe;
6238   PetscScalar   *r;
6239   PetscMPIInt    rank;
6240   PetscInt       dim, cStart, cEnd, c;
6241   PetscErrorCode ierr;
6242 
6243   PetscFunctionBeginUser;
6244   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRQ(ierr);
6245   ierr = DMClone(dm, &rdm);CHKERRQ(ierr);
6246   ierr = DMGetDimension(rdm, &dim);CHKERRQ(ierr);
6247   ierr = PetscFECreateDefault(rdm, dim, 1, PETSC_TRUE, NULL, -1, &fe);CHKERRQ(ierr);
6248   ierr = PetscObjectSetName((PetscObject) fe, "rank");CHKERRQ(ierr);
6249   ierr = DMGetDS(rdm, &prob);CHKERRQ(ierr);
6250   ierr = PetscDSSetDiscretization(prob, 0, (PetscObject) fe);CHKERRQ(ierr);
6251   ierr = PetscFEDestroy(&fe);CHKERRQ(ierr);
6252   ierr = DMPlexGetHeightStratum(rdm, 0, &cStart, &cEnd);CHKERRQ(ierr);
6253   ierr = DMCreateGlobalVector(rdm, ranks);CHKERRQ(ierr);
6254   ierr = PetscObjectSetName((PetscObject) *ranks, "partition");CHKERRQ(ierr);
6255   ierr = VecGetArray(*ranks, &r);CHKERRQ(ierr);
6256   for (c = cStart; c < cEnd; ++c) {
6257     PetscScalar *lr;
6258 
6259     ierr = DMPlexPointGlobalRef(rdm, c, r, &lr);CHKERRQ(ierr);
6260     *lr = rank;
6261   }
6262   ierr = VecRestoreArray(*ranks, &r);CHKERRQ(ierr);
6263   ierr = DMDestroy(&rdm);CHKERRQ(ierr);
6264   PetscFunctionReturn(0);
6265 }
6266 
6267 /*@
6268   DMPlexCheckSymmetry - Check that the adjacency information in the mesh is symmetric.
6269 
6270   Input Parameter:
6271 . dm - The DMPlex object
6272 
6273   Note: This is a useful diagnostic when creating meshes programmatically.
6274 
6275   Level: developer
6276 
6277 .seealso: DMCreate(), DMCheckSkeleton(), DMCheckFaces()
6278 @*/
6279 PetscErrorCode DMPlexCheckSymmetry(DM dm)
6280 {
6281   PetscSection    coneSection, supportSection;
6282   const PetscInt *cone, *support;
6283   PetscInt        coneSize, c, supportSize, s;
6284   PetscInt        pStart, pEnd, p, csize, ssize;
6285   PetscErrorCode  ierr;
6286 
6287   PetscFunctionBegin;
6288   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6289   ierr = DMPlexGetConeSection(dm, &coneSection);CHKERRQ(ierr);
6290   ierr = DMPlexGetSupportSection(dm, &supportSection);CHKERRQ(ierr);
6291   /* Check that point p is found in the support of its cone points, and vice versa */
6292   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
6293   for (p = pStart; p < pEnd; ++p) {
6294     ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
6295     ierr = DMPlexGetCone(dm, p, &cone);CHKERRQ(ierr);
6296     for (c = 0; c < coneSize; ++c) {
6297       PetscBool dup = PETSC_FALSE;
6298       PetscInt  d;
6299       for (d = c-1; d >= 0; --d) {
6300         if (cone[c] == cone[d]) {dup = PETSC_TRUE; break;}
6301       }
6302       ierr = DMPlexGetSupportSize(dm, cone[c], &supportSize);CHKERRQ(ierr);
6303       ierr = DMPlexGetSupport(dm, cone[c], &support);CHKERRQ(ierr);
6304       for (s = 0; s < supportSize; ++s) {
6305         if (support[s] == p) break;
6306       }
6307       if ((s >= supportSize) || (dup && (support[s+1] != p))) {
6308         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %D cone: ", p);CHKERRQ(ierr);
6309         for (s = 0; s < coneSize; ++s) {
6310           ierr = PetscPrintf(PETSC_COMM_SELF, "%D, ", cone[s]);CHKERRQ(ierr);
6311         }
6312         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
6313         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %D support: ", cone[c]);CHKERRQ(ierr);
6314         for (s = 0; s < supportSize; ++s) {
6315           ierr = PetscPrintf(PETSC_COMM_SELF, "%D, ", support[s]);CHKERRQ(ierr);
6316         }
6317         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
6318         if (dup) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %D not repeatedly found in support of repeated cone point %D", p, cone[c]);
6319         else SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %D not found in support of cone point %D", p, cone[c]);
6320       }
6321     }
6322     ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
6323     ierr = DMPlexGetSupport(dm, p, &support);CHKERRQ(ierr);
6324     for (s = 0; s < supportSize; ++s) {
6325       ierr = DMPlexGetConeSize(dm, support[s], &coneSize);CHKERRQ(ierr);
6326       ierr = DMPlexGetCone(dm, support[s], &cone);CHKERRQ(ierr);
6327       for (c = 0; c < coneSize; ++c) {
6328         if (cone[c] == p) break;
6329       }
6330       if (c >= coneSize) {
6331         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %D support: ", p);CHKERRQ(ierr);
6332         for (c = 0; c < supportSize; ++c) {
6333           ierr = PetscPrintf(PETSC_COMM_SELF, "%D, ", support[c]);CHKERRQ(ierr);
6334         }
6335         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
6336         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %D cone: ", support[s]);CHKERRQ(ierr);
6337         for (c = 0; c < coneSize; ++c) {
6338           ierr = PetscPrintf(PETSC_COMM_SELF, "%D, ", cone[c]);CHKERRQ(ierr);
6339         }
6340         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
6341         SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %D not found in cone of support point %D", p, support[s]);
6342       }
6343     }
6344   }
6345   ierr = PetscSectionGetStorageSize(coneSection, &csize);CHKERRQ(ierr);
6346   ierr = PetscSectionGetStorageSize(supportSection, &ssize);CHKERRQ(ierr);
6347   if (csize != ssize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "Total cone size %D != Total support size %D", csize, ssize);
6348   PetscFunctionReturn(0);
6349 }
6350 
6351 /*@
6352   DMPlexCheckSkeleton - Check that each cell has the correct number of vertices
6353 
6354   Input Parameters:
6355 + dm - The DMPlex object
6356 . isSimplex - Are the cells simplices or tensor products
6357 - cellHeight - Normally 0
6358 
6359   Note: This is a useful diagnostic when creating meshes programmatically.
6360 
6361   Level: developer
6362 
6363 .seealso: DMCreate(), DMCheckSymmetry(), DMCheckFaces()
6364 @*/
6365 PetscErrorCode DMPlexCheckSkeleton(DM dm, PetscBool isSimplex, PetscInt cellHeight)
6366 {
6367   PetscInt       dim, numCorners, numHybridCorners, vStart, vEnd, cStart, cEnd, cMax, c;
6368   PetscErrorCode ierr;
6369 
6370   PetscFunctionBegin;
6371   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6372   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
6373   switch (dim) {
6374   case 1: numCorners = isSimplex ? 2 : 2; numHybridCorners = isSimplex ? 2 : 2; break;
6375   case 2: numCorners = isSimplex ? 3 : 4; numHybridCorners = isSimplex ? 4 : 4; break;
6376   case 3: numCorners = isSimplex ? 4 : 8; numHybridCorners = isSimplex ? 6 : 8; break;
6377   default:
6378     SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_OUTOFRANGE, "Cannot handle meshes of dimension %D", dim);
6379   }
6380   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
6381   ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
6382   ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
6383   cMax = cMax >= 0 ? cMax : cEnd;
6384   for (c = cStart; c < cMax; ++c) {
6385     PetscInt *closure = NULL, closureSize, cl, coneSize = 0;
6386 
6387     ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6388     for (cl = 0; cl < closureSize*2; cl += 2) {
6389       const PetscInt p = closure[cl];
6390       if ((p >= vStart) && (p < vEnd)) ++coneSize;
6391     }
6392     ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6393     if (coneSize != numCorners) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %D has  %D vertices != %D", c, coneSize, numCorners);
6394   }
6395   for (c = cMax; c < cEnd; ++c) {
6396     PetscInt *closure = NULL, closureSize, cl, coneSize = 0;
6397 
6398     ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6399     for (cl = 0; cl < closureSize*2; cl += 2) {
6400       const PetscInt p = closure[cl];
6401       if ((p >= vStart) && (p < vEnd)) ++coneSize;
6402     }
6403     ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6404     if (coneSize > numHybridCorners) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Hybrid cell %D has  %D vertices > %D", c, coneSize, numHybridCorners);
6405   }
6406   PetscFunctionReturn(0);
6407 }
6408 
6409 /*@
6410   DMPlexCheckFaces - Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type
6411 
6412   Input Parameters:
6413 + dm - The DMPlex object
6414 . isSimplex - Are the cells simplices or tensor products
6415 - cellHeight - Normally 0
6416 
6417   Note: This is a useful diagnostic when creating meshes programmatically.
6418 
6419   Level: developer
6420 
6421 .seealso: DMCreate(), DMCheckSymmetry(), DMCheckSkeleton()
6422 @*/
6423 PetscErrorCode DMPlexCheckFaces(DM dm, PetscBool isSimplex, PetscInt cellHeight)
6424 {
6425   PetscInt       pMax[4];
6426   PetscInt       dim, vStart, vEnd, cStart, cEnd, c, h;
6427   PetscErrorCode ierr;
6428 
6429   PetscFunctionBegin;
6430   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6431   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
6432   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
6433   ierr = DMPlexGetHybridBounds(dm, &pMax[dim], &pMax[dim-1], &pMax[1], &pMax[0]);CHKERRQ(ierr);
6434   for (h = cellHeight; h < dim; ++h) {
6435     ierr = DMPlexGetHeightStratum(dm, h, &cStart, &cEnd);CHKERRQ(ierr);
6436     for (c = cStart; c < cEnd; ++c) {
6437       const PetscInt *cone, *ornt, *faces;
6438       PetscInt        numFaces, faceSize, coneSize,f;
6439       PetscInt       *closure = NULL, closureSize, cl, numCorners = 0;
6440 
6441       if (pMax[dim-h] >= 0 && c >= pMax[dim-h]) continue;
6442       ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
6443       ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
6444       ierr = DMPlexGetConeOrientation(dm, c, &ornt);CHKERRQ(ierr);
6445       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6446       for (cl = 0; cl < closureSize*2; cl += 2) {
6447         const PetscInt p = closure[cl];
6448         if ((p >= vStart) && (p < vEnd)) closure[numCorners++] = p;
6449       }
6450       ierr = DMPlexGetRawFaces_Internal(dm, dim-h, numCorners, closure, &numFaces, &faceSize, &faces);CHKERRQ(ierr);
6451       if (coneSize != numFaces) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %D has %D faces but should have %D", c, coneSize, numFaces);
6452       for (f = 0; f < numFaces; ++f) {
6453         PetscInt *fclosure = NULL, fclosureSize, cl, fnumCorners = 0, v;
6454 
6455         ierr = DMPlexGetTransitiveClosure_Internal(dm, cone[f], ornt[f], PETSC_TRUE, &fclosureSize, &fclosure);CHKERRQ(ierr);
6456         for (cl = 0; cl < fclosureSize*2; cl += 2) {
6457           const PetscInt p = fclosure[cl];
6458           if ((p >= vStart) && (p < vEnd)) fclosure[fnumCorners++] = p;
6459         }
6460         if (fnumCorners != faceSize) SETERRQ5(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %D (%D) of cell %D has %D vertices but should have %D", cone[f], f, c, fnumCorners, faceSize);
6461         for (v = 0; v < fnumCorners; ++v) {
6462           if (fclosure[v] != faces[f*faceSize+v]) SETERRQ6(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %D (%d) of cell %D vertex %D, %D != %D", cone[f], f, c, v, fclosure[v], faces[f*faceSize+v]);
6463         }
6464         ierr = DMPlexRestoreTransitiveClosure(dm, cone[f], PETSC_TRUE, &fclosureSize, &fclosure);CHKERRQ(ierr);
6465       }
6466       ierr = DMPlexRestoreFaces_Internal(dm, dim, c, &numFaces, &faceSize, &faces);CHKERRQ(ierr);
6467       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6468     }
6469   }
6470   PetscFunctionReturn(0);
6471 }
6472 
6473 /* Pointwise interpolation
6474      Just code FEM for now
6475      u^f = I u^c
6476      sum_k u^f_k phi^f_k = I sum_j u^c_j phi^c_j
6477      u^f_i = sum_j psi^f_i I phi^c_j u^c_j
6478      I_{ij} = psi^f_i phi^c_j
6479 */
6480 PetscErrorCode DMCreateInterpolation_Plex(DM dmCoarse, DM dmFine, Mat *interpolation, Vec *scaling)
6481 {
6482   PetscSection   gsc, gsf;
6483   PetscInt       m, n;
6484   void          *ctx;
6485   DM             cdm;
6486   PetscBool      regular;
6487   PetscErrorCode ierr;
6488 
6489   PetscFunctionBegin;
6490   ierr = DMGetDefaultGlobalSection(dmFine, &gsf);CHKERRQ(ierr);
6491   ierr = PetscSectionGetConstrainedStorageSize(gsf, &m);CHKERRQ(ierr);
6492   ierr = DMGetDefaultGlobalSection(dmCoarse, &gsc);CHKERRQ(ierr);
6493   ierr = PetscSectionGetConstrainedStorageSize(gsc, &n);CHKERRQ(ierr);
6494 
6495   ierr = MatCreate(PetscObjectComm((PetscObject) dmCoarse), interpolation);CHKERRQ(ierr);
6496   ierr = MatSetSizes(*interpolation, m, n, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
6497   ierr = MatSetType(*interpolation, dmCoarse->mattype);CHKERRQ(ierr);
6498   ierr = DMGetApplicationContext(dmFine, &ctx);CHKERRQ(ierr);
6499 
6500   ierr = DMGetCoarseDM(dmFine, &cdm);CHKERRQ(ierr);
6501   ierr = DMPlexGetRegularRefinement(dmFine, &regular);CHKERRQ(ierr);
6502   if (regular && cdm == dmCoarse) {ierr = DMPlexComputeInterpolatorNested(dmCoarse, dmFine, *interpolation, ctx);CHKERRQ(ierr);}
6503   else                            {ierr = DMPlexComputeInterpolatorGeneral(dmCoarse, dmFine, *interpolation, ctx);CHKERRQ(ierr);}
6504   ierr = MatViewFromOptions(*interpolation, NULL, "-interp_mat_view");CHKERRQ(ierr);
6505   /* Use naive scaling */
6506   ierr = DMCreateInterpolationScale(dmCoarse, dmFine, *interpolation, scaling);CHKERRQ(ierr);
6507   PetscFunctionReturn(0);
6508 }
6509 
6510 PetscErrorCode DMCreateInjection_Plex(DM dmCoarse, DM dmFine, Mat *mat)
6511 {
6512   PetscErrorCode ierr;
6513   VecScatter     ctx;
6514 
6515   PetscFunctionBegin;
6516   ierr = DMPlexComputeInjectorFEM(dmCoarse, dmFine, &ctx, NULL);CHKERRQ(ierr);
6517   ierr = MatCreateScatter(PetscObjectComm((PetscObject)ctx), ctx, mat);CHKERRQ(ierr);
6518   ierr = VecScatterDestroy(&ctx);CHKERRQ(ierr);
6519   PetscFunctionReturn(0);
6520 }
6521 
6522 PetscErrorCode DMCreateMassMatrix_Plex(DM dmCoarse, DM dmFine, Mat *mass)
6523 {
6524   PetscSection   gsc, gsf;
6525   PetscInt       m, n;
6526   void          *ctx;
6527   DM             cdm;
6528   PetscBool      regular;
6529   PetscErrorCode ierr;
6530 
6531   PetscFunctionBegin;
6532   ierr = DMGetDefaultGlobalSection(dmFine, &gsf);CHKERRQ(ierr);
6533   ierr = PetscSectionGetConstrainedStorageSize(gsf, &m);CHKERRQ(ierr);
6534   ierr = DMGetDefaultGlobalSection(dmCoarse, &gsc);CHKERRQ(ierr);
6535   ierr = PetscSectionGetConstrainedStorageSize(gsc, &n);CHKERRQ(ierr);
6536 
6537   ierr = MatCreate(PetscObjectComm((PetscObject) dmCoarse), mass);CHKERRQ(ierr);
6538   ierr = MatSetSizes(*mass, m, n, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
6539   ierr = MatSetType(*mass, dmCoarse->mattype);CHKERRQ(ierr);
6540   ierr = DMGetApplicationContext(dmFine, &ctx);CHKERRQ(ierr);
6541 
6542   ierr = DMGetCoarseDM(dmFine, &cdm);CHKERRQ(ierr);
6543   ierr = DMPlexGetRegularRefinement(dmFine, &regular);CHKERRQ(ierr);
6544   if (regular && cdm == dmCoarse) {ierr = DMPlexComputeMassMatrixNested(dmCoarse, dmFine, *mass, ctx);CHKERRQ(ierr);}
6545   else                            {ierr = DMPlexComputeMassMatrixGeneral(dmCoarse, dmFine, *mass, ctx);CHKERRQ(ierr);}
6546   ierr = MatViewFromOptions(*mass, NULL, "-mass_mat_view");CHKERRQ(ierr);
6547   PetscFunctionReturn(0);
6548 }
6549 
6550 PetscErrorCode DMCreateDefaultSection_Plex(DM dm)
6551 {
6552   PetscSection   section;
6553   IS            *bcPoints, *bcComps;
6554   PetscBool     *isFE;
6555   PetscInt      *bcFields, *numComp, *numDof;
6556   PetscInt       depth, dim, numBd, numBC = 0, numFields, bd, bc = 0, f;
6557   PetscInt       cStart, cEnd, cEndInterior;
6558   PetscErrorCode ierr;
6559 
6560   PetscFunctionBegin;
6561   ierr = DMGetNumFields(dm, &numFields);CHKERRQ(ierr);
6562   /* FE and FV boundary conditions are handled slightly differently */
6563   ierr = PetscMalloc1(numFields, &isFE);CHKERRQ(ierr);
6564   for (f = 0; f < numFields; ++f) {
6565     PetscObject  obj;
6566     PetscClassId id;
6567 
6568     ierr = DMGetField(dm, f, &obj);CHKERRQ(ierr);
6569     ierr = PetscObjectGetClassId(obj, &id);CHKERRQ(ierr);
6570     if (id == PETSCFE_CLASSID)      {isFE[f] = PETSC_TRUE;}
6571     else if (id == PETSCFV_CLASSID) {isFE[f] = PETSC_FALSE;}
6572     else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %D", f);
6573   }
6574   /* Allocate boundary point storage for FEM boundaries */
6575   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
6576   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
6577   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
6578   ierr = DMPlexGetHybridBounds(dm, &cEndInterior, NULL, NULL, NULL);CHKERRQ(ierr);
6579   ierr = PetscDSGetNumBoundary(dm->prob, &numBd);CHKERRQ(ierr);
6580   if (!numFields && numBd) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "number of fields is zero and number of boundary conditions is nonzero (this should never happen)");
6581   for (bd = 0; bd < numBd; ++bd) {
6582     PetscInt                field;
6583     DMBoundaryConditionType type;
6584     const char             *labelName;
6585     DMLabel                 label;
6586 
6587     ierr = PetscDSGetBoundary(dm->prob, bd, &type, NULL, &labelName, &field, NULL, NULL, NULL, NULL, NULL, NULL);CHKERRQ(ierr);
6588     ierr = DMGetLabel(dm,labelName,&label);CHKERRQ(ierr);
6589     if (label && isFE[field] && (type & DM_BC_ESSENTIAL)) ++numBC;
6590   }
6591   /* Add ghost cell boundaries for FVM */
6592   for (f = 0; f < numFields; ++f) if (!isFE[f] && cEndInterior >= 0) ++numBC;
6593   ierr = PetscCalloc3(numBC,&bcFields,numBC,&bcPoints,numBC,&bcComps);CHKERRQ(ierr);
6594   /* Constrain ghost cells for FV */
6595   for (f = 0; f < numFields; ++f) {
6596     PetscInt *newidx, c;
6597 
6598     if (isFE[f] || cEndInterior < 0) continue;
6599     ierr = PetscMalloc1(cEnd-cEndInterior,&newidx);CHKERRQ(ierr);
6600     for (c = cEndInterior; c < cEnd; ++c) newidx[c-cEndInterior] = c;
6601     bcFields[bc] = f;
6602     ierr = ISCreateGeneral(PetscObjectComm((PetscObject) dm), cEnd-cEndInterior, newidx, PETSC_OWN_POINTER, &bcPoints[bc++]);CHKERRQ(ierr);
6603   }
6604   /* Handle FEM Dirichlet boundaries */
6605   for (bd = 0; bd < numBd; ++bd) {
6606     const char             *bdLabel;
6607     DMLabel                 label;
6608     const PetscInt         *comps;
6609     const PetscInt         *values;
6610     PetscInt                bd2, field, numComps, numValues;
6611     DMBoundaryConditionType type;
6612     PetscBool               duplicate = PETSC_FALSE;
6613 
6614     ierr = PetscDSGetBoundary(dm->prob, bd, &type, NULL, &bdLabel, &field, &numComps, &comps, NULL, &numValues, &values, NULL);CHKERRQ(ierr);
6615     ierr = DMGetLabel(dm, bdLabel, &label);CHKERRQ(ierr);
6616     if (!isFE[field] || !label) continue;
6617     /* Only want to modify label once */
6618     for (bd2 = 0; bd2 < bd; ++bd2) {
6619       const char *bdname;
6620       ierr = PetscDSGetBoundary(dm->prob, bd2, NULL, NULL, &bdname, NULL, NULL, NULL, NULL, NULL, NULL, NULL);CHKERRQ(ierr);
6621       ierr = PetscStrcmp(bdname, bdLabel, &duplicate);CHKERRQ(ierr);
6622       if (duplicate) break;
6623     }
6624     if (!duplicate && (isFE[field])) {
6625       /* don't complete cells, which are just present to give orientation to the boundary */
6626       ierr = DMPlexLabelComplete(dm, label);CHKERRQ(ierr);
6627     }
6628     /* Filter out cells, if you actually want to constrain cells you need to do things by hand right now */
6629     if (type & DM_BC_ESSENTIAL) {
6630       PetscInt       *newidx;
6631       PetscInt        n, newn = 0, p, v;
6632 
6633       bcFields[bc] = field;
6634       if (numComps) {ierr = ISCreateGeneral(PetscObjectComm((PetscObject) dm), numComps, comps, PETSC_COPY_VALUES, &bcComps[bc]);CHKERRQ(ierr);}
6635       for (v = 0; v < numValues; ++v) {
6636         IS              tmp;
6637         const PetscInt *idx;
6638 
6639         ierr = DMGetStratumIS(dm, bdLabel, values[v], &tmp);CHKERRQ(ierr);
6640         if (!tmp) continue;
6641         ierr = ISGetLocalSize(tmp, &n);CHKERRQ(ierr);
6642         ierr = ISGetIndices(tmp, &idx);CHKERRQ(ierr);
6643         if (isFE[field]) {
6644           for (p = 0; p < n; ++p) if ((idx[p] < cStart) || (idx[p] >= cEnd)) ++newn;
6645         } else {
6646           for (p = 0; p < n; ++p) if ((idx[p] >= cStart) || (idx[p] < cEnd)) ++newn;
6647         }
6648         ierr = ISRestoreIndices(tmp, &idx);CHKERRQ(ierr);
6649         ierr = ISDestroy(&tmp);CHKERRQ(ierr);
6650       }
6651       ierr = PetscMalloc1(newn,&newidx);CHKERRQ(ierr);
6652       newn = 0;
6653       for (v = 0; v < numValues; ++v) {
6654         IS              tmp;
6655         const PetscInt *idx;
6656 
6657         ierr = DMGetStratumIS(dm, bdLabel, values[v], &tmp);CHKERRQ(ierr);
6658         if (!tmp) continue;
6659         ierr = ISGetLocalSize(tmp, &n);CHKERRQ(ierr);
6660         ierr = ISGetIndices(tmp, &idx);CHKERRQ(ierr);
6661         if (isFE[field]) {
6662           for (p = 0; p < n; ++p) if ((idx[p] < cStart) || (idx[p] >= cEnd)) newidx[newn++] = idx[p];
6663         } else {
6664           for (p = 0; p < n; ++p) if ((idx[p] >= cStart) || (idx[p] < cEnd)) newidx[newn++] = idx[p];
6665         }
6666         ierr = ISRestoreIndices(tmp, &idx);CHKERRQ(ierr);
6667         ierr = ISDestroy(&tmp);CHKERRQ(ierr);
6668       }
6669       ierr = ISCreateGeneral(PetscObjectComm((PetscObject) dm), newn, newidx, PETSC_OWN_POINTER, &bcPoints[bc++]);CHKERRQ(ierr);
6670     }
6671   }
6672   /* Handle discretization */
6673   ierr = PetscCalloc2(numFields,&numComp,numFields*(dim+1),&numDof);CHKERRQ(ierr);
6674   for (f = 0; f < numFields; ++f) {
6675     PetscObject obj;
6676 
6677     ierr = DMGetField(dm, f, &obj);CHKERRQ(ierr);
6678     if (isFE[f]) {
6679       PetscFE         fe = (PetscFE) obj;
6680       const PetscInt *numFieldDof;
6681       PetscInt        d;
6682 
6683       ierr = PetscFEGetNumComponents(fe, &numComp[f]);CHKERRQ(ierr);
6684       ierr = PetscFEGetNumDof(fe, &numFieldDof);CHKERRQ(ierr);
6685       for (d = 0; d < dim+1; ++d) numDof[f*(dim+1)+d] = numFieldDof[d];
6686     } else {
6687       PetscFV fv = (PetscFV) obj;
6688 
6689       ierr = PetscFVGetNumComponents(fv, &numComp[f]);CHKERRQ(ierr);
6690       numDof[f*(dim+1)+dim] = numComp[f];
6691     }
6692   }
6693   for (f = 0; f < numFields; ++f) {
6694     PetscInt d;
6695     for (d = 1; d < dim; ++d) {
6696       if ((numDof[f*(dim+1)+d] > 0) && (depth < dim)) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Mesh must be interpolated when unknowns are specified on edges or faces.");
6697     }
6698   }
6699   ierr = DMPlexCreateSection(dm, dim, numFields, numComp, numDof, numBC, bcFields, bcComps, bcPoints, NULL, &section);CHKERRQ(ierr);
6700   for (f = 0; f < numFields; ++f) {
6701     PetscFE     fe;
6702     const char *name;
6703 
6704     ierr = DMGetField(dm, f, (PetscObject *) &fe);CHKERRQ(ierr);
6705     ierr = PetscObjectGetName((PetscObject) fe, &name);CHKERRQ(ierr);
6706     ierr = PetscSectionSetFieldName(section, f, name);CHKERRQ(ierr);
6707   }
6708   ierr = DMSetDefaultSection(dm, section);CHKERRQ(ierr);
6709   ierr = PetscSectionDestroy(&section);CHKERRQ(ierr);
6710   for (bc = 0; bc < numBC; ++bc) {ierr = ISDestroy(&bcPoints[bc]);CHKERRQ(ierr);ierr = ISDestroy(&bcComps[bc]);CHKERRQ(ierr);}
6711   ierr = PetscFree3(bcFields,bcPoints,bcComps);CHKERRQ(ierr);
6712   ierr = PetscFree2(numComp,numDof);CHKERRQ(ierr);
6713   ierr = PetscFree(isFE);CHKERRQ(ierr);
6714   PetscFunctionReturn(0);
6715 }
6716 
6717 /*@
6718   DMPlexGetRegularRefinement - Get the flag indicating that this mesh was obtained by regular refinement from its coarse mesh
6719 
6720   Input Parameter:
6721 . dm - The DMPlex object
6722 
6723   Output Parameter:
6724 . regular - The flag
6725 
6726   Level: intermediate
6727 
6728 .seealso: DMPlexSetRegularRefinement()
6729 @*/
6730 PetscErrorCode DMPlexGetRegularRefinement(DM dm, PetscBool *regular)
6731 {
6732   PetscFunctionBegin;
6733   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6734   PetscValidPointer(regular, 2);
6735   *regular = ((DM_Plex *) dm->data)->regularRefinement;
6736   PetscFunctionReturn(0);
6737 }
6738 
6739 /*@
6740   DMPlexSetRegularRefinement - Set the flag indicating that this mesh was obtained by regular refinement from its coarse mesh
6741 
6742   Input Parameters:
6743 + dm - The DMPlex object
6744 - regular - The flag
6745 
6746   Level: intermediate
6747 
6748 .seealso: DMPlexGetRegularRefinement()
6749 @*/
6750 PetscErrorCode DMPlexSetRegularRefinement(DM dm, PetscBool regular)
6751 {
6752   PetscFunctionBegin;
6753   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6754   ((DM_Plex *) dm->data)->regularRefinement = regular;
6755   PetscFunctionReturn(0);
6756 }
6757 
6758 /* anchors */
6759 /*@
6760   DMPlexGetAnchors - Get the layout of the anchor (point-to-point) constraints.  Typically, the user will not have to
6761   call DMPlexGetAnchors() directly: if there are anchors, then DMPlexGetAnchors() is called during DMGetConstraints().
6762 
6763   not collective
6764 
6765   Input Parameters:
6766 . dm - The DMPlex object
6767 
6768   Output Parameters:
6769 + anchorSection - If not NULL, set to the section describing which points anchor the constrained points.
6770 - anchorIS - If not NULL, set to the list of anchors indexed by anchorSection
6771 
6772 
6773   Level: intermediate
6774 
6775 .seealso: DMPlexSetAnchors(), DMGetConstraints(), DMSetConstraints()
6776 @*/
6777 PetscErrorCode DMPlexGetAnchors(DM dm, PetscSection *anchorSection, IS *anchorIS)
6778 {
6779   DM_Plex *plex = (DM_Plex *)dm->data;
6780   PetscErrorCode ierr;
6781 
6782   PetscFunctionBegin;
6783   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6784   if (!plex->anchorSection && !plex->anchorIS && plex->createanchors) {ierr = (*plex->createanchors)(dm);CHKERRQ(ierr);}
6785   if (anchorSection) *anchorSection = plex->anchorSection;
6786   if (anchorIS) *anchorIS = plex->anchorIS;
6787   PetscFunctionReturn(0);
6788 }
6789 
6790 /*@
6791   DMPlexSetAnchors - Set the layout of the local anchor (point-to-point) constraints.  Unlike boundary conditions,
6792   when a point's degrees of freedom in a section are constrained to an outside value, the anchor constraints set a
6793   point's degrees of freedom to be a linear combination of other points' degrees of freedom.
6794 
6795   After specifying the layout of constraints with DMPlexSetAnchors(), one specifies the constraints by calling
6796   DMGetConstraints() and filling in the entries in the constraint matrix.
6797 
6798   collective on dm
6799 
6800   Input Parameters:
6801 + dm - The DMPlex object
6802 . anchorSection - The section that describes the mapping from constrained points to the anchor points listed in anchorIS.  Must have a local communicator (PETSC_COMM_SELF or derivative).
6803 - anchorIS - The list of all anchor points.  Must have a local communicator (PETSC_COMM_SELF or derivative).
6804 
6805   The reference counts of anchorSection and anchorIS are incremented.
6806 
6807   Level: intermediate
6808 
6809 .seealso: DMPlexGetAnchors(), DMGetConstraints(), DMSetConstraints()
6810 @*/
6811 PetscErrorCode DMPlexSetAnchors(DM dm, PetscSection anchorSection, IS anchorIS)
6812 {
6813   DM_Plex        *plex = (DM_Plex *)dm->data;
6814   PetscMPIInt    result;
6815   PetscErrorCode ierr;
6816 
6817   PetscFunctionBegin;
6818   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6819   if (anchorSection) {
6820     PetscValidHeaderSpecific(anchorSection,PETSC_SECTION_CLASSID,2);
6821     ierr = MPI_Comm_compare(PETSC_COMM_SELF,PetscObjectComm((PetscObject)anchorSection),&result);CHKERRQ(ierr);
6822     if (result != MPI_CONGRUENT && result != MPI_IDENT) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_NOTSAMECOMM,"anchor section must have local communicator");
6823   }
6824   if (anchorIS) {
6825     PetscValidHeaderSpecific(anchorIS,IS_CLASSID,3);
6826     ierr = MPI_Comm_compare(PETSC_COMM_SELF,PetscObjectComm((PetscObject)anchorIS),&result);CHKERRQ(ierr);
6827     if (result != MPI_CONGRUENT && result != MPI_IDENT) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_NOTSAMECOMM,"anchor IS must have local communicator");
6828   }
6829 
6830   ierr = PetscObjectReference((PetscObject)anchorSection);CHKERRQ(ierr);
6831   ierr = PetscSectionDestroy(&plex->anchorSection);CHKERRQ(ierr);
6832   plex->anchorSection = anchorSection;
6833 
6834   ierr = PetscObjectReference((PetscObject)anchorIS);CHKERRQ(ierr);
6835   ierr = ISDestroy(&plex->anchorIS);CHKERRQ(ierr);
6836   plex->anchorIS = anchorIS;
6837 
6838 #if defined(PETSC_USE_DEBUG)
6839   if (anchorIS && anchorSection) {
6840     PetscInt size, a, pStart, pEnd;
6841     const PetscInt *anchors;
6842 
6843     ierr = PetscSectionGetChart(anchorSection,&pStart,&pEnd);CHKERRQ(ierr);
6844     ierr = ISGetLocalSize(anchorIS,&size);CHKERRQ(ierr);
6845     ierr = ISGetIndices(anchorIS,&anchors);CHKERRQ(ierr);
6846     for (a = 0; a < size; a++) {
6847       PetscInt p;
6848 
6849       p = anchors[a];
6850       if (p >= pStart && p < pEnd) {
6851         PetscInt dof;
6852 
6853         ierr = PetscSectionGetDof(anchorSection,p,&dof);CHKERRQ(ierr);
6854         if (dof) {
6855           PetscErrorCode ierr2;
6856 
6857           ierr2 = ISRestoreIndices(anchorIS,&anchors);CHKERRQ(ierr2);
6858           SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_INCOMP,"Point %D cannot be constrained and an anchor",p);
6859         }
6860       }
6861     }
6862     ierr = ISRestoreIndices(anchorIS,&anchors);CHKERRQ(ierr);
6863   }
6864 #endif
6865   /* reset the generic constraints */
6866   ierr = DMSetDefaultConstraints(dm,NULL,NULL);CHKERRQ(ierr);
6867   PetscFunctionReturn(0);
6868 }
6869 
6870 static PetscErrorCode DMPlexCreateConstraintSection_Anchors(DM dm, PetscSection section, PetscSection *cSec)
6871 {
6872   PetscSection anchorSection;
6873   PetscInt pStart, pEnd, sStart, sEnd, p, dof, numFields, f;
6874   PetscErrorCode ierr;
6875 
6876   PetscFunctionBegin;
6877   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6878   ierr = DMPlexGetAnchors(dm,&anchorSection,NULL);CHKERRQ(ierr);
6879   ierr = PetscSectionCreate(PETSC_COMM_SELF,cSec);CHKERRQ(ierr);
6880   ierr = PetscSectionGetNumFields(section,&numFields);CHKERRQ(ierr);
6881   if (numFields) {
6882     PetscInt f;
6883     ierr = PetscSectionSetNumFields(*cSec,numFields);CHKERRQ(ierr);
6884 
6885     for (f = 0; f < numFields; f++) {
6886       PetscInt numComp;
6887 
6888       ierr = PetscSectionGetFieldComponents(section,f,&numComp);CHKERRQ(ierr);
6889       ierr = PetscSectionSetFieldComponents(*cSec,f,numComp);CHKERRQ(ierr);
6890     }
6891   }
6892   ierr = PetscSectionGetChart(anchorSection,&pStart,&pEnd);CHKERRQ(ierr);
6893   ierr = PetscSectionGetChart(section,&sStart,&sEnd);CHKERRQ(ierr);
6894   pStart = PetscMax(pStart,sStart);
6895   pEnd   = PetscMin(pEnd,sEnd);
6896   pEnd   = PetscMax(pStart,pEnd);
6897   ierr = PetscSectionSetChart(*cSec,pStart,pEnd);CHKERRQ(ierr);
6898   for (p = pStart; p < pEnd; p++) {
6899     ierr = PetscSectionGetDof(anchorSection,p,&dof);CHKERRQ(ierr);
6900     if (dof) {
6901       ierr = PetscSectionGetDof(section,p,&dof);CHKERRQ(ierr);
6902       ierr = PetscSectionSetDof(*cSec,p,dof);CHKERRQ(ierr);
6903       for (f = 0; f < numFields; f++) {
6904         ierr = PetscSectionGetFieldDof(section,p,f,&dof);CHKERRQ(ierr);
6905         ierr = PetscSectionSetFieldDof(*cSec,p,f,dof);CHKERRQ(ierr);
6906       }
6907     }
6908   }
6909   ierr = PetscSectionSetUp(*cSec);CHKERRQ(ierr);
6910   PetscFunctionReturn(0);
6911 }
6912 
6913 static PetscErrorCode DMPlexCreateConstraintMatrix_Anchors(DM dm, PetscSection section, PetscSection cSec, Mat *cMat)
6914 {
6915   PetscSection aSec;
6916   PetscInt pStart, pEnd, p, dof, aDof, aOff, off, nnz, annz, m, n, q, a, offset, *i, *j;
6917   const PetscInt *anchors;
6918   PetscInt numFields, f;
6919   IS aIS;
6920   PetscErrorCode ierr;
6921 
6922   PetscFunctionBegin;
6923   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6924   ierr = PetscSectionGetStorageSize(cSec, &m);CHKERRQ(ierr);
6925   ierr = PetscSectionGetStorageSize(section, &n);CHKERRQ(ierr);
6926   ierr = MatCreate(PETSC_COMM_SELF,cMat);CHKERRQ(ierr);
6927   ierr = MatSetSizes(*cMat,m,n,m,n);CHKERRQ(ierr);
6928   ierr = MatSetType(*cMat,MATSEQAIJ);CHKERRQ(ierr);
6929   ierr = DMPlexGetAnchors(dm,&aSec,&aIS);CHKERRQ(ierr);
6930   ierr = ISGetIndices(aIS,&anchors);CHKERRQ(ierr);
6931   /* cSec will be a subset of aSec and section */
6932   ierr = PetscSectionGetChart(cSec,&pStart,&pEnd);CHKERRQ(ierr);
6933   ierr = PetscMalloc1(m+1,&i);CHKERRQ(ierr);
6934   i[0] = 0;
6935   ierr = PetscSectionGetNumFields(section,&numFields);CHKERRQ(ierr);
6936   for (p = pStart; p < pEnd; p++) {
6937     PetscInt rDof, rOff, r;
6938 
6939     ierr = PetscSectionGetDof(aSec,p,&rDof);CHKERRQ(ierr);
6940     if (!rDof) continue;
6941     ierr = PetscSectionGetOffset(aSec,p,&rOff);CHKERRQ(ierr);
6942     if (numFields) {
6943       for (f = 0; f < numFields; f++) {
6944         annz = 0;
6945         for (r = 0; r < rDof; r++) {
6946           a = anchors[rOff + r];
6947           ierr = PetscSectionGetFieldDof(section,a,f,&aDof);CHKERRQ(ierr);
6948           annz += aDof;
6949         }
6950         ierr = PetscSectionGetFieldDof(cSec,p,f,&dof);CHKERRQ(ierr);
6951         ierr = PetscSectionGetFieldOffset(cSec,p,f,&off);CHKERRQ(ierr);
6952         for (q = 0; q < dof; q++) {
6953           i[off + q + 1] = i[off + q] + annz;
6954         }
6955       }
6956     }
6957     else {
6958       annz = 0;
6959       for (q = 0; q < dof; q++) {
6960         a = anchors[off + q];
6961         ierr = PetscSectionGetDof(section,a,&aDof);CHKERRQ(ierr);
6962         annz += aDof;
6963       }
6964       ierr = PetscSectionGetDof(cSec,p,&dof);CHKERRQ(ierr);
6965       ierr = PetscSectionGetOffset(cSec,p,&off);CHKERRQ(ierr);
6966       for (q = 0; q < dof; q++) {
6967         i[off + q + 1] = i[off + q] + annz;
6968       }
6969     }
6970   }
6971   nnz = i[m];
6972   ierr = PetscMalloc1(nnz,&j);CHKERRQ(ierr);
6973   offset = 0;
6974   for (p = pStart; p < pEnd; p++) {
6975     if (numFields) {
6976       for (f = 0; f < numFields; f++) {
6977         ierr = PetscSectionGetFieldDof(cSec,p,f,&dof);CHKERRQ(ierr);
6978         for (q = 0; q < dof; q++) {
6979           PetscInt rDof, rOff, r;
6980           ierr = PetscSectionGetDof(aSec,p,&rDof);CHKERRQ(ierr);
6981           ierr = PetscSectionGetOffset(aSec,p,&rOff);CHKERRQ(ierr);
6982           for (r = 0; r < rDof; r++) {
6983             PetscInt s;
6984 
6985             a = anchors[rOff + r];
6986             ierr = PetscSectionGetFieldDof(section,a,f,&aDof);CHKERRQ(ierr);
6987             ierr = PetscSectionGetFieldOffset(section,a,f,&aOff);CHKERRQ(ierr);
6988             for (s = 0; s < aDof; s++) {
6989               j[offset++] = aOff + s;
6990             }
6991           }
6992         }
6993       }
6994     }
6995     else {
6996       ierr = PetscSectionGetDof(cSec,p,&dof);CHKERRQ(ierr);
6997       for (q = 0; q < dof; q++) {
6998         PetscInt rDof, rOff, r;
6999         ierr = PetscSectionGetDof(aSec,p,&rDof);CHKERRQ(ierr);
7000         ierr = PetscSectionGetOffset(aSec,p,&rOff);CHKERRQ(ierr);
7001         for (r = 0; r < rDof; r++) {
7002           PetscInt s;
7003 
7004           a = anchors[rOff + r];
7005           ierr = PetscSectionGetDof(section,a,&aDof);CHKERRQ(ierr);
7006           ierr = PetscSectionGetOffset(section,a,&aOff);CHKERRQ(ierr);
7007           for (s = 0; s < aDof; s++) {
7008             j[offset++] = aOff + s;
7009           }
7010         }
7011       }
7012     }
7013   }
7014   ierr = MatSeqAIJSetPreallocationCSR(*cMat,i,j,NULL);CHKERRQ(ierr);
7015   ierr = PetscFree(i);CHKERRQ(ierr);
7016   ierr = PetscFree(j);CHKERRQ(ierr);
7017   ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);
7018   PetscFunctionReturn(0);
7019 }
7020 
7021 PetscErrorCode DMCreateDefaultConstraints_Plex(DM dm)
7022 {
7023   DM_Plex        *plex = (DM_Plex *)dm->data;
7024   PetscSection   anchorSection, section, cSec;
7025   Mat            cMat;
7026   PetscErrorCode ierr;
7027 
7028   PetscFunctionBegin;
7029   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7030   ierr = DMPlexGetAnchors(dm,&anchorSection,NULL);CHKERRQ(ierr);
7031   if (anchorSection) {
7032     PetscDS  ds;
7033     PetscInt nf;
7034 
7035     ierr = DMGetDefaultSection(dm,&section);CHKERRQ(ierr);
7036     ierr = DMPlexCreateConstraintSection_Anchors(dm,section,&cSec);CHKERRQ(ierr);
7037     ierr = DMPlexCreateConstraintMatrix_Anchors(dm,section,cSec,&cMat);CHKERRQ(ierr);
7038     ierr = DMGetDS(dm,&ds);CHKERRQ(ierr);
7039     ierr = PetscDSGetNumFields(ds,&nf);CHKERRQ(ierr);
7040     if (nf && plex->computeanchormatrix) {ierr = (*plex->computeanchormatrix)(dm,section,cSec,cMat);CHKERRQ(ierr);}
7041     ierr = DMSetDefaultConstraints(dm,cSec,cMat);CHKERRQ(ierr);
7042     ierr = PetscSectionDestroy(&cSec);CHKERRQ(ierr);
7043     ierr = MatDestroy(&cMat);CHKERRQ(ierr);
7044   }
7045   PetscFunctionReturn(0);
7046 }
7047