xref: /petsc/src/dm/impls/plex/plex.c (revision e7bdbf8ee2d7e05ba48565a98608913b85d23ec9)
1 #include <petsc-private/dmpleximpl.h>   /*I      "petscdmplex.h"   I*/
2 #include <../src/sys/utils/hash.h>
3 #include <petsc-private/isimpl.h>
4 #include <petscsf.h>
5 #include <petscviewerhdf5.h>
6 
7 /* Logging support */
8 PetscLogEvent DMPLEX_Interpolate, DMPLEX_Partition, DMPLEX_Distribute, DMPLEX_DistributeCones, DMPLEX_DistributeLabels, DMPLEX_DistributeSF, DMPLEX_DistributeField, DMPLEX_DistributeData, DMPLEX_Stratify, DMPLEX_Preallocate, DMPLEX_ResidualFEM, DMPLEX_JacobianFEM;
9 
10 PETSC_EXTERN PetscErrorCode VecView_Seq(Vec, PetscViewer);
11 PETSC_EXTERN PetscErrorCode VecView_MPI(Vec, PetscViewer);
12 PETSC_EXTERN PetscErrorCode VecLoad_Default(Vec, PetscViewer);
13 PETSC_EXTERN PetscErrorCode DMTSGetTimeStepNumber(DM,PetscInt*);
14 
15 #undef __FUNCT__
16 #define __FUNCT__ "GetField_Static"
17 static PetscErrorCode GetField_Static(DM dm, PetscSection section, PetscSection sectionGlobal, Vec v, PetscInt field, PetscInt pStart, PetscInt pEnd, IS *is, Vec *subv)
18 {
19   PetscInt      *subIndices;
20   PetscInt       Nc, subSize = 0, subOff = 0, p;
21   PetscErrorCode ierr;
22 
23   PetscFunctionBegin;
24   ierr = PetscSectionGetFieldComponents(section, field, &Nc);CHKERRQ(ierr);
25   for (p = pStart; p < pEnd; ++p) {
26     PetscInt gdof, fdof = 0;
27 
28     ierr = PetscSectionGetDof(sectionGlobal, p, &gdof);CHKERRQ(ierr);
29     if (gdof > 0) {ierr = PetscSectionGetFieldDof(section, p, field, &fdof);CHKERRQ(ierr);}
30     subSize += fdof;
31   }
32   ierr = PetscMalloc1(subSize, &subIndices);CHKERRQ(ierr);
33   for (p = pStart; p < pEnd; ++p) {
34     PetscInt gdof, goff;
35 
36     ierr = PetscSectionGetDof(sectionGlobal, p, &gdof);CHKERRQ(ierr);
37     if (gdof > 0) {
38       PetscInt fdof, fc, f2, poff = 0;
39 
40       ierr = PetscSectionGetOffset(sectionGlobal, p, &goff);CHKERRQ(ierr);
41       /* Can get rid of this loop by storing field information in the global section */
42       for (f2 = 0; f2 < field; ++f2) {
43         ierr  = PetscSectionGetFieldDof(section, p, f2, &fdof);CHKERRQ(ierr);
44         poff += fdof;
45       }
46       ierr = PetscSectionGetFieldDof(section, p, field, &fdof);CHKERRQ(ierr);
47       for (fc = 0; fc < fdof; ++fc, ++subOff) subIndices[subOff] = goff+poff+fc;
48     }
49   }
50   ierr = ISCreateGeneral(PetscObjectComm((PetscObject) dm), subSize, subIndices, PETSC_OWN_POINTER, is);CHKERRQ(ierr);
51   ierr = VecGetSubVector(v, *is, subv);CHKERRQ(ierr);
52   ierr = VecSetBlockSize(*subv, Nc);CHKERRQ(ierr);
53   PetscFunctionReturn(0);
54 }
55 
56 #undef __FUNCT__
57 #define __FUNCT__ "RestoreField_Static"
58 static PetscErrorCode RestoreField_Static(DM dm, PetscSection section, PetscSection sectionGlobal, Vec v, PetscInt field, PetscInt pStart, PetscInt pEnd, IS *is, Vec *subv)
59 {
60   PetscErrorCode ierr;
61 
62   PetscFunctionBegin;
63   ierr = VecRestoreSubVector(v, *is, subv);CHKERRQ(ierr);
64   ierr = ISDestroy(is);CHKERRQ(ierr);
65   PetscFunctionReturn(0);
66 }
67 
68 #undef __FUNCT__
69 #define __FUNCT__ "GetFieldType_Static"
70 static PetscErrorCode GetFieldType_Static(DM dm, PetscSection section, PetscInt field, PetscInt *sStart, PetscInt *sEnd, PetscViewerVTKFieldType *ft)
71 {
72   PetscInt       dim, pStart, pEnd, vStart, vEnd, cStart, cEnd, vdof = 0, cdof = 0;
73   PetscErrorCode ierr;
74 
75   PetscFunctionBegin;
76   *ft  = PETSC_VTK_POINT_FIELD;
77   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
78   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
79   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
80   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
81   if (field >= 0) {
82     if ((vStart >= pStart) && (vStart < pEnd)) {ierr = PetscSectionGetFieldDof(section, vStart, field, &vdof);CHKERRQ(ierr);}
83     if ((cStart >= pStart) && (cStart < pEnd)) {ierr = PetscSectionGetFieldDof(section, cStart, field, &cdof);CHKERRQ(ierr);}
84   } else {
85     if ((vStart >= pStart) && (vStart < pEnd)) {ierr = PetscSectionGetDof(section, vStart, &vdof);CHKERRQ(ierr);}
86     if ((cStart >= pStart) && (cStart < pEnd)) {ierr = PetscSectionGetDof(section, cStart, &cdof);CHKERRQ(ierr);}
87   }
88   if (vdof) {
89     *sStart = vStart;
90     *sEnd   = vEnd;
91     if (vdof == dim) *ft = PETSC_VTK_POINT_VECTOR_FIELD;
92     else             *ft = PETSC_VTK_POINT_FIELD;
93   } else if (cdof) {
94     *sStart = cStart;
95     *sEnd   = cEnd;
96     if (cdof == dim) *ft = PETSC_VTK_CELL_VECTOR_FIELD;
97     else             *ft = PETSC_VTK_CELL_FIELD;
98   } else SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Could not classify input Vec for VTK");
99   PetscFunctionReturn(0);
100 }
101 
102 #if defined(PETSC_HAVE_HDF5)
103 #undef __FUNCT__
104 #define __FUNCT__ "DMSequenceView_HDF5"
105 static PetscErrorCode DMSequenceView_HDF5(DM dm, const char *seqname, PetscInt seqnum, PetscScalar value, PetscViewer viewer)
106 {
107   Vec            stamp;
108   PetscMPIInt    rank;
109   PetscErrorCode ierr;
110 
111   PetscFunctionBegin;
112   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) viewer), &rank);CHKERRQ(ierr);
113   ierr = VecCreateMPI(PetscObjectComm((PetscObject) viewer), rank ? 0 : 1, 1, &stamp);CHKERRQ(ierr);
114   ierr = VecSetBlockSize(stamp, 1);CHKERRQ(ierr);
115   ierr = PetscObjectSetName((PetscObject) stamp, seqname);CHKERRQ(ierr);
116   if (!rank) {
117     PetscReal timeScale;
118     PetscBool istime;
119 
120     ierr = PetscStrncmp(seqname, "time", 5, &istime);CHKERRQ(ierr);
121     if (istime) {ierr = DMPlexGetScale(dm, PETSC_UNIT_TIME, &timeScale);CHKERRQ(ierr); value *= timeScale;}
122     ierr = VecSetValue(stamp, 0, value, INSERT_VALUES);CHKERRQ(ierr);
123   }
124   ierr = VecAssemblyBegin(stamp);CHKERRQ(ierr);
125   ierr = VecAssemblyEnd(stamp);CHKERRQ(ierr);
126   ierr = PetscViewerHDF5PushGroup(viewer, "/");CHKERRQ(ierr);
127   ierr = PetscViewerHDF5SetTimestep(viewer, seqnum);CHKERRQ(ierr);
128   ierr = VecView(stamp, viewer);CHKERRQ(ierr);
129   ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
130   ierr = VecDestroy(&stamp);CHKERRQ(ierr);
131   PetscFunctionReturn(0);
132 }
133 
134 #undef __FUNCT__
135 #define __FUNCT__ "VecView_Plex_Local_HDF5"
136 static PetscErrorCode VecView_Plex_Local_HDF5(Vec v, PetscViewer viewer)
137 {
138   DM                      dm;
139   DM                      dmBC;
140   PetscSection            section, sectionGlobal;
141   Vec                     gv;
142   const char             *name;
143   PetscViewerVTKFieldType ft;
144   PetscViewerFormat       format;
145   PetscInt                seqnum;
146   PetscBool               isseq;
147   PetscErrorCode          ierr;
148 
149   PetscFunctionBegin;
150   ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
151   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
152   ierr = DMGetDefaultSection(dm, &section);CHKERRQ(ierr);
153   ierr = DMGetOutputSequenceNumber(dm, &seqnum);CHKERRQ(ierr);
154   ierr = PetscViewerHDF5SetTimestep(viewer, seqnum);CHKERRQ(ierr);
155   ierr = DMSequenceView_HDF5(dm, "time", seqnum, 0.0, viewer);CHKERRQ(ierr);
156   ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
157   ierr = DMGetOutputDM(dm, &dmBC);CHKERRQ(ierr);
158   ierr = DMGetDefaultGlobalSection(dmBC, &sectionGlobal);CHKERRQ(ierr);
159   ierr = DMGetGlobalVector(dmBC, &gv);CHKERRQ(ierr);
160   ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
161   ierr = PetscObjectSetName((PetscObject) gv, name);CHKERRQ(ierr);
162   ierr = DMLocalToGlobalBegin(dmBC, v, INSERT_VALUES, gv);CHKERRQ(ierr);
163   ierr = DMLocalToGlobalEnd(dmBC, v, INSERT_VALUES, gv);CHKERRQ(ierr);
164   ierr = PetscObjectTypeCompare((PetscObject) gv, VECSEQ, &isseq);CHKERRQ(ierr);
165   if (isseq) {ierr = VecView_Seq(gv, viewer);CHKERRQ(ierr);}
166   else       {ierr = VecView_MPI(gv, viewer);CHKERRQ(ierr);}
167   if (format == PETSC_VIEWER_HDF5_VIZ) {
168     /* Output visualization representation */
169     PetscInt numFields, f;
170 
171     ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
172     for (f = 0; f < numFields; ++f) {
173       Vec         subv;
174       IS          is;
175       const char *fname, *fgroup;
176       char        group[PETSC_MAX_PATH_LEN];
177       PetscInt    pStart, pEnd;
178 
179       ierr = GetFieldType_Static(dm, section, f, &pStart, &pEnd, &ft);CHKERRQ(ierr);
180       fgroup = (ft == PETSC_VTK_POINT_VECTOR_FIELD) || (ft == PETSC_VTK_POINT_FIELD) ? "/vertex_fields" : "/cell_fields";
181       ierr = PetscSectionGetFieldName(section, f, &fname);CHKERRQ(ierr);
182       ierr = PetscViewerHDF5PushGroup(viewer, fgroup);CHKERRQ(ierr);
183       ierr = GetField_Static(dmBC, section, sectionGlobal, gv, f, pStart, pEnd, &is, &subv);CHKERRQ(ierr);
184       ierr = PetscObjectSetName((PetscObject) subv, fname);CHKERRQ(ierr);
185       if (isseq) {ierr = VecView_Seq(subv, viewer);CHKERRQ(ierr);}
186       else       {ierr = VecView_MPI(subv, viewer);CHKERRQ(ierr);}
187       ierr = RestoreField_Static(dmBC, section, sectionGlobal, gv, f, pStart, pEnd, &is, &subv);CHKERRQ(ierr);
188       ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
189       ierr = PetscSNPrintf(group, PETSC_MAX_PATH_LEN, "%s/%s", fgroup, fname);CHKERRQ(ierr);
190       if ((ft == PETSC_VTK_POINT_VECTOR_FIELD) || (ft == PETSC_VTK_CELL_VECTOR_FIELD)) {
191         ierr = PetscViewerHDF5WriteAttribute(viewer, group, "vector_field_type", PETSC_STRING, "vector");CHKERRQ(ierr);
192       } else {
193         ierr = PetscViewerHDF5WriteAttribute(viewer, group, "vector_field_type", PETSC_STRING, "scalar");CHKERRQ(ierr);
194       }
195     }
196   }
197   ierr = DMRestoreGlobalVector(dmBC, &gv);CHKERRQ(ierr);
198   PetscFunctionReturn(0);
199 }
200 #endif
201 
202 #undef __FUNCT__
203 #define __FUNCT__ "VecView_Plex_Local"
204 PetscErrorCode VecView_Plex_Local(Vec v, PetscViewer viewer)
205 {
206   DM             dm;
207   PetscBool      isvtk, ishdf5, isseq;
208   PetscErrorCode ierr;
209 
210   PetscFunctionBegin;
211   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
212   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
213   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,  &isvtk);CHKERRQ(ierr);
214   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
215   ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
216   if (isvtk || ishdf5) {ierr = DMPlexInsertBoundaryValues(dm, v);CHKERRQ(ierr);}
217   if (isvtk) {
218     PetscSection            section;
219     PetscViewerVTKFieldType ft;
220     PetscInt                pStart, pEnd;
221 
222     ierr = DMGetDefaultSection(dm, &section);CHKERRQ(ierr);
223     ierr = GetFieldType_Static(dm, section, PETSC_DETERMINE, &pStart, &pEnd, &ft);CHKERRQ(ierr);
224     ierr = PetscObjectReference((PetscObject) dm);CHKERRQ(ierr); /* viewer drops reference */
225     ierr = PetscObjectReference((PetscObject) v);CHKERRQ(ierr);  /* viewer drops reference */
226     ierr = PetscViewerVTKAddField(viewer, (PetscObject) dm, DMPlexVTKWriteAll, ft, (PetscObject) v);CHKERRQ(ierr);
227   } else if (ishdf5) {
228 #if defined(PETSC_HAVE_HDF5)
229     ierr = VecView_Plex_Local_HDF5(v, viewer);CHKERRQ(ierr);
230 #else
231     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
232 #endif
233   } else {
234     if (isseq) {ierr = VecView_Seq(v, viewer);CHKERRQ(ierr);}
235     else       {ierr = VecView_MPI(v, viewer);CHKERRQ(ierr);}
236   }
237   PetscFunctionReturn(0);
238 }
239 
240 #if defined(PETSC_HAVE_HDF5)
241 #undef __FUNCT__
242 #define __FUNCT__ "VecView_Plex_HDF5"
243 static PetscErrorCode VecView_Plex_HDF5(Vec v, PetscViewer viewer)
244 {
245   DM             dm;
246   Vec            locv;
247   const char    *name;
248   PetscErrorCode ierr;
249 
250   PetscFunctionBegin;
251   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
252   ierr = DMGetLocalVector(dm, &locv);CHKERRQ(ierr);
253   ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
254   ierr = PetscObjectSetName((PetscObject) locv, name);CHKERRQ(ierr);
255   ierr = DMGlobalToLocalBegin(dm, v, INSERT_VALUES, locv);CHKERRQ(ierr);
256   ierr = DMGlobalToLocalEnd(dm, v, INSERT_VALUES, locv);CHKERRQ(ierr);
257   ierr = PetscViewerHDF5PushGroup(viewer, "/fields");CHKERRQ(ierr);
258   ierr = PetscViewerPushFormat(viewer, PETSC_VIEWER_HDF5_VIZ);CHKERRQ(ierr);
259   ierr = VecView_Plex_Local(locv, viewer);CHKERRQ(ierr);
260   ierr = PetscViewerPopFormat(viewer);CHKERRQ(ierr);
261   ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
262   ierr = DMRestoreLocalVector(dm, &locv);CHKERRQ(ierr);
263   PetscFunctionReturn(0);
264 }
265 #endif
266 
267 #undef __FUNCT__
268 #define __FUNCT__ "VecView_Plex"
269 PetscErrorCode VecView_Plex(Vec v, PetscViewer viewer)
270 {
271   DM             dm;
272   PetscBool      isvtk, ishdf5, isseq;
273   PetscErrorCode ierr;
274 
275   PetscFunctionBegin;
276   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
277   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
278   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,  &isvtk);CHKERRQ(ierr);
279   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
280   ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
281   if (isvtk) {
282     Vec         locv;
283     const char *name;
284 
285     ierr = DMGetLocalVector(dm, &locv);CHKERRQ(ierr);
286     ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
287     ierr = PetscObjectSetName((PetscObject) locv, name);CHKERRQ(ierr);
288     ierr = DMGlobalToLocalBegin(dm, v, INSERT_VALUES, locv);CHKERRQ(ierr);
289     ierr = DMGlobalToLocalEnd(dm, v, INSERT_VALUES, locv);CHKERRQ(ierr);
290     ierr = VecView_Plex_Local(locv, viewer);CHKERRQ(ierr);
291     ierr = DMRestoreLocalVector(dm, &locv);CHKERRQ(ierr);
292   } else if (ishdf5) {
293 #if defined(PETSC_HAVE_HDF5)
294     ierr = VecView_Plex_HDF5(v, viewer);CHKERRQ(ierr);
295 #else
296     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
297 #endif
298   } else {
299     if (isseq) {ierr = VecView_Seq(v, viewer);CHKERRQ(ierr);}
300     else       {ierr = VecView_MPI(v, viewer);CHKERRQ(ierr);}
301   }
302   PetscFunctionReturn(0);
303 }
304 
305 #undef __FUNCT__
306 #define __FUNCT__ "VecLoad_Plex_Local"
307 PetscErrorCode VecLoad_Plex_Local(Vec v, PetscViewer viewer)
308 {
309   DM             dm;
310   PetscBool      ishdf5;
311   PetscErrorCode ierr;
312 
313   PetscFunctionBegin;
314   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
315   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
316   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
317   if (ishdf5) {
318     DM          dmBC;
319     Vec         gv;
320     const char *name;
321 
322     ierr = DMGetOutputDM(dm, &dmBC);CHKERRQ(ierr);
323     ierr = DMGetGlobalVector(dmBC, &gv);CHKERRQ(ierr);
324     ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
325     ierr = PetscObjectSetName((PetscObject) gv, name);CHKERRQ(ierr);
326     ierr = VecLoad_Default(gv, viewer);CHKERRQ(ierr);
327     ierr = DMGlobalToLocalBegin(dmBC, gv, INSERT_VALUES, v);CHKERRQ(ierr);
328     ierr = DMGlobalToLocalEnd(dmBC, gv, INSERT_VALUES, v);CHKERRQ(ierr);
329     ierr = DMRestoreGlobalVector(dmBC, &gv);CHKERRQ(ierr);
330   } else {
331     ierr = VecLoad_Default(v, viewer);CHKERRQ(ierr);
332   }
333   PetscFunctionReturn(0);
334 }
335 
336 #if defined(PETSC_HAVE_HDF5)
337 #undef __FUNCT__
338 #define __FUNCT__ "VecLoad_Plex_HDF5"
339 static PetscErrorCode VecLoad_Plex_HDF5(Vec v, PetscViewer viewer)
340 {
341   DM             dm;
342   Vec            locv;
343   const char    *name;
344   PetscInt       seqnum;
345   PetscErrorCode ierr;
346 
347   PetscFunctionBegin;
348   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
349   ierr = DMGetLocalVector(dm, &locv);CHKERRQ(ierr);
350   ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
351   ierr = PetscObjectSetName((PetscObject) locv, name);CHKERRQ(ierr);
352   ierr = DMGetOutputSequenceNumber(dm, &seqnum);CHKERRQ(ierr);
353   ierr = PetscViewerHDF5SetTimestep(viewer, seqnum);CHKERRQ(ierr);
354   ierr = PetscViewerHDF5PushGroup(viewer, "/fields");CHKERRQ(ierr);
355   ierr = VecLoad_Plex_Local(locv, viewer);CHKERRQ(ierr);
356   ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
357   ierr = DMLocalToGlobalBegin(dm, locv, INSERT_VALUES, v);CHKERRQ(ierr);
358   ierr = DMLocalToGlobalEnd(dm, locv, INSERT_VALUES, v);CHKERRQ(ierr);
359   ierr = DMRestoreLocalVector(dm, &locv);CHKERRQ(ierr);
360   PetscFunctionReturn(0);
361 }
362 #endif
363 
364 #undef __FUNCT__
365 #define __FUNCT__ "VecLoad_Plex"
366 PetscErrorCode VecLoad_Plex(Vec v, PetscViewer viewer)
367 {
368   DM             dm;
369   PetscBool      ishdf5;
370   PetscErrorCode ierr;
371 
372   PetscFunctionBegin;
373   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
374   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
375   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
376   if (ishdf5) {
377 #if defined(PETSC_HAVE_HDF5)
378     ierr = VecLoad_Plex_HDF5(v, viewer);CHKERRQ(ierr);
379 #else
380     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
381 #endif
382   } else {
383     ierr = VecLoad_Default(v, viewer);CHKERRQ(ierr);
384   }
385   PetscFunctionReturn(0);
386 }
387 
388 #undef __FUNCT__
389 #define __FUNCT__ "DMPlexView_Ascii"
390 PetscErrorCode DMPlexView_Ascii(DM dm, PetscViewer viewer)
391 {
392   DM_Plex          *mesh = (DM_Plex*) dm->data;
393   DM                cdm;
394   DMLabel           markers;
395   PetscSection      coordSection;
396   Vec               coordinates;
397   PetscViewerFormat format;
398   PetscErrorCode    ierr;
399 
400   PetscFunctionBegin;
401   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
402   ierr = DMGetDefaultSection(cdm, &coordSection);CHKERRQ(ierr);
403   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
404   ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
405   if (format == PETSC_VIEWER_ASCII_INFO_DETAIL) {
406     const char *name;
407     PetscInt    maxConeSize, maxSupportSize;
408     PetscInt    pStart, pEnd, p;
409     PetscMPIInt rank, size;
410 
411     ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRQ(ierr);
412     ierr = MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size);CHKERRQ(ierr);
413     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
414     ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
415     ierr = DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize);CHKERRQ(ierr);
416     ierr = PetscViewerASCIISynchronizedAllow(viewer, PETSC_TRUE);CHKERRQ(ierr);
417     ierr = PetscViewerASCIIPrintf(viewer, "Mesh '%s':\n", name);CHKERRQ(ierr);
418     ierr = PetscViewerASCIISynchronizedPrintf(viewer, "Max sizes cone: %D support: %D\n", maxConeSize, maxSupportSize);CHKERRQ(ierr);
419     ierr = PetscViewerASCIIPrintf(viewer, "orientation is missing\n", name);CHKERRQ(ierr);
420     ierr = PetscViewerASCIIPrintf(viewer, "cap --> base:\n", name);CHKERRQ(ierr);
421     for (p = pStart; p < pEnd; ++p) {
422       PetscInt dof, off, s;
423 
424       ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
425       ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
426       for (s = off; s < off+dof; ++s) {
427         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%D]: %D ----> %D\n", rank, p, mesh->supports[s]);CHKERRQ(ierr);
428       }
429     }
430     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
431     ierr = PetscViewerASCIIPrintf(viewer, "base <-- cap:\n", name);CHKERRQ(ierr);
432     for (p = pStart; p < pEnd; ++p) {
433       PetscInt dof, off, c;
434 
435       ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
436       ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
437       for (c = off; c < off+dof; ++c) {
438         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%D]: %D <---- %D (%D)\n", rank, p, mesh->cones[c], mesh->coneOrientations[c]);CHKERRQ(ierr);
439       }
440     }
441     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
442     ierr = PetscSectionGetChart(coordSection, &pStart, NULL);CHKERRQ(ierr);
443     if (pStart >= 0) {ierr = PetscSectionVecView(coordSection, coordinates, viewer);CHKERRQ(ierr);}
444     ierr = DMPlexGetLabel(dm, "marker", &markers);CHKERRQ(ierr);
445     ierr = DMLabelView(markers,viewer);CHKERRQ(ierr);
446     if (size > 1) {
447       PetscSF sf;
448 
449       ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr);
450       ierr = PetscSFView(sf, viewer);CHKERRQ(ierr);
451     }
452     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
453   } else if (format == PETSC_VIEWER_ASCII_LATEX) {
454     const char  *name;
455     const char  *colors[3] = {"red", "blue", "green"};
456     const int    numColors  = 3;
457     PetscReal    scale      = 2.0;
458     PetscScalar *coords;
459     PetscInt     depth, cStart, cEnd, c, vStart, vEnd, v, eStart = 0, eEnd = 0, e, p;
460     PetscMPIInt  rank, size;
461 
462     ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
463     ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRQ(ierr);
464     ierr = MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size);CHKERRQ(ierr);
465     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
466     ierr = PetscViewerASCIISynchronizedAllow(viewer, PETSC_TRUE);CHKERRQ(ierr);
467     ierr = PetscViewerASCIIPrintf(viewer, "\
468 \\documentclass[crop,multi=false]{standalone}\n\n\
469 \\usepackage{tikz}\n\
470 \\usepackage{pgflibraryshapes}\n\
471 \\usetikzlibrary{backgrounds}\n\
472 \\usetikzlibrary{arrows}\n\
473 \\begin{document}\n\
474 \\section{%s}\n\
475 \\begin{center}\n", name, 8.0/scale);CHKERRQ(ierr);
476     ierr = PetscViewerASCIIPrintf(viewer, "Mesh for process ");CHKERRQ(ierr);
477     for (p = 0; p < size; ++p) {
478       if (p > 0 && p == size-1) {
479         ierr = PetscViewerASCIIPrintf(viewer, ", and ", colors[p%numColors], p);CHKERRQ(ierr);
480       } else if (p > 0) {
481         ierr = PetscViewerASCIIPrintf(viewer, ", ", colors[p%numColors], p);CHKERRQ(ierr);
482       }
483       ierr = PetscViewerASCIIPrintf(viewer, "{\\textcolor{%s}%D}", colors[p%numColors], p);CHKERRQ(ierr);
484     }
485     ierr = PetscViewerASCIIPrintf(viewer, ".\n\n\n\
486 \\begin{tikzpicture}[scale = %g,font=\\fontsize{8}{8}\\selectfont]\n");CHKERRQ(ierr);
487     /* Plot vertices */
488     ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
489     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
490     ierr = PetscViewerASCIIPrintf(viewer, "\\path\n");CHKERRQ(ierr);
491     for (v = vStart; v < vEnd; ++v) {
492       PetscInt off, dof, d;
493 
494       ierr = PetscSectionGetDof(coordSection, v, &dof);CHKERRQ(ierr);
495       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
496       ierr = PetscViewerASCIISynchronizedPrintf(viewer, "(");CHKERRQ(ierr);
497       for (d = 0; d < dof; ++d) {
498         if (d > 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, ",");CHKERRQ(ierr);}
499         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "%g", (double)(scale*PetscRealPart(coords[off+d])));CHKERRQ(ierr);
500       }
501       ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%D) [draw,shape=circle,color=%s] {%D} --\n", v, rank, colors[rank%numColors], v);CHKERRQ(ierr);
502     }
503     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
504     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
505     ierr = PetscViewerASCIIPrintf(viewer, "(0,0);\n");CHKERRQ(ierr);
506     /* Plot edges */
507     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
508     ierr = PetscViewerASCIIPrintf(viewer, "\\path\n");CHKERRQ(ierr);
509     if (depth > 1) {ierr = DMPlexGetDepthStratum(dm, 1, &eStart, &eEnd);CHKERRQ(ierr);}
510     for (e = eStart; e < eEnd; ++e) {
511       const PetscInt *cone;
512       PetscInt        coneSize, offA, offB, dof, d;
513 
514       ierr = DMPlexGetConeSize(dm, e, &coneSize);CHKERRQ(ierr);
515       if (coneSize != 2) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Edge %d cone should have two vertices, not %d", e, coneSize);
516       ierr = DMPlexGetCone(dm, e, &cone);CHKERRQ(ierr);
517       ierr = PetscSectionGetDof(coordSection, cone[0], &dof);CHKERRQ(ierr);
518       ierr = PetscSectionGetOffset(coordSection, cone[0], &offA);CHKERRQ(ierr);
519       ierr = PetscSectionGetOffset(coordSection, cone[1], &offB);CHKERRQ(ierr);
520       ierr = PetscViewerASCIISynchronizedPrintf(viewer, "(");CHKERRQ(ierr);
521       for (d = 0; d < dof; ++d) {
522         if (d > 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, ",");CHKERRQ(ierr);}
523         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "%g", (double)(scale*0.5*PetscRealPart(coords[offA+d]+coords[offB+d])));CHKERRQ(ierr);
524       }
525       ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%D) [draw,shape=circle,color=%s] {%D} --\n", e, rank, colors[rank%numColors], e);CHKERRQ(ierr);
526     }
527     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
528     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
529     ierr = PetscViewerASCIIPrintf(viewer, "(0,0);\n");CHKERRQ(ierr);
530     /* Plot cells */
531     ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
532     for (c = cStart; c < cEnd; ++c) {
533       PetscInt *closure = NULL;
534       PetscInt  closureSize, firstPoint = -1;
535 
536       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
537       ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\draw[color=%s] ", colors[rank%numColors]);CHKERRQ(ierr);
538       for (p = 0; p < closureSize*2; p += 2) {
539         const PetscInt point = closure[p];
540 
541         if ((point < vStart) || (point >= vEnd)) continue;
542         if (firstPoint >= 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- ");CHKERRQ(ierr);}
543         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "(%D_%D)", point, rank);CHKERRQ(ierr);
544         if (firstPoint < 0) firstPoint = point;
545       }
546       /* Why doesn't this work? ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- cycle;\n");CHKERRQ(ierr); */
547       ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- (%D_%D);\n", firstPoint, rank);CHKERRQ(ierr);
548       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
549     }
550     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
551     ierr = PetscViewerASCIIPrintf(viewer, "\\end{tikzpicture}\n\\end{center}\n");CHKERRQ(ierr);
552     ierr = PetscViewerASCIIPrintf(viewer, "\\end{document}\n", name);CHKERRQ(ierr);
553   } else {
554     MPI_Comm    comm;
555     PetscInt   *sizes, *hybsizes;
556     PetscInt    locDepth, depth, dim, d, pMax[4];
557     PetscInt    pStart, pEnd, p;
558     PetscInt    numLabels, l;
559     const char *name;
560     PetscMPIInt size;
561 
562     ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
563     ierr = MPI_Comm_size(comm, &size);CHKERRQ(ierr);
564     ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
565     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
566     if (name) {ierr = PetscViewerASCIIPrintf(viewer, "%s in %D dimensions:\n", name, dim);CHKERRQ(ierr);}
567     else      {ierr = PetscViewerASCIIPrintf(viewer, "Mesh in %D dimensions:\n", dim);CHKERRQ(ierr);}
568     ierr = DMPlexGetDepth(dm, &locDepth);CHKERRQ(ierr);
569     ierr = MPI_Allreduce(&locDepth, &depth, 1, MPIU_INT, MPI_MAX, comm);CHKERRQ(ierr);
570     ierr = DMPlexGetHybridBounds(dm, &pMax[depth], &pMax[depth-1], &pMax[1], &pMax[0]);CHKERRQ(ierr);
571     ierr = PetscMalloc2(size,&sizes,size,&hybsizes);CHKERRQ(ierr);
572     if (depth == 1) {
573       ierr = DMPlexGetDepthStratum(dm, 0, &pStart, &pEnd);CHKERRQ(ierr);
574       pEnd = pEnd - pStart;
575       ierr = MPI_Gather(&pEnd, 1, MPIU_INT, sizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
576       ierr = PetscViewerASCIIPrintf(viewer, "  %D-cells:", 0);CHKERRQ(ierr);
577       for (p = 0; p < size; ++p) {ierr = PetscViewerASCIIPrintf(viewer, " %D", sizes[p]);CHKERRQ(ierr);}
578       ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
579       ierr = DMPlexGetHeightStratum(dm, 0, &pStart, &pEnd);CHKERRQ(ierr);
580       pEnd = pEnd - pStart;
581       ierr = MPI_Gather(&pEnd, 1, MPIU_INT, sizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
582       ierr = PetscViewerASCIIPrintf(viewer, "  %D-cells:", dim);CHKERRQ(ierr);
583       for (p = 0; p < size; ++p) {ierr = PetscViewerASCIIPrintf(viewer, " %D", sizes[p]);CHKERRQ(ierr);}
584       ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
585     } else {
586       for (d = 0; d <= dim; d++) {
587         ierr = DMPlexGetDepthStratum(dm, d, &pStart, &pEnd);CHKERRQ(ierr);
588         pEnd    -= pStart;
589         pMax[d] -= pStart;
590         ierr = MPI_Gather(&pEnd, 1, MPIU_INT, sizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
591         ierr = MPI_Gather(&pMax[d], 1, MPIU_INT, hybsizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
592         ierr = PetscViewerASCIIPrintf(viewer, "  %D-cells:", d);CHKERRQ(ierr);
593         for (p = 0; p < size; ++p) {
594           if (hybsizes[p] >= 0) {ierr = PetscViewerASCIIPrintf(viewer, " %D (%D)", sizes[p], sizes[p] - hybsizes[p]);CHKERRQ(ierr);}
595           else                  {ierr = PetscViewerASCIIPrintf(viewer, " %D", sizes[p]);CHKERRQ(ierr);}
596         }
597         ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
598       }
599     }
600     ierr = PetscFree2(sizes,hybsizes);CHKERRQ(ierr);
601     ierr = DMPlexGetNumLabels(dm, &numLabels);CHKERRQ(ierr);
602     if (numLabels) {ierr = PetscViewerASCIIPrintf(viewer, "Labels:\n");CHKERRQ(ierr);}
603     for (l = 0; l < numLabels; ++l) {
604       DMLabel         label;
605       const char     *name;
606       IS              valueIS;
607       const PetscInt *values;
608       PetscInt        numValues, v;
609 
610       ierr = DMPlexGetLabelName(dm, l, &name);CHKERRQ(ierr);
611       ierr = DMPlexGetLabel(dm, name, &label);CHKERRQ(ierr);
612       ierr = DMLabelGetNumValues(label, &numValues);CHKERRQ(ierr);
613       ierr = PetscViewerASCIIPrintf(viewer, "  %s: %d strata of sizes (", name, numValues);CHKERRQ(ierr);
614       ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
615       ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
616       for (v = 0; v < numValues; ++v) {
617         PetscInt size;
618 
619         ierr = DMLabelGetStratumSize(label, values[v], &size);CHKERRQ(ierr);
620         if (v > 0) {ierr = PetscViewerASCIIPrintf(viewer, ", ");CHKERRQ(ierr);}
621         ierr = PetscViewerASCIIPrintf(viewer, "%d", size);CHKERRQ(ierr);
622       }
623       ierr = PetscViewerASCIIPrintf(viewer, ")\n");CHKERRQ(ierr);
624       ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
625       ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
626     }
627   }
628   PetscFunctionReturn(0);
629 }
630 
631 #if defined(PETSC_HAVE_HDF5)
632 #undef __FUNCT__
633 #define __FUNCT__ "DMPlexView_HDF5"
634 /* We only write cells and vertices. Does this screw up parallel reading? */
635 static PetscErrorCode DMPlexView_HDF5(DM dm, PetscViewer viewer)
636 {
637   DM              cdm;
638   Vec             coordinates, newcoords;
639   Vec             coneVec, cellVec;
640   IS              globalVertexNumbers;
641   const PetscInt *gvertex;
642   PetscScalar    *sizes, *vertices;
643   PetscReal       lengthScale;
644   const char     *label   = NULL;
645   PetscInt        labelId = 0, dim;
646   char            group[PETSC_MAX_PATH_LEN];
647   PetscInt        vStart, vEnd, v, cellHeight, cStart, cEnd, cMax, cell, conesSize = 0, numCornersLocal = 0, numCorners, numLabels, l;
648   hid_t           fileId, groupId;
649   herr_t          status;
650   PetscErrorCode  ierr;
651 
652   PetscFunctionBegin;
653   /* Write coordinates */
654   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
655   ierr = DMPlexGetScale(dm, PETSC_UNIT_LENGTH, &lengthScale);CHKERRQ(ierr);
656   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
657   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
658   ierr = VecDuplicate(coordinates, &newcoords);CHKERRQ(ierr);
659   ierr = PetscObjectSetName((PetscObject) newcoords, "vertices");CHKERRQ(ierr);
660   ierr = VecCopy(coordinates, newcoords);CHKERRQ(ierr);
661   ierr = VecScale(newcoords, lengthScale);CHKERRQ(ierr);
662   /* Use the local version to bypass the default group setting */
663   ierr = PetscViewerHDF5PushGroup(viewer, "/geometry");CHKERRQ(ierr);
664   ierr = VecView(newcoords, viewer);CHKERRQ(ierr);
665   ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
666   ierr = VecDestroy(&newcoords);CHKERRQ(ierr);
667   /* Write toplogy */
668   ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
669   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
670   ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
671   ierr = DMPlexGetHybridBounds(dm, &cMax, PETSC_NULL, PETSC_NULL, PETSC_NULL);CHKERRQ(ierr);
672   if (cMax >= 0) cEnd = PetscMin(cEnd, cMax);
673 
674   ierr = VecCreate(PetscObjectComm((PetscObject) dm), &coneVec);CHKERRQ(ierr);
675   ierr = VecSetSizes(coneVec, cEnd-cStart, PETSC_DETERMINE);CHKERRQ(ierr);
676   ierr = VecSetBlockSize(coneVec, 1);CHKERRQ(ierr);
677   ierr = VecSetFromOptions(coneVec);CHKERRQ(ierr);
678   ierr = PetscObjectSetName((PetscObject) coneVec, "coneSize");CHKERRQ(ierr);
679   ierr = VecGetArray(coneVec, &sizes);CHKERRQ(ierr);
680   for (cell = cStart; cell < cEnd; ++cell) {
681     PetscInt *closure = NULL;
682     PetscInt  closureSize, v, Nc = 0;
683 
684     if (label) {
685       PetscInt value;
686       ierr = DMPlexGetLabelValue(dm, label, cell, &value);CHKERRQ(ierr);
687       if (value == labelId) continue;
688     }
689     ierr = DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
690     for (v = 0; v < closureSize*2; v += 2) {
691       if ((closure[v] >= vStart) && (closure[v] < vEnd)) ++Nc;
692     }
693     ierr = DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
694     conesSize += Nc;
695     if (!numCornersLocal)           numCornersLocal = Nc;
696     else if (numCornersLocal != Nc) numCornersLocal = 1;
697   }
698   ierr = VecRestoreArray(coneVec, &sizes);CHKERRQ(ierr);
699   ierr = MPI_Allreduce(&numCornersLocal, &numCorners, 1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject) dm));CHKERRQ(ierr);
700   if (numCornersLocal && numCornersLocal != numCorners) numCorners = 1;
701 
702   ierr = DMPlexGetVertexNumbering(dm, &globalVertexNumbers);CHKERRQ(ierr);
703   ierr = ISGetIndices(globalVertexNumbers, &gvertex);CHKERRQ(ierr);
704   ierr = VecCreate(PetscObjectComm((PetscObject) dm), &cellVec);CHKERRQ(ierr);
705   ierr = VecSetSizes(cellVec, conesSize, PETSC_DETERMINE);CHKERRQ(ierr);
706   ierr = VecSetBlockSize(cellVec, numCorners);CHKERRQ(ierr);
707   ierr = VecSetFromOptions(cellVec);CHKERRQ(ierr);
708   ierr = PetscObjectSetName((PetscObject) cellVec, "cells");CHKERRQ(ierr);
709   ierr = VecGetArray(cellVec, &vertices);CHKERRQ(ierr);
710   for (cell = cStart, v = 0; cell < cEnd; ++cell) {
711     PetscInt *closure = NULL;
712     PetscInt  closureSize, Nc = 0, p;
713 
714     if (label) {
715       PetscInt value;
716       ierr = DMPlexGetLabelValue(dm, label, cell, &value);CHKERRQ(ierr);
717       if (value == labelId) continue;
718     }
719     ierr = DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
720     for (p = 0; p < closureSize*2; p += 2) {
721       if ((closure[p] >= vStart) && (closure[p] < vEnd)) {
722         closure[Nc++] = closure[p];
723         }
724     }
725     ierr = DMPlexInvertCell(dim, Nc, closure);CHKERRQ(ierr);
726     for (p = 0; p < Nc; ++p) {
727       const PetscInt gv = gvertex[closure[p] - vStart];
728       vertices[v++] = gv < 0 ? -(gv+1) : gv;
729     }
730     ierr = DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
731   }
732   if (v != conesSize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_LIB, "Total number of cell vertices %d != %d", v, conesSize);
733   ierr = VecRestoreArray(cellVec, &vertices);CHKERRQ(ierr);
734   ierr = PetscViewerHDF5PushGroup(viewer, "/topology");CHKERRQ(ierr);
735   ierr = VecView(cellVec, viewer);CHKERRQ(ierr);
736   if (numCorners == 1) {
737     ierr = VecView(coneVec, viewer);CHKERRQ(ierr);
738   } else {
739     ierr = PetscViewerHDF5WriteAttribute(viewer, "/topology/cells", "cell_corners", PETSC_INT, (void *) &numCorners);CHKERRQ(ierr);
740   }
741   ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
742   ierr = VecDestroy(&cellVec);CHKERRQ(ierr);
743   ierr = VecDestroy(&coneVec);CHKERRQ(ierr);
744   ierr = ISRestoreIndices(globalVertexNumbers, &gvertex);CHKERRQ(ierr);
745 
746   ierr = PetscViewerHDF5WriteAttribute(viewer, "/topology/cells", "cell_dim", PETSC_INT, (void *) &dim);CHKERRQ(ierr);
747   /* Write Labels*/
748   ierr = PetscViewerHDF5PushGroup(viewer, "/labels");CHKERRQ(ierr);
749   ierr = PetscViewerHDF5OpenGroup(viewer, &fileId, &groupId);CHKERRQ(ierr);
750   if (groupId != fileId) {status = H5Gclose(groupId);CHKERRQ(status);}
751   ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
752   ierr = DMPlexGetNumLabels(dm, &numLabels);CHKERRQ(ierr);
753   for (l = 0; l < numLabels; ++l) {
754     DMLabel         label;
755     const char     *name;
756     IS              valueIS;
757     const PetscInt *values;
758     PetscInt        numValues, v;
759     PetscBool       isDepth;
760 
761     ierr = DMPlexGetLabelName(dm, l, &name);CHKERRQ(ierr);
762     ierr = PetscStrncmp(name, "depth", 10, &isDepth);CHKERRQ(ierr);
763     if (isDepth) continue;
764     ierr = DMPlexGetLabel(dm, name, &label);CHKERRQ(ierr);
765     ierr = DMLabelGetNumValues(label, &numValues);CHKERRQ(ierr);
766     ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
767     ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
768     ierr = PetscSNPrintf(group, PETSC_MAX_PATH_LEN, "/labels/%s", name);CHKERRQ(ierr);
769     ierr = PetscViewerHDF5PushGroup(viewer, group);CHKERRQ(ierr);
770     ierr = PetscViewerHDF5OpenGroup(viewer, &fileId, &groupId);CHKERRQ(ierr);
771     if (groupId != fileId) {status = H5Gclose(groupId);CHKERRQ(status);}
772     ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
773     /* TODO: Need to actually loop over the union of label values, ISAllGather() */
774     for (v = 0; v < numValues; ++v) {
775       IS   stratumIS;
776 
777       ierr = PetscSNPrintf(group, PETSC_MAX_PATH_LEN, "/labels/%s/%d", name, values[v]);CHKERRQ(ierr);
778       ierr = DMLabelGetStratumIS(label, values[v], &stratumIS);CHKERRQ(ierr);
779       /* TODO: Need to globalize point names and remove unowned points */
780       ierr = PetscViewerHDF5PushGroup(viewer, group);CHKERRQ(ierr);
781       ierr = ISView(stratumIS, viewer);CHKERRQ(ierr);
782       ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
783       ierr = ISDestroy(&stratumIS);CHKERRQ(ierr);
784     }
785     ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
786     ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
787   }
788   PetscFunctionReturn(0);
789 }
790 #endif
791 
792 #undef __FUNCT__
793 #define __FUNCT__ "DMView_Plex"
794 PetscErrorCode DMView_Plex(DM dm, PetscViewer viewer)
795 {
796   PetscBool      iascii, ishdf5;
797   PetscErrorCode ierr;
798 
799   PetscFunctionBegin;
800   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
801   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
802   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERASCII, &iascii);CHKERRQ(ierr);
803   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,  &ishdf5);CHKERRQ(ierr);
804   if (iascii) {
805     ierr = DMPlexView_Ascii(dm, viewer);CHKERRQ(ierr);
806   } else if (ishdf5) {
807 #if defined(PETSC_HAVE_HDF5)
808     ierr = DMPlexView_HDF5(dm, viewer);CHKERRQ(ierr);
809 #else
810     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
811 #endif
812   }
813   PetscFunctionReturn(0);
814 }
815 
816 #if defined(PETSC_HAVE_HDF5)
817 typedef struct {
818   DM          dm;
819   PetscViewer viewer;
820   DMLabel     label;
821 } LabelCtx;
822 
823 static herr_t ReadLabelStratumHDF5_Static(hid_t g_id, const char *name, const H5L_info_t *info, void *op_data)
824 {
825   PetscViewer     viewer = ((LabelCtx *) op_data)->viewer;
826   DMLabel         label  = ((LabelCtx *) op_data)->label;
827   IS              stratumIS;
828   const PetscInt *ind;
829   PetscInt        value, N, i;
830   const char     *lname;
831   char            group[PETSC_MAX_PATH_LEN];
832   PetscErrorCode  ierr;
833 
834   ierr = PetscOptionsStringToInt(name, &value);
835   ierr = ISCreate(PetscObjectComm((PetscObject) viewer), &stratumIS);
836   ierr = PetscObjectSetName((PetscObject) stratumIS, "indices");
837   ierr = DMLabelGetName(label, &lname);
838   ierr = PetscSNPrintf(group, PETSC_MAX_PATH_LEN, "/labels/%s/%s", lname, name);CHKERRQ(ierr);
839   ierr = PetscViewerHDF5PushGroup(viewer, group);CHKERRQ(ierr);
840   ierr = ISLoad(stratumIS, viewer);
841   ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
842   ierr = ISGetSize(stratumIS, &N);
843   ierr = ISGetIndices(stratumIS, &ind);
844   for (i = 0; i < N; ++i) {ierr = DMLabelSetValue(label, ind[i], value);}
845   ierr = ISRestoreIndices(stratumIS, &ind);
846   ierr = ISDestroy(&stratumIS);
847   return 0;
848 }
849 
850 static herr_t ReadLabelHDF5_Static(hid_t g_id, const char *name, const H5L_info_t *info, void *op_data)
851 {
852   DM             dm = ((LabelCtx *) op_data)->dm;
853   hsize_t        idx;
854   herr_t         status;
855   PetscErrorCode ierr;
856 
857   ierr = DMPlexCreateLabel(dm, name); if (ierr) return (herr_t) ierr;
858   ierr = DMPlexGetLabel(dm, name, &((LabelCtx *) op_data)->label); if (ierr) return (herr_t) ierr;
859   status = H5Literate_by_name(g_id, name, H5_INDEX_NAME, H5_ITER_NATIVE, &idx, ReadLabelStratumHDF5_Static, op_data, 0);
860   return status;
861 }
862 
863 #undef __FUNCT__
864 #define __FUNCT__ "DMPlexLoad_HDF5"
865 /* The first version will read everything onto proc 0, letting the user distribute
866    The next will create a naive partition, and then rebalance after reading
867 */
868 static PetscErrorCode DMPlexLoad_HDF5(DM dm, PetscViewer viewer)
869 {
870   LabelCtx        ctx;
871   PetscSection    coordSection;
872   Vec             coordinates;
873   Vec             cellVec;
874   PetscScalar    *cells;
875   PetscReal       lengthScale;
876   PetscInt       *cone;
877   PetscInt        dim, spatialDim, numVertices, v, numCorners, numCells, cell, c;
878   hid_t           fileId, groupId;
879   hsize_t         idx;
880   herr_t          status;
881   PetscErrorCode  ierr;
882 
883   PetscFunctionBegin;
884   /* Read geometry */
885   ierr = PetscViewerHDF5PushGroup(viewer, "/geometry");CHKERRQ(ierr);
886   ierr = VecCreate(PetscObjectComm((PetscObject) dm), &coordinates);CHKERRQ(ierr);
887   ierr = PetscObjectSetName((PetscObject) coordinates, "vertices");CHKERRQ(ierr);
888   ierr = VecLoad(coordinates, viewer);CHKERRQ(ierr);
889   ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
890   ierr = DMPlexGetScale(dm, PETSC_UNIT_LENGTH, &lengthScale);CHKERRQ(ierr);
891   ierr = VecScale(coordinates, 1.0/lengthScale);CHKERRQ(ierr);
892   ierr = VecGetSize(coordinates, &numVertices);CHKERRQ(ierr);
893   ierr = VecGetBlockSize(coordinates, &spatialDim);CHKERRQ(ierr);
894   numVertices /= spatialDim;
895   /* Read toplogy */
896   ierr = PetscViewerHDF5ReadAttribute(viewer, "/topology/cells", "cell_dim", PETSC_INT, (void *) &dim);CHKERRQ(ierr);
897   ierr = DMPlexSetDimension(dm, dim);CHKERRQ(ierr);
898   /*   TODO Check for coneSize vector rather than this attribute */
899   ierr = PetscViewerHDF5ReadAttribute(viewer, "/topology/cells", "cell_corners", PETSC_INT, (void *) &numCorners);CHKERRQ(ierr);
900   ierr = PetscViewerHDF5PushGroup(viewer, "/topology");CHKERRQ(ierr);
901   ierr = VecCreate(PetscObjectComm((PetscObject) dm), &cellVec);CHKERRQ(ierr);
902   ierr = VecSetBlockSize(cellVec, numCorners);CHKERRQ(ierr);
903   ierr = PetscObjectSetName((PetscObject) cellVec, "cells");CHKERRQ(ierr);
904   ierr = VecLoad(cellVec, viewer);CHKERRQ(ierr);
905   ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
906   ierr = VecGetSize(cellVec, &numCells);CHKERRQ(ierr);
907   numCells /= numCorners;
908   /* Create Plex */
909   ierr = DMPlexSetChart(dm, 0, numCells+numVertices);CHKERRQ(ierr);
910   for (cell = 0; cell < numCells; ++cell) {ierr = DMPlexSetConeSize(dm, cell, numCorners);CHKERRQ(ierr);}
911   ierr = DMSetUp(dm);CHKERRQ(ierr);
912   ierr = PetscMalloc1(numCorners,&cone);CHKERRQ(ierr);
913   ierr = VecGetArray(cellVec, &cells);CHKERRQ(ierr);
914   for (cell = 0; cell < numCells; ++cell) {
915     for (c = 0; c < numCorners; ++c) {cone[c] = numCells + cells[cell*numCorners+c];}
916     ierr = DMPlexSetCone(dm, cell, cone);CHKERRQ(ierr);
917   }
918   ierr = VecRestoreArray(cellVec, &cells);CHKERRQ(ierr);
919   ierr = PetscFree(cone);CHKERRQ(ierr);
920   ierr = VecDestroy(&cellVec);CHKERRQ(ierr);
921   ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
922   ierr = DMPlexStratify(dm);CHKERRQ(ierr);
923   /* Create coordinates */
924   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
925   ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
926   ierr = PetscSectionSetFieldComponents(coordSection, 0, spatialDim);CHKERRQ(ierr);
927   ierr = PetscSectionSetChart(coordSection, numCells, numCells+numVertices);CHKERRQ(ierr);
928   for (v = numCells; v < numCells+numVertices; ++v) {
929     ierr = PetscSectionSetDof(coordSection, v, spatialDim);CHKERRQ(ierr);
930     ierr = PetscSectionSetFieldDof(coordSection, v, 0, spatialDim);CHKERRQ(ierr);
931   }
932   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
933   ierr = DMSetCoordinates(dm, coordinates);CHKERRQ(ierr);
934   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
935   /* Read Labels*/
936   ctx.dm     = dm;
937   ctx.viewer = viewer;
938   ierr = PetscViewerHDF5PushGroup(viewer, "/labels");CHKERRQ(ierr);
939   ierr = PetscViewerHDF5OpenGroup(viewer, &fileId, &groupId);CHKERRQ(ierr);
940   status = H5Literate(groupId, H5_INDEX_NAME, H5_ITER_NATIVE, &idx, ReadLabelHDF5_Static, &ctx);CHKERRQ(status);
941   status = H5Gclose(groupId);CHKERRQ(status);
942   ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
943   PetscFunctionReturn(0);
944 }
945 #endif
946 
947 #undef __FUNCT__
948 #define __FUNCT__ "DMLoad_Plex"
949 PetscErrorCode DMLoad_Plex(DM dm, PetscViewer viewer)
950 {
951   PetscBool      isbinary, ishdf5;
952   PetscErrorCode ierr;
953 
954   PetscFunctionBegin;
955   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
956   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
957   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERBINARY, &isbinary);CHKERRQ(ierr);
958   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,   &ishdf5);CHKERRQ(ierr);
959   if (isbinary) {SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Do not yet support binary viewers");}
960   else if (ishdf5) {
961 #if defined(PETSC_HAVE_HDF5)
962     DM odm;
963     ierr = DMCreate(PetscObjectComm((PetscObject) dm), &odm);CHKERRQ(ierr);
964     ierr = DMSetType(odm, DMPLEX);CHKERRQ(ierr);
965     ierr = DMPlexLoad_HDF5(odm, viewer);CHKERRQ(ierr);
966     ierr = DMPlexInterpolate(odm, &dm);CHKERRQ(ierr);
967     ierr = DMPlexCopyCoordinates(odm, dm);CHKERRQ(ierr);
968     ierr = DMPlexCopyLabels(odm, dm);CHKERRQ(ierr);
969     ierr = DMDestroy(&odm);CHKERRQ(ierr);
970 #else
971     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
972 #endif
973   }
974   PetscFunctionReturn(0);
975 }
976 
977 #undef __FUNCT__
978 #define __FUNCT__ "BoundaryDestroy"
979 static PetscErrorCode BoundaryDestroy(DMBoundary *boundary)
980 {
981   DMBoundary     b, next;
982   PetscErrorCode ierr;
983 
984   PetscFunctionBeginUser;
985   if (!boundary) PetscFunctionReturn(0);
986   b = *boundary;
987   *boundary = NULL;
988   for (; b; b = next) {
989     next = b->next;
990     ierr = PetscFree(b->ids);CHKERRQ(ierr);
991     ierr = PetscFree(b->name);CHKERRQ(ierr);
992     ierr = PetscFree(b);CHKERRQ(ierr);
993   }
994   PetscFunctionReturn(0);
995 }
996 
997 #undef __FUNCT__
998 #define __FUNCT__ "DMDestroy_Plex"
999 PetscErrorCode DMDestroy_Plex(DM dm)
1000 {
1001   DM_Plex       *mesh = (DM_Plex*) dm->data;
1002   DMLabel        next  = mesh->labels;
1003   PetscErrorCode ierr;
1004 
1005   PetscFunctionBegin;
1006   if (--mesh->refct > 0) PetscFunctionReturn(0);
1007   ierr = PetscSectionDestroy(&mesh->coneSection);CHKERRQ(ierr);
1008   ierr = PetscFree(mesh->cones);CHKERRQ(ierr);
1009   ierr = PetscFree(mesh->coneOrientations);CHKERRQ(ierr);
1010   ierr = PetscSectionDestroy(&mesh->supportSection);CHKERRQ(ierr);
1011   ierr = PetscFree(mesh->supports);CHKERRQ(ierr);
1012   ierr = PetscFree(mesh->facesTmp);CHKERRQ(ierr);
1013   while (next) {
1014     DMLabel tmp = next->next;
1015 
1016     ierr = DMLabelDestroy(&next);CHKERRQ(ierr);
1017     next = tmp;
1018   }
1019   ierr = DMDestroy(&mesh->coarseMesh);CHKERRQ(ierr);
1020   ierr = DMLabelDestroy(&mesh->subpointMap);CHKERRQ(ierr);
1021   ierr = ISDestroy(&mesh->globalVertexNumbers);CHKERRQ(ierr);
1022   ierr = ISDestroy(&mesh->globalCellNumbers);CHKERRQ(ierr);
1023   ierr = BoundaryDestroy(&mesh->boundary);CHKERRQ(ierr);
1024   /* This was originally freed in DMDestroy(), but that prevents reference counting of backend objects */
1025   ierr = PetscFree(mesh);CHKERRQ(ierr);
1026   PetscFunctionReturn(0);
1027 }
1028 
1029 #undef __FUNCT__
1030 #define __FUNCT__ "DMCreateMatrix_Plex"
1031 PetscErrorCode DMCreateMatrix_Plex(DM dm, Mat *J)
1032 {
1033   PetscSection   section, sectionGlobal;
1034   PetscInt       bs = -1;
1035   PetscInt       localSize;
1036   PetscBool      isShell, isBlock, isSeqBlock, isMPIBlock, isSymBlock, isSymSeqBlock, isSymMPIBlock;
1037   PetscErrorCode ierr;
1038   MatType        mtype;
1039 
1040   PetscFunctionBegin;
1041   ierr = MatInitializePackage();CHKERRQ(ierr);
1042   mtype = dm->mattype;
1043   ierr = DMGetDefaultSection(dm, &section);CHKERRQ(ierr);
1044   ierr = DMGetDefaultGlobalSection(dm, &sectionGlobal);CHKERRQ(ierr);
1045   /* ierr = PetscSectionGetStorageSize(sectionGlobal, &localSize);CHKERRQ(ierr); */
1046   ierr = PetscSectionGetConstrainedStorageSize(sectionGlobal, &localSize);CHKERRQ(ierr);
1047   ierr = MatCreate(PetscObjectComm((PetscObject)dm), J);CHKERRQ(ierr);
1048   ierr = MatSetSizes(*J, localSize, localSize, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
1049   ierr = MatSetType(*J, mtype);CHKERRQ(ierr);
1050   ierr = MatSetFromOptions(*J);CHKERRQ(ierr);
1051   ierr = PetscStrcmp(mtype, MATSHELL, &isShell);CHKERRQ(ierr);
1052   ierr = PetscStrcmp(mtype, MATBAIJ, &isBlock);CHKERRQ(ierr);
1053   ierr = PetscStrcmp(mtype, MATSEQBAIJ, &isSeqBlock);CHKERRQ(ierr);
1054   ierr = PetscStrcmp(mtype, MATMPIBAIJ, &isMPIBlock);CHKERRQ(ierr);
1055   ierr = PetscStrcmp(mtype, MATSBAIJ, &isSymBlock);CHKERRQ(ierr);
1056   ierr = PetscStrcmp(mtype, MATSEQSBAIJ, &isSymSeqBlock);CHKERRQ(ierr);
1057   ierr = PetscStrcmp(mtype, MATMPISBAIJ, &isSymMPIBlock);CHKERRQ(ierr);
1058   if (!isShell) {
1059     PetscBool fillMatrix = (PetscBool) !dm->prealloc_only;
1060     PetscInt *dnz, *onz, *dnzu, *onzu, bsLocal, bsMax, bsMin;
1061 
1062     if (bs < 0) {
1063       if (isBlock || isSeqBlock || isMPIBlock || isSymBlock || isSymSeqBlock || isSymMPIBlock) {
1064         PetscInt pStart, pEnd, p, dof, cdof;
1065 
1066         ierr = PetscSectionGetChart(sectionGlobal, &pStart, &pEnd);CHKERRQ(ierr);
1067         for (p = pStart; p < pEnd; ++p) {
1068           ierr = PetscSectionGetDof(sectionGlobal, p, &dof);CHKERRQ(ierr);
1069           ierr = PetscSectionGetConstraintDof(sectionGlobal, p, &cdof);CHKERRQ(ierr);
1070           if (dof-cdof) {
1071             if (bs < 0) {
1072               bs = dof-cdof;
1073             } else if (bs != dof-cdof) {
1074               /* Layout does not admit a pointwise block size */
1075               bs = 1;
1076               break;
1077             }
1078           }
1079         }
1080         /* Must have same blocksize on all procs (some might have no points) */
1081         bsLocal = bs;
1082         ierr = MPI_Allreduce(&bsLocal, &bsMax, 1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
1083         bsLocal = bs < 0 ? bsMax : bs;
1084         ierr = MPI_Allreduce(&bsLocal, &bsMin, 1, MPIU_INT, MPI_MIN, PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
1085         if (bsMin != bsMax) {
1086           bs = 1;
1087         } else {
1088           bs = bsMax;
1089         }
1090       } else {
1091         bs = 1;
1092       }
1093     }
1094     ierr = PetscCalloc4(localSize/bs, &dnz, localSize/bs, &onz, localSize/bs, &dnzu, localSize/bs, &onzu);CHKERRQ(ierr);
1095     ierr = DMPlexPreallocateOperator(dm, bs, section, sectionGlobal, dnz, onz, dnzu, onzu, *J, fillMatrix);CHKERRQ(ierr);
1096     ierr = PetscFree4(dnz, onz, dnzu, onzu);CHKERRQ(ierr);
1097   }
1098   PetscFunctionReturn(0);
1099 }
1100 
1101 #undef __FUNCT__
1102 #define __FUNCT__ "DMPlexGetDimension"
1103 /*@
1104   DMPlexGetDimension - Return the topological mesh dimension
1105 
1106   Not collective
1107 
1108   Input Parameter:
1109 . mesh - The DMPlex
1110 
1111   Output Parameter:
1112 . dim - The topological mesh dimension
1113 
1114   Level: beginner
1115 
1116 .seealso: DMPlexCreate()
1117 @*/
1118 PetscErrorCode DMPlexGetDimension(DM dm, PetscInt *dim)
1119 {
1120   DM_Plex *mesh = (DM_Plex*) dm->data;
1121 
1122   PetscFunctionBegin;
1123   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1124   PetscValidPointer(dim, 2);
1125   *dim = mesh->dim;
1126   PetscFunctionReturn(0);
1127 }
1128 
1129 #undef __FUNCT__
1130 #define __FUNCT__ "DMPlexSetDimension"
1131 /*@
1132   DMPlexSetDimension - Set the topological mesh dimension
1133 
1134   Collective on mesh
1135 
1136   Input Parameters:
1137 + mesh - The DMPlex
1138 - dim - The topological mesh dimension
1139 
1140   Level: beginner
1141 
1142 .seealso: DMPlexCreate()
1143 @*/
1144 PetscErrorCode DMPlexSetDimension(DM dm, PetscInt dim)
1145 {
1146   DM_Plex *mesh = (DM_Plex*) dm->data;
1147 
1148   PetscFunctionBegin;
1149   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1150   PetscValidLogicalCollectiveInt(dm, dim, 2);
1151   mesh->dim = dim;
1152   PetscFunctionReturn(0);
1153 }
1154 
1155 #undef __FUNCT__
1156 #define __FUNCT__ "DMPlexGetChart"
1157 /*@
1158   DMPlexGetChart - Return the interval for all mesh points [pStart, pEnd)
1159 
1160   Not collective
1161 
1162   Input Parameter:
1163 . mesh - The DMPlex
1164 
1165   Output Parameters:
1166 + pStart - The first mesh point
1167 - pEnd   - The upper bound for mesh points
1168 
1169   Level: beginner
1170 
1171 .seealso: DMPlexCreate(), DMPlexSetChart()
1172 @*/
1173 PetscErrorCode DMPlexGetChart(DM dm, PetscInt *pStart, PetscInt *pEnd)
1174 {
1175   DM_Plex       *mesh = (DM_Plex*) dm->data;
1176   PetscErrorCode ierr;
1177 
1178   PetscFunctionBegin;
1179   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1180   ierr = PetscSectionGetChart(mesh->coneSection, pStart, pEnd);CHKERRQ(ierr);
1181   PetscFunctionReturn(0);
1182 }
1183 
1184 #undef __FUNCT__
1185 #define __FUNCT__ "DMPlexSetChart"
1186 /*@
1187   DMPlexSetChart - Set the interval for all mesh points [pStart, pEnd)
1188 
1189   Not collective
1190 
1191   Input Parameters:
1192 + mesh - The DMPlex
1193 . pStart - The first mesh point
1194 - pEnd   - The upper bound for mesh points
1195 
1196   Output Parameters:
1197 
1198   Level: beginner
1199 
1200 .seealso: DMPlexCreate(), DMPlexGetChart()
1201 @*/
1202 PetscErrorCode DMPlexSetChart(DM dm, PetscInt pStart, PetscInt pEnd)
1203 {
1204   DM_Plex       *mesh = (DM_Plex*) dm->data;
1205   PetscErrorCode ierr;
1206 
1207   PetscFunctionBegin;
1208   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1209   ierr = PetscSectionSetChart(mesh->coneSection, pStart, pEnd);CHKERRQ(ierr);
1210   ierr = PetscSectionSetChart(mesh->supportSection, pStart, pEnd);CHKERRQ(ierr);
1211   PetscFunctionReturn(0);
1212 }
1213 
1214 #undef __FUNCT__
1215 #define __FUNCT__ "DMPlexGetConeSize"
1216 /*@
1217   DMPlexGetConeSize - Return the number of in-edges for this point in the Sieve DAG
1218 
1219   Not collective
1220 
1221   Input Parameters:
1222 + mesh - The DMPlex
1223 - p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1224 
1225   Output Parameter:
1226 . size - The cone size for point p
1227 
1228   Level: beginner
1229 
1230 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart()
1231 @*/
1232 PetscErrorCode DMPlexGetConeSize(DM dm, PetscInt p, PetscInt *size)
1233 {
1234   DM_Plex       *mesh = (DM_Plex*) dm->data;
1235   PetscErrorCode ierr;
1236 
1237   PetscFunctionBegin;
1238   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1239   PetscValidPointer(size, 3);
1240   ierr = PetscSectionGetDof(mesh->coneSection, p, size);CHKERRQ(ierr);
1241   PetscFunctionReturn(0);
1242 }
1243 
1244 #undef __FUNCT__
1245 #define __FUNCT__ "DMPlexSetConeSize"
1246 /*@
1247   DMPlexSetConeSize - Set the number of in-edges for this point in the Sieve DAG
1248 
1249   Not collective
1250 
1251   Input Parameters:
1252 + mesh - The DMPlex
1253 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1254 - size - The cone size for point p
1255 
1256   Output Parameter:
1257 
1258   Note:
1259   This should be called after DMPlexSetChart().
1260 
1261   Level: beginner
1262 
1263 .seealso: DMPlexCreate(), DMPlexGetConeSize(), DMPlexSetChart()
1264 @*/
1265 PetscErrorCode DMPlexSetConeSize(DM dm, PetscInt p, PetscInt size)
1266 {
1267   DM_Plex       *mesh = (DM_Plex*) dm->data;
1268   PetscErrorCode ierr;
1269 
1270   PetscFunctionBegin;
1271   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1272   ierr = PetscSectionSetDof(mesh->coneSection, p, size);CHKERRQ(ierr);
1273 
1274   mesh->maxConeSize = PetscMax(mesh->maxConeSize, size);
1275   PetscFunctionReturn(0);
1276 }
1277 
1278 #undef __FUNCT__
1279 #define __FUNCT__ "DMPlexGetCone"
1280 /*@C
1281   DMPlexGetCone - Return the points on the in-edges for this point in the Sieve DAG
1282 
1283   Not collective
1284 
1285   Input Parameters:
1286 + mesh - The DMPlex
1287 - p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1288 
1289   Output Parameter:
1290 . cone - An array of points which are on the in-edges for point p
1291 
1292   Level: beginner
1293 
1294   Fortran Notes:
1295   Since it returns an array, this routine is only available in Fortran 90, and you must
1296   include petsc.h90 in your code.
1297 
1298   You must also call DMPlexRestoreCone() after you finish using the returned array.
1299 
1300 .seealso: DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart()
1301 @*/
1302 PetscErrorCode DMPlexGetCone(DM dm, PetscInt p, const PetscInt *cone[])
1303 {
1304   DM_Plex       *mesh = (DM_Plex*) dm->data;
1305   PetscInt       off;
1306   PetscErrorCode ierr;
1307 
1308   PetscFunctionBegin;
1309   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1310   PetscValidPointer(cone, 3);
1311   ierr  = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1312   *cone = &mesh->cones[off];
1313   PetscFunctionReturn(0);
1314 }
1315 
1316 #undef __FUNCT__
1317 #define __FUNCT__ "DMPlexSetCone"
1318 /*@
1319   DMPlexSetCone - Set the points on the in-edges for this point in the Sieve DAG
1320 
1321   Not collective
1322 
1323   Input Parameters:
1324 + mesh - The DMPlex
1325 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1326 - cone - An array of points which are on the in-edges for point p
1327 
1328   Output Parameter:
1329 
1330   Note:
1331   This should be called after all calls to DMPlexSetConeSize() and DMSetUp().
1332 
1333   Level: beginner
1334 
1335 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
1336 @*/
1337 PetscErrorCode DMPlexSetCone(DM dm, PetscInt p, const PetscInt cone[])
1338 {
1339   DM_Plex       *mesh = (DM_Plex*) dm->data;
1340   PetscInt       pStart, pEnd;
1341   PetscInt       dof, off, c;
1342   PetscErrorCode ierr;
1343 
1344   PetscFunctionBegin;
1345   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1346   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
1347   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
1348   if (dof) PetscValidPointer(cone, 3);
1349   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1350   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
1351   for (c = 0; c < dof; ++c) {
1352     if ((cone[c] < pStart) || (cone[c] >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone point %D is not in the valid range [%D, %D)", cone[c], pStart, pEnd);
1353     mesh->cones[off+c] = cone[c];
1354   }
1355   PetscFunctionReturn(0);
1356 }
1357 
1358 #undef __FUNCT__
1359 #define __FUNCT__ "DMPlexGetConeOrientation"
1360 /*@C
1361   DMPlexGetConeOrientation - Return the orientations on the in-edges for this point in the Sieve DAG
1362 
1363   Not collective
1364 
1365   Input Parameters:
1366 + mesh - The DMPlex
1367 - p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1368 
1369   Output Parameter:
1370 . coneOrientation - An array of orientations which are on the in-edges for point p. An orientation is an
1371                     integer giving the prescription for cone traversal. If it is negative, the cone is
1372                     traversed in the opposite direction. Its value 'o', or if negative '-(o+1)', gives
1373                     the index of the cone point on which to start.
1374 
1375   Level: beginner
1376 
1377   Fortran Notes:
1378   Since it returns an array, this routine is only available in Fortran 90, and you must
1379   include petsc.h90 in your code.
1380 
1381   You must also call DMPlexRestoreConeOrientation() after you finish using the returned array.
1382 
1383 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetCone(), DMPlexSetChart()
1384 @*/
1385 PetscErrorCode DMPlexGetConeOrientation(DM dm, PetscInt p, const PetscInt *coneOrientation[])
1386 {
1387   DM_Plex       *mesh = (DM_Plex*) dm->data;
1388   PetscInt       off;
1389   PetscErrorCode ierr;
1390 
1391   PetscFunctionBegin;
1392   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1393 #if defined(PETSC_USE_DEBUG)
1394   {
1395     PetscInt dof;
1396     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
1397     if (dof) PetscValidPointer(coneOrientation, 3);
1398   }
1399 #endif
1400   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1401 
1402   *coneOrientation = &mesh->coneOrientations[off];
1403   PetscFunctionReturn(0);
1404 }
1405 
1406 #undef __FUNCT__
1407 #define __FUNCT__ "DMPlexSetConeOrientation"
1408 /*@
1409   DMPlexSetConeOrientation - Set the orientations on the in-edges for this point in the Sieve DAG
1410 
1411   Not collective
1412 
1413   Input Parameters:
1414 + mesh - The DMPlex
1415 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1416 - coneOrientation - An array of orientations which are on the in-edges for point p. An orientation is an
1417                     integer giving the prescription for cone traversal. If it is negative, the cone is
1418                     traversed in the opposite direction. Its value 'o', or if negative '-(o+1)', gives
1419                     the index of the cone point on which to start.
1420 
1421   Output Parameter:
1422 
1423   Note:
1424   This should be called after all calls to DMPlexSetConeSize() and DMSetUp().
1425 
1426   Level: beginner
1427 
1428 .seealso: DMPlexCreate(), DMPlexGetConeOrientation(), DMPlexSetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
1429 @*/
1430 PetscErrorCode DMPlexSetConeOrientation(DM dm, PetscInt p, const PetscInt coneOrientation[])
1431 {
1432   DM_Plex       *mesh = (DM_Plex*) dm->data;
1433   PetscInt       pStart, pEnd;
1434   PetscInt       dof, off, c;
1435   PetscErrorCode ierr;
1436 
1437   PetscFunctionBegin;
1438   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1439   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
1440   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
1441   if (dof) PetscValidPointer(coneOrientation, 3);
1442   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1443   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
1444   for (c = 0; c < dof; ++c) {
1445     PetscInt cdof, o = coneOrientation[c];
1446 
1447     ierr = PetscSectionGetDof(mesh->coneSection, mesh->cones[off+c], &cdof);CHKERRQ(ierr);
1448     if (o && ((o < -(cdof+1)) || (o >= cdof))) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone orientation %D is not in the valid range [%D. %D)", o, -(cdof+1), cdof);
1449     mesh->coneOrientations[off+c] = o;
1450   }
1451   PetscFunctionReturn(0);
1452 }
1453 
1454 #undef __FUNCT__
1455 #define __FUNCT__ "DMPlexInsertCone"
1456 PetscErrorCode DMPlexInsertCone(DM dm, PetscInt p, PetscInt conePos, PetscInt conePoint)
1457 {
1458   DM_Plex       *mesh = (DM_Plex*) dm->data;
1459   PetscInt       pStart, pEnd;
1460   PetscInt       dof, off;
1461   PetscErrorCode ierr;
1462 
1463   PetscFunctionBegin;
1464   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1465   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
1466   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
1467   if ((conePoint < pStart) || (conePoint >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone point %D is not in the valid range [%D, %D)", conePoint, pStart, pEnd);
1468   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
1469   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1470   if ((conePos < 0) || (conePos >= dof)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone position %D of point %D is not in the valid range [0, %D)", conePos, p, dof);
1471   mesh->cones[off+conePos] = conePoint;
1472   PetscFunctionReturn(0);
1473 }
1474 
1475 #undef __FUNCT__
1476 #define __FUNCT__ "DMPlexInsertConeOrientation"
1477 PetscErrorCode DMPlexInsertConeOrientation(DM dm, PetscInt p, PetscInt conePos, PetscInt coneOrientation)
1478 {
1479   DM_Plex       *mesh = (DM_Plex*) dm->data;
1480   PetscInt       pStart, pEnd;
1481   PetscInt       dof, off;
1482   PetscErrorCode ierr;
1483 
1484   PetscFunctionBegin;
1485   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1486   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
1487   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
1488   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
1489   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1490   if ((conePos < 0) || (conePos >= dof)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone position %D of point %D is not in the valid range [0, %D)", conePos, p, dof);
1491   mesh->coneOrientations[off+conePos] = coneOrientation;
1492   PetscFunctionReturn(0);
1493 }
1494 
1495 #undef __FUNCT__
1496 #define __FUNCT__ "DMPlexGetSupportSize"
1497 /*@
1498   DMPlexGetSupportSize - Return the number of out-edges for this point in the Sieve DAG
1499 
1500   Not collective
1501 
1502   Input Parameters:
1503 + mesh - The DMPlex
1504 - p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1505 
1506   Output Parameter:
1507 . size - The support size for point p
1508 
1509   Level: beginner
1510 
1511 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart(), DMPlexGetConeSize()
1512 @*/
1513 PetscErrorCode DMPlexGetSupportSize(DM dm, PetscInt p, PetscInt *size)
1514 {
1515   DM_Plex       *mesh = (DM_Plex*) dm->data;
1516   PetscErrorCode ierr;
1517 
1518   PetscFunctionBegin;
1519   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1520   PetscValidPointer(size, 3);
1521   ierr = PetscSectionGetDof(mesh->supportSection, p, size);CHKERRQ(ierr);
1522   PetscFunctionReturn(0);
1523 }
1524 
1525 #undef __FUNCT__
1526 #define __FUNCT__ "DMPlexSetSupportSize"
1527 /*@
1528   DMPlexSetSupportSize - Set the number of out-edges for this point in the Sieve DAG
1529 
1530   Not collective
1531 
1532   Input Parameters:
1533 + mesh - The DMPlex
1534 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1535 - size - The support size for point p
1536 
1537   Output Parameter:
1538 
1539   Note:
1540   This should be called after DMPlexSetChart().
1541 
1542   Level: beginner
1543 
1544 .seealso: DMPlexCreate(), DMPlexGetSupportSize(), DMPlexSetChart()
1545 @*/
1546 PetscErrorCode DMPlexSetSupportSize(DM dm, PetscInt p, PetscInt size)
1547 {
1548   DM_Plex       *mesh = (DM_Plex*) dm->data;
1549   PetscErrorCode ierr;
1550 
1551   PetscFunctionBegin;
1552   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1553   ierr = PetscSectionSetDof(mesh->supportSection, p, size);CHKERRQ(ierr);
1554 
1555   mesh->maxSupportSize = PetscMax(mesh->maxSupportSize, size);
1556   PetscFunctionReturn(0);
1557 }
1558 
1559 #undef __FUNCT__
1560 #define __FUNCT__ "DMPlexGetSupport"
1561 /*@C
1562   DMPlexGetSupport - Return the points on the out-edges for this point in the Sieve DAG
1563 
1564   Not collective
1565 
1566   Input Parameters:
1567 + mesh - The DMPlex
1568 - p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1569 
1570   Output Parameter:
1571 . support - An array of points which are on the out-edges for point p
1572 
1573   Level: beginner
1574 
1575   Fortran Notes:
1576   Since it returns an array, this routine is only available in Fortran 90, and you must
1577   include petsc.h90 in your code.
1578 
1579   You must also call DMPlexRestoreSupport() after you finish using the returned array.
1580 
1581 .seealso: DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
1582 @*/
1583 PetscErrorCode DMPlexGetSupport(DM dm, PetscInt p, const PetscInt *support[])
1584 {
1585   DM_Plex       *mesh = (DM_Plex*) dm->data;
1586   PetscInt       off;
1587   PetscErrorCode ierr;
1588 
1589   PetscFunctionBegin;
1590   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1591   PetscValidPointer(support, 3);
1592   ierr     = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
1593   *support = &mesh->supports[off];
1594   PetscFunctionReturn(0);
1595 }
1596 
1597 #undef __FUNCT__
1598 #define __FUNCT__ "DMPlexSetSupport"
1599 /*@
1600   DMPlexSetSupport - Set the points on the out-edges for this point in the Sieve DAG
1601 
1602   Not collective
1603 
1604   Input Parameters:
1605 + mesh - The DMPlex
1606 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1607 - support - An array of points which are on the in-edges for point p
1608 
1609   Output Parameter:
1610 
1611   Note:
1612   This should be called after all calls to DMPlexSetSupportSize() and DMSetUp().
1613 
1614   Level: beginner
1615 
1616 .seealso: DMPlexCreate(), DMPlexGetSupport(), DMPlexSetChart(), DMPlexSetSupportSize(), DMSetUp()
1617 @*/
1618 PetscErrorCode DMPlexSetSupport(DM dm, PetscInt p, const PetscInt support[])
1619 {
1620   DM_Plex       *mesh = (DM_Plex*) dm->data;
1621   PetscInt       pStart, pEnd;
1622   PetscInt       dof, off, c;
1623   PetscErrorCode ierr;
1624 
1625   PetscFunctionBegin;
1626   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1627   ierr = PetscSectionGetChart(mesh->supportSection, &pStart, &pEnd);CHKERRQ(ierr);
1628   ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
1629   if (dof) PetscValidPointer(support, 3);
1630   ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
1631   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
1632   for (c = 0; c < dof; ++c) {
1633     if ((support[c] < pStart) || (support[c] >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support point %D is not in the valid range [%D, %D)", support[c], pStart, pEnd);
1634     mesh->supports[off+c] = support[c];
1635   }
1636   PetscFunctionReturn(0);
1637 }
1638 
1639 #undef __FUNCT__
1640 #define __FUNCT__ "DMPlexInsertSupport"
1641 PetscErrorCode DMPlexInsertSupport(DM dm, PetscInt p, PetscInt supportPos, PetscInt supportPoint)
1642 {
1643   DM_Plex       *mesh = (DM_Plex*) dm->data;
1644   PetscInt       pStart, pEnd;
1645   PetscInt       dof, off;
1646   PetscErrorCode ierr;
1647 
1648   PetscFunctionBegin;
1649   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1650   ierr = PetscSectionGetChart(mesh->supportSection, &pStart, &pEnd);CHKERRQ(ierr);
1651   ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
1652   ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
1653   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
1654   if ((supportPoint < pStart) || (supportPoint >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support point %D is not in the valid range [%D, %D)", supportPoint, pStart, pEnd);
1655   if (supportPos >= dof) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support position %D of point %D is not in the valid range [0, %D)", supportPos, p, dof);
1656   mesh->supports[off+supportPos] = supportPoint;
1657   PetscFunctionReturn(0);
1658 }
1659 
1660 #undef __FUNCT__
1661 #define __FUNCT__ "DMPlexGetTransitiveClosure"
1662 /*@C
1663   DMPlexGetTransitiveClosure - Return the points on the transitive closure of the in-edges or out-edges for this point in the Sieve DAG
1664 
1665   Not collective
1666 
1667   Input Parameters:
1668 + mesh - The DMPlex
1669 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1670 . useCone - PETSC_TRUE for in-edges,  otherwise use out-edges
1671 - points - If points is NULL on input, internal storage will be returned, otherwise the provided array is used
1672 
1673   Output Parameters:
1674 + numPoints - The number of points in the closure, so points[] is of size 2*numPoints
1675 - points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...]
1676 
1677   Note:
1678   If using internal storage (points is NULL on input), each call overwrites the last output.
1679 
1680   Fortran Notes:
1681   Since it returns an array, this routine is only available in Fortran 90, and you must
1682   include petsc.h90 in your code.
1683 
1684   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
1685 
1686   Level: beginner
1687 
1688 .seealso: DMPlexRestoreTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
1689 @*/
1690 PetscErrorCode DMPlexGetTransitiveClosure(DM dm, PetscInt p, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
1691 {
1692   DM_Plex        *mesh = (DM_Plex*) dm->data;
1693   PetscInt       *closure, *fifo;
1694   const PetscInt *tmp = NULL, *tmpO = NULL;
1695   PetscInt        tmpSize, t;
1696   PetscInt        depth       = 0, maxSize;
1697   PetscInt        closureSize = 2, fifoSize = 0, fifoStart = 0;
1698   PetscErrorCode  ierr;
1699 
1700   PetscFunctionBegin;
1701   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1702   ierr    = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
1703   /* This is only 1-level */
1704   if (useCone) {
1705     ierr = DMPlexGetConeSize(dm, p, &tmpSize);CHKERRQ(ierr);
1706     ierr = DMPlexGetCone(dm, p, &tmp);CHKERRQ(ierr);
1707     ierr = DMPlexGetConeOrientation(dm, p, &tmpO);CHKERRQ(ierr);
1708   } else {
1709     ierr = DMPlexGetSupportSize(dm, p, &tmpSize);CHKERRQ(ierr);
1710     ierr = DMPlexGetSupport(dm, p, &tmp);CHKERRQ(ierr);
1711   }
1712   if (depth == 1) {
1713     if (*points) {
1714       closure = *points;
1715     } else {
1716       maxSize = 2*(PetscMax(mesh->maxConeSize, mesh->maxSupportSize)+1);
1717       ierr = DMGetWorkArray(dm, maxSize, PETSC_INT, &closure);CHKERRQ(ierr);
1718     }
1719     closure[0] = p; closure[1] = 0;
1720     for (t = 0; t < tmpSize; ++t, closureSize += 2) {
1721       closure[closureSize]   = tmp[t];
1722       closure[closureSize+1] = tmpO ? tmpO[t] : 0;
1723     }
1724     if (numPoints) *numPoints = closureSize/2;
1725     if (points)    *points    = closure;
1726     PetscFunctionReturn(0);
1727   }
1728   maxSize = 2*PetscMax(PetscMax(PetscPowInt(mesh->maxConeSize,depth+1),PetscPowInt(mesh->maxSupportSize,depth+1)),depth+1);
1729   ierr    = DMGetWorkArray(dm, maxSize, PETSC_INT, &fifo);CHKERRQ(ierr);
1730   if (*points) {
1731     closure = *points;
1732   } else {
1733     ierr = DMGetWorkArray(dm, maxSize, PETSC_INT, &closure);CHKERRQ(ierr);
1734   }
1735   closure[0] = p; closure[1] = 0;
1736   for (t = 0; t < tmpSize; ++t, closureSize += 2, fifoSize += 2) {
1737     const PetscInt cp = tmp[t];
1738     const PetscInt co = tmpO ? tmpO[t] : 0;
1739 
1740     closure[closureSize]   = cp;
1741     closure[closureSize+1] = co;
1742     fifo[fifoSize]         = cp;
1743     fifo[fifoSize+1]       = co;
1744   }
1745   /* Should kick out early when depth is reached, rather than checking all vertices for empty cones */
1746   while (fifoSize - fifoStart) {
1747     const PetscInt q   = fifo[fifoStart];
1748     const PetscInt o   = fifo[fifoStart+1];
1749     const PetscInt rev = o >= 0 ? 0 : 1;
1750     const PetscInt off = rev ? -(o+1) : o;
1751 
1752     if (useCone) {
1753       ierr = DMPlexGetConeSize(dm, q, &tmpSize);CHKERRQ(ierr);
1754       ierr = DMPlexGetCone(dm, q, &tmp);CHKERRQ(ierr);
1755       ierr = DMPlexGetConeOrientation(dm, q, &tmpO);CHKERRQ(ierr);
1756     } else {
1757       ierr = DMPlexGetSupportSize(dm, q, &tmpSize);CHKERRQ(ierr);
1758       ierr = DMPlexGetSupport(dm, q, &tmp);CHKERRQ(ierr);
1759       tmpO = NULL;
1760     }
1761     for (t = 0; t < tmpSize; ++t) {
1762       const PetscInt i  = ((rev ? tmpSize-t : t) + off)%tmpSize;
1763       const PetscInt cp = tmp[i];
1764       /* Must propogate orientation: When we reverse orientation, we both reverse the direction of iteration and start at the other end of the chain. */
1765       /* HACK: It is worse to get the size here, than to change the interpretation of -(*+1)
1766        const PetscInt co = tmpO ? (rev ? -(tmpO[i]+1) : tmpO[i]) : 0; */
1767       PetscInt       co = tmpO ? tmpO[i] : 0;
1768       PetscInt       c;
1769 
1770       if (rev) {
1771         PetscInt childSize, coff;
1772         ierr = DMPlexGetConeSize(dm, cp, &childSize);CHKERRQ(ierr);
1773         coff = tmpO[i] < 0 ? -(tmpO[i]+1) : tmpO[i];
1774         co   = childSize ? -(((coff+childSize-1)%childSize)+1) : 0;
1775       }
1776       /* Check for duplicate */
1777       for (c = 0; c < closureSize; c += 2) {
1778         if (closure[c] == cp) break;
1779       }
1780       if (c == closureSize) {
1781         closure[closureSize]   = cp;
1782         closure[closureSize+1] = co;
1783         fifo[fifoSize]         = cp;
1784         fifo[fifoSize+1]       = co;
1785         closureSize           += 2;
1786         fifoSize              += 2;
1787       }
1788     }
1789     fifoStart += 2;
1790   }
1791   if (numPoints) *numPoints = closureSize/2;
1792   if (points)    *points    = closure;
1793   ierr = DMRestoreWorkArray(dm, maxSize, PETSC_INT, &fifo);CHKERRQ(ierr);
1794   PetscFunctionReturn(0);
1795 }
1796 
1797 #undef __FUNCT__
1798 #define __FUNCT__ "DMPlexGetTransitiveClosure_Internal"
1799 /*@C
1800   DMPlexGetTransitiveClosure_Internal - Return the points on the transitive closure of the in-edges or out-edges for this point in the Sieve DAG with a specified initial orientation
1801 
1802   Not collective
1803 
1804   Input Parameters:
1805 + mesh - The DMPlex
1806 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1807 . orientation - The orientation of the point
1808 . useCone - PETSC_TRUE for in-edges,  otherwise use out-edges
1809 - points - If points is NULL on input, internal storage will be returned, otherwise the provided array is used
1810 
1811   Output Parameters:
1812 + numPoints - The number of points in the closure, so points[] is of size 2*numPoints
1813 - points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...]
1814 
1815   Note:
1816   If using internal storage (points is NULL on input), each call overwrites the last output.
1817 
1818   Fortran Notes:
1819   Since it returns an array, this routine is only available in Fortran 90, and you must
1820   include petsc.h90 in your code.
1821 
1822   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
1823 
1824   Level: beginner
1825 
1826 .seealso: DMPlexRestoreTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
1827 @*/
1828 PetscErrorCode DMPlexGetTransitiveClosure_Internal(DM dm, PetscInt p, PetscInt ornt, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
1829 {
1830   DM_Plex        *mesh = (DM_Plex*) dm->data;
1831   PetscInt       *closure, *fifo;
1832   const PetscInt *tmp = NULL, *tmpO = NULL;
1833   PetscInt        tmpSize, t;
1834   PetscInt        depth       = 0, maxSize;
1835   PetscInt        closureSize = 2, fifoSize = 0, fifoStart = 0;
1836   PetscErrorCode  ierr;
1837 
1838   PetscFunctionBegin;
1839   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1840   ierr    = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
1841   /* This is only 1-level */
1842   if (useCone) {
1843     ierr = DMPlexGetConeSize(dm, p, &tmpSize);CHKERRQ(ierr);
1844     ierr = DMPlexGetCone(dm, p, &tmp);CHKERRQ(ierr);
1845     ierr = DMPlexGetConeOrientation(dm, p, &tmpO);CHKERRQ(ierr);
1846   } else {
1847     ierr = DMPlexGetSupportSize(dm, p, &tmpSize);CHKERRQ(ierr);
1848     ierr = DMPlexGetSupport(dm, p, &tmp);CHKERRQ(ierr);
1849   }
1850   if (depth == 1) {
1851     if (*points) {
1852       closure = *points;
1853     } else {
1854       maxSize = 2*(PetscMax(mesh->maxConeSize, mesh->maxSupportSize)+1);
1855       ierr = DMGetWorkArray(dm, maxSize, PETSC_INT, &closure);CHKERRQ(ierr);
1856     }
1857     closure[0] = p; closure[1] = ornt;
1858     for (t = 0; t < tmpSize; ++t, closureSize += 2) {
1859       const PetscInt i = ornt >= 0 ? (t+ornt)%tmpSize : (-(ornt+1) + tmpSize-t)%tmpSize;
1860       closure[closureSize]   = tmp[i];
1861       closure[closureSize+1] = tmpO ? tmpO[i] : 0;
1862     }
1863     if (numPoints) *numPoints = closureSize/2;
1864     if (points)    *points    = closure;
1865     PetscFunctionReturn(0);
1866   }
1867   maxSize = 2*PetscMax(PetscMax(PetscPowInt(mesh->maxConeSize,depth+1),PetscPowInt(mesh->maxSupportSize,depth+1)),depth+1);
1868   ierr    = DMGetWorkArray(dm, maxSize, PETSC_INT, &fifo);CHKERRQ(ierr);
1869   if (*points) {
1870     closure = *points;
1871   } else {
1872     ierr = DMGetWorkArray(dm, maxSize, PETSC_INT, &closure);CHKERRQ(ierr);
1873   }
1874   closure[0] = p; closure[1] = ornt;
1875   for (t = 0; t < tmpSize; ++t, closureSize += 2, fifoSize += 2) {
1876     const PetscInt i  = ornt >= 0 ? (t+ornt)%tmpSize : (-(ornt+1) + tmpSize-t)%tmpSize;
1877     const PetscInt cp = tmp[i];
1878     PetscInt       co = tmpO ? tmpO[i] : 0;
1879 
1880     if (ornt < 0) {
1881       PetscInt childSize, coff;
1882       ierr = DMPlexGetConeSize(dm, cp, &childSize);CHKERRQ(ierr);
1883       coff = tmpO[i] < 0 ? -(tmpO[i]+1) : tmpO[i];
1884       co   = childSize ? -(((coff+childSize-1)%childSize)+1) : 0;
1885     }
1886     closure[closureSize]   = cp;
1887     closure[closureSize+1] = co;
1888     fifo[fifoSize]         = cp;
1889     fifo[fifoSize+1]       = co;
1890   }
1891   /* Should kick out early when depth is reached, rather than checking all vertices for empty cones */
1892   while (fifoSize - fifoStart) {
1893     const PetscInt q   = fifo[fifoStart];
1894     const PetscInt o   = fifo[fifoStart+1];
1895     const PetscInt rev = o >= 0 ? 0 : 1;
1896     const PetscInt off = rev ? -(o+1) : o;
1897 
1898     if (useCone) {
1899       ierr = DMPlexGetConeSize(dm, q, &tmpSize);CHKERRQ(ierr);
1900       ierr = DMPlexGetCone(dm, q, &tmp);CHKERRQ(ierr);
1901       ierr = DMPlexGetConeOrientation(dm, q, &tmpO);CHKERRQ(ierr);
1902     } else {
1903       ierr = DMPlexGetSupportSize(dm, q, &tmpSize);CHKERRQ(ierr);
1904       ierr = DMPlexGetSupport(dm, q, &tmp);CHKERRQ(ierr);
1905       tmpO = NULL;
1906     }
1907     for (t = 0; t < tmpSize; ++t) {
1908       const PetscInt i  = ((rev ? tmpSize-t : t) + off)%tmpSize;
1909       const PetscInt cp = tmp[i];
1910       /* Must propogate orientation: When we reverse orientation, we both reverse the direction of iteration and start at the other end of the chain. */
1911       /* HACK: It is worse to get the size here, than to change the interpretation of -(*+1)
1912        const PetscInt co = tmpO ? (rev ? -(tmpO[i]+1) : tmpO[i]) : 0; */
1913       PetscInt       co = tmpO ? tmpO[i] : 0;
1914       PetscInt       c;
1915 
1916       if (rev) {
1917         PetscInt childSize, coff;
1918         ierr = DMPlexGetConeSize(dm, cp, &childSize);CHKERRQ(ierr);
1919         coff = tmpO[i] < 0 ? -(tmpO[i]+1) : tmpO[i];
1920         co   = childSize ? -(((coff+childSize-1)%childSize)+1) : 0;
1921       }
1922       /* Check for duplicate */
1923       for (c = 0; c < closureSize; c += 2) {
1924         if (closure[c] == cp) break;
1925       }
1926       if (c == closureSize) {
1927         closure[closureSize]   = cp;
1928         closure[closureSize+1] = co;
1929         fifo[fifoSize]         = cp;
1930         fifo[fifoSize+1]       = co;
1931         closureSize           += 2;
1932         fifoSize              += 2;
1933       }
1934     }
1935     fifoStart += 2;
1936   }
1937   if (numPoints) *numPoints = closureSize/2;
1938   if (points)    *points    = closure;
1939   ierr = DMRestoreWorkArray(dm, maxSize, PETSC_INT, &fifo);CHKERRQ(ierr);
1940   PetscFunctionReturn(0);
1941 }
1942 
1943 #undef __FUNCT__
1944 #define __FUNCT__ "DMPlexRestoreTransitiveClosure"
1945 /*@C
1946   DMPlexRestoreTransitiveClosure - Restore the array of points on the transitive closure of the in-edges or out-edges for this point in the Sieve DAG
1947 
1948   Not collective
1949 
1950   Input Parameters:
1951 + mesh - The DMPlex
1952 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1953 . useCone - PETSC_TRUE for in-edges,  otherwise use out-edges
1954 . numPoints - The number of points in the closure, so points[] is of size 2*numPoints, zeroed on exit
1955 - points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...], zeroed on exit
1956 
1957   Note:
1958   If not using internal storage (points is not NULL on input), this call is unnecessary
1959 
1960   Fortran Notes:
1961   Since it returns an array, this routine is only available in Fortran 90, and you must
1962   include petsc.h90 in your code.
1963 
1964   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
1965 
1966   Level: beginner
1967 
1968 .seealso: DMPlexGetTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
1969 @*/
1970 PetscErrorCode DMPlexRestoreTransitiveClosure(DM dm, PetscInt p, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
1971 {
1972   PetscErrorCode ierr;
1973 
1974   PetscFunctionBegin;
1975   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1976   if (numPoints) PetscValidIntPointer(numPoints,4);
1977   if (points) PetscValidPointer(points,5);
1978   ierr = DMRestoreWorkArray(dm, 0, PETSC_INT, points);CHKERRQ(ierr);
1979   if (numPoints) *numPoints = 0;
1980   PetscFunctionReturn(0);
1981 }
1982 
1983 #undef __FUNCT__
1984 #define __FUNCT__ "DMPlexGetMaxSizes"
1985 /*@
1986   DMPlexGetMaxSizes - Return the maximum number of in-edges (cone) and out-edges (support) for any point in the Sieve DAG
1987 
1988   Not collective
1989 
1990   Input Parameter:
1991 . mesh - The DMPlex
1992 
1993   Output Parameters:
1994 + maxConeSize - The maximum number of in-edges
1995 - maxSupportSize - The maximum number of out-edges
1996 
1997   Level: beginner
1998 
1999 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart()
2000 @*/
2001 PetscErrorCode DMPlexGetMaxSizes(DM dm, PetscInt *maxConeSize, PetscInt *maxSupportSize)
2002 {
2003   DM_Plex *mesh = (DM_Plex*) dm->data;
2004 
2005   PetscFunctionBegin;
2006   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2007   if (maxConeSize)    *maxConeSize    = mesh->maxConeSize;
2008   if (maxSupportSize) *maxSupportSize = mesh->maxSupportSize;
2009   PetscFunctionReturn(0);
2010 }
2011 
2012 #undef __FUNCT__
2013 #define __FUNCT__ "DMSetUp_Plex"
2014 PetscErrorCode DMSetUp_Plex(DM dm)
2015 {
2016   DM_Plex       *mesh = (DM_Plex*) dm->data;
2017   PetscInt       size;
2018   PetscErrorCode ierr;
2019 
2020   PetscFunctionBegin;
2021   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2022   ierr = PetscSectionSetUp(mesh->coneSection);CHKERRQ(ierr);
2023   ierr = PetscSectionGetStorageSize(mesh->coneSection, &size);CHKERRQ(ierr);
2024   ierr = PetscMalloc1(size, &mesh->cones);CHKERRQ(ierr);
2025   ierr = PetscCalloc1(size, &mesh->coneOrientations);CHKERRQ(ierr);
2026   if (mesh->maxSupportSize) {
2027     ierr = PetscSectionSetUp(mesh->supportSection);CHKERRQ(ierr);
2028     ierr = PetscSectionGetStorageSize(mesh->supportSection, &size);CHKERRQ(ierr);
2029     ierr = PetscMalloc1(size, &mesh->supports);CHKERRQ(ierr);
2030   }
2031   PetscFunctionReturn(0);
2032 }
2033 
2034 #undef __FUNCT__
2035 #define __FUNCT__ "DMCreateSubDM_Plex"
2036 PetscErrorCode DMCreateSubDM_Plex(DM dm, PetscInt numFields, PetscInt fields[], IS *is, DM *subdm)
2037 {
2038   PetscErrorCode ierr;
2039 
2040   PetscFunctionBegin;
2041   if (subdm) {ierr = DMClone(dm, subdm);CHKERRQ(ierr);}
2042   ierr = DMCreateSubDM_Section_Private(dm, numFields, fields, is, subdm);CHKERRQ(ierr);
2043   PetscFunctionReturn(0);
2044 }
2045 
2046 #undef __FUNCT__
2047 #define __FUNCT__ "DMPlexSymmetrize"
2048 /*@
2049   DMPlexSymmetrize - Creates support (out-edge) information from cone (in-edge) inoformation
2050 
2051   Not collective
2052 
2053   Input Parameter:
2054 . mesh - The DMPlex
2055 
2056   Output Parameter:
2057 
2058   Note:
2059   This should be called after all calls to DMPlexSetCone()
2060 
2061   Level: beginner
2062 
2063 .seealso: DMPlexCreate(), DMPlexSetChart(), DMPlexSetConeSize(), DMPlexSetCone()
2064 @*/
2065 PetscErrorCode DMPlexSymmetrize(DM dm)
2066 {
2067   DM_Plex       *mesh = (DM_Plex*) dm->data;
2068   PetscInt      *offsets;
2069   PetscInt       supportSize;
2070   PetscInt       pStart, pEnd, p;
2071   PetscErrorCode ierr;
2072 
2073   PetscFunctionBegin;
2074   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2075   if (mesh->supports) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONGSTATE, "Supports were already setup in this DMPlex");
2076   /* Calculate support sizes */
2077   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
2078   for (p = pStart; p < pEnd; ++p) {
2079     PetscInt dof, off, c;
2080 
2081     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2082     ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2083     for (c = off; c < off+dof; ++c) {
2084       ierr = PetscSectionAddDof(mesh->supportSection, mesh->cones[c], 1);CHKERRQ(ierr);
2085     }
2086   }
2087   for (p = pStart; p < pEnd; ++p) {
2088     PetscInt dof;
2089 
2090     ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
2091 
2092     mesh->maxSupportSize = PetscMax(mesh->maxSupportSize, dof);
2093   }
2094   ierr = PetscSectionSetUp(mesh->supportSection);CHKERRQ(ierr);
2095   /* Calculate supports */
2096   ierr = PetscSectionGetStorageSize(mesh->supportSection, &supportSize);CHKERRQ(ierr);
2097   ierr = PetscMalloc1(supportSize, &mesh->supports);CHKERRQ(ierr);
2098   ierr = PetscCalloc1(pEnd - pStart, &offsets);CHKERRQ(ierr);
2099   for (p = pStart; p < pEnd; ++p) {
2100     PetscInt dof, off, c;
2101 
2102     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2103     ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2104     for (c = off; c < off+dof; ++c) {
2105       const PetscInt q = mesh->cones[c];
2106       PetscInt       offS;
2107 
2108       ierr = PetscSectionGetOffset(mesh->supportSection, q, &offS);CHKERRQ(ierr);
2109 
2110       mesh->supports[offS+offsets[q]] = p;
2111       ++offsets[q];
2112     }
2113   }
2114   ierr = PetscFree(offsets);CHKERRQ(ierr);
2115   PetscFunctionReturn(0);
2116 }
2117 
2118 #undef __FUNCT__
2119 #define __FUNCT__ "DMPlexStratify"
2120 /*@
2121   DMPlexStratify - The Sieve DAG for most topologies is a graded poset (http://en.wikipedia.org/wiki/Graded_poset), and
2122   can be illustrated by Hasse Diagram (a http://en.wikipedia.org/wiki/Hasse_diagram). The strata group all points of the
2123   same grade, and this function calculates the strata. This grade can be seen as the height (or depth) of the point in
2124   the DAG.
2125 
2126   Not collective
2127 
2128   Input Parameter:
2129 . mesh - The DMPlex
2130 
2131   Output Parameter:
2132 
2133   Notes:
2134   The normal association for the point grade is element dimension (or co-dimension). For instance, all vertices would
2135   have depth 0, and all edges depth 1. Likewise, all cells heights would have height 0, and all faces height 1.
2136 
2137   This should be called after all calls to DMPlexSymmetrize()
2138 
2139   Level: beginner
2140 
2141 .seealso: DMPlexCreate(), DMPlexSymmetrize()
2142 @*/
2143 PetscErrorCode DMPlexStratify(DM dm)
2144 {
2145   DMLabel        label;
2146   PetscInt       pStart, pEnd, p;
2147   PetscInt       numRoots = 0, numLeaves = 0;
2148   PetscErrorCode ierr;
2149 
2150   PetscFunctionBegin;
2151   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2152   ierr = PetscLogEventBegin(DMPLEX_Stratify,dm,0,0,0);CHKERRQ(ierr);
2153   /* Calculate depth */
2154   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
2155   ierr = DMPlexCreateLabel(dm, "depth");CHKERRQ(ierr);
2156   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
2157   /* Initialize roots and count leaves */
2158   for (p = pStart; p < pEnd; ++p) {
2159     PetscInt coneSize, supportSize;
2160 
2161     ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
2162     ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
2163     if (!coneSize && supportSize) {
2164       ++numRoots;
2165       ierr = DMLabelSetValue(label, p, 0);CHKERRQ(ierr);
2166     } else if (!supportSize && coneSize) {
2167       ++numLeaves;
2168     } else if (!supportSize && !coneSize) {
2169       /* Isolated points */
2170       ierr = DMLabelSetValue(label, p, 0);CHKERRQ(ierr);
2171     }
2172   }
2173   if (numRoots + numLeaves == (pEnd - pStart)) {
2174     for (p = pStart; p < pEnd; ++p) {
2175       PetscInt coneSize, supportSize;
2176 
2177       ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
2178       ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
2179       if (!supportSize && coneSize) {
2180         ierr = DMLabelSetValue(label, p, 1);CHKERRQ(ierr);
2181       }
2182     }
2183   } else {
2184     IS       pointIS;
2185     PetscInt numPoints = 0, level = 0;
2186 
2187     ierr = DMLabelGetStratumIS(label, level, &pointIS);CHKERRQ(ierr);
2188     if (pointIS) {ierr = ISGetLocalSize(pointIS, &numPoints);CHKERRQ(ierr);}
2189     while (numPoints) {
2190       const PetscInt *points;
2191       const PetscInt  newLevel = level+1;
2192 
2193       ierr = ISGetIndices(pointIS, &points);CHKERRQ(ierr);
2194       for (p = 0; p < numPoints; ++p) {
2195         const PetscInt  point = points[p];
2196         const PetscInt *support;
2197         PetscInt        supportSize, s;
2198 
2199         ierr = DMPlexGetSupportSize(dm, point, &supportSize);CHKERRQ(ierr);
2200         ierr = DMPlexGetSupport(dm, point, &support);CHKERRQ(ierr);
2201         for (s = 0; s < supportSize; ++s) {
2202           ierr = DMLabelSetValue(label, support[s], newLevel);CHKERRQ(ierr);
2203         }
2204       }
2205       ierr = ISRestoreIndices(pointIS, &points);CHKERRQ(ierr);
2206       ++level;
2207       ierr = ISDestroy(&pointIS);CHKERRQ(ierr);
2208       ierr = DMLabelGetStratumIS(label, level, &pointIS);CHKERRQ(ierr);
2209       if (pointIS) {ierr = ISGetLocalSize(pointIS, &numPoints);CHKERRQ(ierr);}
2210       else         {numPoints = 0;}
2211     }
2212     ierr = ISDestroy(&pointIS);CHKERRQ(ierr);
2213   }
2214   ierr = PetscLogEventEnd(DMPLEX_Stratify,dm,0,0,0);CHKERRQ(ierr);
2215   PetscFunctionReturn(0);
2216 }
2217 
2218 #undef __FUNCT__
2219 #define __FUNCT__ "DMPlexGetJoin"
2220 /*@C
2221   DMPlexGetJoin - Get an array for the join of the set of points
2222 
2223   Not Collective
2224 
2225   Input Parameters:
2226 + dm - The DMPlex object
2227 . numPoints - The number of input points for the join
2228 - points - The input points
2229 
2230   Output Parameters:
2231 + numCoveredPoints - The number of points in the join
2232 - coveredPoints - The points in the join
2233 
2234   Level: intermediate
2235 
2236   Note: Currently, this is restricted to a single level join
2237 
2238   Fortran Notes:
2239   Since it returns an array, this routine is only available in Fortran 90, and you must
2240   include petsc.h90 in your code.
2241 
2242   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2243 
2244 .keywords: mesh
2245 .seealso: DMPlexRestoreJoin(), DMPlexGetMeet()
2246 @*/
2247 PetscErrorCode DMPlexGetJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
2248 {
2249   DM_Plex       *mesh = (DM_Plex*) dm->data;
2250   PetscInt      *join[2];
2251   PetscInt       joinSize, i = 0;
2252   PetscInt       dof, off, p, c, m;
2253   PetscErrorCode ierr;
2254 
2255   PetscFunctionBegin;
2256   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2257   PetscValidPointer(points, 2);
2258   PetscValidPointer(numCoveredPoints, 3);
2259   PetscValidPointer(coveredPoints, 4);
2260   ierr = DMGetWorkArray(dm, mesh->maxSupportSize, PETSC_INT, &join[0]);CHKERRQ(ierr);
2261   ierr = DMGetWorkArray(dm, mesh->maxSupportSize, PETSC_INT, &join[1]);CHKERRQ(ierr);
2262   /* Copy in support of first point */
2263   ierr = PetscSectionGetDof(mesh->supportSection, points[0], &dof);CHKERRQ(ierr);
2264   ierr = PetscSectionGetOffset(mesh->supportSection, points[0], &off);CHKERRQ(ierr);
2265   for (joinSize = 0; joinSize < dof; ++joinSize) {
2266     join[i][joinSize] = mesh->supports[off+joinSize];
2267   }
2268   /* Check each successive support */
2269   for (p = 1; p < numPoints; ++p) {
2270     PetscInt newJoinSize = 0;
2271 
2272     ierr = PetscSectionGetDof(mesh->supportSection, points[p], &dof);CHKERRQ(ierr);
2273     ierr = PetscSectionGetOffset(mesh->supportSection, points[p], &off);CHKERRQ(ierr);
2274     for (c = 0; c < dof; ++c) {
2275       const PetscInt point = mesh->supports[off+c];
2276 
2277       for (m = 0; m < joinSize; ++m) {
2278         if (point == join[i][m]) {
2279           join[1-i][newJoinSize++] = point;
2280           break;
2281         }
2282       }
2283     }
2284     joinSize = newJoinSize;
2285     i        = 1-i;
2286   }
2287   *numCoveredPoints = joinSize;
2288   *coveredPoints    = join[i];
2289   ierr              = DMRestoreWorkArray(dm, mesh->maxSupportSize, PETSC_INT, &join[1-i]);CHKERRQ(ierr);
2290   PetscFunctionReturn(0);
2291 }
2292 
2293 #undef __FUNCT__
2294 #define __FUNCT__ "DMPlexRestoreJoin"
2295 /*@C
2296   DMPlexRestoreJoin - Restore an array for the join of the set of points
2297 
2298   Not Collective
2299 
2300   Input Parameters:
2301 + dm - The DMPlex object
2302 . numPoints - The number of input points for the join
2303 - points - The input points
2304 
2305   Output Parameters:
2306 + numCoveredPoints - The number of points in the join
2307 - coveredPoints - The points in the join
2308 
2309   Fortran Notes:
2310   Since it returns an array, this routine is only available in Fortran 90, and you must
2311   include petsc.h90 in your code.
2312 
2313   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2314 
2315   Level: intermediate
2316 
2317 .keywords: mesh
2318 .seealso: DMPlexGetJoin(), DMPlexGetFullJoin(), DMPlexGetMeet()
2319 @*/
2320 PetscErrorCode DMPlexRestoreJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
2321 {
2322   PetscErrorCode ierr;
2323 
2324   PetscFunctionBegin;
2325   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2326   if (points) PetscValidIntPointer(points,3);
2327   if (numCoveredPoints) PetscValidIntPointer(numCoveredPoints,4);
2328   PetscValidPointer(coveredPoints, 5);
2329   ierr = DMRestoreWorkArray(dm, 0, PETSC_INT, (void*) coveredPoints);CHKERRQ(ierr);
2330   if (numCoveredPoints) *numCoveredPoints = 0;
2331   PetscFunctionReturn(0);
2332 }
2333 
2334 #undef __FUNCT__
2335 #define __FUNCT__ "DMPlexGetFullJoin"
2336 /*@C
2337   DMPlexGetFullJoin - Get an array for the join of the set of points
2338 
2339   Not Collective
2340 
2341   Input Parameters:
2342 + dm - The DMPlex object
2343 . numPoints - The number of input points for the join
2344 - points - The input points
2345 
2346   Output Parameters:
2347 + numCoveredPoints - The number of points in the join
2348 - coveredPoints - The points in the join
2349 
2350   Fortran Notes:
2351   Since it returns an array, this routine is only available in Fortran 90, and you must
2352   include petsc.h90 in your code.
2353 
2354   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2355 
2356   Level: intermediate
2357 
2358 .keywords: mesh
2359 .seealso: DMPlexGetJoin(), DMPlexRestoreJoin(), DMPlexGetMeet()
2360 @*/
2361 PetscErrorCode DMPlexGetFullJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
2362 {
2363   DM_Plex       *mesh = (DM_Plex*) dm->data;
2364   PetscInt      *offsets, **closures;
2365   PetscInt      *join[2];
2366   PetscInt       depth = 0, maxSize, joinSize = 0, i = 0;
2367   PetscInt       p, d, c, m;
2368   PetscErrorCode ierr;
2369 
2370   PetscFunctionBegin;
2371   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2372   PetscValidPointer(points, 2);
2373   PetscValidPointer(numCoveredPoints, 3);
2374   PetscValidPointer(coveredPoints, 4);
2375 
2376   ierr    = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
2377   ierr    = PetscCalloc1(numPoints, &closures);CHKERRQ(ierr);
2378   ierr    = DMGetWorkArray(dm, numPoints*(depth+2), PETSC_INT, &offsets);CHKERRQ(ierr);
2379   maxSize = PetscPowInt(mesh->maxSupportSize,depth+1);
2380   ierr    = DMGetWorkArray(dm, maxSize, PETSC_INT, &join[0]);CHKERRQ(ierr);
2381   ierr    = DMGetWorkArray(dm, maxSize, PETSC_INT, &join[1]);CHKERRQ(ierr);
2382 
2383   for (p = 0; p < numPoints; ++p) {
2384     PetscInt closureSize;
2385 
2386     ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_FALSE, &closureSize, &closures[p]);CHKERRQ(ierr);
2387 
2388     offsets[p*(depth+2)+0] = 0;
2389     for (d = 0; d < depth+1; ++d) {
2390       PetscInt pStart, pEnd, i;
2391 
2392       ierr = DMPlexGetDepthStratum(dm, d, &pStart, &pEnd);CHKERRQ(ierr);
2393       for (i = offsets[p*(depth+2)+d]; i < closureSize; ++i) {
2394         if ((pStart > closures[p][i*2]) || (pEnd <= closures[p][i*2])) {
2395           offsets[p*(depth+2)+d+1] = i;
2396           break;
2397         }
2398       }
2399       if (i == closureSize) offsets[p*(depth+2)+d+1] = i;
2400     }
2401     if (offsets[p*(depth+2)+depth+1] != closureSize) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Total size of closure %D should be %D", offsets[p*(depth+2)+depth+1], closureSize);
2402   }
2403   for (d = 0; d < depth+1; ++d) {
2404     PetscInt dof;
2405 
2406     /* Copy in support of first point */
2407     dof = offsets[d+1] - offsets[d];
2408     for (joinSize = 0; joinSize < dof; ++joinSize) {
2409       join[i][joinSize] = closures[0][(offsets[d]+joinSize)*2];
2410     }
2411     /* Check each successive cone */
2412     for (p = 1; p < numPoints && joinSize; ++p) {
2413       PetscInt newJoinSize = 0;
2414 
2415       dof = offsets[p*(depth+2)+d+1] - offsets[p*(depth+2)+d];
2416       for (c = 0; c < dof; ++c) {
2417         const PetscInt point = closures[p][(offsets[p*(depth+2)+d]+c)*2];
2418 
2419         for (m = 0; m < joinSize; ++m) {
2420           if (point == join[i][m]) {
2421             join[1-i][newJoinSize++] = point;
2422             break;
2423           }
2424         }
2425       }
2426       joinSize = newJoinSize;
2427       i        = 1-i;
2428     }
2429     if (joinSize) break;
2430   }
2431   *numCoveredPoints = joinSize;
2432   *coveredPoints    = join[i];
2433   for (p = 0; p < numPoints; ++p) {
2434     ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_FALSE, NULL, &closures[p]);CHKERRQ(ierr);
2435   }
2436   ierr = PetscFree(closures);CHKERRQ(ierr);
2437   ierr = DMRestoreWorkArray(dm, numPoints*(depth+2), PETSC_INT, &offsets);CHKERRQ(ierr);
2438   ierr = DMRestoreWorkArray(dm, mesh->maxSupportSize, PETSC_INT, &join[1-i]);CHKERRQ(ierr);
2439   PetscFunctionReturn(0);
2440 }
2441 
2442 #undef __FUNCT__
2443 #define __FUNCT__ "DMPlexGetMeet"
2444 /*@C
2445   DMPlexGetMeet - Get an array for the meet of the set of points
2446 
2447   Not Collective
2448 
2449   Input Parameters:
2450 + dm - The DMPlex object
2451 . numPoints - The number of input points for the meet
2452 - points - The input points
2453 
2454   Output Parameters:
2455 + numCoveredPoints - The number of points in the meet
2456 - coveredPoints - The points in the meet
2457 
2458   Level: intermediate
2459 
2460   Note: Currently, this is restricted to a single level meet
2461 
2462   Fortran Notes:
2463   Since it returns an array, this routine is only available in Fortran 90, and you must
2464   include petsc.h90 in your code.
2465 
2466   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2467 
2468 .keywords: mesh
2469 .seealso: DMPlexRestoreMeet(), DMPlexGetJoin()
2470 @*/
2471 PetscErrorCode DMPlexGetMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveringPoints, const PetscInt **coveringPoints)
2472 {
2473   DM_Plex       *mesh = (DM_Plex*) dm->data;
2474   PetscInt      *meet[2];
2475   PetscInt       meetSize, i = 0;
2476   PetscInt       dof, off, p, c, m;
2477   PetscErrorCode ierr;
2478 
2479   PetscFunctionBegin;
2480   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2481   PetscValidPointer(points, 2);
2482   PetscValidPointer(numCoveringPoints, 3);
2483   PetscValidPointer(coveringPoints, 4);
2484   ierr = DMGetWorkArray(dm, mesh->maxConeSize, PETSC_INT, &meet[0]);CHKERRQ(ierr);
2485   ierr = DMGetWorkArray(dm, mesh->maxConeSize, PETSC_INT, &meet[1]);CHKERRQ(ierr);
2486   /* Copy in cone of first point */
2487   ierr = PetscSectionGetDof(mesh->coneSection, points[0], &dof);CHKERRQ(ierr);
2488   ierr = PetscSectionGetOffset(mesh->coneSection, points[0], &off);CHKERRQ(ierr);
2489   for (meetSize = 0; meetSize < dof; ++meetSize) {
2490     meet[i][meetSize] = mesh->cones[off+meetSize];
2491   }
2492   /* Check each successive cone */
2493   for (p = 1; p < numPoints; ++p) {
2494     PetscInt newMeetSize = 0;
2495 
2496     ierr = PetscSectionGetDof(mesh->coneSection, points[p], &dof);CHKERRQ(ierr);
2497     ierr = PetscSectionGetOffset(mesh->coneSection, points[p], &off);CHKERRQ(ierr);
2498     for (c = 0; c < dof; ++c) {
2499       const PetscInt point = mesh->cones[off+c];
2500 
2501       for (m = 0; m < meetSize; ++m) {
2502         if (point == meet[i][m]) {
2503           meet[1-i][newMeetSize++] = point;
2504           break;
2505         }
2506       }
2507     }
2508     meetSize = newMeetSize;
2509     i        = 1-i;
2510   }
2511   *numCoveringPoints = meetSize;
2512   *coveringPoints    = meet[i];
2513   ierr               = DMRestoreWorkArray(dm, mesh->maxConeSize, PETSC_INT, &meet[1-i]);CHKERRQ(ierr);
2514   PetscFunctionReturn(0);
2515 }
2516 
2517 #undef __FUNCT__
2518 #define __FUNCT__ "DMPlexRestoreMeet"
2519 /*@C
2520   DMPlexRestoreMeet - Restore an array for the meet of the set of points
2521 
2522   Not Collective
2523 
2524   Input Parameters:
2525 + dm - The DMPlex object
2526 . numPoints - The number of input points for the meet
2527 - points - The input points
2528 
2529   Output Parameters:
2530 + numCoveredPoints - The number of points in the meet
2531 - coveredPoints - The points in the meet
2532 
2533   Level: intermediate
2534 
2535   Fortran Notes:
2536   Since it returns an array, this routine is only available in Fortran 90, and you must
2537   include petsc.h90 in your code.
2538 
2539   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2540 
2541 .keywords: mesh
2542 .seealso: DMPlexGetMeet(), DMPlexGetFullMeet(), DMPlexGetJoin()
2543 @*/
2544 PetscErrorCode DMPlexRestoreMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
2545 {
2546   PetscErrorCode ierr;
2547 
2548   PetscFunctionBegin;
2549   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2550   if (points) PetscValidIntPointer(points,3);
2551   if (numCoveredPoints) PetscValidIntPointer(numCoveredPoints,4);
2552   PetscValidPointer(coveredPoints,5);
2553   ierr = DMRestoreWorkArray(dm, 0, PETSC_INT, (void*) coveredPoints);CHKERRQ(ierr);
2554   if (numCoveredPoints) *numCoveredPoints = 0;
2555   PetscFunctionReturn(0);
2556 }
2557 
2558 #undef __FUNCT__
2559 #define __FUNCT__ "DMPlexGetFullMeet"
2560 /*@C
2561   DMPlexGetFullMeet - Get an array for the meet of the set of points
2562 
2563   Not Collective
2564 
2565   Input Parameters:
2566 + dm - The DMPlex object
2567 . numPoints - The number of input points for the meet
2568 - points - The input points
2569 
2570   Output Parameters:
2571 + numCoveredPoints - The number of points in the meet
2572 - coveredPoints - The points in the meet
2573 
2574   Level: intermediate
2575 
2576   Fortran Notes:
2577   Since it returns an array, this routine is only available in Fortran 90, and you must
2578   include petsc.h90 in your code.
2579 
2580   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2581 
2582 .keywords: mesh
2583 .seealso: DMPlexGetMeet(), DMPlexRestoreMeet(), DMPlexGetJoin()
2584 @*/
2585 PetscErrorCode DMPlexGetFullMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
2586 {
2587   DM_Plex       *mesh = (DM_Plex*) dm->data;
2588   PetscInt      *offsets, **closures;
2589   PetscInt      *meet[2];
2590   PetscInt       height = 0, maxSize, meetSize = 0, i = 0;
2591   PetscInt       p, h, c, m;
2592   PetscErrorCode ierr;
2593 
2594   PetscFunctionBegin;
2595   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2596   PetscValidPointer(points, 2);
2597   PetscValidPointer(numCoveredPoints, 3);
2598   PetscValidPointer(coveredPoints, 4);
2599 
2600   ierr    = DMPlexGetDepth(dm, &height);CHKERRQ(ierr);
2601   ierr    = PetscMalloc1(numPoints, &closures);CHKERRQ(ierr);
2602   ierr    = DMGetWorkArray(dm, numPoints*(height+2), PETSC_INT, &offsets);CHKERRQ(ierr);
2603   maxSize = PetscPowInt(mesh->maxConeSize,height+1);
2604   ierr    = DMGetWorkArray(dm, maxSize, PETSC_INT, &meet[0]);CHKERRQ(ierr);
2605   ierr    = DMGetWorkArray(dm, maxSize, PETSC_INT, &meet[1]);CHKERRQ(ierr);
2606 
2607   for (p = 0; p < numPoints; ++p) {
2608     PetscInt closureSize;
2609 
2610     ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_TRUE, &closureSize, &closures[p]);CHKERRQ(ierr);
2611 
2612     offsets[p*(height+2)+0] = 0;
2613     for (h = 0; h < height+1; ++h) {
2614       PetscInt pStart, pEnd, i;
2615 
2616       ierr = DMPlexGetHeightStratum(dm, h, &pStart, &pEnd);CHKERRQ(ierr);
2617       for (i = offsets[p*(height+2)+h]; i < closureSize; ++i) {
2618         if ((pStart > closures[p][i*2]) || (pEnd <= closures[p][i*2])) {
2619           offsets[p*(height+2)+h+1] = i;
2620           break;
2621         }
2622       }
2623       if (i == closureSize) offsets[p*(height+2)+h+1] = i;
2624     }
2625     if (offsets[p*(height+2)+height+1] != closureSize) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Total size of closure %D should be %D", offsets[p*(height+2)+height+1], closureSize);
2626   }
2627   for (h = 0; h < height+1; ++h) {
2628     PetscInt dof;
2629 
2630     /* Copy in cone of first point */
2631     dof = offsets[h+1] - offsets[h];
2632     for (meetSize = 0; meetSize < dof; ++meetSize) {
2633       meet[i][meetSize] = closures[0][(offsets[h]+meetSize)*2];
2634     }
2635     /* Check each successive cone */
2636     for (p = 1; p < numPoints && meetSize; ++p) {
2637       PetscInt newMeetSize = 0;
2638 
2639       dof = offsets[p*(height+2)+h+1] - offsets[p*(height+2)+h];
2640       for (c = 0; c < dof; ++c) {
2641         const PetscInt point = closures[p][(offsets[p*(height+2)+h]+c)*2];
2642 
2643         for (m = 0; m < meetSize; ++m) {
2644           if (point == meet[i][m]) {
2645             meet[1-i][newMeetSize++] = point;
2646             break;
2647           }
2648         }
2649       }
2650       meetSize = newMeetSize;
2651       i        = 1-i;
2652     }
2653     if (meetSize) break;
2654   }
2655   *numCoveredPoints = meetSize;
2656   *coveredPoints    = meet[i];
2657   for (p = 0; p < numPoints; ++p) {
2658     ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_TRUE, NULL, &closures[p]);CHKERRQ(ierr);
2659   }
2660   ierr = PetscFree(closures);CHKERRQ(ierr);
2661   ierr = DMRestoreWorkArray(dm, numPoints*(height+2), PETSC_INT, &offsets);CHKERRQ(ierr);
2662   ierr = DMRestoreWorkArray(dm, mesh->maxConeSize, PETSC_INT, &meet[1-i]);CHKERRQ(ierr);
2663   PetscFunctionReturn(0);
2664 }
2665 
2666 #undef __FUNCT__
2667 #define __FUNCT__ "DMPlexEqual"
2668 /*@C
2669   DMPlexEqual - Determine if two DMs have the same topology
2670 
2671   Not Collective
2672 
2673   Input Parameters:
2674 + dmA - A DMPlex object
2675 - dmB - A DMPlex object
2676 
2677   Output Parameters:
2678 . equal - PETSC_TRUE if the topologies are identical
2679 
2680   Level: intermediate
2681 
2682   Notes:
2683   We are not solving graph isomorphism, so we do not permutation.
2684 
2685 .keywords: mesh
2686 .seealso: DMPlexGetCone()
2687 @*/
2688 PetscErrorCode DMPlexEqual(DM dmA, DM dmB, PetscBool *equal)
2689 {
2690   PetscInt       depth, depthB, pStart, pEnd, pStartB, pEndB, p;
2691   PetscErrorCode ierr;
2692 
2693   PetscFunctionBegin;
2694   PetscValidHeaderSpecific(dmA, DM_CLASSID, 1);
2695   PetscValidHeaderSpecific(dmB, DM_CLASSID, 2);
2696   PetscValidPointer(equal, 3);
2697 
2698   *equal = PETSC_FALSE;
2699   ierr = DMPlexGetDepth(dmA, &depth);CHKERRQ(ierr);
2700   ierr = DMPlexGetDepth(dmB, &depthB);CHKERRQ(ierr);
2701   if (depth != depthB) PetscFunctionReturn(0);
2702   ierr = DMPlexGetChart(dmA, &pStart,  &pEnd);CHKERRQ(ierr);
2703   ierr = DMPlexGetChart(dmB, &pStartB, &pEndB);CHKERRQ(ierr);
2704   if ((pStart != pStartB) || (pEnd != pEndB)) PetscFunctionReturn(0);
2705   for (p = pStart; p < pEnd; ++p) {
2706     const PetscInt *cone, *coneB, *ornt, *orntB, *support, *supportB;
2707     PetscInt        coneSize, coneSizeB, c, supportSize, supportSizeB, s;
2708 
2709     ierr = DMPlexGetConeSize(dmA, p, &coneSize);CHKERRQ(ierr);
2710     ierr = DMPlexGetCone(dmA, p, &cone);CHKERRQ(ierr);
2711     ierr = DMPlexGetConeOrientation(dmA, p, &ornt);CHKERRQ(ierr);
2712     ierr = DMPlexGetConeSize(dmB, p, &coneSizeB);CHKERRQ(ierr);
2713     ierr = DMPlexGetCone(dmB, p, &coneB);CHKERRQ(ierr);
2714     ierr = DMPlexGetConeOrientation(dmB, p, &orntB);CHKERRQ(ierr);
2715     if (coneSize != coneSizeB) PetscFunctionReturn(0);
2716     for (c = 0; c < coneSize; ++c) {
2717       if (cone[c] != coneB[c]) PetscFunctionReturn(0);
2718       if (ornt[c] != orntB[c]) PetscFunctionReturn(0);
2719     }
2720     ierr = DMPlexGetSupportSize(dmA, p, &supportSize);CHKERRQ(ierr);
2721     ierr = DMPlexGetSupport(dmA, p, &support);CHKERRQ(ierr);
2722     ierr = DMPlexGetSupportSize(dmB, p, &supportSizeB);CHKERRQ(ierr);
2723     ierr = DMPlexGetSupport(dmB, p, &supportB);CHKERRQ(ierr);
2724     if (supportSize != supportSizeB) PetscFunctionReturn(0);
2725     for (s = 0; s < supportSize; ++s) {
2726       if (support[s] != supportB[s]) PetscFunctionReturn(0);
2727     }
2728   }
2729   *equal = PETSC_TRUE;
2730   PetscFunctionReturn(0);
2731 }
2732 
2733 #undef __FUNCT__
2734 #define __FUNCT__ "DMPlexGetNumFaceVertices"
2735 PetscErrorCode DMPlexGetNumFaceVertices(DM dm, PetscInt cellDim, PetscInt numCorners, PetscInt *numFaceVertices)
2736 {
2737   MPI_Comm       comm;
2738   PetscErrorCode ierr;
2739 
2740   PetscFunctionBegin;
2741   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
2742   PetscValidPointer(numFaceVertices,3);
2743   switch (cellDim) {
2744   case 0:
2745     *numFaceVertices = 0;
2746     break;
2747   case 1:
2748     *numFaceVertices = 1;
2749     break;
2750   case 2:
2751     switch (numCorners) {
2752     case 3: /* triangle */
2753       *numFaceVertices = 2; /* Edge has 2 vertices */
2754       break;
2755     case 4: /* quadrilateral */
2756       *numFaceVertices = 2; /* Edge has 2 vertices */
2757       break;
2758     case 6: /* quadratic triangle, tri and quad cohesive Lagrange cells */
2759       *numFaceVertices = 3; /* Edge has 3 vertices */
2760       break;
2761     case 9: /* quadratic quadrilateral, quadratic quad cohesive Lagrange cells */
2762       *numFaceVertices = 3; /* Edge has 3 vertices */
2763       break;
2764     default:
2765       SETERRQ2(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid number of face corners %d for dimension %d", numCorners, cellDim);
2766     }
2767     break;
2768   case 3:
2769     switch (numCorners) {
2770     case 4: /* tetradehdron */
2771       *numFaceVertices = 3; /* Face has 3 vertices */
2772       break;
2773     case 6: /* tet cohesive cells */
2774       *numFaceVertices = 4; /* Face has 4 vertices */
2775       break;
2776     case 8: /* hexahedron */
2777       *numFaceVertices = 4; /* Face has 4 vertices */
2778       break;
2779     case 9: /* tet cohesive Lagrange cells */
2780       *numFaceVertices = 6; /* Face has 6 vertices */
2781       break;
2782     case 10: /* quadratic tetrahedron */
2783       *numFaceVertices = 6; /* Face has 6 vertices */
2784       break;
2785     case 12: /* hex cohesive Lagrange cells */
2786       *numFaceVertices = 6; /* Face has 6 vertices */
2787       break;
2788     case 18: /* quadratic tet cohesive Lagrange cells */
2789       *numFaceVertices = 6; /* Face has 6 vertices */
2790       break;
2791     case 27: /* quadratic hexahedron, quadratic hex cohesive Lagrange cells */
2792       *numFaceVertices = 9; /* Face has 9 vertices */
2793       break;
2794     default:
2795       SETERRQ2(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid number of face corners %d for dimension %d", numCorners, cellDim);
2796     }
2797     break;
2798   default:
2799     SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid cell dimension %d", cellDim);
2800   }
2801   PetscFunctionReturn(0);
2802 }
2803 
2804 #undef __FUNCT__
2805 #define __FUNCT__ "DMPlexOrient"
2806 /* Trys to give the mesh a consistent orientation */
2807 PetscErrorCode DMPlexOrient(DM dm)
2808 {
2809   PetscBT        seenCells, flippedCells, seenFaces;
2810   PetscInt      *faceFIFO, fTop, fBottom;
2811   PetscInt       dim, h, cStart, cEnd, c, fStart, fEnd, face, maxConeSize, *revcone, *revconeO;
2812   PetscErrorCode ierr;
2813 
2814   PetscFunctionBegin;
2815   /* Truth Table
2816      mismatch    flips   do action   mismatch   flipA ^ flipB   action
2817          F       0 flips     no         F             F           F
2818          F       1 flip      yes        F             T           T
2819          F       2 flips     no         T             F           T
2820          T       0 flips     yes        T             T           F
2821          T       1 flip      no
2822          T       2 flips     yes
2823   */
2824   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
2825   ierr = DMPlexGetVTKCellHeight(dm, &h);CHKERRQ(ierr);
2826   ierr = DMPlexGetHeightStratum(dm, h,   &cStart, &cEnd);CHKERRQ(ierr);
2827   ierr = DMPlexGetHeightStratum(dm, h+1, &fStart, &fEnd);CHKERRQ(ierr);
2828   ierr = PetscBTCreate(cEnd - cStart, &seenCells);CHKERRQ(ierr);
2829   ierr = PetscBTMemzero(cEnd - cStart, seenCells);CHKERRQ(ierr);
2830   ierr = PetscBTCreate(cEnd - cStart, &flippedCells);CHKERRQ(ierr);
2831   ierr = PetscBTMemzero(cEnd - cStart, flippedCells);CHKERRQ(ierr);
2832   ierr = PetscBTCreate(fEnd - fStart, &seenFaces);CHKERRQ(ierr);
2833   ierr = PetscBTMemzero(fEnd - fStart, seenFaces);CHKERRQ(ierr);
2834   ierr = PetscMalloc1((fEnd - fStart), &faceFIFO);CHKERRQ(ierr);
2835   fTop = fBottom = 0;
2836   /* Initialize FIFO with first cell */
2837   if (cEnd > cStart) {
2838     const PetscInt *cone;
2839     PetscInt        coneSize;
2840 
2841     ierr = DMPlexGetConeSize(dm, cStart, &coneSize);CHKERRQ(ierr);
2842     ierr = DMPlexGetCone(dm, cStart, &cone);CHKERRQ(ierr);
2843     for (c = 0; c < coneSize; ++c) {
2844       faceFIFO[fBottom++] = cone[c];
2845       ierr = PetscBTSet(seenFaces, cone[c]-fStart);CHKERRQ(ierr);
2846     }
2847   }
2848   /* Consider each face in FIFO */
2849   while (fTop < fBottom) {
2850     const PetscInt *support, *coneA, *coneB, *coneOA, *coneOB;
2851     PetscInt        supportSize, coneSizeA, coneSizeB, posA = -1, posB = -1;
2852     PetscInt        seenA, flippedA, seenB, flippedB, mismatch;
2853 
2854     face = faceFIFO[fTop++];
2855     ierr = DMPlexGetSupportSize(dm, face, &supportSize);CHKERRQ(ierr);
2856     ierr = DMPlexGetSupport(dm, face, &support);CHKERRQ(ierr);
2857     if (supportSize < 2) continue;
2858     if (supportSize != 2) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Faces should separate only two cells, not %d", supportSize);
2859     seenA    = PetscBTLookup(seenCells,    support[0]-cStart);
2860     flippedA = PetscBTLookup(flippedCells, support[0]-cStart) ? 1 : 0;
2861     seenB    = PetscBTLookup(seenCells,    support[1]-cStart);
2862     flippedB = PetscBTLookup(flippedCells, support[1]-cStart) ? 1 : 0;
2863 
2864     ierr = DMPlexGetConeSize(dm, support[0], &coneSizeA);CHKERRQ(ierr);
2865     ierr = DMPlexGetConeSize(dm, support[1], &coneSizeB);CHKERRQ(ierr);
2866     ierr = DMPlexGetCone(dm, support[0], &coneA);CHKERRQ(ierr);
2867     ierr = DMPlexGetCone(dm, support[1], &coneB);CHKERRQ(ierr);
2868     ierr = DMPlexGetConeOrientation(dm, support[0], &coneOA);CHKERRQ(ierr);
2869     ierr = DMPlexGetConeOrientation(dm, support[1], &coneOB);CHKERRQ(ierr);
2870     for (c = 0; c < coneSizeA; ++c) {
2871       if (!PetscBTLookup(seenFaces, coneA[c]-fStart)) {
2872         faceFIFO[fBottom++] = coneA[c];
2873         ierr = PetscBTSet(seenFaces, coneA[c]-fStart);CHKERRQ(ierr);
2874       }
2875       if (coneA[c] == face) posA = c;
2876       if (fBottom > fEnd-fStart) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Face %d was pushed exceeding capacity %d > %d", coneA[c], fBottom, fEnd-fStart);
2877     }
2878     if (posA < 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %d could not be located in cell %d", face, support[0]);
2879     for (c = 0; c < coneSizeB; ++c) {
2880       if (!PetscBTLookup(seenFaces, coneB[c]-fStart)) {
2881         faceFIFO[fBottom++] = coneB[c];
2882         ierr = PetscBTSet(seenFaces, coneB[c]-fStart);CHKERRQ(ierr);
2883       }
2884       if (coneB[c] == face) posB = c;
2885       if (fBottom > fEnd-fStart) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Face %d was pushed exceeding capacity %d > %d", coneA[c], fBottom, fEnd-fStart);
2886     }
2887     if (posB < 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %d could not be located in cell %d", face, support[1]);
2888 
2889     if (dim == 1) {
2890       mismatch = posA == posB;
2891     } else {
2892       mismatch = coneOA[posA] == coneOB[posB];
2893     }
2894 
2895     if (mismatch ^ (flippedA ^ flippedB)) {
2896       if (seenA && seenB) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Previously seen cells %d and %d do not match: Fault mesh is non-orientable", support[0], support[1]);
2897       if (!seenA && !flippedA) {
2898         ierr = PetscBTSet(flippedCells, support[0]-cStart);CHKERRQ(ierr);
2899       } else if (!seenB && !flippedB) {
2900         ierr = PetscBTSet(flippedCells, support[1]-cStart);CHKERRQ(ierr);
2901       } else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Inconsistent mesh orientation: Fault mesh is non-orientable");
2902     } else if (mismatch && flippedA && flippedB) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Attempt to flip already flipped cell: Fault mesh is non-orientable");
2903     ierr = PetscBTSet(seenCells, support[0]-cStart);CHKERRQ(ierr);
2904     ierr = PetscBTSet(seenCells, support[1]-cStart);CHKERRQ(ierr);
2905   }
2906   /* Now all subdomains are oriented, but we need a consistent parallel orientation */
2907   {
2908     /* Find a representative face (edge) separating pairs of procs */
2909     PetscSF            sf;
2910     const PetscInt    *lpoints;
2911     const PetscSFNode *rpoints;
2912     PetscInt          *neighbors, *nranks;
2913     PetscInt           numLeaves, numRoots, numNeighbors = 0, l, n;
2914 
2915     ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr);
2916     ierr = PetscSFGetGraph(sf, &numRoots, &numLeaves, &lpoints, &rpoints);CHKERRQ(ierr);
2917     if (numLeaves >= 0) {
2918       const PetscInt *cone, *ornt, *support;
2919       PetscInt        coneSize, supportSize;
2920       int            *rornt, *lornt; /* PetscSF cannot handle smaller than int */
2921       PetscBool      *match, flipped = PETSC_FALSE;
2922 
2923       ierr = PetscMalloc1(numLeaves,&neighbors);CHKERRQ(ierr);
2924       /* I know this is p^2 time in general, but for bounded degree its alright */
2925       for (l = 0; l < numLeaves; ++l) {
2926         const PetscInt face = lpoints[l];
2927         if ((face >= fStart) && (face < fEnd)) {
2928           const PetscInt rank = rpoints[l].rank;
2929           for (n = 0; n < numNeighbors; ++n) if (rank == rpoints[neighbors[n]].rank) break;
2930           if (n >= numNeighbors) {
2931             PetscInt supportSize;
2932             ierr = DMPlexGetSupportSize(dm, face, &supportSize);CHKERRQ(ierr);
2933             if (supportSize != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Boundary faces should see one cell, not %d", supportSize);
2934             neighbors[numNeighbors++] = l;
2935           }
2936         }
2937       }
2938       ierr = PetscCalloc4(numNeighbors,&match,numNeighbors,&nranks,numRoots,&rornt,numRoots,&lornt);CHKERRQ(ierr);
2939       for (face = fStart; face < fEnd; ++face) {
2940         ierr = DMPlexGetSupportSize(dm, face, &supportSize);CHKERRQ(ierr);
2941         if (supportSize != 1) continue;
2942         ierr = DMPlexGetSupport(dm, face, &support);CHKERRQ(ierr);
2943 
2944         ierr = DMPlexGetCone(dm, support[0], &cone);CHKERRQ(ierr);
2945         ierr = DMPlexGetConeSize(dm, support[0], &coneSize);CHKERRQ(ierr);
2946         ierr = DMPlexGetConeOrientation(dm, support[0], &ornt);CHKERRQ(ierr);
2947         for (c = 0; c < coneSize; ++c) if (cone[c] == face) break;
2948         if (dim == 1) {
2949           /* Use cone position instead, shifted to -1 or 1 */
2950           rornt[face] = c*2-1;
2951         } else {
2952           if (PetscBTLookup(flippedCells, support[0]-cStart)) rornt[face] = ornt[c] < 0 ? -1 :  1;
2953           else                                                rornt[face] = ornt[c] < 0 ?  1 : -1;
2954         }
2955       }
2956       /* Mark each edge with match or nomatch */
2957       ierr = PetscSFBcastBegin(sf, MPI_INT, rornt, lornt);CHKERRQ(ierr);
2958       ierr = PetscSFBcastEnd(sf, MPI_INT, rornt, lornt);CHKERRQ(ierr);
2959       for (n = 0; n < numNeighbors; ++n) {
2960         const PetscInt face = lpoints[neighbors[n]];
2961 
2962         if (rornt[face]*lornt[face] < 0) match[n] = PETSC_TRUE;
2963         else                             match[n] = PETSC_FALSE;
2964         nranks[n] = rpoints[neighbors[n]].rank;
2965       }
2966       /* Collect the graph on 0 */
2967       {
2968         MPI_Comm     comm = PetscObjectComm((PetscObject) sf);
2969         PetscBT      seenProcs, flippedProcs;
2970         PetscInt    *procFIFO, pTop, pBottom;
2971         PetscInt    *adj = NULL;
2972         PetscBool   *val = NULL;
2973         PetscMPIInt *recvcounts = NULL, *displs = NULL, p;
2974         PetscMPIInt  N = numNeighbors, numProcs = 0, rank;
2975         PetscInt     debug = 0;
2976 
2977         ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
2978         if (!rank) {ierr = MPI_Comm_size(comm, &numProcs);CHKERRQ(ierr);}
2979         ierr = PetscCalloc2(numProcs,&recvcounts,numProcs+1,&displs);CHKERRQ(ierr);
2980         ierr = MPI_Gather(&N, 1, MPI_INT, recvcounts, 1, MPI_INT, 0, comm);CHKERRQ(ierr);
2981         for (p = 0; p < numProcs; ++p) {
2982           displs[p+1] = displs[p] + recvcounts[p];
2983         }
2984         if (!rank) {ierr = PetscMalloc2(displs[numProcs],&adj,displs[numProcs],&val);CHKERRQ(ierr);}
2985         ierr = MPI_Gatherv(nranks, numNeighbors, MPIU_INT, adj, recvcounts, displs, MPIU_INT, 0, comm);CHKERRQ(ierr);
2986         ierr = MPI_Gatherv(match, numNeighbors, MPIU_BOOL, val, recvcounts, displs, MPIU_BOOL, 0, comm);CHKERRQ(ierr);
2987         if (debug) {
2988           for (p = 0; p < numProcs; ++p) {
2989             ierr = PetscPrintf(comm, "Proc %d:\n", p);
2990             for (n = 0; n < recvcounts[p]; ++n) {
2991               ierr = PetscPrintf(comm, "  edge %d (%d):\n", adj[displs[p]+n], val[displs[p]+n]);
2992             }
2993           }
2994         }
2995         ierr = PetscBTCreate(numProcs, &seenProcs);CHKERRQ(ierr);
2996         ierr = PetscBTMemzero(numProcs, seenProcs);CHKERRQ(ierr);
2997         ierr = PetscBTCreate(numProcs, &flippedProcs);CHKERRQ(ierr);
2998         ierr = PetscBTMemzero(numProcs, flippedProcs);CHKERRQ(ierr);
2999         ierr = PetscMalloc1(numProcs,&procFIFO);CHKERRQ(ierr);
3000         pTop = pBottom = 0;
3001         for (p = 0; p < numProcs; ++p) {
3002           if (PetscBTLookup(seenProcs, p)) continue;
3003           /* Initialize FIFO with next proc */
3004           procFIFO[pBottom++] = p;
3005           ierr = PetscBTSet(seenProcs, p);CHKERRQ(ierr);
3006           /* Consider each proc in FIFO */
3007           while (pTop < pBottom) {
3008             PetscInt proc, nproc, seen, flippedA, flippedB, mismatch;
3009 
3010             proc     = procFIFO[pTop++];
3011             flippedA = PetscBTLookup(flippedProcs, proc) ? 1 : 0;
3012             /* Loop over neighboring procs */
3013             for (n = 0; n < recvcounts[proc]; ++n) {
3014               nproc    = adj[displs[proc]+n];
3015               mismatch = val[displs[proc]+n] ? 0 : 1;
3016               seen     = PetscBTLookup(seenProcs, nproc);
3017               flippedB = PetscBTLookup(flippedProcs, nproc) ? 1 : 0;
3018 
3019               if (mismatch ^ (flippedA ^ flippedB)) {
3020                 if (seen) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Previously seen procs %d and %d do not match: Fault mesh is non-orientable", proc, nproc);
3021                 if (!flippedB) {
3022                   ierr = PetscBTSet(flippedProcs, nproc);CHKERRQ(ierr);
3023               } else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Inconsistent mesh orientation: Fault mesh is non-orientable");
3024               } else if (mismatch && flippedA && flippedB) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Attempt to flip already flipped cell: Fault mesh is non-orientable");
3025               if (!seen) {
3026                 procFIFO[pBottom++] = nproc;
3027                 ierr = PetscBTSet(seenProcs, nproc);CHKERRQ(ierr);
3028               }
3029             }
3030           }
3031         }
3032         ierr = PetscFree(procFIFO);CHKERRQ(ierr);
3033 
3034         ierr = PetscFree2(recvcounts,displs);CHKERRQ(ierr);
3035         ierr = PetscFree2(adj,val);CHKERRQ(ierr);
3036         {
3037           PetscBool *flips;
3038 
3039           ierr = PetscMalloc1(numProcs,&flips);CHKERRQ(ierr);
3040           for (p = 0; p < numProcs; ++p) {
3041             flips[p] = PetscBTLookup(flippedProcs, p) ? PETSC_TRUE : PETSC_FALSE;
3042             if (debug && flips[p]) {ierr = PetscPrintf(comm, "Flipping Proc %d:\n", p);}
3043           }
3044           ierr = MPI_Scatter(flips, 1, MPIU_BOOL, &flipped, 1, MPIU_BOOL, 0, comm);CHKERRQ(ierr);
3045           ierr = PetscFree(flips);CHKERRQ(ierr);
3046         }
3047         ierr = PetscBTDestroy(&seenProcs);CHKERRQ(ierr);
3048         ierr = PetscBTDestroy(&flippedProcs);CHKERRQ(ierr);
3049       }
3050       ierr = PetscFree4(match,nranks,rornt,lornt);CHKERRQ(ierr);
3051       ierr = PetscFree(neighbors);CHKERRQ(ierr);
3052       if (flipped) {for (c = cStart; c < cEnd; ++c) {ierr = PetscBTNegate(flippedCells, c-cStart);CHKERRQ(ierr);}}
3053     }
3054   }
3055   /* Reverse flipped cells in the mesh */
3056   ierr = DMPlexGetMaxSizes(dm, &maxConeSize, NULL);CHKERRQ(ierr);
3057   ierr = DMGetWorkArray(dm, maxConeSize, PETSC_INT, &revcone);CHKERRQ(ierr);
3058   ierr = DMGetWorkArray(dm, maxConeSize, PETSC_INT, &revconeO);CHKERRQ(ierr);
3059   for (c = cStart; c < cEnd; ++c) {
3060     const PetscInt *cone, *coneO, *support;
3061     PetscInt        coneSize, supportSize, faceSize, cp, sp;
3062 
3063     if (!PetscBTLookup(flippedCells, c-cStart)) continue;
3064     ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
3065     ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
3066     ierr = DMPlexGetConeOrientation(dm, c, &coneO);CHKERRQ(ierr);
3067     for (cp = 0; cp < coneSize; ++cp) {
3068       const PetscInt rcp = coneSize-cp-1;
3069 
3070       ierr = DMPlexGetConeSize(dm, cone[rcp], &faceSize);CHKERRQ(ierr);
3071       revcone[cp]  = cone[rcp];
3072       revconeO[cp] = coneO[rcp] >= 0 ? -(faceSize-coneO[rcp]) : faceSize+coneO[rcp];
3073     }
3074     ierr = DMPlexSetCone(dm, c, revcone);CHKERRQ(ierr);
3075     ierr = DMPlexSetConeOrientation(dm, c, revconeO);CHKERRQ(ierr);
3076     /* Reverse orientations of support */
3077     faceSize = coneSize;
3078     ierr = DMPlexGetSupportSize(dm, c, &supportSize);CHKERRQ(ierr);
3079     ierr = DMPlexGetSupport(dm, c, &support);CHKERRQ(ierr);
3080     for (sp = 0; sp < supportSize; ++sp) {
3081       ierr = DMPlexGetConeSize(dm, support[sp], &coneSize);CHKERRQ(ierr);
3082       ierr = DMPlexGetCone(dm, support[sp], &cone);CHKERRQ(ierr);
3083       ierr = DMPlexGetConeOrientation(dm, support[sp], &coneO);CHKERRQ(ierr);
3084       for (cp = 0; cp < coneSize; ++cp) {
3085         if (cone[cp] != c) continue;
3086         ierr = DMPlexInsertConeOrientation(dm, support[sp], cp, coneO[cp] >= 0 ? -(faceSize-coneO[cp]) : faceSize+coneO[cp]);CHKERRQ(ierr);
3087       }
3088     }
3089   }
3090   ierr = DMRestoreWorkArray(dm, maxConeSize, PETSC_INT, &revcone);CHKERRQ(ierr);
3091   ierr = DMRestoreWorkArray(dm, maxConeSize, PETSC_INT, &revconeO);CHKERRQ(ierr);
3092   ierr = PetscBTDestroy(&seenCells);CHKERRQ(ierr);
3093   ierr = PetscBTDestroy(&flippedCells);CHKERRQ(ierr);
3094   ierr = PetscBTDestroy(&seenFaces);CHKERRQ(ierr);
3095   ierr = PetscFree(faceFIFO);CHKERRQ(ierr);
3096   PetscFunctionReturn(0);
3097 }
3098 
3099 #undef __FUNCT__
3100 #define __FUNCT__ "DMPlexInvertCell"
3101 /*@C
3102   DMPlexInvertCell - This flips tetrahedron and hexahedron orientation since Plex stores them internally with outward normals. Other cells are left untouched.
3103 
3104   Input Parameters:
3105 + numCorners - The number of vertices in a cell
3106 - cone - The incoming cone
3107 
3108   Output Parameter:
3109 . cone - The inverted cone (in-place)
3110 
3111   Level: developer
3112 
3113 .seealso: DMPlexGenerate()
3114 @*/
3115 PetscErrorCode DMPlexInvertCell(PetscInt dim, PetscInt numCorners, int cone[])
3116 {
3117   int tmpc;
3118 
3119   PetscFunctionBegin;
3120   if (dim != 3) PetscFunctionReturn(0);
3121   switch (numCorners) {
3122   case 4:
3123     tmpc    = cone[0];
3124     cone[0] = cone[1];
3125     cone[1] = tmpc;
3126     break;
3127   case 8:
3128     tmpc    = cone[1];
3129     cone[1] = cone[3];
3130     cone[3] = tmpc;
3131     break;
3132   default: break;
3133   }
3134   PetscFunctionReturn(0);
3135 }
3136 
3137 #undef __FUNCT__
3138 #define __FUNCT__ "DMPlexInvertCells_Internal"
3139 /* This is to fix the tetrahedron orientation from TetGen */
3140 PETSC_UNUSED static PetscErrorCode DMPlexInvertCells_Internal(PetscInt dim, PetscInt numCells, PetscInt numCorners, int cells[])
3141 {
3142   PetscInt       bound = numCells*numCorners, coff;
3143   PetscErrorCode ierr;
3144 
3145   PetscFunctionBegin;
3146   for (coff = 0; coff < bound; coff += numCorners) {
3147     ierr = DMPlexInvertCell(dim, numCorners, &cells[coff]);CHKERRQ(ierr);
3148   }
3149   PetscFunctionReturn(0);
3150 }
3151 
3152 #if defined(PETSC_HAVE_TRIANGLE)
3153 #include <triangle.h>
3154 
3155 #undef __FUNCT__
3156 #define __FUNCT__ "InitInput_Triangle"
3157 PetscErrorCode InitInput_Triangle(struct triangulateio *inputCtx)
3158 {
3159   PetscFunctionBegin;
3160   inputCtx->numberofpoints             = 0;
3161   inputCtx->numberofpointattributes    = 0;
3162   inputCtx->pointlist                  = NULL;
3163   inputCtx->pointattributelist         = NULL;
3164   inputCtx->pointmarkerlist            = NULL;
3165   inputCtx->numberofsegments           = 0;
3166   inputCtx->segmentlist                = NULL;
3167   inputCtx->segmentmarkerlist          = NULL;
3168   inputCtx->numberoftriangleattributes = 0;
3169   inputCtx->trianglelist               = NULL;
3170   inputCtx->numberofholes              = 0;
3171   inputCtx->holelist                   = NULL;
3172   inputCtx->numberofregions            = 0;
3173   inputCtx->regionlist                 = NULL;
3174   PetscFunctionReturn(0);
3175 }
3176 
3177 #undef __FUNCT__
3178 #define __FUNCT__ "InitOutput_Triangle"
3179 PetscErrorCode InitOutput_Triangle(struct triangulateio *outputCtx)
3180 {
3181   PetscFunctionBegin;
3182   outputCtx->numberofpoints        = 0;
3183   outputCtx->pointlist             = NULL;
3184   outputCtx->pointattributelist    = NULL;
3185   outputCtx->pointmarkerlist       = NULL;
3186   outputCtx->numberoftriangles     = 0;
3187   outputCtx->trianglelist          = NULL;
3188   outputCtx->triangleattributelist = NULL;
3189   outputCtx->neighborlist          = NULL;
3190   outputCtx->segmentlist           = NULL;
3191   outputCtx->segmentmarkerlist     = NULL;
3192   outputCtx->numberofedges         = 0;
3193   outputCtx->edgelist              = NULL;
3194   outputCtx->edgemarkerlist        = NULL;
3195   PetscFunctionReturn(0);
3196 }
3197 
3198 #undef __FUNCT__
3199 #define __FUNCT__ "FiniOutput_Triangle"
3200 PetscErrorCode FiniOutput_Triangle(struct triangulateio *outputCtx)
3201 {
3202   PetscFunctionBegin;
3203   free(outputCtx->pointlist);
3204   free(outputCtx->pointmarkerlist);
3205   free(outputCtx->segmentlist);
3206   free(outputCtx->segmentmarkerlist);
3207   free(outputCtx->edgelist);
3208   free(outputCtx->edgemarkerlist);
3209   free(outputCtx->trianglelist);
3210   free(outputCtx->neighborlist);
3211   PetscFunctionReturn(0);
3212 }
3213 
3214 #undef __FUNCT__
3215 #define __FUNCT__ "DMPlexGenerate_Triangle"
3216 PetscErrorCode DMPlexGenerate_Triangle(DM boundary, PetscBool interpolate, DM *dm)
3217 {
3218   MPI_Comm             comm;
3219   PetscInt             dim              = 2;
3220   const PetscBool      createConvexHull = PETSC_FALSE;
3221   const PetscBool      constrained      = PETSC_FALSE;
3222   struct triangulateio in;
3223   struct triangulateio out;
3224   PetscInt             vStart, vEnd, v, eStart, eEnd, e;
3225   PetscMPIInt          rank;
3226   PetscErrorCode       ierr;
3227 
3228   PetscFunctionBegin;
3229   ierr = PetscObjectGetComm((PetscObject)boundary,&comm);CHKERRQ(ierr);
3230   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
3231   ierr = InitInput_Triangle(&in);CHKERRQ(ierr);
3232   ierr = InitOutput_Triangle(&out);CHKERRQ(ierr);
3233   ierr = DMPlexGetDepthStratum(boundary, 0, &vStart, &vEnd);CHKERRQ(ierr);
3234 
3235   in.numberofpoints = vEnd - vStart;
3236   if (in.numberofpoints > 0) {
3237     PetscSection coordSection;
3238     Vec          coordinates;
3239     PetscScalar *array;
3240 
3241     ierr = PetscMalloc1(in.numberofpoints*dim, &in.pointlist);CHKERRQ(ierr);
3242     ierr = PetscMalloc1(in.numberofpoints, &in.pointmarkerlist);CHKERRQ(ierr);
3243     ierr = DMGetCoordinatesLocal(boundary, &coordinates);CHKERRQ(ierr);
3244     ierr = DMGetCoordinateSection(boundary, &coordSection);CHKERRQ(ierr);
3245     ierr = VecGetArray(coordinates, &array);CHKERRQ(ierr);
3246     for (v = vStart; v < vEnd; ++v) {
3247       const PetscInt idx = v - vStart;
3248       PetscInt       off, d;
3249 
3250       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
3251       for (d = 0; d < dim; ++d) {
3252         in.pointlist[idx*dim + d] = PetscRealPart(array[off+d]);
3253       }
3254       ierr = DMPlexGetLabelValue(boundary, "marker", v, &in.pointmarkerlist[idx]);CHKERRQ(ierr);
3255     }
3256     ierr = VecRestoreArray(coordinates, &array);CHKERRQ(ierr);
3257   }
3258   ierr  = DMPlexGetHeightStratum(boundary, 0, &eStart, &eEnd);CHKERRQ(ierr);
3259   in.numberofsegments = eEnd - eStart;
3260   if (in.numberofsegments > 0) {
3261     ierr = PetscMalloc1(in.numberofsegments*2, &in.segmentlist);CHKERRQ(ierr);
3262     ierr = PetscMalloc1(in.numberofsegments, &in.segmentmarkerlist);CHKERRQ(ierr);
3263     for (e = eStart; e < eEnd; ++e) {
3264       const PetscInt  idx = e - eStart;
3265       const PetscInt *cone;
3266 
3267       ierr = DMPlexGetCone(boundary, e, &cone);CHKERRQ(ierr);
3268 
3269       in.segmentlist[idx*2+0] = cone[0] - vStart;
3270       in.segmentlist[idx*2+1] = cone[1] - vStart;
3271 
3272       ierr = DMPlexGetLabelValue(boundary, "marker", e, &in.segmentmarkerlist[idx]);CHKERRQ(ierr);
3273     }
3274   }
3275 #if 0 /* Do not currently support holes */
3276   PetscReal *holeCoords;
3277   PetscInt   h, d;
3278 
3279   ierr = DMPlexGetHoles(boundary, &in.numberofholes, &holeCords);CHKERRQ(ierr);
3280   if (in.numberofholes > 0) {
3281     ierr = PetscMalloc1(in.numberofholes*dim, &in.holelist);CHKERRQ(ierr);
3282     for (h = 0; h < in.numberofholes; ++h) {
3283       for (d = 0; d < dim; ++d) {
3284         in.holelist[h*dim+d] = holeCoords[h*dim+d];
3285       }
3286     }
3287   }
3288 #endif
3289   if (!rank) {
3290     char args[32];
3291 
3292     /* Take away 'Q' for verbose output */
3293     ierr = PetscStrcpy(args, "pqezQ");CHKERRQ(ierr);
3294     if (createConvexHull) {
3295       ierr = PetscStrcat(args, "c");CHKERRQ(ierr);
3296     }
3297     if (constrained) {
3298       ierr = PetscStrcpy(args, "zepDQ");CHKERRQ(ierr);
3299     }
3300     triangulate(args, &in, &out, NULL);
3301   }
3302   ierr = PetscFree(in.pointlist);CHKERRQ(ierr);
3303   ierr = PetscFree(in.pointmarkerlist);CHKERRQ(ierr);
3304   ierr = PetscFree(in.segmentlist);CHKERRQ(ierr);
3305   ierr = PetscFree(in.segmentmarkerlist);CHKERRQ(ierr);
3306   ierr = PetscFree(in.holelist);CHKERRQ(ierr);
3307 
3308   {
3309     const PetscInt numCorners  = 3;
3310     const PetscInt numCells    = out.numberoftriangles;
3311     const PetscInt numVertices = out.numberofpoints;
3312     const int     *cells      = out.trianglelist;
3313     const double  *meshCoords = out.pointlist;
3314 
3315     ierr = DMPlexCreateFromCellList(comm, dim, numCells, numVertices, numCorners, interpolate, cells, dim, meshCoords, dm);CHKERRQ(ierr);
3316     /* Set labels */
3317     for (v = 0; v < numVertices; ++v) {
3318       if (out.pointmarkerlist[v]) {
3319         ierr = DMPlexSetLabelValue(*dm, "marker", v+numCells, out.pointmarkerlist[v]);CHKERRQ(ierr);
3320       }
3321     }
3322     if (interpolate) {
3323       for (e = 0; e < out.numberofedges; e++) {
3324         if (out.edgemarkerlist[e]) {
3325           const PetscInt  vertices[2] = {out.edgelist[e*2+0]+numCells, out.edgelist[e*2+1]+numCells};
3326           const PetscInt *edges;
3327           PetscInt        numEdges;
3328 
3329           ierr = DMPlexGetJoin(*dm, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3330           if (numEdges != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Two vertices must cover only one edge, not %D", numEdges);
3331           ierr = DMPlexSetLabelValue(*dm, "marker", edges[0], out.edgemarkerlist[e]);CHKERRQ(ierr);
3332           ierr = DMPlexRestoreJoin(*dm, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3333         }
3334       }
3335     }
3336     ierr = DMPlexSetRefinementUniform(*dm, PETSC_FALSE);CHKERRQ(ierr);
3337   }
3338 #if 0 /* Do not currently support holes */
3339   ierr = DMPlexCopyHoles(*dm, boundary);CHKERRQ(ierr);
3340 #endif
3341   ierr = FiniOutput_Triangle(&out);CHKERRQ(ierr);
3342   PetscFunctionReturn(0);
3343 }
3344 
3345 #undef __FUNCT__
3346 #define __FUNCT__ "DMPlexRefine_Triangle"
3347 PetscErrorCode DMPlexRefine_Triangle(DM dm, double *maxVolumes, DM *dmRefined)
3348 {
3349   MPI_Comm             comm;
3350   PetscInt             dim  = 2;
3351   struct triangulateio in;
3352   struct triangulateio out;
3353   PetscInt             vStart, vEnd, v, cStart, cEnd, c, depth, depthGlobal;
3354   PetscMPIInt          rank;
3355   PetscErrorCode       ierr;
3356 
3357   PetscFunctionBegin;
3358   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
3359   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
3360   ierr = InitInput_Triangle(&in);CHKERRQ(ierr);
3361   ierr = InitOutput_Triangle(&out);CHKERRQ(ierr);
3362   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3363   ierr = MPI_Allreduce(&depth, &depthGlobal, 1, MPIU_INT, MPI_MAX, comm);CHKERRQ(ierr);
3364   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
3365 
3366   in.numberofpoints = vEnd - vStart;
3367   if (in.numberofpoints > 0) {
3368     PetscSection coordSection;
3369     Vec          coordinates;
3370     PetscScalar *array;
3371 
3372     ierr = PetscMalloc1(in.numberofpoints*dim, &in.pointlist);CHKERRQ(ierr);
3373     ierr = PetscMalloc1(in.numberofpoints, &in.pointmarkerlist);CHKERRQ(ierr);
3374     ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
3375     ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
3376     ierr = VecGetArray(coordinates, &array);CHKERRQ(ierr);
3377     for (v = vStart; v < vEnd; ++v) {
3378       const PetscInt idx = v - vStart;
3379       PetscInt       off, d;
3380 
3381       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
3382       for (d = 0; d < dim; ++d) {
3383         in.pointlist[idx*dim + d] = PetscRealPart(array[off+d]);
3384       }
3385       ierr = DMPlexGetLabelValue(dm, "marker", v, &in.pointmarkerlist[idx]);CHKERRQ(ierr);
3386     }
3387     ierr = VecRestoreArray(coordinates, &array);CHKERRQ(ierr);
3388   }
3389   ierr  = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
3390 
3391   in.numberofcorners   = 3;
3392   in.numberoftriangles = cEnd - cStart;
3393 
3394   in.trianglearealist  = (double*) maxVolumes;
3395   if (in.numberoftriangles > 0) {
3396     ierr = PetscMalloc1(in.numberoftriangles*in.numberofcorners, &in.trianglelist);CHKERRQ(ierr);
3397     for (c = cStart; c < cEnd; ++c) {
3398       const PetscInt idx      = c - cStart;
3399       PetscInt      *closure = NULL;
3400       PetscInt       closureSize;
3401 
3402       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
3403       if ((closureSize != 4) && (closureSize != 7)) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Mesh has cell which is not a triangle, %D vertices in closure", closureSize);
3404       for (v = 0; v < 3; ++v) {
3405         in.trianglelist[idx*in.numberofcorners + v] = closure[(v+closureSize-3)*2] - vStart;
3406       }
3407       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
3408     }
3409   }
3410   /* TODO: Segment markers are missing on input */
3411 #if 0 /* Do not currently support holes */
3412   PetscReal *holeCoords;
3413   PetscInt   h, d;
3414 
3415   ierr = DMPlexGetHoles(boundary, &in.numberofholes, &holeCords);CHKERRQ(ierr);
3416   if (in.numberofholes > 0) {
3417     ierr = PetscMalloc1(in.numberofholes*dim, &in.holelist);CHKERRQ(ierr);
3418     for (h = 0; h < in.numberofholes; ++h) {
3419       for (d = 0; d < dim; ++d) {
3420         in.holelist[h*dim+d] = holeCoords[h*dim+d];
3421       }
3422     }
3423   }
3424 #endif
3425   if (!rank) {
3426     char args[32];
3427 
3428     /* Take away 'Q' for verbose output */
3429     ierr = PetscStrcpy(args, "pqezQra");CHKERRQ(ierr);
3430     triangulate(args, &in, &out, NULL);
3431   }
3432   ierr = PetscFree(in.pointlist);CHKERRQ(ierr);
3433   ierr = PetscFree(in.pointmarkerlist);CHKERRQ(ierr);
3434   ierr = PetscFree(in.segmentlist);CHKERRQ(ierr);
3435   ierr = PetscFree(in.segmentmarkerlist);CHKERRQ(ierr);
3436   ierr = PetscFree(in.trianglelist);CHKERRQ(ierr);
3437 
3438   {
3439     const PetscInt numCorners  = 3;
3440     const PetscInt numCells    = out.numberoftriangles;
3441     const PetscInt numVertices = out.numberofpoints;
3442     const int     *cells      = out.trianglelist;
3443     const double  *meshCoords = out.pointlist;
3444     PetscBool      interpolate = depthGlobal > 1 ? PETSC_TRUE : PETSC_FALSE;
3445 
3446     ierr = DMPlexCreateFromCellList(comm, dim, numCells, numVertices, numCorners, interpolate, cells, dim, meshCoords, dmRefined);CHKERRQ(ierr);
3447     /* Set labels */
3448     for (v = 0; v < numVertices; ++v) {
3449       if (out.pointmarkerlist[v]) {
3450         ierr = DMPlexSetLabelValue(*dmRefined, "marker", v+numCells, out.pointmarkerlist[v]);CHKERRQ(ierr);
3451       }
3452     }
3453     if (interpolate) {
3454       PetscInt e;
3455 
3456       for (e = 0; e < out.numberofedges; e++) {
3457         if (out.edgemarkerlist[e]) {
3458           const PetscInt  vertices[2] = {out.edgelist[e*2+0]+numCells, out.edgelist[e*2+1]+numCells};
3459           const PetscInt *edges;
3460           PetscInt        numEdges;
3461 
3462           ierr = DMPlexGetJoin(*dmRefined, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3463           if (numEdges != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Two vertices must cover only one edge, not %D", numEdges);
3464           ierr = DMPlexSetLabelValue(*dmRefined, "marker", edges[0], out.edgemarkerlist[e]);CHKERRQ(ierr);
3465           ierr = DMPlexRestoreJoin(*dmRefined, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3466         }
3467       }
3468     }
3469     ierr = DMPlexSetRefinementUniform(*dmRefined, PETSC_FALSE);CHKERRQ(ierr);
3470   }
3471 #if 0 /* Do not currently support holes */
3472   ierr = DMPlexCopyHoles(*dm, boundary);CHKERRQ(ierr);
3473 #endif
3474   ierr = FiniOutput_Triangle(&out);CHKERRQ(ierr);
3475   PetscFunctionReturn(0);
3476 }
3477 #endif
3478 
3479 #if defined(PETSC_HAVE_TETGEN)
3480 #include <tetgen.h>
3481 #undef __FUNCT__
3482 #define __FUNCT__ "DMPlexGenerate_Tetgen"
3483 PetscErrorCode DMPlexGenerate_Tetgen(DM boundary, PetscBool interpolate, DM *dm)
3484 {
3485   MPI_Comm       comm;
3486   const PetscInt dim  = 3;
3487   ::tetgenio     in;
3488   ::tetgenio     out;
3489   PetscInt       vStart, vEnd, v, fStart, fEnd, f;
3490   PetscMPIInt    rank;
3491   PetscErrorCode ierr;
3492 
3493   PetscFunctionBegin;
3494   ierr              = PetscObjectGetComm((PetscObject)boundary,&comm);CHKERRQ(ierr);
3495   ierr              = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
3496   ierr              = DMPlexGetDepthStratum(boundary, 0, &vStart, &vEnd);CHKERRQ(ierr);
3497   in.numberofpoints = vEnd - vStart;
3498   if (in.numberofpoints > 0) {
3499     PetscSection coordSection;
3500     Vec          coordinates;
3501     PetscScalar *array;
3502 
3503     in.pointlist       = new double[in.numberofpoints*dim];
3504     in.pointmarkerlist = new int[in.numberofpoints];
3505 
3506     ierr = DMGetCoordinatesLocal(boundary, &coordinates);CHKERRQ(ierr);
3507     ierr = DMGetCoordinateSection(boundary, &coordSection);CHKERRQ(ierr);
3508     ierr = VecGetArray(coordinates, &array);CHKERRQ(ierr);
3509     for (v = vStart; v < vEnd; ++v) {
3510       const PetscInt idx = v - vStart;
3511       PetscInt       off, d;
3512 
3513       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
3514       for (d = 0; d < dim; ++d) in.pointlist[idx*dim + d] = array[off+d];
3515       ierr = DMPlexGetLabelValue(boundary, "marker", v, &in.pointmarkerlist[idx]);CHKERRQ(ierr);
3516     }
3517     ierr = VecRestoreArray(coordinates, &array);CHKERRQ(ierr);
3518   }
3519   ierr  = DMPlexGetHeightStratum(boundary, 0, &fStart, &fEnd);CHKERRQ(ierr);
3520 
3521   in.numberoffacets = fEnd - fStart;
3522   if (in.numberoffacets > 0) {
3523     in.facetlist       = new tetgenio::facet[in.numberoffacets];
3524     in.facetmarkerlist = new int[in.numberoffacets];
3525     for (f = fStart; f < fEnd; ++f) {
3526       const PetscInt idx     = f - fStart;
3527       PetscInt      *points = NULL, numPoints, p, numVertices = 0, v;
3528 
3529       in.facetlist[idx].numberofpolygons = 1;
3530       in.facetlist[idx].polygonlist      = new tetgenio::polygon[in.facetlist[idx].numberofpolygons];
3531       in.facetlist[idx].numberofholes    = 0;
3532       in.facetlist[idx].holelist         = NULL;
3533 
3534       ierr = DMPlexGetTransitiveClosure(boundary, f, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
3535       for (p = 0; p < numPoints*2; p += 2) {
3536         const PetscInt point = points[p];
3537         if ((point >= vStart) && (point < vEnd)) points[numVertices++] = point;
3538       }
3539 
3540       tetgenio::polygon *poly = in.facetlist[idx].polygonlist;
3541       poly->numberofvertices = numVertices;
3542       poly->vertexlist       = new int[poly->numberofvertices];
3543       for (v = 0; v < numVertices; ++v) {
3544         const PetscInt vIdx = points[v] - vStart;
3545         poly->vertexlist[v] = vIdx;
3546       }
3547       ierr = DMPlexGetLabelValue(boundary, "marker", f, &in.facetmarkerlist[idx]);CHKERRQ(ierr);
3548       ierr = DMPlexRestoreTransitiveClosure(boundary, f, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
3549     }
3550   }
3551   if (!rank) {
3552     char args[32];
3553 
3554     /* Take away 'Q' for verbose output */
3555     ierr = PetscStrcpy(args, "pqezQ");CHKERRQ(ierr);
3556     ::tetrahedralize(args, &in, &out);
3557   }
3558   {
3559     const PetscInt numCorners  = 4;
3560     const PetscInt numCells    = out.numberoftetrahedra;
3561     const PetscInt numVertices = out.numberofpoints;
3562     const double   *meshCoords = out.pointlist;
3563     int            *cells      = out.tetrahedronlist;
3564 
3565     ierr = DMPlexInvertCells_Internal(dim, numCells, numCorners, cells);CHKERRQ(ierr);
3566     ierr = DMPlexCreateFromCellList(comm, dim, numCells, numVertices, numCorners, interpolate, cells, dim, meshCoords, dm);CHKERRQ(ierr);
3567     /* Set labels */
3568     for (v = 0; v < numVertices; ++v) {
3569       if (out.pointmarkerlist[v]) {
3570         ierr = DMPlexSetLabelValue(*dm, "marker", v+numCells, out.pointmarkerlist[v]);CHKERRQ(ierr);
3571       }
3572     }
3573     if (interpolate) {
3574       PetscInt e;
3575 
3576       for (e = 0; e < out.numberofedges; e++) {
3577         if (out.edgemarkerlist[e]) {
3578           const PetscInt  vertices[2] = {out.edgelist[e*2+0]+numCells, out.edgelist[e*2+1]+numCells};
3579           const PetscInt *edges;
3580           PetscInt        numEdges;
3581 
3582           ierr = DMPlexGetJoin(*dm, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3583           if (numEdges != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Two vertices must cover only one edge, not %D", numEdges);
3584           ierr = DMPlexSetLabelValue(*dm, "marker", edges[0], out.edgemarkerlist[e]);CHKERRQ(ierr);
3585           ierr = DMPlexRestoreJoin(*dm, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3586         }
3587       }
3588       for (f = 0; f < out.numberoftrifaces; f++) {
3589         if (out.trifacemarkerlist[f]) {
3590           const PetscInt  vertices[3] = {out.trifacelist[f*3+0]+numCells, out.trifacelist[f*3+1]+numCells, out.trifacelist[f*3+2]+numCells};
3591           const PetscInt *faces;
3592           PetscInt        numFaces;
3593 
3594           ierr = DMPlexGetJoin(*dm, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
3595           if (numFaces != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Three vertices must cover only one face, not %D", numFaces);
3596           ierr = DMPlexSetLabelValue(*dm, "marker", faces[0], out.trifacemarkerlist[f]);CHKERRQ(ierr);
3597           ierr = DMPlexRestoreJoin(*dm, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
3598         }
3599       }
3600     }
3601     ierr = DMPlexSetRefinementUniform(*dm, PETSC_FALSE);CHKERRQ(ierr);
3602   }
3603   PetscFunctionReturn(0);
3604 }
3605 
3606 #undef __FUNCT__
3607 #define __FUNCT__ "DMPlexRefine_Tetgen"
3608 PetscErrorCode DMPlexRefine_Tetgen(DM dm, double *maxVolumes, DM *dmRefined)
3609 {
3610   MPI_Comm       comm;
3611   const PetscInt dim  = 3;
3612   ::tetgenio     in;
3613   ::tetgenio     out;
3614   PetscInt       vStart, vEnd, v, cStart, cEnd, c, depth, depthGlobal;
3615   PetscMPIInt    rank;
3616   PetscErrorCode ierr;
3617 
3618   PetscFunctionBegin;
3619   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
3620   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
3621   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3622   ierr = MPI_Allreduce(&depth, &depthGlobal, 1, MPIU_INT, MPI_MAX, comm);CHKERRQ(ierr);
3623   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
3624 
3625   in.numberofpoints = vEnd - vStart;
3626   if (in.numberofpoints > 0) {
3627     PetscSection coordSection;
3628     Vec          coordinates;
3629     PetscScalar *array;
3630 
3631     in.pointlist       = new double[in.numberofpoints*dim];
3632     in.pointmarkerlist = new int[in.numberofpoints];
3633 
3634     ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
3635     ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
3636     ierr = VecGetArray(coordinates, &array);CHKERRQ(ierr);
3637     for (v = vStart; v < vEnd; ++v) {
3638       const PetscInt idx = v - vStart;
3639       PetscInt       off, d;
3640 
3641       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
3642       for (d = 0; d < dim; ++d) in.pointlist[idx*dim + d] = array[off+d];
3643       ierr = DMPlexGetLabelValue(dm, "marker", v, &in.pointmarkerlist[idx]);CHKERRQ(ierr);
3644     }
3645     ierr = VecRestoreArray(coordinates, &array);CHKERRQ(ierr);
3646   }
3647   ierr  = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
3648 
3649   in.numberofcorners       = 4;
3650   in.numberoftetrahedra    = cEnd - cStart;
3651   in.tetrahedronvolumelist = (double*) maxVolumes;
3652   if (in.numberoftetrahedra > 0) {
3653     in.tetrahedronlist = new int[in.numberoftetrahedra*in.numberofcorners];
3654     for (c = cStart; c < cEnd; ++c) {
3655       const PetscInt idx      = c - cStart;
3656       PetscInt      *closure = NULL;
3657       PetscInt       closureSize;
3658 
3659       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
3660       if ((closureSize != 5) && (closureSize != 15)) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Mesh has cell which is not a tetrahedron, %D vertices in closure", closureSize);
3661       for (v = 0; v < 4; ++v) {
3662         in.tetrahedronlist[idx*in.numberofcorners + v] = closure[(v+closureSize-4)*2] - vStart;
3663       }
3664       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
3665     }
3666   }
3667   /* TODO: Put in boundary faces with markers */
3668   if (!rank) {
3669     char args[32];
3670 
3671     /* Take away 'Q' for verbose output */
3672     /*ierr = PetscStrcpy(args, "qezQra");CHKERRQ(ierr); */
3673     ierr = PetscStrcpy(args, "qezraVVVV");CHKERRQ(ierr);
3674     ::tetrahedralize(args, &in, &out);
3675   }
3676   in.tetrahedronvolumelist = NULL;
3677 
3678   {
3679     const PetscInt numCorners  = 4;
3680     const PetscInt numCells    = out.numberoftetrahedra;
3681     const PetscInt numVertices = out.numberofpoints;
3682     const double   *meshCoords = out.pointlist;
3683     int            *cells      = out.tetrahedronlist;
3684 
3685     PetscBool      interpolate = depthGlobal > 1 ? PETSC_TRUE : PETSC_FALSE;
3686 
3687     ierr = DMPlexInvertCells_Internal(dim, numCells, numCorners, cells);CHKERRQ(ierr);
3688     ierr = DMPlexCreateFromCellList(comm, dim, numCells, numVertices, numCorners, interpolate, cells, dim, meshCoords, dmRefined);CHKERRQ(ierr);
3689     /* Set labels */
3690     for (v = 0; v < numVertices; ++v) {
3691       if (out.pointmarkerlist[v]) {
3692         ierr = DMPlexSetLabelValue(*dmRefined, "marker", v+numCells, out.pointmarkerlist[v]);CHKERRQ(ierr);
3693       }
3694     }
3695     if (interpolate) {
3696       PetscInt e, f;
3697 
3698       for (e = 0; e < out.numberofedges; e++) {
3699         if (out.edgemarkerlist[e]) {
3700           const PetscInt  vertices[2] = {out.edgelist[e*2+0]+numCells, out.edgelist[e*2+1]+numCells};
3701           const PetscInt *edges;
3702           PetscInt        numEdges;
3703 
3704           ierr = DMPlexGetJoin(*dmRefined, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3705           if (numEdges != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Two vertices must cover only one edge, not %D", numEdges);
3706           ierr = DMPlexSetLabelValue(*dmRefined, "marker", edges[0], out.edgemarkerlist[e]);CHKERRQ(ierr);
3707           ierr = DMPlexRestoreJoin(*dmRefined, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3708         }
3709       }
3710       for (f = 0; f < out.numberoftrifaces; f++) {
3711         if (out.trifacemarkerlist[f]) {
3712           const PetscInt  vertices[3] = {out.trifacelist[f*3+0]+numCells, out.trifacelist[f*3+1]+numCells, out.trifacelist[f*3+2]+numCells};
3713           const PetscInt *faces;
3714           PetscInt        numFaces;
3715 
3716           ierr = DMPlexGetJoin(*dmRefined, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
3717           if (numFaces != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Three vertices must cover only one face, not %D", numFaces);
3718           ierr = DMPlexSetLabelValue(*dmRefined, "marker", faces[0], out.trifacemarkerlist[f]);CHKERRQ(ierr);
3719           ierr = DMPlexRestoreJoin(*dmRefined, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
3720         }
3721       }
3722     }
3723     ierr = DMPlexSetRefinementUniform(*dmRefined, PETSC_FALSE);CHKERRQ(ierr);
3724   }
3725   PetscFunctionReturn(0);
3726 }
3727 #endif
3728 
3729 #if defined(PETSC_HAVE_CTETGEN)
3730 #include <ctetgen.h>
3731 
3732 #undef __FUNCT__
3733 #define __FUNCT__ "DMPlexGenerate_CTetgen"
3734 PetscErrorCode DMPlexGenerate_CTetgen(DM boundary, PetscBool interpolate, DM *dm)
3735 {
3736   MPI_Comm       comm;
3737   const PetscInt dim  = 3;
3738   PLC           *in, *out;
3739   PetscInt       verbose = 0, vStart, vEnd, v, fStart, fEnd, f;
3740   PetscMPIInt    rank;
3741   PetscErrorCode ierr;
3742 
3743   PetscFunctionBegin;
3744   ierr = PetscObjectGetComm((PetscObject)boundary,&comm);CHKERRQ(ierr);
3745   ierr = PetscOptionsGetInt(((PetscObject) boundary)->prefix, "-ctetgen_verbose", &verbose, NULL);CHKERRQ(ierr);
3746   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
3747   ierr = DMPlexGetDepthStratum(boundary, 0, &vStart, &vEnd);CHKERRQ(ierr);
3748   ierr = PLCCreate(&in);CHKERRQ(ierr);
3749   ierr = PLCCreate(&out);CHKERRQ(ierr);
3750 
3751   in->numberofpoints = vEnd - vStart;
3752   if (in->numberofpoints > 0) {
3753     PetscSection coordSection;
3754     Vec          coordinates;
3755     PetscScalar *array;
3756 
3757     ierr = PetscMalloc1(in->numberofpoints*dim, &in->pointlist);CHKERRQ(ierr);
3758     ierr = PetscMalloc1(in->numberofpoints,       &in->pointmarkerlist);CHKERRQ(ierr);
3759     ierr = DMGetCoordinatesLocal(boundary, &coordinates);CHKERRQ(ierr);
3760     ierr = DMGetCoordinateSection(boundary, &coordSection);CHKERRQ(ierr);
3761     ierr = VecGetArray(coordinates, &array);CHKERRQ(ierr);
3762     for (v = vStart; v < vEnd; ++v) {
3763       const PetscInt idx = v - vStart;
3764       PetscInt       off, d, m;
3765 
3766       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
3767       for (d = 0; d < dim; ++d) {
3768         in->pointlist[idx*dim + d] = PetscRealPart(array[off+d]);
3769       }
3770       ierr = DMPlexGetLabelValue(boundary, "marker", v, &m);CHKERRQ(ierr);
3771 
3772       in->pointmarkerlist[idx] = (int) m;
3773     }
3774     ierr = VecRestoreArray(coordinates, &array);CHKERRQ(ierr);
3775   }
3776   ierr  = DMPlexGetHeightStratum(boundary, 0, &fStart, &fEnd);CHKERRQ(ierr);
3777 
3778   in->numberoffacets = fEnd - fStart;
3779   if (in->numberoffacets > 0) {
3780     ierr = PetscMalloc1(in->numberoffacets, &in->facetlist);CHKERRQ(ierr);
3781     ierr = PetscMalloc1(in->numberoffacets,   &in->facetmarkerlist);CHKERRQ(ierr);
3782     for (f = fStart; f < fEnd; ++f) {
3783       const PetscInt idx     = f - fStart;
3784       PetscInt      *points = NULL, numPoints, p, numVertices = 0, v, m;
3785       polygon       *poly;
3786 
3787       in->facetlist[idx].numberofpolygons = 1;
3788 
3789       ierr = PetscMalloc1(in->facetlist[idx].numberofpolygons, &in->facetlist[idx].polygonlist);CHKERRQ(ierr);
3790 
3791       in->facetlist[idx].numberofholes    = 0;
3792       in->facetlist[idx].holelist         = NULL;
3793 
3794       ierr = DMPlexGetTransitiveClosure(boundary, f, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
3795       for (p = 0; p < numPoints*2; p += 2) {
3796         const PetscInt point = points[p];
3797         if ((point >= vStart) && (point < vEnd)) points[numVertices++] = point;
3798       }
3799 
3800       poly                   = in->facetlist[idx].polygonlist;
3801       poly->numberofvertices = numVertices;
3802       ierr                   = PetscMalloc1(poly->numberofvertices, &poly->vertexlist);CHKERRQ(ierr);
3803       for (v = 0; v < numVertices; ++v) {
3804         const PetscInt vIdx = points[v] - vStart;
3805         poly->vertexlist[v] = vIdx;
3806       }
3807       ierr                     = DMPlexGetLabelValue(boundary, "marker", f, &m);CHKERRQ(ierr);
3808       in->facetmarkerlist[idx] = (int) m;
3809       ierr                     = DMPlexRestoreTransitiveClosure(boundary, f, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
3810     }
3811   }
3812   if (!rank) {
3813     TetGenOpts t;
3814 
3815     ierr        = TetGenOptsInitialize(&t);CHKERRQ(ierr);
3816     t.in        = boundary; /* Should go away */
3817     t.plc       = 1;
3818     t.quality   = 1;
3819     t.edgesout  = 1;
3820     t.zeroindex = 1;
3821     t.quiet     = 1;
3822     t.verbose   = verbose;
3823     ierr        = TetGenCheckOpts(&t);CHKERRQ(ierr);
3824     ierr        = TetGenTetrahedralize(&t, in, out);CHKERRQ(ierr);
3825   }
3826   {
3827     const PetscInt numCorners  = 4;
3828     const PetscInt numCells    = out->numberoftetrahedra;
3829     const PetscInt numVertices = out->numberofpoints;
3830     const double   *meshCoords = out->pointlist;
3831     int            *cells      = out->tetrahedronlist;
3832 
3833     ierr = DMPlexInvertCells_Internal(dim, numCells, numCorners, cells);CHKERRQ(ierr);
3834     ierr = DMPlexCreateFromCellList(comm, dim, numCells, numVertices, numCorners, interpolate, cells, dim, meshCoords, dm);CHKERRQ(ierr);
3835     /* Set labels */
3836     for (v = 0; v < numVertices; ++v) {
3837       if (out->pointmarkerlist[v]) {
3838         ierr = DMPlexSetLabelValue(*dm, "marker", v+numCells, out->pointmarkerlist[v]);CHKERRQ(ierr);
3839       }
3840     }
3841     if (interpolate) {
3842       PetscInt e;
3843 
3844       for (e = 0; e < out->numberofedges; e++) {
3845         if (out->edgemarkerlist[e]) {
3846           const PetscInt  vertices[2] = {out->edgelist[e*2+0]+numCells, out->edgelist[e*2+1]+numCells};
3847           const PetscInt *edges;
3848           PetscInt        numEdges;
3849 
3850           ierr = DMPlexGetJoin(*dm, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3851           if (numEdges != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Two vertices must cover only one edge, not %D", numEdges);
3852           ierr = DMPlexSetLabelValue(*dm, "marker", edges[0], out->edgemarkerlist[e]);CHKERRQ(ierr);
3853           ierr = DMPlexRestoreJoin(*dm, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3854         }
3855       }
3856       for (f = 0; f < out->numberoftrifaces; f++) {
3857         if (out->trifacemarkerlist[f]) {
3858           const PetscInt  vertices[3] = {out->trifacelist[f*3+0]+numCells, out->trifacelist[f*3+1]+numCells, out->trifacelist[f*3+2]+numCells};
3859           const PetscInt *faces;
3860           PetscInt        numFaces;
3861 
3862           ierr = DMPlexGetFullJoin(*dm, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
3863           if (numFaces != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Three vertices must cover only one face, not %D", numFaces);
3864           ierr = DMPlexSetLabelValue(*dm, "marker", faces[0], out->trifacemarkerlist[f]);CHKERRQ(ierr);
3865           ierr = DMPlexRestoreJoin(*dm, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
3866         }
3867       }
3868     }
3869     ierr = DMPlexSetRefinementUniform(*dm, PETSC_FALSE);CHKERRQ(ierr);
3870   }
3871 
3872   ierr = PLCDestroy(&in);CHKERRQ(ierr);
3873   ierr = PLCDestroy(&out);CHKERRQ(ierr);
3874   PetscFunctionReturn(0);
3875 }
3876 
3877 #undef __FUNCT__
3878 #define __FUNCT__ "DMPlexRefine_CTetgen"
3879 PetscErrorCode DMPlexRefine_CTetgen(DM dm, PetscReal *maxVolumes, DM *dmRefined)
3880 {
3881   MPI_Comm       comm;
3882   const PetscInt dim  = 3;
3883   PLC           *in, *out;
3884   PetscInt       verbose = 0, vStart, vEnd, v, cStart, cEnd, c, depth, depthGlobal;
3885   PetscMPIInt    rank;
3886   PetscErrorCode ierr;
3887 
3888   PetscFunctionBegin;
3889   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
3890   ierr = PetscOptionsGetInt(((PetscObject) dm)->prefix, "-ctetgen_verbose", &verbose, NULL);CHKERRQ(ierr);
3891   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
3892   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3893   ierr = MPI_Allreduce(&depth, &depthGlobal, 1, MPIU_INT, MPI_MAX, comm);CHKERRQ(ierr);
3894   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
3895   ierr = PLCCreate(&in);CHKERRQ(ierr);
3896   ierr = PLCCreate(&out);CHKERRQ(ierr);
3897 
3898   in->numberofpoints = vEnd - vStart;
3899   if (in->numberofpoints > 0) {
3900     PetscSection coordSection;
3901     Vec          coordinates;
3902     PetscScalar *array;
3903 
3904     ierr = PetscMalloc1(in->numberofpoints*dim, &in->pointlist);CHKERRQ(ierr);
3905     ierr = PetscMalloc1(in->numberofpoints,       &in->pointmarkerlist);CHKERRQ(ierr);
3906     ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
3907     ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
3908     ierr = VecGetArray(coordinates, &array);CHKERRQ(ierr);
3909     for (v = vStart; v < vEnd; ++v) {
3910       const PetscInt idx = v - vStart;
3911       PetscInt       off, d, m;
3912 
3913       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
3914       for (d = 0; d < dim; ++d) {
3915         in->pointlist[idx*dim + d] = PetscRealPart(array[off+d]);
3916       }
3917       ierr = DMPlexGetLabelValue(dm, "marker", v, &m);CHKERRQ(ierr);
3918 
3919       in->pointmarkerlist[idx] = (int) m;
3920     }
3921     ierr = VecRestoreArray(coordinates, &array);CHKERRQ(ierr);
3922   }
3923   ierr  = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
3924 
3925   in->numberofcorners       = 4;
3926   in->numberoftetrahedra    = cEnd - cStart;
3927   in->tetrahedronvolumelist = maxVolumes;
3928   if (in->numberoftetrahedra > 0) {
3929     ierr = PetscMalloc1(in->numberoftetrahedra*in->numberofcorners, &in->tetrahedronlist);CHKERRQ(ierr);
3930     for (c = cStart; c < cEnd; ++c) {
3931       const PetscInt idx      = c - cStart;
3932       PetscInt      *closure = NULL;
3933       PetscInt       closureSize;
3934 
3935       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
3936       if ((closureSize != 5) && (closureSize != 15)) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Mesh has cell which is not a tetrahedron, %D vertices in closure", closureSize);
3937       for (v = 0; v < 4; ++v) {
3938         in->tetrahedronlist[idx*in->numberofcorners + v] = closure[(v+closureSize-4)*2] - vStart;
3939       }
3940       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
3941     }
3942   }
3943   if (!rank) {
3944     TetGenOpts t;
3945 
3946     ierr = TetGenOptsInitialize(&t);CHKERRQ(ierr);
3947 
3948     t.in        = dm; /* Should go away */
3949     t.refine    = 1;
3950     t.varvolume = 1;
3951     t.quality   = 1;
3952     t.edgesout  = 1;
3953     t.zeroindex = 1;
3954     t.quiet     = 1;
3955     t.verbose   = verbose; /* Change this */
3956 
3957     ierr = TetGenCheckOpts(&t);CHKERRQ(ierr);
3958     ierr = TetGenTetrahedralize(&t, in, out);CHKERRQ(ierr);
3959   }
3960   {
3961     const PetscInt numCorners  = 4;
3962     const PetscInt numCells    = out->numberoftetrahedra;
3963     const PetscInt numVertices = out->numberofpoints;
3964     const double   *meshCoords = out->pointlist;
3965     int            *cells      = out->tetrahedronlist;
3966     PetscBool      interpolate = depthGlobal > 1 ? PETSC_TRUE : PETSC_FALSE;
3967 
3968     ierr = DMPlexInvertCells_Internal(dim, numCells, numCorners, cells);CHKERRQ(ierr);
3969     ierr = DMPlexCreateFromCellList(comm, dim, numCells, numVertices, numCorners, interpolate, cells, dim, meshCoords, dmRefined);CHKERRQ(ierr);
3970     /* Set labels */
3971     for (v = 0; v < numVertices; ++v) {
3972       if (out->pointmarkerlist[v]) {
3973         ierr = DMPlexSetLabelValue(*dmRefined, "marker", v+numCells, out->pointmarkerlist[v]);CHKERRQ(ierr);
3974       }
3975     }
3976     if (interpolate) {
3977       PetscInt e, f;
3978 
3979       for (e = 0; e < out->numberofedges; e++) {
3980         if (out->edgemarkerlist[e]) {
3981           const PetscInt  vertices[2] = {out->edgelist[e*2+0]+numCells, out->edgelist[e*2+1]+numCells};
3982           const PetscInt *edges;
3983           PetscInt        numEdges;
3984 
3985           ierr = DMPlexGetJoin(*dmRefined, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3986           if (numEdges != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Two vertices must cover only one edge, not %D", numEdges);
3987           ierr = DMPlexSetLabelValue(*dmRefined, "marker", edges[0], out->edgemarkerlist[e]);CHKERRQ(ierr);
3988           ierr = DMPlexRestoreJoin(*dmRefined, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3989         }
3990       }
3991       for (f = 0; f < out->numberoftrifaces; f++) {
3992         if (out->trifacemarkerlist[f]) {
3993           const PetscInt  vertices[3] = {out->trifacelist[f*3+0]+numCells, out->trifacelist[f*3+1]+numCells, out->trifacelist[f*3+2]+numCells};
3994           const PetscInt *faces;
3995           PetscInt        numFaces;
3996 
3997           ierr = DMPlexGetFullJoin(*dmRefined, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
3998           if (numFaces != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Three vertices must cover only one face, not %D", numFaces);
3999           ierr = DMPlexSetLabelValue(*dmRefined, "marker", faces[0], out->trifacemarkerlist[f]);CHKERRQ(ierr);
4000           ierr = DMPlexRestoreJoin(*dmRefined, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
4001         }
4002       }
4003     }
4004     ierr = DMPlexSetRefinementUniform(*dmRefined, PETSC_FALSE);CHKERRQ(ierr);
4005   }
4006   ierr = PLCDestroy(&in);CHKERRQ(ierr);
4007   ierr = PLCDestroy(&out);CHKERRQ(ierr);
4008   PetscFunctionReturn(0);
4009 }
4010 #endif
4011 
4012 #undef __FUNCT__
4013 #define __FUNCT__ "DMPlexGenerate"
4014 /*@C
4015   DMPlexGenerate - Generates a mesh.
4016 
4017   Not Collective
4018 
4019   Input Parameters:
4020 + boundary - The DMPlex boundary object
4021 . name - The mesh generation package name
4022 - interpolate - Flag to create intermediate mesh elements
4023 
4024   Output Parameter:
4025 . mesh - The DMPlex object
4026 
4027   Level: intermediate
4028 
4029 .keywords: mesh, elements
4030 .seealso: DMPlexCreate(), DMRefine()
4031 @*/
4032 PetscErrorCode DMPlexGenerate(DM boundary, const char name[], PetscBool interpolate, DM *mesh)
4033 {
4034   PetscInt       dim;
4035   char           genname[1024];
4036   PetscBool      isTriangle = PETSC_FALSE, isTetgen = PETSC_FALSE, isCTetgen = PETSC_FALSE, flg;
4037   PetscErrorCode ierr;
4038 
4039   PetscFunctionBegin;
4040   PetscValidHeaderSpecific(boundary, DM_CLASSID, 1);
4041   PetscValidLogicalCollectiveBool(boundary, interpolate, 2);
4042   ierr = DMPlexGetDimension(boundary, &dim);CHKERRQ(ierr);
4043   ierr = PetscOptionsGetString(((PetscObject) boundary)->prefix, "-dm_plex_generator", genname, 1024, &flg);CHKERRQ(ierr);
4044   if (flg) name = genname;
4045   if (name) {
4046     ierr = PetscStrcmp(name, "triangle", &isTriangle);CHKERRQ(ierr);
4047     ierr = PetscStrcmp(name, "tetgen",   &isTetgen);CHKERRQ(ierr);
4048     ierr = PetscStrcmp(name, "ctetgen",  &isCTetgen);CHKERRQ(ierr);
4049   }
4050   switch (dim) {
4051   case 1:
4052     if (!name || isTriangle) {
4053 #if defined(PETSC_HAVE_TRIANGLE)
4054       ierr = DMPlexGenerate_Triangle(boundary, interpolate, mesh);CHKERRQ(ierr);
4055 #else
4056       SETERRQ(PetscObjectComm((PetscObject)boundary), PETSC_ERR_SUP, "Mesh generation needs external package support.\nPlease reconfigure with --download-triangle.");
4057 #endif
4058     } else SETERRQ1(PetscObjectComm((PetscObject)boundary), PETSC_ERR_SUP, "Unknown 2D mesh generation package %s", name);
4059     break;
4060   case 2:
4061     if (!name || isCTetgen) {
4062 #if defined(PETSC_HAVE_CTETGEN)
4063       ierr = DMPlexGenerate_CTetgen(boundary, interpolate, mesh);CHKERRQ(ierr);
4064 #else
4065       SETERRQ(PetscObjectComm((PetscObject)boundary), PETSC_ERR_SUP, "CTetgen needs external package support.\nPlease reconfigure with --download-ctetgen.");
4066 #endif
4067     } else if (isTetgen) {
4068 #if defined(PETSC_HAVE_TETGEN)
4069       ierr = DMPlexGenerate_Tetgen(boundary, interpolate, mesh);CHKERRQ(ierr);
4070 #else
4071       SETERRQ(PetscObjectComm((PetscObject)boundary), PETSC_ERR_SUP, "Tetgen needs external package support.\nPlease reconfigure with --with-c-language=cxx --download-tetgen.");
4072 #endif
4073     } else SETERRQ1(PetscObjectComm((PetscObject)boundary), PETSC_ERR_SUP, "Unknown 3D mesh generation package %s", name);
4074     break;
4075   default:
4076     SETERRQ1(PetscObjectComm((PetscObject)boundary), PETSC_ERR_SUP, "Mesh generation for a dimension %d boundary is not supported.", dim);
4077   }
4078   PetscFunctionReturn(0);
4079 }
4080 
4081 #undef __FUNCT__
4082 #define __FUNCT__ "DMRefine_Plex"
4083 PetscErrorCode DMRefine_Plex(DM dm, MPI_Comm comm, DM *dmRefined)
4084 {
4085   PetscReal      refinementLimit;
4086   PetscInt       dim, cStart, cEnd;
4087   char           genname[1024], *name = NULL;
4088   PetscBool      isUniform, isTriangle = PETSC_FALSE, isTetgen = PETSC_FALSE, isCTetgen = PETSC_FALSE, flg;
4089   PetscErrorCode ierr;
4090 
4091   PetscFunctionBegin;
4092   ierr = DMPlexGetRefinementUniform(dm, &isUniform);CHKERRQ(ierr);
4093   if (isUniform) {
4094     CellRefiner cellRefiner;
4095 
4096     ierr = DMPlexGetCellRefiner_Internal(dm, &cellRefiner);CHKERRQ(ierr);
4097     ierr = DMPlexRefineUniform_Internal(dm, cellRefiner, dmRefined);CHKERRQ(ierr);
4098     PetscFunctionReturn(0);
4099   }
4100   ierr = DMPlexGetRefinementLimit(dm, &refinementLimit);CHKERRQ(ierr);
4101   if (refinementLimit == 0.0) PetscFunctionReturn(0);
4102   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
4103   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
4104   ierr = PetscOptionsGetString(((PetscObject) dm)->prefix, "-dm_plex_generator", genname, 1024, &flg);CHKERRQ(ierr);
4105   if (flg) name = genname;
4106   if (name) {
4107     ierr = PetscStrcmp(name, "triangle", &isTriangle);CHKERRQ(ierr);
4108     ierr = PetscStrcmp(name, "tetgen",   &isTetgen);CHKERRQ(ierr);
4109     ierr = PetscStrcmp(name, "ctetgen",  &isCTetgen);CHKERRQ(ierr);
4110   }
4111   switch (dim) {
4112   case 2:
4113     if (!name || isTriangle) {
4114 #if defined(PETSC_HAVE_TRIANGLE)
4115       double  *maxVolumes;
4116       PetscInt c;
4117 
4118       ierr = PetscMalloc1((cEnd - cStart), &maxVolumes);CHKERRQ(ierr);
4119       for (c = 0; c < cEnd-cStart; ++c) maxVolumes[c] = refinementLimit;
4120       ierr = DMPlexRefine_Triangle(dm, maxVolumes, dmRefined);CHKERRQ(ierr);
4121       ierr = PetscFree(maxVolumes);CHKERRQ(ierr);
4122 #else
4123       SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Mesh refinement needs external package support.\nPlease reconfigure with --download-triangle.");
4124 #endif
4125     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Unknown 2D mesh generation package %s", name);
4126     break;
4127   case 3:
4128     if (!name || isCTetgen) {
4129 #if defined(PETSC_HAVE_CTETGEN)
4130       PetscReal *maxVolumes;
4131       PetscInt   c;
4132 
4133       ierr = PetscMalloc1((cEnd - cStart), &maxVolumes);CHKERRQ(ierr);
4134       for (c = 0; c < cEnd-cStart; ++c) maxVolumes[c] = refinementLimit;
4135       ierr = DMPlexRefine_CTetgen(dm, maxVolumes, dmRefined);CHKERRQ(ierr);
4136 #else
4137       SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "CTetgen needs external package support.\nPlease reconfigure with --download-ctetgen.");
4138 #endif
4139     } else if (isTetgen) {
4140 #if defined(PETSC_HAVE_TETGEN)
4141       double  *maxVolumes;
4142       PetscInt c;
4143 
4144       ierr = PetscMalloc1((cEnd - cStart), &maxVolumes);CHKERRQ(ierr);
4145       for (c = 0; c < cEnd-cStart; ++c) maxVolumes[c] = refinementLimit;
4146       ierr = DMPlexRefine_Tetgen(dm, maxVolumes, dmRefined);CHKERRQ(ierr);
4147 #else
4148       SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Tetgen needs external package support.\nPlease reconfigure with --with-c-language=cxx --download-tetgen.");
4149 #endif
4150     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Unknown 3D mesh generation package %s", name);
4151     break;
4152   default:
4153     SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Mesh refinement in dimension %d is not supported.", dim);
4154   }
4155   PetscFunctionReturn(0);
4156 }
4157 
4158 #undef __FUNCT__
4159 #define __FUNCT__ "DMRefineHierarchy_Plex"
4160 PetscErrorCode DMRefineHierarchy_Plex(DM dm, PetscInt nlevels, DM dmRefined[])
4161 {
4162   DM             cdm = dm;
4163   PetscInt       r;
4164   PetscBool      isUniform;
4165   PetscErrorCode ierr;
4166 
4167   PetscFunctionBegin;
4168   ierr = DMPlexGetRefinementUniform(dm, &isUniform);CHKERRQ(ierr);
4169   if (!isUniform) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Non-uniform refinement is incompatible with the hierarchy");
4170   for (r = 0; r < nlevels; ++r) {
4171     CellRefiner cellRefiner;
4172 
4173     ierr = DMPlexGetCellRefiner_Internal(cdm, &cellRefiner);CHKERRQ(ierr);
4174     ierr = DMPlexRefineUniform_Internal(cdm, cellRefiner, &dmRefined[r]);CHKERRQ(ierr);
4175     ierr = DMPlexSetCoarseDM(dmRefined[r], cdm);CHKERRQ(ierr);
4176     cdm  = dmRefined[r];
4177   }
4178   PetscFunctionReturn(0);
4179 }
4180 
4181 #undef __FUNCT__
4182 #define __FUNCT__ "DMCoarsen_Plex"
4183 PetscErrorCode DMCoarsen_Plex(DM dm, MPI_Comm comm, DM *dmCoarsened)
4184 {
4185   DM_Plex       *mesh = (DM_Plex*) dm->data;
4186   PetscErrorCode ierr;
4187 
4188   PetscFunctionBegin;
4189   ierr = PetscObjectReference((PetscObject) mesh->coarseMesh);CHKERRQ(ierr);
4190   *dmCoarsened = mesh->coarseMesh;
4191   PetscFunctionReturn(0);
4192 }
4193 
4194 #undef __FUNCT__
4195 #define __FUNCT__ "DMPlexGetDepthLabel"
4196 /*@
4197   DMPlexGetDepthLabel - Get the DMLabel recording the depth of each point
4198 
4199   Not Collective
4200 
4201   Input Parameter:
4202 . dm    - The DMPlex object
4203 
4204   Output Parameter:
4205 . depthLabel - The DMLabel recording point depth
4206 
4207   Level: developer
4208 
4209 .keywords: mesh, points
4210 .seealso: DMPlexGetDepth(), DMPlexGetHeightStratum(), DMPlexGetDepthStratum()
4211 @*/
4212 PetscErrorCode DMPlexGetDepthLabel(DM dm, DMLabel *depthLabel)
4213 {
4214   DM_Plex       *mesh = (DM_Plex*) dm->data;
4215   PetscErrorCode ierr;
4216 
4217   PetscFunctionBegin;
4218   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4219   PetscValidPointer(depthLabel, 2);
4220   if (!mesh->depthLabel) {ierr = DMPlexGetLabel(dm, "depth", &mesh->depthLabel);CHKERRQ(ierr);}
4221   *depthLabel = mesh->depthLabel;
4222   PetscFunctionReturn(0);
4223 }
4224 
4225 #undef __FUNCT__
4226 #define __FUNCT__ "DMPlexGetDepth"
4227 /*@
4228   DMPlexGetDepth - Get the depth of the DAG representing this mesh
4229 
4230   Not Collective
4231 
4232   Input Parameter:
4233 . dm    - The DMPlex object
4234 
4235   Output Parameter:
4236 . depth - The number of strata (breadth first levels) in the DAG
4237 
4238   Level: developer
4239 
4240 .keywords: mesh, points
4241 .seealso: DMPlexGetDepthLabel(), DMPlexGetHeightStratum(), DMPlexGetDepthStratum()
4242 @*/
4243 PetscErrorCode DMPlexGetDepth(DM dm, PetscInt *depth)
4244 {
4245   DMLabel        label;
4246   PetscInt       d = 0;
4247   PetscErrorCode ierr;
4248 
4249   PetscFunctionBegin;
4250   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4251   PetscValidPointer(depth, 2);
4252   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
4253   if (label) {ierr = DMLabelGetNumValues(label, &d);CHKERRQ(ierr);}
4254   *depth = d-1;
4255   PetscFunctionReturn(0);
4256 }
4257 
4258 #undef __FUNCT__
4259 #define __FUNCT__ "DMPlexGetDepthStratum"
4260 /*@
4261   DMPlexGetDepthStratum - Get the bounds [start, end) for all points at a certain depth.
4262 
4263   Not Collective
4264 
4265   Input Parameters:
4266 + dm           - The DMPlex object
4267 - stratumValue - The requested depth
4268 
4269   Output Parameters:
4270 + start - The first point at this depth
4271 - end   - One beyond the last point at this depth
4272 
4273   Level: developer
4274 
4275 .keywords: mesh, points
4276 .seealso: DMPlexGetHeightStratum(), DMPlexGetDepth()
4277 @*/
4278 PetscErrorCode DMPlexGetDepthStratum(DM dm, PetscInt stratumValue, PetscInt *start, PetscInt *end)
4279 {
4280   DMLabel        label;
4281   PetscInt       pStart, pEnd;
4282   PetscErrorCode ierr;
4283 
4284   PetscFunctionBegin;
4285   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4286   if (start) {PetscValidPointer(start, 3); *start = 0;}
4287   if (end)   {PetscValidPointer(end,   4); *end   = 0;}
4288   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
4289   if (pStart == pEnd) PetscFunctionReturn(0);
4290   if (stratumValue < 0) {
4291     if (start) *start = pStart;
4292     if (end)   *end   = pEnd;
4293     PetscFunctionReturn(0);
4294   }
4295   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
4296   if (!label) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "No label named depth was found");
4297   ierr = DMLabelGetStratumBounds(label, stratumValue, start, end);CHKERRQ(ierr);
4298   PetscFunctionReturn(0);
4299 }
4300 
4301 #undef __FUNCT__
4302 #define __FUNCT__ "DMPlexGetHeightStratum"
4303 /*@
4304   DMPlexGetHeightStratum - Get the bounds [start, end) for all points at a certain height.
4305 
4306   Not Collective
4307 
4308   Input Parameters:
4309 + dm           - The DMPlex object
4310 - stratumValue - The requested height
4311 
4312   Output Parameters:
4313 + start - The first point at this height
4314 - end   - One beyond the last point at this height
4315 
4316   Level: developer
4317 
4318 .keywords: mesh, points
4319 .seealso: DMPlexGetDepthStratum(), DMPlexGetDepth()
4320 @*/
4321 PetscErrorCode DMPlexGetHeightStratum(DM dm, PetscInt stratumValue, PetscInt *start, PetscInt *end)
4322 {
4323   DMLabel        label;
4324   PetscInt       depth, pStart, pEnd;
4325   PetscErrorCode ierr;
4326 
4327   PetscFunctionBegin;
4328   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4329   if (start) {PetscValidPointer(start, 3); *start = 0;}
4330   if (end)   {PetscValidPointer(end,   4); *end   = 0;}
4331   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
4332   if (pStart == pEnd) PetscFunctionReturn(0);
4333   if (stratumValue < 0) {
4334     if (start) *start = pStart;
4335     if (end)   *end   = pEnd;
4336     PetscFunctionReturn(0);
4337   }
4338   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
4339   if (!label) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "No label named depth was found");CHKERRQ(ierr);
4340   ierr = DMLabelGetNumValues(label, &depth);CHKERRQ(ierr);
4341   ierr = DMLabelGetStratumBounds(label, depth-1-stratumValue, start, end);CHKERRQ(ierr);
4342   PetscFunctionReturn(0);
4343 }
4344 
4345 #undef __FUNCT__
4346 #define __FUNCT__ "DMPlexCreateSectionInitial"
4347 /* Set the number of dof on each point and separate by fields */
4348 PetscErrorCode DMPlexCreateSectionInitial(DM dm, PetscInt dim, PetscInt numFields,const PetscInt numComp[],const PetscInt numDof[], PetscSection *section)
4349 {
4350   PetscInt      *numDofTot;
4351   PetscInt       depth, pStart = 0, pEnd = 0;
4352   PetscInt       p, d, dep, f;
4353   PetscErrorCode ierr;
4354 
4355   PetscFunctionBegin;
4356   ierr = PetscMalloc1((dim+1), &numDofTot);CHKERRQ(ierr);
4357   for (d = 0; d <= dim; ++d) {
4358     numDofTot[d] = 0;
4359     for (f = 0; f < numFields; ++f) numDofTot[d] += numDof[f*(dim+1)+d];
4360   }
4361   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), section);CHKERRQ(ierr);
4362   if (numFields > 0) {
4363     ierr = PetscSectionSetNumFields(*section, numFields);CHKERRQ(ierr);
4364     if (numComp) {
4365       for (f = 0; f < numFields; ++f) {
4366         ierr = PetscSectionSetFieldComponents(*section, f, numComp[f]);CHKERRQ(ierr);
4367       }
4368     }
4369   }
4370   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
4371   ierr = PetscSectionSetChart(*section, pStart, pEnd);CHKERRQ(ierr);
4372   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
4373   for (dep = 0; dep <= depth; ++dep) {
4374     d    = dim == depth ? dep : (!dep ? 0 : dim);
4375     ierr = DMPlexGetDepthStratum(dm, dep, &pStart, &pEnd);CHKERRQ(ierr);
4376     for (p = pStart; p < pEnd; ++p) {
4377       for (f = 0; f < numFields; ++f) {
4378         ierr = PetscSectionSetFieldDof(*section, p, f, numDof[f*(dim+1)+d]);CHKERRQ(ierr);
4379       }
4380       ierr = PetscSectionSetDof(*section, p, numDofTot[d]);CHKERRQ(ierr);
4381     }
4382   }
4383   ierr = PetscFree(numDofTot);CHKERRQ(ierr);
4384   PetscFunctionReturn(0);
4385 }
4386 
4387 #undef __FUNCT__
4388 #define __FUNCT__ "DMPlexCreateSectionBCDof"
4389 /* Set the number of dof on each point and separate by fields
4390    If constDof is PETSC_DETERMINE, constrain every dof on the point
4391 */
4392 PetscErrorCode DMPlexCreateSectionBCDof(DM dm, PetscInt numBC,const PetscInt bcField[],const IS bcPoints[], PetscInt constDof, PetscSection section)
4393 {
4394   PetscInt       numFields;
4395   PetscInt       bc;
4396   PetscErrorCode ierr;
4397 
4398   PetscFunctionBegin;
4399   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
4400   for (bc = 0; bc < numBC; ++bc) {
4401     PetscInt        field = 0;
4402     const PetscInt *idx;
4403     PetscInt        n, i;
4404 
4405     if (numFields) field = bcField[bc];
4406     ierr = ISGetLocalSize(bcPoints[bc], &n);CHKERRQ(ierr);
4407     ierr = ISGetIndices(bcPoints[bc], &idx);CHKERRQ(ierr);
4408     for (i = 0; i < n; ++i) {
4409       const PetscInt p        = idx[i];
4410       PetscInt       numConst = constDof;
4411 
4412       /* Constrain every dof on the point */
4413       if (numConst < 0) {
4414         if (numFields) {
4415           ierr = PetscSectionGetFieldDof(section, p, field, &numConst);CHKERRQ(ierr);
4416         } else {
4417           ierr = PetscSectionGetDof(section, p, &numConst);CHKERRQ(ierr);
4418         }
4419       }
4420       if (numFields) {
4421         ierr = PetscSectionAddFieldConstraintDof(section, p, field, numConst);CHKERRQ(ierr);
4422       }
4423       ierr = PetscSectionAddConstraintDof(section, p, numConst);CHKERRQ(ierr);
4424     }
4425     ierr = ISRestoreIndices(bcPoints[bc], &idx);CHKERRQ(ierr);
4426   }
4427   PetscFunctionReturn(0);
4428 }
4429 
4430 #undef __FUNCT__
4431 #define __FUNCT__ "DMPlexCreateSectionBCIndicesAll"
4432 /* Set the constrained indices on each point and separate by fields */
4433 PetscErrorCode DMPlexCreateSectionBCIndicesAll(DM dm, PetscSection section)
4434 {
4435   PetscInt      *maxConstraints;
4436   PetscInt       numFields, f, pStart = 0, pEnd = 0, p;
4437   PetscErrorCode ierr;
4438 
4439   PetscFunctionBegin;
4440   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
4441   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
4442   ierr = PetscMalloc1((numFields+1), &maxConstraints);CHKERRQ(ierr);
4443   for (f = 0; f <= numFields; ++f) maxConstraints[f] = 0;
4444   for (p = pStart; p < pEnd; ++p) {
4445     PetscInt cdof;
4446 
4447     if (numFields) {
4448       for (f = 0; f < numFields; ++f) {
4449         ierr              = PetscSectionGetFieldConstraintDof(section, p, f, &cdof);CHKERRQ(ierr);
4450         maxConstraints[f] = PetscMax(maxConstraints[f], cdof);
4451       }
4452     } else {
4453       ierr              = PetscSectionGetConstraintDof(section, p, &cdof);CHKERRQ(ierr);
4454       maxConstraints[0] = PetscMax(maxConstraints[0], cdof);
4455     }
4456   }
4457   for (f = 0; f < numFields; ++f) {
4458     maxConstraints[numFields] += maxConstraints[f];
4459   }
4460   if (maxConstraints[numFields]) {
4461     PetscInt *indices;
4462 
4463     ierr = PetscMalloc1(maxConstraints[numFields], &indices);CHKERRQ(ierr);
4464     for (p = pStart; p < pEnd; ++p) {
4465       PetscInt cdof, d;
4466 
4467       ierr = PetscSectionGetConstraintDof(section, p, &cdof);CHKERRQ(ierr);
4468       if (cdof) {
4469         if (cdof > maxConstraints[numFields]) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_LIB, "Likely memory corruption, point %D cDof %D > maxConstraints %D", p, cdof, maxConstraints[numFields]);
4470         if (numFields) {
4471           PetscInt numConst = 0, foff = 0;
4472 
4473           for (f = 0; f < numFields; ++f) {
4474             PetscInt cfdof, fdof;
4475 
4476             ierr = PetscSectionGetFieldDof(section, p, f, &fdof);CHKERRQ(ierr);
4477             ierr = PetscSectionGetFieldConstraintDof(section, p, f, &cfdof);CHKERRQ(ierr);
4478             /* Change constraint numbering from absolute local dof number to field relative local dof number */
4479             for (d = 0; d < cfdof; ++d) indices[numConst+d] = d;
4480             ierr = PetscSectionSetFieldConstraintIndices(section, p, f, &indices[numConst]);CHKERRQ(ierr);
4481             for (d = 0; d < cfdof; ++d) indices[numConst+d] += foff;
4482             numConst += cfdof;
4483             foff     += fdof;
4484           }
4485           if (cdof != numConst) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_LIB, "Total number of field constraints %D should be %D", numConst, cdof);
4486         } else {
4487           for (d = 0; d < cdof; ++d) indices[d] = d;
4488         }
4489         ierr = PetscSectionSetConstraintIndices(section, p, indices);CHKERRQ(ierr);
4490       }
4491     }
4492     ierr = PetscFree(indices);CHKERRQ(ierr);
4493   }
4494   ierr = PetscFree(maxConstraints);CHKERRQ(ierr);
4495   PetscFunctionReturn(0);
4496 }
4497 
4498 #undef __FUNCT__
4499 #define __FUNCT__ "DMPlexCreateSectionBCIndicesField"
4500 /* Set the constrained field indices on each point */
4501 PetscErrorCode DMPlexCreateSectionBCIndicesField(DM dm, PetscInt field, IS bcPoints, IS constraintIndices, PetscSection section)
4502 {
4503   const PetscInt *points, *indices;
4504   PetscInt        numFields, maxDof, numPoints, p, numConstraints;
4505   PetscErrorCode  ierr;
4506 
4507   PetscFunctionBegin;
4508   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
4509   if ((field < 0) || (field >= numFields)) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Section field %d should be in [%d, %d)", field, 0, numFields);
4510 
4511   ierr = ISGetLocalSize(bcPoints, &numPoints);CHKERRQ(ierr);
4512   ierr = ISGetIndices(bcPoints, &points);CHKERRQ(ierr);
4513   if (!constraintIndices) {
4514     PetscInt *idx, i;
4515 
4516     ierr = PetscSectionGetMaxDof(section, &maxDof);CHKERRQ(ierr);
4517     ierr = PetscMalloc1(maxDof, &idx);CHKERRQ(ierr);
4518     for (i = 0; i < maxDof; ++i) idx[i] = i;
4519     for (p = 0; p < numPoints; ++p) {
4520       ierr = PetscSectionSetFieldConstraintIndices(section, points[p], field, idx);CHKERRQ(ierr);
4521     }
4522     ierr = PetscFree(idx);CHKERRQ(ierr);
4523   } else {
4524     ierr = ISGetLocalSize(constraintIndices, &numConstraints);CHKERRQ(ierr);
4525     ierr = ISGetIndices(constraintIndices, &indices);CHKERRQ(ierr);
4526     for (p = 0; p < numPoints; ++p) {
4527       PetscInt fcdof;
4528 
4529       ierr = PetscSectionGetFieldConstraintDof(section, points[p], field, &fcdof);CHKERRQ(ierr);
4530       if (fcdof != numConstraints) SETERRQ4(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Section point %d field %d has %d constraints, but yo ugave %d indices", p, field, fcdof, numConstraints);
4531       ierr = PetscSectionSetFieldConstraintIndices(section, points[p], field, indices);CHKERRQ(ierr);
4532     }
4533     ierr = ISRestoreIndices(constraintIndices, &indices);CHKERRQ(ierr);
4534   }
4535   ierr = ISRestoreIndices(bcPoints, &points);CHKERRQ(ierr);
4536   PetscFunctionReturn(0);
4537 }
4538 
4539 #undef __FUNCT__
4540 #define __FUNCT__ "DMPlexCreateSectionBCIndices"
4541 /* Set the constrained indices on each point and separate by fields */
4542 PetscErrorCode DMPlexCreateSectionBCIndices(DM dm, PetscSection section)
4543 {
4544   PetscInt      *indices;
4545   PetscInt       numFields, maxDof, f, pStart = 0, pEnd = 0, p;
4546   PetscErrorCode ierr;
4547 
4548   PetscFunctionBegin;
4549   ierr = PetscSectionGetMaxDof(section, &maxDof);CHKERRQ(ierr);
4550   ierr = PetscMalloc1(maxDof, &indices);CHKERRQ(ierr);
4551   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
4552   if (!numFields) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "This function only works after users have set field constraint indices.");
4553   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
4554   for (p = pStart; p < pEnd; ++p) {
4555     PetscInt cdof, d;
4556 
4557     ierr = PetscSectionGetConstraintDof(section, p, &cdof);CHKERRQ(ierr);
4558     if (cdof) {
4559       PetscInt numConst = 0, foff = 0;
4560 
4561       for (f = 0; f < numFields; ++f) {
4562         const PetscInt *fcind;
4563         PetscInt        fdof, fcdof;
4564 
4565         ierr = PetscSectionGetFieldDof(section, p, f, &fdof);CHKERRQ(ierr);
4566         ierr = PetscSectionGetFieldConstraintDof(section, p, f, &fcdof);CHKERRQ(ierr);
4567         if (fcdof) {ierr = PetscSectionGetFieldConstraintIndices(section, p, f, &fcind);CHKERRQ(ierr);}
4568         /* Change constraint numbering from field relative local dof number to absolute local dof number */
4569         for (d = 0; d < fcdof; ++d) indices[numConst+d] = fcind[d]+foff;
4570         foff     += fdof;
4571         numConst += fcdof;
4572       }
4573       if (cdof != numConst) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_LIB, "Total number of field constraints %D should be %D", numConst, cdof);
4574       ierr = PetscSectionSetConstraintIndices(section, p, indices);CHKERRQ(ierr);
4575     }
4576   }
4577   ierr = PetscFree(indices);CHKERRQ(ierr);
4578   PetscFunctionReturn(0);
4579 }
4580 
4581 #undef __FUNCT__
4582 #define __FUNCT__ "DMPlexCreateSection"
4583 /*@C
4584   DMPlexCreateSection - Create a PetscSection based upon the dof layout specification provided.
4585 
4586   Not Collective
4587 
4588   Input Parameters:
4589 + dm        - The DMPlex object
4590 . dim       - The spatial dimension of the problem
4591 . numFields - The number of fields in the problem
4592 . numComp   - An array of size numFields that holds the number of components for each field
4593 . numDof    - An array of size numFields*(dim+1) which holds the number of dof for each field on a mesh piece of dimension d
4594 . numBC     - The number of boundary conditions
4595 . bcField   - An array of size numBC giving the field number for each boundry condition
4596 - bcPoints  - An array of size numBC giving an IS holding the sieve points to which each boundary condition applies
4597 
4598   Output Parameter:
4599 . section - The PetscSection object
4600 
4601   Notes: numDof[f*(dim+1)+d] gives the number of dof for field f on sieve points of dimension d. For instance, numDof[1] is the
4602   nubmer of dof for field 0 on each edge.
4603 
4604   Level: developer
4605 
4606   Fortran Notes:
4607   A Fortran 90 version is available as DMPlexCreateSectionF90()
4608 
4609 .keywords: mesh, elements
4610 .seealso: DMPlexCreate(), PetscSectionCreate()
4611 @*/
4612 PetscErrorCode DMPlexCreateSection(DM dm, PetscInt dim, PetscInt numFields,const PetscInt numComp[],const PetscInt numDof[], PetscInt numBC,const PetscInt bcField[],const IS bcPoints[], PetscSection *section)
4613 {
4614   PetscErrorCode ierr;
4615 
4616   PetscFunctionBegin;
4617   ierr = DMPlexCreateSectionInitial(dm, dim, numFields, numComp, numDof, section);CHKERRQ(ierr);
4618   ierr = DMPlexCreateSectionBCDof(dm, numBC, bcField, bcPoints, PETSC_DETERMINE, *section);CHKERRQ(ierr);
4619   ierr = PetscSectionSetUp(*section);CHKERRQ(ierr);
4620   if (numBC) {ierr = DMPlexCreateSectionBCIndicesAll(dm, *section);CHKERRQ(ierr);}
4621   ierr = PetscSectionViewFromOptions(*section,NULL,"-section_view");CHKERRQ(ierr);
4622   PetscFunctionReturn(0);
4623 }
4624 
4625 #undef __FUNCT__
4626 #define __FUNCT__ "DMCreateCoordinateDM_Plex"
4627 PetscErrorCode DMCreateCoordinateDM_Plex(DM dm, DM *cdm)
4628 {
4629   PetscSection   section;
4630   PetscErrorCode ierr;
4631 
4632   PetscFunctionBegin;
4633   ierr = DMClone(dm, cdm);CHKERRQ(ierr);
4634   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &section);CHKERRQ(ierr);
4635   ierr = DMSetDefaultSection(*cdm, section);CHKERRQ(ierr);
4636   ierr = PetscSectionDestroy(&section);CHKERRQ(ierr);
4637   PetscFunctionReturn(0);
4638 }
4639 
4640 #undef __FUNCT__
4641 #define __FUNCT__ "DMPlexGetConeSection"
4642 PetscErrorCode DMPlexGetConeSection(DM dm, PetscSection *section)
4643 {
4644   DM_Plex *mesh = (DM_Plex*) dm->data;
4645 
4646   PetscFunctionBegin;
4647   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4648   if (section) *section = mesh->coneSection;
4649   PetscFunctionReturn(0);
4650 }
4651 
4652 #undef __FUNCT__
4653 #define __FUNCT__ "DMPlexGetSupportSection"
4654 PetscErrorCode DMPlexGetSupportSection(DM dm, PetscSection *section)
4655 {
4656   DM_Plex *mesh = (DM_Plex*) dm->data;
4657 
4658   PetscFunctionBegin;
4659   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4660   if (section) *section = mesh->supportSection;
4661   PetscFunctionReturn(0);
4662 }
4663 
4664 #undef __FUNCT__
4665 #define __FUNCT__ "DMPlexGetCones"
4666 PetscErrorCode DMPlexGetCones(DM dm, PetscInt *cones[])
4667 {
4668   DM_Plex *mesh = (DM_Plex*) dm->data;
4669 
4670   PetscFunctionBegin;
4671   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4672   if (cones) *cones = mesh->cones;
4673   PetscFunctionReturn(0);
4674 }
4675 
4676 #undef __FUNCT__
4677 #define __FUNCT__ "DMPlexGetConeOrientations"
4678 PetscErrorCode DMPlexGetConeOrientations(DM dm, PetscInt *coneOrientations[])
4679 {
4680   DM_Plex *mesh = (DM_Plex*) dm->data;
4681 
4682   PetscFunctionBegin;
4683   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4684   if (coneOrientations) *coneOrientations = mesh->coneOrientations;
4685   PetscFunctionReturn(0);
4686 }
4687 
4688 /******************************** FEM Support **********************************/
4689 
4690 #undef __FUNCT__
4691 #define __FUNCT__ "DMPlexVecGetClosure_Depth1_Static"
4692 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecGetClosure_Depth1_Static(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
4693 {
4694   PetscScalar    *array, *vArray;
4695   const PetscInt *cone, *coneO;
4696   PetscInt        pStart, pEnd, p, numPoints, size = 0, offset = 0;
4697   PetscErrorCode  ierr;
4698 
4699   PetscFunctionBeginHot;
4700   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
4701   ierr = DMPlexGetConeSize(dm, point, &numPoints);CHKERRQ(ierr);
4702   ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
4703   ierr = DMPlexGetConeOrientation(dm, point, &coneO);CHKERRQ(ierr);
4704   if (!values || !*values) {
4705     if ((point >= pStart) && (point < pEnd)) {
4706       PetscInt dof;
4707 
4708       ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4709       size += dof;
4710     }
4711     for (p = 0; p < numPoints; ++p) {
4712       const PetscInt cp = cone[p];
4713       PetscInt       dof;
4714 
4715       if ((cp < pStart) || (cp >= pEnd)) continue;
4716       ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
4717       size += dof;
4718     }
4719     if (!values) {
4720       if (csize) *csize = size;
4721       PetscFunctionReturn(0);
4722     }
4723     ierr = DMGetWorkArray(dm, size, PETSC_SCALAR, &array);CHKERRQ(ierr);
4724   } else {
4725     array = *values;
4726   }
4727   size = 0;
4728   ierr = VecGetArray(v, &vArray);CHKERRQ(ierr);
4729   if ((point >= pStart) && (point < pEnd)) {
4730     PetscInt     dof, off, d;
4731     PetscScalar *varr;
4732 
4733     ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4734     ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
4735     varr = &vArray[off];
4736     for (d = 0; d < dof; ++d, ++offset) {
4737       array[offset] = varr[d];
4738     }
4739     size += dof;
4740   }
4741   for (p = 0; p < numPoints; ++p) {
4742     const PetscInt cp = cone[p];
4743     PetscInt       o  = coneO[p];
4744     PetscInt       dof, off, d;
4745     PetscScalar   *varr;
4746 
4747     if ((cp < pStart) || (cp >= pEnd)) continue;
4748     ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
4749     ierr = PetscSectionGetOffset(section, cp, &off);CHKERRQ(ierr);
4750     varr = &vArray[off];
4751     if (o >= 0) {
4752       for (d = 0; d < dof; ++d, ++offset) {
4753         array[offset] = varr[d];
4754       }
4755     } else {
4756       for (d = dof-1; d >= 0; --d, ++offset) {
4757         array[offset] = varr[d];
4758       }
4759     }
4760     size += dof;
4761   }
4762   ierr = VecRestoreArray(v, &vArray);CHKERRQ(ierr);
4763   if (!*values) {
4764     if (csize) *csize = size;
4765     *values = array;
4766   } else {
4767     if (size > *csize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Size of input array %d < actual size %d", *csize, size);
4768     *csize = size;
4769   }
4770   PetscFunctionReturn(0);
4771 }
4772 
4773 #undef __FUNCT__
4774 #define __FUNCT__ "DMPlexVecGetClosure_Static"
4775 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecGetClosure_Static(PetscSection section, PetscInt numPoints, const PetscInt points[], const PetscScalar vArray[], PetscInt *size, PetscScalar array[])
4776 {
4777   PetscInt       offset = 0, p;
4778   PetscErrorCode ierr;
4779 
4780   PetscFunctionBeginHot;
4781   *size = 0;
4782   for (p = 0; p < numPoints*2; p += 2) {
4783     const PetscInt point = points[p];
4784     const PetscInt o     = points[p+1];
4785     PetscInt       dof, off, d;
4786     const PetscScalar *varr;
4787 
4788     ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4789     ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
4790     varr = &vArray[off];
4791     if (o >= 0) {
4792       for (d = 0; d < dof; ++d, ++offset)    array[offset] = varr[d];
4793     } else {
4794       for (d = dof-1; d >= 0; --d, ++offset) array[offset] = varr[d];
4795     }
4796   }
4797   *size = offset;
4798   PetscFunctionReturn(0);
4799 }
4800 
4801 #undef __FUNCT__
4802 #define __FUNCT__ "DMPlexVecGetClosure_Fields_Static"
4803 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecGetClosure_Fields_Static(PetscSection section, PetscInt numPoints, const PetscInt points[], PetscInt numFields, const PetscScalar vArray[], PetscInt *size, PetscScalar array[])
4804 {
4805   PetscInt       offset = 0, f;
4806   PetscErrorCode ierr;
4807 
4808   PetscFunctionBeginHot;
4809   *size = 0;
4810   for (f = 0; f < numFields; ++f) {
4811     PetscInt fcomp, p;
4812 
4813     ierr = PetscSectionGetFieldComponents(section, f, &fcomp);CHKERRQ(ierr);
4814     for (p = 0; p < numPoints*2; p += 2) {
4815       const PetscInt point = points[p];
4816       const PetscInt o     = points[p+1];
4817       PetscInt       fdof, foff, d, c;
4818       const PetscScalar *varr;
4819 
4820       ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
4821       ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
4822       varr = &vArray[foff];
4823       if (o >= 0) {
4824         for (d = 0; d < fdof; ++d, ++offset) array[offset] = varr[d];
4825       } else {
4826         for (d = fdof/fcomp-1; d >= 0; --d) {
4827           for (c = 0; c < fcomp; ++c, ++offset) {
4828             array[offset] = varr[d*fcomp+c];
4829           }
4830         }
4831       }
4832     }
4833   }
4834   *size = offset;
4835   PetscFunctionReturn(0);
4836 }
4837 
4838 #undef __FUNCT__
4839 #define __FUNCT__ "DMPlexVecGetClosure"
4840 /*@C
4841   DMPlexVecGetClosure - Get an array of the values on the closure of 'point'
4842 
4843   Not collective
4844 
4845   Input Parameters:
4846 + dm - The DM
4847 . section - The section describing the layout in v, or NULL to use the default section
4848 . v - The local vector
4849 - point - The sieve point in the DM
4850 
4851   Output Parameters:
4852 + csize - The number of values in the closure, or NULL
4853 - values - The array of values, which is a borrowed array and should not be freed
4854 
4855   Fortran Notes:
4856   Since it returns an array, this routine is only available in Fortran 90, and you must
4857   include petsc.h90 in your code.
4858 
4859   The csize argument is not present in the Fortran 90 binding since it is internal to the array.
4860 
4861   Level: intermediate
4862 
4863 .seealso DMPlexVecRestoreClosure(), DMPlexVecSetClosure(), DMPlexMatSetClosure()
4864 @*/
4865 PetscErrorCode DMPlexVecGetClosure(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
4866 {
4867   PetscSection    clSection;
4868   IS              clPoints;
4869   PetscScalar    *array, *vArray;
4870   PetscInt       *points = NULL;
4871   const PetscInt *clp;
4872   PetscInt        depth, numFields, numPoints, size;
4873   PetscErrorCode  ierr;
4874 
4875   PetscFunctionBeginHot;
4876   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4877   if (!section) {ierr = DMGetDefaultSection(dm, &section);CHKERRQ(ierr);}
4878   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
4879   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
4880   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
4881   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
4882   if (depth == 1 && numFields < 2) {
4883     ierr = DMPlexVecGetClosure_Depth1_Static(dm, section, v, point, csize, values);CHKERRQ(ierr);
4884     PetscFunctionReturn(0);
4885   }
4886   /* Get points */
4887   ierr = PetscSectionGetClosureIndex(section, (PetscObject) dm, &clSection, &clPoints);CHKERRQ(ierr);
4888   if (!clPoints) {
4889     PetscInt pStart, pEnd, p, q;
4890 
4891     ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
4892     ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
4893     /* Compress out points not in the section */
4894     for (p = 0, q = 0; p < numPoints*2; p += 2) {
4895       if ((points[p] >= pStart) && (points[p] < pEnd)) {
4896         points[q*2]   = points[p];
4897         points[q*2+1] = points[p+1];
4898         ++q;
4899       }
4900     }
4901     numPoints = q;
4902   } else {
4903     PetscInt dof, off;
4904 
4905     ierr = PetscSectionGetDof(clSection, point, &dof);CHKERRQ(ierr);
4906     ierr = PetscSectionGetOffset(clSection, point, &off);CHKERRQ(ierr);
4907     ierr = ISGetIndices(clPoints, &clp);CHKERRQ(ierr);
4908     numPoints = dof/2;
4909     points    = (PetscInt *) &clp[off];
4910   }
4911   /* Get array */
4912   if (!values || !*values) {
4913     PetscInt asize = 0, dof, p;
4914 
4915     for (p = 0; p < numPoints*2; p += 2) {
4916       ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
4917       asize += dof;
4918     }
4919     if (!values) {
4920       if (!clPoints) {ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);}
4921       else           {ierr = ISRestoreIndices(clPoints, &clp);CHKERRQ(ierr);}
4922       if (csize) *csize = asize;
4923       PetscFunctionReturn(0);
4924     }
4925     ierr = DMGetWorkArray(dm, asize, PETSC_SCALAR, &array);CHKERRQ(ierr);
4926   } else {
4927     array = *values;
4928   }
4929   ierr = VecGetArray(v, &vArray);CHKERRQ(ierr);
4930   /* Get values */
4931   if (numFields > 0) {ierr = DMPlexVecGetClosure_Fields_Static(section, numPoints, points, numFields, vArray, &size, array);CHKERRQ(ierr);}
4932   else               {ierr = DMPlexVecGetClosure_Static(section, numPoints, points, vArray, &size, array);CHKERRQ(ierr);}
4933   /* Cleanup points */
4934   if (!clPoints) {ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);}
4935   else           {ierr = ISRestoreIndices(clPoints, &clp);CHKERRQ(ierr);}
4936   /* Cleanup array */
4937   ierr = VecRestoreArray(v, &vArray);CHKERRQ(ierr);
4938   if (!*values) {
4939     if (csize) *csize = size;
4940     *values = array;
4941   } else {
4942     if (size > *csize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Size of input array %d < actual size %d", *csize, size);
4943     *csize = size;
4944   }
4945   PetscFunctionReturn(0);
4946 }
4947 
4948 #undef __FUNCT__
4949 #define __FUNCT__ "DMPlexVecRestoreClosure"
4950 /*@C
4951   DMPlexVecRestoreClosure - Restore the array of the values on the closure of 'point'
4952 
4953   Not collective
4954 
4955   Input Parameters:
4956 + dm - The DM
4957 . section - The section describing the layout in v, or NULL to use the default section
4958 . v - The local vector
4959 . point - The sieve point in the DM
4960 . csize - The number of values in the closure, or NULL
4961 - values - The array of values, which is a borrowed array and should not be freed
4962 
4963   Fortran Notes:
4964   Since it returns an array, this routine is only available in Fortran 90, and you must
4965   include petsc.h90 in your code.
4966 
4967   The csize argument is not present in the Fortran 90 binding since it is internal to the array.
4968 
4969   Level: intermediate
4970 
4971 .seealso DMPlexVecGetClosure(), DMPlexVecSetClosure(), DMPlexMatSetClosure()
4972 @*/
4973 PetscErrorCode DMPlexVecRestoreClosure(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
4974 {
4975   PetscInt       size = 0;
4976   PetscErrorCode ierr;
4977 
4978   PetscFunctionBegin;
4979   /* Should work without recalculating size */
4980   ierr = DMRestoreWorkArray(dm, size, PETSC_SCALAR, (void*) values);CHKERRQ(ierr);
4981   PetscFunctionReturn(0);
4982 }
4983 
4984 PETSC_STATIC_INLINE void add   (PetscScalar *x, PetscScalar y) {*x += y;}
4985 PETSC_STATIC_INLINE void insert(PetscScalar *x, PetscScalar y) {*x  = y;}
4986 
4987 #undef __FUNCT__
4988 #define __FUNCT__ "updatePoint_private"
4989 PETSC_STATIC_INLINE PetscErrorCode updatePoint_private(PetscSection section, PetscInt point, PetscInt dof, void (*fuse)(PetscScalar*, PetscScalar), PetscBool setBC, PetscInt orientation, const PetscScalar values[], PetscScalar array[])
4990 {
4991   PetscInt        cdof;   /* The number of constraints on this point */
4992   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
4993   PetscScalar    *a;
4994   PetscInt        off, cind = 0, k;
4995   PetscErrorCode  ierr;
4996 
4997   PetscFunctionBegin;
4998   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
4999   ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
5000   a    = &array[off];
5001   if (!cdof || setBC) {
5002     if (orientation >= 0) {
5003       for (k = 0; k < dof; ++k) {
5004         fuse(&a[k], values[k]);
5005       }
5006     } else {
5007       for (k = 0; k < dof; ++k) {
5008         fuse(&a[k], values[dof-k-1]);
5009       }
5010     }
5011   } else {
5012     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
5013     if (orientation >= 0) {
5014       for (k = 0; k < dof; ++k) {
5015         if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
5016         fuse(&a[k], values[k]);
5017       }
5018     } else {
5019       for (k = 0; k < dof; ++k) {
5020         if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
5021         fuse(&a[k], values[dof-k-1]);
5022       }
5023     }
5024   }
5025   PetscFunctionReturn(0);
5026 }
5027 
5028 #undef __FUNCT__
5029 #define __FUNCT__ "updatePointBC_private"
5030 PETSC_STATIC_INLINE PetscErrorCode updatePointBC_private(PetscSection section, PetscInt point, PetscInt dof, void (*fuse)(PetscScalar*, PetscScalar), PetscInt orientation, const PetscScalar values[], PetscScalar array[])
5031 {
5032   PetscInt        cdof;   /* The number of constraints on this point */
5033   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
5034   PetscScalar    *a;
5035   PetscInt        off, cind = 0, k;
5036   PetscErrorCode  ierr;
5037 
5038   PetscFunctionBegin;
5039   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
5040   ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
5041   a    = &array[off];
5042   if (cdof) {
5043     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
5044     if (orientation >= 0) {
5045       for (k = 0; k < dof; ++k) {
5046         if ((cind < cdof) && (k == cdofs[cind])) {
5047           fuse(&a[k], values[k]);
5048           ++cind;
5049         }
5050       }
5051     } else {
5052       for (k = 0; k < dof; ++k) {
5053         if ((cind < cdof) && (k == cdofs[cind])) {
5054           fuse(&a[k], values[dof-k-1]);
5055           ++cind;
5056         }
5057       }
5058     }
5059   }
5060   PetscFunctionReturn(0);
5061 }
5062 
5063 #undef __FUNCT__
5064 #define __FUNCT__ "updatePointFields_private"
5065 PETSC_STATIC_INLINE PetscErrorCode updatePointFields_private(PetscSection section, PetscInt point, PetscInt o, PetscInt f, PetscInt fcomp, void (*fuse)(PetscScalar*, PetscScalar), PetscBool setBC, const PetscScalar values[], PetscInt *offset, PetscScalar array[])
5066 {
5067   PetscScalar    *a;
5068   PetscInt        fdof, foff, fcdof, foffset = *offset;
5069   const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
5070   PetscInt        cind = 0, k, c;
5071   PetscErrorCode  ierr;
5072 
5073   PetscFunctionBegin;
5074   ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
5075   ierr = PetscSectionGetFieldConstraintDof(section, point, f, &fcdof);CHKERRQ(ierr);
5076   ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
5077   a    = &array[foff];
5078   if (!fcdof || setBC) {
5079     if (o >= 0) {
5080       for (k = 0; k < fdof; ++k) fuse(&a[k], values[foffset+k]);
5081     } else {
5082       for (k = fdof/fcomp-1; k >= 0; --k) {
5083         for (c = 0; c < fcomp; ++c) {
5084           fuse(&a[(fdof/fcomp-1-k)*fcomp+c], values[foffset+k*fcomp+c]);
5085         }
5086       }
5087     }
5088   } else {
5089     ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
5090     if (o >= 0) {
5091       for (k = 0; k < fdof; ++k) {
5092         if ((cind < fcdof) && (k == fcdofs[cind])) {++cind; continue;}
5093         fuse(&a[k], values[foffset+k]);
5094       }
5095     } else {
5096       for (k = fdof/fcomp-1; k >= 0; --k) {
5097         for (c = 0; c < fcomp; ++c) {
5098           if ((cind < fcdof) && (k*fcomp+c == fcdofs[cind])) {++cind; continue;}
5099           fuse(&a[(fdof/fcomp-1-k)*fcomp+c], values[foffset+k*fcomp+c]);
5100         }
5101       }
5102     }
5103   }
5104   *offset += fdof;
5105   PetscFunctionReturn(0);
5106 }
5107 
5108 #undef __FUNCT__
5109 #define __FUNCT__ "updatePointFieldsBC_private"
5110 PETSC_STATIC_INLINE PetscErrorCode updatePointFieldsBC_private(PetscSection section, PetscInt point, PetscInt o, PetscInt f, PetscInt fcomp, void (*fuse)(PetscScalar*, PetscScalar), const PetscScalar values[], PetscInt *offset, PetscScalar array[])
5111 {
5112   PetscScalar    *a;
5113   PetscInt        fdof, foff, fcdof, foffset = *offset;
5114   const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
5115   PetscInt        cind = 0, k, c;
5116   PetscErrorCode  ierr;
5117 
5118   PetscFunctionBegin;
5119   ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
5120   ierr = PetscSectionGetFieldConstraintDof(section, point, f, &fcdof);CHKERRQ(ierr);
5121   ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
5122   a    = &array[foff];
5123   if (fcdof) {
5124     ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
5125     if (o >= 0) {
5126       for (k = 0; k < fdof; ++k) {
5127         if ((cind < fcdof) && (k == fcdofs[cind])) {
5128           fuse(&a[k], values[foffset+k]);
5129           ++cind;
5130         }
5131       }
5132     } else {
5133       for (k = fdof/fcomp-1; k >= 0; --k) {
5134         for (c = 0; c < fcomp; ++c) {
5135           if ((cind < fcdof) && (k*fcomp+c == fcdofs[cind])) {
5136             fuse(&a[(fdof/fcomp-1-k)*fcomp+c], values[foffset+k*fcomp+c]);
5137             ++cind;
5138           }
5139         }
5140       }
5141     }
5142   }
5143   *offset += fdof;
5144   PetscFunctionReturn(0);
5145 }
5146 
5147 #undef __FUNCT__
5148 #define __FUNCT__ "DMPlexVecSetClosure_Static"
5149 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecSetClosure_Static(DM dm, PetscSection section, Vec v, PetscInt point, const PetscScalar values[], InsertMode mode)
5150 {
5151   PetscScalar    *array;
5152   const PetscInt *cone, *coneO;
5153   PetscInt        pStart, pEnd, p, numPoints, off, dof;
5154   PetscErrorCode  ierr;
5155 
5156   PetscFunctionBeginHot;
5157   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
5158   ierr = DMPlexGetConeSize(dm, point, &numPoints);CHKERRQ(ierr);
5159   ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
5160   ierr = DMPlexGetConeOrientation(dm, point, &coneO);CHKERRQ(ierr);
5161   ierr = VecGetArray(v, &array);CHKERRQ(ierr);
5162   for (p = 0, off = 0; p <= numPoints; ++p, off += dof) {
5163     const PetscInt cp = !p ? point : cone[p-1];
5164     const PetscInt o  = !p ? 0     : coneO[p-1];
5165 
5166     if ((cp < pStart) || (cp >= pEnd)) {dof = 0; continue;}
5167     ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
5168     /* ADD_VALUES */
5169     {
5170       const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
5171       PetscScalar    *a;
5172       PetscInt        cdof, coff, cind = 0, k;
5173 
5174       ierr = PetscSectionGetConstraintDof(section, cp, &cdof);CHKERRQ(ierr);
5175       ierr = PetscSectionGetOffset(section, cp, &coff);CHKERRQ(ierr);
5176       a    = &array[coff];
5177       if (!cdof) {
5178         if (o >= 0) {
5179           for (k = 0; k < dof; ++k) {
5180             a[k] += values[off+k];
5181           }
5182         } else {
5183           for (k = 0; k < dof; ++k) {
5184             a[k] += values[off+dof-k-1];
5185           }
5186         }
5187       } else {
5188         ierr = PetscSectionGetConstraintIndices(section, cp, &cdofs);CHKERRQ(ierr);
5189         if (o >= 0) {
5190           for (k = 0; k < dof; ++k) {
5191             if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
5192             a[k] += values[off+k];
5193           }
5194         } else {
5195           for (k = 0; k < dof; ++k) {
5196             if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
5197             a[k] += values[off+dof-k-1];
5198           }
5199         }
5200       }
5201     }
5202   }
5203   ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
5204   PetscFunctionReturn(0);
5205 }
5206 
5207 #undef __FUNCT__
5208 #define __FUNCT__ "DMPlexVecSetClosure"
5209 /*@C
5210   DMPlexVecSetClosure - Set an array of the values on the closure of 'point'
5211 
5212   Not collective
5213 
5214   Input Parameters:
5215 + dm - The DM
5216 . section - The section describing the layout in v, or NULL to use the default section
5217 . v - The local vector
5218 . point - The sieve point in the DM
5219 . values - The array of values
5220 - mode - The insert mode, where INSERT_ALL_VALUES and ADD_ALL_VALUES also overwrite boundary conditions
5221 
5222   Fortran Notes:
5223   This routine is only available in Fortran 90, and you must include petsc.h90 in your code.
5224 
5225   Level: intermediate
5226 
5227 .seealso DMPlexVecGetClosure(), DMPlexMatSetClosure()
5228 @*/
5229 PetscErrorCode DMPlexVecSetClosure(DM dm, PetscSection section, Vec v, PetscInt point, const PetscScalar values[], InsertMode mode)
5230 {
5231   PetscSection    clSection;
5232   IS              clPoints;
5233   PetscScalar    *array;
5234   PetscInt       *points = NULL;
5235   const PetscInt *clp;
5236   PetscInt        depth, numFields, numPoints, p;
5237   PetscErrorCode  ierr;
5238 
5239   PetscFunctionBeginHot;
5240   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5241   if (!section) {ierr = DMGetDefaultSection(dm, &section);CHKERRQ(ierr);}
5242   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
5243   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
5244   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
5245   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5246   if (depth == 1 && numFields < 2 && mode == ADD_VALUES) {
5247     ierr = DMPlexVecSetClosure_Static(dm, section, v, point, values, mode);CHKERRQ(ierr);
5248     PetscFunctionReturn(0);
5249   }
5250   /* Get points */
5251   ierr = PetscSectionGetClosureIndex(section, (PetscObject) dm, &clSection, &clPoints);CHKERRQ(ierr);
5252   if (!clPoints) {
5253     PetscInt pStart, pEnd, q;
5254 
5255     ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
5256     ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
5257     /* Compress out points not in the section */
5258     for (p = 0, q = 0; p < numPoints*2; p += 2) {
5259       if ((points[p] >= pStart) && (points[p] < pEnd)) {
5260         points[q*2]   = points[p];
5261         points[q*2+1] = points[p+1];
5262         ++q;
5263       }
5264     }
5265     numPoints = q;
5266   } else {
5267     PetscInt dof, off;
5268 
5269     ierr = PetscSectionGetDof(clSection, point, &dof);CHKERRQ(ierr);
5270     ierr = PetscSectionGetOffset(clSection, point, &off);CHKERRQ(ierr);
5271     ierr = ISGetIndices(clPoints, &clp);CHKERRQ(ierr);
5272     numPoints = dof/2;
5273     points    = (PetscInt *) &clp[off];
5274   }
5275   /* Get array */
5276   ierr = VecGetArray(v, &array);CHKERRQ(ierr);
5277   /* Get values */
5278   if (numFields > 0) {
5279     PetscInt offset = 0, fcomp, f;
5280     for (f = 0; f < numFields; ++f) {
5281       ierr = PetscSectionGetFieldComponents(section, f, &fcomp);CHKERRQ(ierr);
5282       switch (mode) {
5283       case INSERT_VALUES:
5284         for (p = 0; p < numPoints*2; p += 2) {
5285           const PetscInt point = points[p];
5286           const PetscInt o     = points[p+1];
5287           updatePointFields_private(section, point, o, f, fcomp, insert, PETSC_FALSE, values, &offset, array);
5288         } break;
5289       case INSERT_ALL_VALUES:
5290         for (p = 0; p < numPoints*2; p += 2) {
5291           const PetscInt point = points[p];
5292           const PetscInt o     = points[p+1];
5293           updatePointFields_private(section, point, o, f, fcomp, insert, PETSC_TRUE, values, &offset, array);
5294         } break;
5295       case INSERT_BC_VALUES:
5296         for (p = 0; p < numPoints*2; p += 2) {
5297           const PetscInt point = points[p];
5298           const PetscInt o     = points[p+1];
5299           updatePointFieldsBC_private(section, point, o, f, fcomp, insert, values, &offset, array);
5300         } break;
5301       case ADD_VALUES:
5302         for (p = 0; p < numPoints*2; p += 2) {
5303           const PetscInt point = points[p];
5304           const PetscInt o     = points[p+1];
5305           updatePointFields_private(section, point, o, f, fcomp, add, PETSC_FALSE, values, &offset, array);
5306         } break;
5307       case ADD_ALL_VALUES:
5308         for (p = 0; p < numPoints*2; p += 2) {
5309           const PetscInt point = points[p];
5310           const PetscInt o     = points[p+1];
5311           updatePointFields_private(section, point, o, f, fcomp, add, PETSC_TRUE, values, &offset, array);
5312         } break;
5313       default:
5314         SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %D", mode);
5315       }
5316     }
5317   } else {
5318     PetscInt dof, off;
5319 
5320     switch (mode) {
5321     case INSERT_VALUES:
5322       for (p = 0, off = 0; p < numPoints*2; p += 2, off += dof) {
5323         PetscInt o = points[p+1];
5324         ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5325         updatePoint_private(section, points[p], dof, insert, PETSC_FALSE, o, &values[off], array);
5326       } break;
5327     case INSERT_ALL_VALUES:
5328       for (p = 0, off = 0; p < numPoints*2; p += 2, off += dof) {
5329         PetscInt o = points[p+1];
5330         ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5331         updatePoint_private(section, points[p], dof, insert, PETSC_TRUE,  o, &values[off], array);
5332       } break;
5333     case INSERT_BC_VALUES:
5334       for (p = 0, off = 0; p < numPoints*2; p += 2, off += dof) {
5335         PetscInt o = points[p+1];
5336         ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5337         updatePointBC_private(section, points[p], dof, insert,  o, &values[off], array);
5338       } break;
5339     case ADD_VALUES:
5340       for (p = 0, off = 0; p < numPoints*2; p += 2, off += dof) {
5341         PetscInt o = points[p+1];
5342         ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5343         updatePoint_private(section, points[p], dof, add,    PETSC_FALSE, o, &values[off], array);
5344       } break;
5345     case ADD_ALL_VALUES:
5346       for (p = 0, off = 0; p < numPoints*2; p += 2, off += dof) {
5347         PetscInt o = points[p+1];
5348         ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5349         updatePoint_private(section, points[p], dof, add,    PETSC_TRUE,  o, &values[off], array);
5350       } break;
5351     default:
5352       SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %D", mode);
5353     }
5354   }
5355   /* Cleanup points */
5356   if (!clPoints) {ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);}
5357   else           {ierr = ISRestoreIndices(clPoints, &clp);CHKERRQ(ierr);}
5358   /* Cleanup array */
5359   ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
5360   PetscFunctionReturn(0);
5361 }
5362 
5363 #undef __FUNCT__
5364 #define __FUNCT__ "DMPlexPrintMatSetValues"
5365 PetscErrorCode DMPlexPrintMatSetValues(PetscViewer viewer, Mat A, PetscInt point, PetscInt numRIndices, const PetscInt rindices[], PetscInt numCIndices, const PetscInt cindices[], const PetscScalar values[])
5366 {
5367   PetscMPIInt    rank;
5368   PetscInt       i, j;
5369   PetscErrorCode ierr;
5370 
5371   PetscFunctionBegin;
5372   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRQ(ierr);
5373   ierr = PetscViewerASCIIPrintf(viewer, "[%D]mat for sieve point %D\n", rank, point);CHKERRQ(ierr);
5374   for (i = 0; i < numRIndices; i++) {ierr = PetscViewerASCIIPrintf(viewer, "[%D]mat row indices[%D] = %D\n", rank, i, rindices[i]);CHKERRQ(ierr);}
5375   for (i = 0; i < numCIndices; i++) {ierr = PetscViewerASCIIPrintf(viewer, "[%D]mat col indices[%D] = %D\n", rank, i, cindices[i]);CHKERRQ(ierr);}
5376   numCIndices = numCIndices ? numCIndices : numRIndices;
5377   for (i = 0; i < numRIndices; i++) {
5378     ierr = PetscViewerASCIIPrintf(viewer, "[%D]", rank);CHKERRQ(ierr);
5379     for (j = 0; j < numCIndices; j++) {
5380 #if defined(PETSC_USE_COMPLEX)
5381       ierr = PetscViewerASCIIPrintf(viewer, " (%g,%g)", (double)PetscRealPart(values[i*numCIndices+j]), (double)PetscImaginaryPart(values[i*numCIndices+j]));CHKERRQ(ierr);
5382 #else
5383       ierr = PetscViewerASCIIPrintf(viewer, " %g", (double)values[i*numCIndices+j]);CHKERRQ(ierr);
5384 #endif
5385     }
5386     ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
5387   }
5388   PetscFunctionReturn(0);
5389 }
5390 
5391 #undef __FUNCT__
5392 #define __FUNCT__ "indicesPoint_private"
5393 /* . off - The global offset of this point */
5394 PetscErrorCode indicesPoint_private(PetscSection section, PetscInt point, PetscInt off, PetscInt *loff, PetscBool setBC, PetscInt orientation, PetscInt indices[])
5395 {
5396   PetscInt        dof;    /* The number of unknowns on this point */
5397   PetscInt        cdof;   /* The number of constraints on this point */
5398   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
5399   PetscInt        cind = 0, k;
5400   PetscErrorCode  ierr;
5401 
5402   PetscFunctionBegin;
5403   ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
5404   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
5405   if (!cdof || setBC) {
5406     if (orientation >= 0) {
5407       for (k = 0; k < dof; ++k) indices[*loff+k] = off+k;
5408     } else {
5409       for (k = 0; k < dof; ++k) indices[*loff+dof-k-1] = off+k;
5410     }
5411   } else {
5412     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
5413     if (orientation >= 0) {
5414       for (k = 0; k < dof; ++k) {
5415         if ((cind < cdof) && (k == cdofs[cind])) {
5416           /* Insert check for returning constrained indices */
5417           indices[*loff+k] = -(off+k+1);
5418           ++cind;
5419         } else {
5420           indices[*loff+k] = off+k-cind;
5421         }
5422       }
5423     } else {
5424       for (k = 0; k < dof; ++k) {
5425         if ((cind < cdof) && (k == cdofs[cind])) {
5426           /* Insert check for returning constrained indices */
5427           indices[*loff+dof-k-1] = -(off+k+1);
5428           ++cind;
5429         } else {
5430           indices[*loff+dof-k-1] = off+k-cind;
5431         }
5432       }
5433     }
5434   }
5435   *loff += dof;
5436   PetscFunctionReturn(0);
5437 }
5438 
5439 #undef __FUNCT__
5440 #define __FUNCT__ "indicesPointFields_private"
5441 /* . off - The global offset of this point */
5442 PetscErrorCode indicesPointFields_private(PetscSection section, PetscInt point, PetscInt off, PetscInt foffs[], PetscBool setBC, PetscInt orientation, PetscInt indices[])
5443 {
5444   PetscInt       numFields, foff, f;
5445   PetscErrorCode ierr;
5446 
5447   PetscFunctionBegin;
5448   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5449   for (f = 0, foff = 0; f < numFields; ++f) {
5450     PetscInt        fdof, fcomp, cfdof;
5451     const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
5452     PetscInt        cind = 0, k, c;
5453 
5454     ierr = PetscSectionGetFieldComponents(section, f, &fcomp);CHKERRQ(ierr);
5455     ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
5456     ierr = PetscSectionGetFieldConstraintDof(section, point, f, &cfdof);CHKERRQ(ierr);
5457     if (!cfdof || setBC) {
5458       if (orientation >= 0) {
5459         for (k = 0; k < fdof; ++k) indices[foffs[f]+k] = off+foff+k;
5460       } else {
5461         for (k = fdof/fcomp-1; k >= 0; --k) {
5462           for (c = 0; c < fcomp; ++c) {
5463             indices[foffs[f]+k*fcomp+c] = off+foff+(fdof/fcomp-1-k)*fcomp+c;
5464           }
5465         }
5466       }
5467     } else {
5468       ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
5469       if (orientation >= 0) {
5470         for (k = 0; k < fdof; ++k) {
5471           if ((cind < cfdof) && (k == fcdofs[cind])) {
5472             indices[foffs[f]+k] = -(off+foff+k+1);
5473             ++cind;
5474           } else {
5475             indices[foffs[f]+k] = off+foff+k-cind;
5476           }
5477         }
5478       } else {
5479         for (k = fdof/fcomp-1; k >= 0; --k) {
5480           for (c = 0; c < fcomp; ++c) {
5481             if ((cind < cfdof) && ((fdof/fcomp-1-k)*fcomp+c == fcdofs[cind])) {
5482               indices[foffs[f]+k*fcomp+c] = -(off+foff+(fdof/fcomp-1-k)*fcomp+c+1);
5483               ++cind;
5484             } else {
5485               indices[foffs[f]+k*fcomp+c] = off+foff+(fdof/fcomp-1-k)*fcomp+c-cind;
5486             }
5487           }
5488         }
5489       }
5490     }
5491     foff     += fdof - cfdof;
5492     foffs[f] += fdof;
5493   }
5494   PetscFunctionReturn(0);
5495 }
5496 
5497 #undef __FUNCT__
5498 #define __FUNCT__ "DMPlexMatSetClosure"
5499 /*@C
5500   DMPlexMatSetClosure - Set an array of the values on the closure of 'point'
5501 
5502   Not collective
5503 
5504   Input Parameters:
5505 + dm - The DM
5506 . section - The section describing the layout in v, or NULL to use the default section
5507 . globalSection - The section describing the layout in v, or NULL to use the default global section
5508 . A - The matrix
5509 . point - The sieve point in the DM
5510 . values - The array of values
5511 - mode - The insert mode, where INSERT_ALL_VALUES and ADD_ALL_VALUES also overwrite boundary conditions
5512 
5513   Fortran Notes:
5514   This routine is only available in Fortran 90, and you must include petsc.h90 in your code.
5515 
5516   Level: intermediate
5517 
5518 .seealso DMPlexVecGetClosure(), DMPlexVecSetClosure()
5519 @*/
5520 PetscErrorCode DMPlexMatSetClosure(DM dm, PetscSection section, PetscSection globalSection, Mat A, PetscInt point, const PetscScalar values[], InsertMode mode)
5521 {
5522   DM_Plex        *mesh   = (DM_Plex*) dm->data;
5523   PetscSection    clSection;
5524   IS              clPoints;
5525   PetscInt       *points = NULL;
5526   const PetscInt *clp;
5527   PetscInt       *indices;
5528   PetscInt        offsets[32];
5529   PetscInt        numFields, numPoints, numIndices, dof, off, globalOff, pStart, pEnd, p, q, f;
5530   PetscErrorCode  ierr;
5531 
5532   PetscFunctionBegin;
5533   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5534   if (!section) {ierr = DMGetDefaultSection(dm, &section);CHKERRQ(ierr);}
5535   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
5536   if (!globalSection) {ierr = DMGetDefaultGlobalSection(dm, &globalSection);CHKERRQ(ierr);}
5537   PetscValidHeaderSpecific(globalSection, PETSC_SECTION_CLASSID, 3);
5538   PetscValidHeaderSpecific(A, MAT_CLASSID, 4);
5539   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5540   if (numFields > 31) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", numFields);
5541   ierr = PetscMemzero(offsets, 32 * sizeof(PetscInt));CHKERRQ(ierr);
5542   ierr = PetscSectionGetClosureIndex(section, (PetscObject) dm, &clSection, &clPoints);CHKERRQ(ierr);
5543   if (!clPoints) {
5544     ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
5545     /* Compress out points not in the section */
5546     ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
5547     for (p = 0, q = 0; p < numPoints*2; p += 2) {
5548       if ((points[p] >= pStart) && (points[p] < pEnd)) {
5549         points[q*2]   = points[p];
5550         points[q*2+1] = points[p+1];
5551         ++q;
5552       }
5553     }
5554     numPoints = q;
5555   } else {
5556     PetscInt dof, off;
5557 
5558     ierr = PetscSectionGetDof(clSection, point, &dof);CHKERRQ(ierr);
5559     numPoints = dof/2;
5560     ierr = PetscSectionGetOffset(clSection, point, &off);CHKERRQ(ierr);
5561     ierr = ISGetIndices(clPoints, &clp);CHKERRQ(ierr);
5562     points = (PetscInt *) &clp[off];
5563   }
5564   for (p = 0, numIndices = 0; p < numPoints*2; p += 2) {
5565     PetscInt fdof;
5566 
5567     ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5568     for (f = 0; f < numFields; ++f) {
5569       ierr          = PetscSectionGetFieldDof(section, points[p], f, &fdof);CHKERRQ(ierr);
5570       offsets[f+1] += fdof;
5571     }
5572     numIndices += dof;
5573   }
5574   for (f = 1; f < numFields; ++f) offsets[f+1] += offsets[f];
5575 
5576   if (numFields && offsets[numFields] != numIndices) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid size for closure %d should be %d", offsets[numFields], numIndices);
5577   ierr = DMGetWorkArray(dm, numIndices, PETSC_INT, &indices);CHKERRQ(ierr);
5578   if (numFields) {
5579     for (p = 0; p < numPoints*2; p += 2) {
5580       PetscInt o = points[p+1];
5581       ierr = PetscSectionGetOffset(globalSection, points[p], &globalOff);CHKERRQ(ierr);
5582       indicesPointFields_private(section, points[p], globalOff < 0 ? -(globalOff+1) : globalOff, offsets, PETSC_FALSE, o, indices);
5583     }
5584   } else {
5585     for (p = 0, off = 0; p < numPoints*2; p += 2) {
5586       PetscInt o = points[p+1];
5587       ierr = PetscSectionGetOffset(globalSection, points[p], &globalOff);CHKERRQ(ierr);
5588       indicesPoint_private(section, points[p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, o, indices);
5589     }
5590   }
5591   if (mesh->printSetValues) {ierr = DMPlexPrintMatSetValues(PETSC_VIEWER_STDOUT_SELF, A, point, numIndices, indices, 0, NULL, values);CHKERRQ(ierr);}
5592   ierr = MatSetValues(A, numIndices, indices, numIndices, indices, values, mode);
5593   if (ierr) {
5594     PetscMPIInt    rank;
5595     PetscErrorCode ierr2;
5596 
5597     ierr2 = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRQ(ierr2);
5598     ierr2 = (*PetscErrorPrintf)("[%D]ERROR in DMPlexMatSetClosure\n", rank);CHKERRQ(ierr2);
5599     ierr2 = DMPlexPrintMatSetValues(PETSC_VIEWER_STDERR_SELF, A, point, numIndices, indices, 0, NULL, values);CHKERRQ(ierr2);
5600     ierr2 = DMRestoreWorkArray(dm, numIndices, PETSC_INT, &indices);CHKERRQ(ierr2);
5601     CHKERRQ(ierr);
5602   }
5603   if (!clPoints) {
5604     ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
5605   } else {
5606     ierr = ISRestoreIndices(clPoints, &clp);CHKERRQ(ierr);
5607   }
5608   ierr = DMRestoreWorkArray(dm, numIndices, PETSC_INT, &indices);CHKERRQ(ierr);
5609   PetscFunctionReturn(0);
5610 }
5611 
5612 #undef __FUNCT__
5613 #define __FUNCT__ "DMPlexMatSetClosureRefined"
5614 PetscErrorCode DMPlexMatSetClosureRefined(DM dmf, PetscSection fsection, PetscSection globalFSection, DM dmc, PetscSection csection, PetscSection globalCSection, Mat A, PetscInt point, const PetscScalar values[], InsertMode mode)
5615 {
5616   DM_Plex        *mesh   = (DM_Plex*) dmf->data;
5617   PetscInt       *fpoints = NULL, *ftotpoints = NULL;
5618   PetscInt       *cpoints = NULL;
5619   PetscInt       *findices, *cindices;
5620   PetscInt        foffsets[32], coffsets[32];
5621   CellRefiner     cellRefiner;
5622   PetscInt        numFields, numSubcells, maxFPoints, numFPoints, numCPoints, numFIndices, numCIndices, dof, off, globalOff, pStart, pEnd, p, q, r, s, f;
5623   PetscErrorCode  ierr;
5624 
5625   PetscFunctionBegin;
5626   PetscValidHeaderSpecific(dmf, DM_CLASSID, 1);
5627   PetscValidHeaderSpecific(dmc, DM_CLASSID, 4);
5628   if (!fsection) {ierr = DMGetDefaultSection(dmf, &fsection);CHKERRQ(ierr);}
5629   PetscValidHeaderSpecific(fsection, PETSC_SECTION_CLASSID, 2);
5630   if (!csection) {ierr = DMGetDefaultSection(dmc, &csection);CHKERRQ(ierr);}
5631   PetscValidHeaderSpecific(csection, PETSC_SECTION_CLASSID, 5);
5632   if (!globalFSection) {ierr = DMGetDefaultGlobalSection(dmf, &globalFSection);CHKERRQ(ierr);}
5633   PetscValidHeaderSpecific(globalFSection, PETSC_SECTION_CLASSID, 3);
5634   if (!globalCSection) {ierr = DMGetDefaultGlobalSection(dmc, &globalCSection);CHKERRQ(ierr);}
5635   PetscValidHeaderSpecific(globalCSection, PETSC_SECTION_CLASSID, 6);
5636   PetscValidHeaderSpecific(A, MAT_CLASSID, 7);
5637   ierr = PetscSectionGetNumFields(fsection, &numFields);CHKERRQ(ierr);
5638   if (numFields > 31) SETERRQ1(PetscObjectComm((PetscObject)dmf), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", numFields);
5639   ierr = PetscMemzero(foffsets, 32 * sizeof(PetscInt));CHKERRQ(ierr);
5640   ierr = PetscMemzero(coffsets, 32 * sizeof(PetscInt));CHKERRQ(ierr);
5641   /* Column indices */
5642   ierr = DMPlexGetTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
5643   maxFPoints = numCPoints;
5644   /* Compress out points not in the section */
5645   /*   TODO: Squeeze out points with 0 dof as well */
5646   ierr = PetscSectionGetChart(csection, &pStart, &pEnd);CHKERRQ(ierr);
5647   for (p = 0, q = 0; p < numCPoints*2; p += 2) {
5648     if ((cpoints[p] >= pStart) && (cpoints[p] < pEnd)) {
5649       cpoints[q*2]   = cpoints[p];
5650       cpoints[q*2+1] = cpoints[p+1];
5651       ++q;
5652     }
5653   }
5654   numCPoints = q;
5655   for (p = 0, numCIndices = 0; p < numCPoints*2; p += 2) {
5656     PetscInt fdof;
5657 
5658     ierr = PetscSectionGetDof(csection, cpoints[p], &dof);CHKERRQ(ierr);
5659     if (!dof) continue;
5660     for (f = 0; f < numFields; ++f) {
5661       ierr           = PetscSectionGetFieldDof(csection, cpoints[p], f, &fdof);CHKERRQ(ierr);
5662       coffsets[f+1] += fdof;
5663     }
5664     numCIndices += dof;
5665   }
5666   for (f = 1; f < numFields; ++f) coffsets[f+1] += coffsets[f];
5667   /* Row indices */
5668   ierr = DMPlexGetCellRefiner_Internal(dmc, &cellRefiner);CHKERRQ(ierr);
5669   ierr = CellRefinerGetAffineTransforms_Internal(cellRefiner, &numSubcells, NULL, NULL, NULL);CHKERRQ(ierr);
5670   ierr = DMGetWorkArray(dmf, maxFPoints*2*numSubcells, PETSC_INT, &ftotpoints);CHKERRQ(ierr);
5671   for (r = 0, q = 0; r < numSubcells; ++r) {
5672     /* TODO Map from coarse to fine cells */
5673     ierr = DMPlexGetTransitiveClosure(dmf, point*numSubcells + r, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
5674     /* Compress out points not in the section */
5675     ierr = PetscSectionGetChart(fsection, &pStart, &pEnd);CHKERRQ(ierr);
5676     for (p = 0; p < numFPoints*2; p += 2) {
5677       if ((fpoints[p] >= pStart) && (fpoints[p] < pEnd)) {
5678         ierr = PetscSectionGetDof(fsection, fpoints[p], &dof);CHKERRQ(ierr);
5679         if (!dof) continue;
5680         for (s = 0; s < q; ++s) if (fpoints[p] == ftotpoints[s*2]) break;
5681         if (s < q) continue;
5682         ftotpoints[q*2]   = fpoints[p];
5683         ftotpoints[q*2+1] = fpoints[p+1];
5684         ++q;
5685       }
5686     }
5687     ierr = DMPlexRestoreTransitiveClosure(dmf, point, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
5688   }
5689   numFPoints = q;
5690   for (p = 0, numFIndices = 0; p < numFPoints*2; p += 2) {
5691     PetscInt fdof;
5692 
5693     ierr = PetscSectionGetDof(fsection, ftotpoints[p], &dof);CHKERRQ(ierr);
5694     if (!dof) continue;
5695     for (f = 0; f < numFields; ++f) {
5696       ierr           = PetscSectionGetFieldDof(fsection, ftotpoints[p], f, &fdof);CHKERRQ(ierr);
5697       foffsets[f+1] += fdof;
5698     }
5699     numFIndices += dof;
5700   }
5701   for (f = 1; f < numFields; ++f) foffsets[f+1] += foffsets[f];
5702 
5703   if (numFields && foffsets[numFields] != numFIndices) SETERRQ2(PetscObjectComm((PetscObject)dmf), PETSC_ERR_PLIB, "Invalid size for closure %d should be %d", foffsets[numFields], numFIndices);
5704   if (numFields && coffsets[numFields] != numCIndices) SETERRQ2(PetscObjectComm((PetscObject)dmc), PETSC_ERR_PLIB, "Invalid size for closure %d should be %d", coffsets[numFields], numCIndices);
5705   ierr = DMGetWorkArray(dmf, numFIndices, PETSC_INT, &findices);CHKERRQ(ierr);
5706   ierr = DMGetWorkArray(dmc, numCIndices, PETSC_INT, &cindices);CHKERRQ(ierr);
5707   if (numFields) {
5708     for (p = 0; p < numFPoints*2; p += 2) {
5709       PetscInt o = ftotpoints[p+1];
5710       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[p], &globalOff);CHKERRQ(ierr);
5711       indicesPointFields_private(fsection, ftotpoints[p], globalOff < 0 ? -(globalOff+1) : globalOff, foffsets, PETSC_FALSE, o, findices);
5712     }
5713     for (p = 0; p < numCPoints*2; p += 2) {
5714       PetscInt o = cpoints[p+1];
5715       ierr = PetscSectionGetOffset(globalCSection, cpoints[p], &globalOff);CHKERRQ(ierr);
5716       indicesPointFields_private(csection, cpoints[p], globalOff < 0 ? -(globalOff+1) : globalOff, coffsets, PETSC_FALSE, o, cindices);
5717     }
5718   } else {
5719     for (p = 0, off = 0; p < numFPoints*2; p += 2) {
5720       PetscInt o = ftotpoints[p+1];
5721       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[p], &globalOff);CHKERRQ(ierr);
5722       indicesPoint_private(fsection, ftotpoints[p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, o, findices);
5723     }
5724     for (p = 0, off = 0; p < numCPoints*2; p += 2) {
5725       PetscInt o = cpoints[p+1];
5726       ierr = PetscSectionGetOffset(globalCSection, cpoints[p], &globalOff);CHKERRQ(ierr);
5727       indicesPoint_private(csection, cpoints[p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, o, cindices);
5728     }
5729   }
5730   if (mesh->printSetValues) {ierr = DMPlexPrintMatSetValues(PETSC_VIEWER_STDOUT_SELF, A, point, numFIndices, findices, numCIndices, cindices, values);CHKERRQ(ierr);}
5731   ierr = MatSetValues(A, numFIndices, findices, numCIndices, cindices, values, mode);
5732   if (ierr) {
5733     PetscMPIInt    rank;
5734     PetscErrorCode ierr2;
5735 
5736     ierr2 = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRQ(ierr2);
5737     ierr2 = (*PetscErrorPrintf)("[%D]ERROR in DMPlexMatSetClosure\n", rank);CHKERRQ(ierr2);
5738     ierr2 = DMPlexPrintMatSetValues(PETSC_VIEWER_STDERR_SELF, A, point, numFIndices, findices, numCIndices, cindices, values);CHKERRQ(ierr2);
5739     ierr2 = DMRestoreWorkArray(dmf, numFIndices, PETSC_INT, &findices);CHKERRQ(ierr2);
5740     ierr2 = DMRestoreWorkArray(dmc, numCIndices, PETSC_INT, &cindices);CHKERRQ(ierr2);
5741     CHKERRQ(ierr);
5742   }
5743   ierr = DMRestoreWorkArray(dmf, numCPoints*2*4, PETSC_INT, &ftotpoints);CHKERRQ(ierr);
5744   ierr = DMPlexRestoreTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
5745   ierr = DMRestoreWorkArray(dmf, numFIndices, PETSC_INT, &findices);CHKERRQ(ierr);
5746   ierr = DMRestoreWorkArray(dmc, numCIndices, PETSC_INT, &cindices);CHKERRQ(ierr);
5747   PetscFunctionReturn(0);
5748 }
5749 
5750 #undef __FUNCT__
5751 #define __FUNCT__ "DMPlexGetHybridBounds"
5752 /*@
5753   DMPlexGetHybridBounds - Get the first mesh point of each dimension which is a hybrid
5754 
5755   Input Parameter:
5756 . dm - The DMPlex object
5757 
5758   Output Parameters:
5759 + cMax - The first hybrid cell
5760 . cMax - The first hybrid face
5761 . cMax - The first hybrid edge
5762 - cMax - The first hybrid vertex
5763 
5764   Level: developer
5765 
5766 .seealso DMPlexCreateHybridMesh()
5767 @*/
5768 PetscErrorCode DMPlexGetHybridBounds(DM dm, PetscInt *cMax, PetscInt *fMax, PetscInt *eMax, PetscInt *vMax)
5769 {
5770   DM_Plex       *mesh = (DM_Plex*) dm->data;
5771   PetscInt       dim;
5772   PetscErrorCode ierr;
5773 
5774   PetscFunctionBegin;
5775   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5776   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
5777   if (cMax) *cMax = mesh->hybridPointMax[dim];
5778   if (fMax) *fMax = mesh->hybridPointMax[dim-1];
5779   if (eMax) *eMax = mesh->hybridPointMax[1];
5780   if (vMax) *vMax = mesh->hybridPointMax[0];
5781   PetscFunctionReturn(0);
5782 }
5783 
5784 #undef __FUNCT__
5785 #define __FUNCT__ "DMPlexSetHybridBounds"
5786 PetscErrorCode DMPlexSetHybridBounds(DM dm, PetscInt cMax, PetscInt fMax, PetscInt eMax, PetscInt vMax)
5787 {
5788   DM_Plex       *mesh = (DM_Plex*) dm->data;
5789   PetscInt       dim;
5790   PetscErrorCode ierr;
5791 
5792   PetscFunctionBegin;
5793   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5794   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
5795   if (cMax >= 0) mesh->hybridPointMax[dim]   = cMax;
5796   if (fMax >= 0) mesh->hybridPointMax[dim-1] = fMax;
5797   if (eMax >= 0) mesh->hybridPointMax[1]     = eMax;
5798   if (vMax >= 0) mesh->hybridPointMax[0]     = vMax;
5799   PetscFunctionReturn(0);
5800 }
5801 
5802 #undef __FUNCT__
5803 #define __FUNCT__ "DMPlexGetVTKCellHeight"
5804 PetscErrorCode DMPlexGetVTKCellHeight(DM dm, PetscInt *cellHeight)
5805 {
5806   DM_Plex *mesh = (DM_Plex*) dm->data;
5807 
5808   PetscFunctionBegin;
5809   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5810   PetscValidPointer(cellHeight, 2);
5811   *cellHeight = mesh->vtkCellHeight;
5812   PetscFunctionReturn(0);
5813 }
5814 
5815 #undef __FUNCT__
5816 #define __FUNCT__ "DMPlexSetVTKCellHeight"
5817 PetscErrorCode DMPlexSetVTKCellHeight(DM dm, PetscInt cellHeight)
5818 {
5819   DM_Plex *mesh = (DM_Plex*) dm->data;
5820 
5821   PetscFunctionBegin;
5822   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5823   mesh->vtkCellHeight = cellHeight;
5824   PetscFunctionReturn(0);
5825 }
5826 
5827 #undef __FUNCT__
5828 #define __FUNCT__ "DMPlexCreateNumbering_Private"
5829 /* We can easily have a form that takes an IS instead */
5830 PetscErrorCode DMPlexCreateNumbering_Private(DM dm, PetscInt pStart, PetscInt pEnd, PetscSF sf, IS *numbering)
5831 {
5832   PetscSection   section, globalSection;
5833   PetscInt      *numbers, p;
5834   PetscErrorCode ierr;
5835 
5836   PetscFunctionBegin;
5837   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &section);CHKERRQ(ierr);
5838   ierr = PetscSectionSetChart(section, pStart, pEnd);CHKERRQ(ierr);
5839   for (p = pStart; p < pEnd; ++p) {
5840     ierr = PetscSectionSetDof(section, p, 1);CHKERRQ(ierr);
5841   }
5842   ierr = PetscSectionSetUp(section);CHKERRQ(ierr);
5843   ierr = PetscSectionCreateGlobalSection(section, sf, PETSC_FALSE, &globalSection);CHKERRQ(ierr);
5844   ierr = PetscMalloc1((pEnd - pStart), &numbers);CHKERRQ(ierr);
5845   for (p = pStart; p < pEnd; ++p) {
5846     ierr = PetscSectionGetOffset(globalSection, p, &numbers[p-pStart]);CHKERRQ(ierr);
5847   }
5848   ierr = ISCreateGeneral(PetscObjectComm((PetscObject)dm), pEnd - pStart, numbers, PETSC_OWN_POINTER, numbering);CHKERRQ(ierr);
5849   ierr = PetscSectionDestroy(&section);CHKERRQ(ierr);
5850   ierr = PetscSectionDestroy(&globalSection);CHKERRQ(ierr);
5851   PetscFunctionReturn(0);
5852 }
5853 
5854 #undef __FUNCT__
5855 #define __FUNCT__ "DMPlexGetCellNumbering"
5856 PetscErrorCode DMPlexGetCellNumbering(DM dm, IS *globalCellNumbers)
5857 {
5858   DM_Plex       *mesh = (DM_Plex*) dm->data;
5859   PetscInt       cellHeight, cStart, cEnd, cMax;
5860   PetscErrorCode ierr;
5861 
5862   PetscFunctionBegin;
5863   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5864   if (!mesh->globalCellNumbers) {
5865     ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
5866     ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
5867     ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
5868     if (cMax >= 0) cEnd = PetscMin(cEnd, cMax);
5869     ierr = DMPlexCreateNumbering_Private(dm, cStart, cEnd, dm->sf, &mesh->globalCellNumbers);CHKERRQ(ierr);
5870   }
5871   *globalCellNumbers = mesh->globalCellNumbers;
5872   PetscFunctionReturn(0);
5873 }
5874 
5875 #undef __FUNCT__
5876 #define __FUNCT__ "DMPlexGetVertexNumbering"
5877 PetscErrorCode DMPlexGetVertexNumbering(DM dm, IS *globalVertexNumbers)
5878 {
5879   DM_Plex       *mesh = (DM_Plex*) dm->data;
5880   PetscInt       vStart, vEnd, vMax;
5881   PetscErrorCode ierr;
5882 
5883   PetscFunctionBegin;
5884   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5885   if (!mesh->globalVertexNumbers) {
5886     ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
5887     ierr = DMPlexGetHybridBounds(dm, NULL, NULL, NULL, &vMax);CHKERRQ(ierr);
5888     if (vMax >= 0) vEnd = PetscMin(vEnd, vMax);
5889     ierr = DMPlexCreateNumbering_Private(dm, vStart, vEnd, dm->sf, &mesh->globalVertexNumbers);CHKERRQ(ierr);
5890   }
5891   *globalVertexNumbers = mesh->globalVertexNumbers;
5892   PetscFunctionReturn(0);
5893 }
5894 
5895 
5896 #undef __FUNCT__
5897 #define __FUNCT__ "PetscSectionCreateGlobalSectionLabel"
5898 /*@C
5899   PetscSectionCreateGlobalSectionLabel - Create a section describing the global field layout using
5900   the local section and an SF describing the section point overlap.
5901 
5902   Input Parameters:
5903   + s - The PetscSection for the local field layout
5904   . sf - The SF describing parallel layout of the section points
5905   . includeConstraints - By default this is PETSC_FALSE, meaning that the global field vector will not possess constrained dofs
5906   . label - The label specifying the points
5907   - labelValue - The label stratum specifying the points
5908 
5909   Output Parameter:
5910   . gsection - The PetscSection for the global field layout
5911 
5912   Note: This gives negative sizes and offsets to points not owned by this process
5913 
5914   Level: developer
5915 
5916 .seealso: PetscSectionCreate()
5917 @*/
5918 PetscErrorCode PetscSectionCreateGlobalSectionLabel(PetscSection s, PetscSF sf, PetscBool includeConstraints, DMLabel label, PetscInt labelValue, PetscSection *gsection)
5919 {
5920   PetscInt      *neg = NULL, *tmpOff = NULL;
5921   PetscInt       pStart, pEnd, p, dof, cdof, off, globalOff = 0, nroots;
5922   PetscErrorCode ierr;
5923 
5924   PetscFunctionBegin;
5925   ierr = PetscSectionCreate(s->atlasLayout.comm, gsection);CHKERRQ(ierr);
5926   ierr = PetscSectionGetChart(s, &pStart, &pEnd);CHKERRQ(ierr);
5927   ierr = PetscSectionSetChart(*gsection, pStart, pEnd);CHKERRQ(ierr);
5928   ierr = PetscSFGetGraph(sf, &nroots, NULL, NULL, NULL);CHKERRQ(ierr);
5929   if (nroots >= 0) {
5930     if (nroots < pEnd-pStart) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "PetscSF nroots %d < %d section size", nroots, pEnd-pStart);
5931     ierr = PetscCalloc1(nroots, &neg);CHKERRQ(ierr);
5932     if (nroots > pEnd-pStart) {
5933       ierr = PetscCalloc1(nroots, &tmpOff);CHKERRQ(ierr);
5934     } else {
5935       tmpOff = &(*gsection)->atlasDof[-pStart];
5936     }
5937   }
5938   /* Mark ghost points with negative dof */
5939   for (p = pStart; p < pEnd; ++p) {
5940     PetscInt value;
5941 
5942     ierr = DMLabelGetValue(label, p, &value);CHKERRQ(ierr);
5943     if (value != labelValue) continue;
5944     ierr = PetscSectionGetDof(s, p, &dof);CHKERRQ(ierr);
5945     ierr = PetscSectionSetDof(*gsection, p, dof);CHKERRQ(ierr);
5946     ierr = PetscSectionGetConstraintDof(s, p, &cdof);CHKERRQ(ierr);
5947     if (!includeConstraints && cdof > 0) {ierr = PetscSectionSetConstraintDof(*gsection, p, cdof);CHKERRQ(ierr);}
5948     if (neg) neg[p] = -(dof+1);
5949   }
5950   ierr = PetscSectionSetUpBC(*gsection);CHKERRQ(ierr);
5951   if (nroots >= 0) {
5952     ierr = PetscSFBcastBegin(sf, MPIU_INT, neg, tmpOff);CHKERRQ(ierr);
5953     ierr = PetscSFBcastEnd(sf, MPIU_INT, neg, tmpOff);CHKERRQ(ierr);
5954     if (nroots > pEnd-pStart) {
5955       for (p = pStart; p < pEnd; ++p) {if (tmpOff[p] < 0) (*gsection)->atlasDof[p-pStart] = tmpOff[p];}
5956     }
5957   }
5958   /* Calculate new sizes, get proccess offset, and calculate point offsets */
5959   for (p = 0, off = 0; p < pEnd-pStart; ++p) {
5960     cdof = (!includeConstraints && s->bc) ? s->bc->atlasDof[p] : 0;
5961     (*gsection)->atlasOff[p] = off;
5962     off += (*gsection)->atlasDof[p] > 0 ? (*gsection)->atlasDof[p]-cdof : 0;
5963   }
5964   ierr       = MPI_Scan(&off, &globalOff, 1, MPIU_INT, MPI_SUM, s->atlasLayout.comm);CHKERRQ(ierr);
5965   globalOff -= off;
5966   for (p = 0, off = 0; p < pEnd-pStart; ++p) {
5967     (*gsection)->atlasOff[p] += globalOff;
5968     if (neg) neg[p] = -((*gsection)->atlasOff[p]+1);
5969   }
5970   /* Put in negative offsets for ghost points */
5971   if (nroots >= 0) {
5972     ierr = PetscSFBcastBegin(sf, MPIU_INT, neg, tmpOff);CHKERRQ(ierr);
5973     ierr = PetscSFBcastEnd(sf, MPIU_INT, neg, tmpOff);CHKERRQ(ierr);
5974     if (nroots > pEnd-pStart) {
5975       for (p = pStart; p < pEnd; ++p) {if (tmpOff[p] < 0) (*gsection)->atlasOff[p-pStart] = tmpOff[p];}
5976     }
5977   }
5978   if (nroots >= 0 && nroots > pEnd-pStart) {ierr = PetscFree(tmpOff);CHKERRQ(ierr);}
5979   ierr = PetscFree(neg);CHKERRQ(ierr);
5980   PetscFunctionReturn(0);
5981 }
5982 
5983 #undef __FUNCT__
5984 #define __FUNCT__ "DMPlexCheckSymmetry"
5985 /*@
5986   DMPlexCheckSymmetry - Check that the adjacency information in the mesh is symmetric.
5987 
5988   Input Parameters:
5989   + dm - The DMPlex object
5990 
5991   Note: This is a useful diagnostic when creating meshes programmatically.
5992 
5993   Level: developer
5994 
5995 .seealso: DMCreate(), DMCheckSkeleton(), DMCheckFaces()
5996 @*/
5997 PetscErrorCode DMPlexCheckSymmetry(DM dm)
5998 {
5999   PetscSection    coneSection, supportSection;
6000   const PetscInt *cone, *support;
6001   PetscInt        coneSize, c, supportSize, s;
6002   PetscInt        pStart, pEnd, p, csize, ssize;
6003   PetscErrorCode  ierr;
6004 
6005   PetscFunctionBegin;
6006   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6007   ierr = DMPlexGetConeSection(dm, &coneSection);CHKERRQ(ierr);
6008   ierr = DMPlexGetSupportSection(dm, &supportSection);CHKERRQ(ierr);
6009   /* Check that point p is found in the support of its cone points, and vice versa */
6010   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
6011   for (p = pStart; p < pEnd; ++p) {
6012     ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
6013     ierr = DMPlexGetCone(dm, p, &cone);CHKERRQ(ierr);
6014     for (c = 0; c < coneSize; ++c) {
6015       PetscBool dup = PETSC_FALSE;
6016       PetscInt  d;
6017       for (d = c-1; d >= 0; --d) {
6018         if (cone[c] == cone[d]) {dup = PETSC_TRUE; break;}
6019       }
6020       ierr = DMPlexGetSupportSize(dm, cone[c], &supportSize);CHKERRQ(ierr);
6021       ierr = DMPlexGetSupport(dm, cone[c], &support);CHKERRQ(ierr);
6022       for (s = 0; s < supportSize; ++s) {
6023         if (support[s] == p) break;
6024       }
6025       if ((s >= supportSize) || (dup && (support[s+1] != p))) {
6026         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %d cone: ", p);
6027         for (s = 0; s < coneSize; ++s) {
6028           ierr = PetscPrintf(PETSC_COMM_SELF, "%d, ", cone[s]);
6029         }
6030         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");
6031         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %d support: ", cone[c]);
6032         for (s = 0; s < supportSize; ++s) {
6033           ierr = PetscPrintf(PETSC_COMM_SELF, "%d, ", support[s]);
6034         }
6035         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");
6036         if (dup) {
6037           SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %d not repeatedly found in support of repeated cone point %d", p, cone[c]);
6038         } else {
6039           SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %d not found in support of cone point %d", p, cone[c]);
6040         }
6041       }
6042     }
6043     ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
6044     ierr = DMPlexGetSupport(dm, p, &support);CHKERRQ(ierr);
6045     for (s = 0; s < supportSize; ++s) {
6046       ierr = DMPlexGetConeSize(dm, support[s], &coneSize);CHKERRQ(ierr);
6047       ierr = DMPlexGetCone(dm, support[s], &cone);CHKERRQ(ierr);
6048       for (c = 0; c < coneSize; ++c) {
6049         if (cone[c] == p) break;
6050       }
6051       if (c >= coneSize) {
6052         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %d support: ", p);
6053         for (c = 0; c < supportSize; ++c) {
6054           ierr = PetscPrintf(PETSC_COMM_SELF, "%d, ", support[c]);
6055         }
6056         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");
6057         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %d cone: ", support[s]);
6058         for (c = 0; c < coneSize; ++c) {
6059           ierr = PetscPrintf(PETSC_COMM_SELF, "%d, ", cone[c]);
6060         }
6061         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");
6062         SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %d not found in cone of support point %d", p, support[s]);
6063       }
6064     }
6065   }
6066   ierr = PetscSectionGetStorageSize(coneSection, &csize);CHKERRQ(ierr);
6067   ierr = PetscSectionGetStorageSize(supportSection, &ssize);CHKERRQ(ierr);
6068   if (csize != ssize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "Total cone size %d != Total support size %d", csize, ssize);
6069   PetscFunctionReturn(0);
6070 }
6071 
6072 #undef __FUNCT__
6073 #define __FUNCT__ "DMPlexCheckSkeleton"
6074 /*@
6075   DMPlexCheckSkeleton - Check that each cell has the correct number of vertices
6076 
6077   Input Parameters:
6078 + dm - The DMPlex object
6079 . isSimplex - Are the cells simplices or tensor products
6080 - cellHeight - Normally 0
6081 
6082   Note: This is a useful diagnostic when creating meshes programmatically.
6083 
6084   Level: developer
6085 
6086 .seealso: DMCreate(), DMCheckSymmetry(), DMCheckFaces()
6087 @*/
6088 PetscErrorCode DMPlexCheckSkeleton(DM dm, PetscBool isSimplex, PetscInt cellHeight)
6089 {
6090   PetscInt       dim, numCorners, numHybridCorners, vStart, vEnd, cStart, cEnd, cMax, c;
6091   PetscErrorCode ierr;
6092 
6093   PetscFunctionBegin;
6094   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6095   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
6096   switch (dim) {
6097   case 1: numCorners = isSimplex ? 2 : 2; numHybridCorners = isSimplex ? 2 : 2; break;
6098   case 2: numCorners = isSimplex ? 3 : 4; numHybridCorners = isSimplex ? 4 : 4; break;
6099   case 3: numCorners = isSimplex ? 4 : 8; numHybridCorners = isSimplex ? 6 : 8; break;
6100   default:
6101     SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_OUTOFRANGE, "Cannot handle meshes of dimension %d", dim);
6102   }
6103   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
6104   ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
6105   ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
6106   cMax = cMax >= 0 ? cMax : cEnd;
6107   for (c = cStart; c < cMax; ++c) {
6108     PetscInt *closure = NULL, closureSize, cl, coneSize = 0;
6109 
6110     ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6111     for (cl = 0; cl < closureSize*2; cl += 2) {
6112       const PetscInt p = closure[cl];
6113       if ((p >= vStart) && (p < vEnd)) ++coneSize;
6114     }
6115     ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6116     if (coneSize != numCorners) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %d has  %d vertices != %d", c, coneSize, numCorners);
6117   }
6118   for (c = cMax; c < cEnd; ++c) {
6119     PetscInt *closure = NULL, closureSize, cl, coneSize = 0;
6120 
6121     ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6122     for (cl = 0; cl < closureSize*2; cl += 2) {
6123       const PetscInt p = closure[cl];
6124       if ((p >= vStart) && (p < vEnd)) ++coneSize;
6125     }
6126     ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6127     if (coneSize > numHybridCorners) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Hybrid cell %d has  %d vertices > %d", c, coneSize, numHybridCorners);
6128   }
6129   PetscFunctionReturn(0);
6130 }
6131 
6132 #undef __FUNCT__
6133 #define __FUNCT__ "DMPlexCheckFaces"
6134 /*@
6135   DMPlexCheckFaces - Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type
6136 
6137   Input Parameters:
6138 + dm - The DMPlex object
6139 . isSimplex - Are the cells simplices or tensor products
6140 - cellHeight - Normally 0
6141 
6142   Note: This is a useful diagnostic when creating meshes programmatically.
6143 
6144   Level: developer
6145 
6146 .seealso: DMCreate(), DMCheckSymmetry(), DMCheckSkeleton()
6147 @*/
6148 PetscErrorCode DMPlexCheckFaces(DM dm, PetscBool isSimplex, PetscInt cellHeight)
6149 {
6150   PetscInt       pMax[4];
6151   PetscInt       dim, vStart, vEnd, cStart, cEnd, c, h;
6152   PetscErrorCode ierr;
6153 
6154   PetscFunctionBegin;
6155   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6156   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
6157   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
6158   ierr = DMPlexGetHybridBounds(dm, &pMax[dim], &pMax[dim-1], &pMax[1], &pMax[0]);CHKERRQ(ierr);
6159   for (h = cellHeight; h < dim; ++h) {
6160     ierr = DMPlexGetHeightStratum(dm, h, &cStart, &cEnd);CHKERRQ(ierr);
6161     for (c = cStart; c < cEnd; ++c) {
6162       const PetscInt *cone, *ornt, *faces;
6163       PetscInt        numFaces, faceSize, coneSize,f;
6164       PetscInt       *closure = NULL, closureSize, cl, numCorners = 0;
6165 
6166       if (pMax[dim-h] >= 0 && c >= pMax[dim-h]) continue;
6167       ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
6168       ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
6169       ierr = DMPlexGetConeOrientation(dm, c, &ornt);CHKERRQ(ierr);
6170       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6171       for (cl = 0; cl < closureSize*2; cl += 2) {
6172         const PetscInt p = closure[cl];
6173         if ((p >= vStart) && (p < vEnd)) closure[numCorners++] = p;
6174       }
6175       ierr = DMPlexGetRawFaces_Internal(dm, dim-h, numCorners, closure, &numFaces, &faceSize, &faces);CHKERRQ(ierr);
6176       if (coneSize != numFaces) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %d has %d faces but should have %d", c, coneSize, numFaces);
6177       for (f = 0; f < numFaces; ++f) {
6178         PetscInt *fclosure = NULL, fclosureSize, cl, fnumCorners = 0, v;
6179 
6180         ierr = DMPlexGetTransitiveClosure_Internal(dm, cone[f], ornt[f], PETSC_TRUE, &fclosureSize, &fclosure);CHKERRQ(ierr);
6181         for (cl = 0; cl < fclosureSize*2; cl += 2) {
6182           const PetscInt p = fclosure[cl];
6183           if ((p >= vStart) && (p < vEnd)) fclosure[fnumCorners++] = p;
6184         }
6185         if (fnumCorners != faceSize) SETERRQ5(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %d (%d) of cell %d has %d vertices but should have %d", cone[f], f, c, fnumCorners, faceSize);
6186         for (v = 0; v < fnumCorners; ++v) {
6187           if (fclosure[v] != faces[f*faceSize+v]) SETERRQ6(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %d (%d) of cell %d vertex %d, %d != %d", cone[f], f, c, v, fclosure[v], faces[f*faceSize+v]);
6188         }
6189         ierr = DMPlexRestoreTransitiveClosure(dm, cone[f], PETSC_TRUE, &fclosureSize, &fclosure);CHKERRQ(ierr);
6190       }
6191       ierr = DMPlexRestoreFaces_Internal(dm, dim, c, &numFaces, &faceSize, &faces);CHKERRQ(ierr);
6192       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6193     }
6194   }
6195   PetscFunctionReturn(0);
6196 }
6197 
6198 #undef __FUNCT__
6199 #define __FUNCT__ "DMCreateInterpolation_Plex"
6200 /* Pointwise interpolation
6201      Just code FEM for now
6202      u^f = I u^c
6203      sum_k u^f_k phi^f_k = I sum_j u^c_j phi^c_j
6204      u^f_i = sum_j psi^f_i I phi^c_j u^c_j
6205      I_{ij} = psi^f_i phi^c_j
6206 */
6207 PetscErrorCode DMCreateInterpolation_Plex(DM dmCoarse, DM dmFine, Mat *interpolation, Vec *scaling)
6208 {
6209   PetscSection   gsc, gsf;
6210   PetscInt       m, n;
6211   void          *ctx;
6212   PetscErrorCode ierr;
6213 
6214   PetscFunctionBegin;
6215   /*
6216   Loop over coarse cells
6217     Loop over coarse basis functions
6218       Loop over fine cells in coarse cell
6219         Loop over fine dual basis functions
6220           Evaluate coarse basis on fine dual basis quad points
6221           Sum
6222           Update local element matrix
6223     Accumulate to interpolation matrix
6224 
6225    Can extend PetscFEIntegrateJacobian_Basic() to do a specialized cell loop
6226   */
6227   ierr = DMGetDefaultGlobalSection(dmFine, &gsf);CHKERRQ(ierr);
6228   ierr = PetscSectionGetConstrainedStorageSize(gsf, &m);CHKERRQ(ierr);
6229   ierr = DMGetDefaultGlobalSection(dmCoarse, &gsc);CHKERRQ(ierr);
6230   ierr = PetscSectionGetConstrainedStorageSize(gsc, &n);CHKERRQ(ierr);
6231   /* We need to preallocate properly */
6232   ierr = MatCreate(PetscObjectComm((PetscObject) dmCoarse), interpolation);CHKERRQ(ierr);
6233   ierr = MatSetSizes(*interpolation, m, n, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
6234   ierr = MatSetType(*interpolation, dmCoarse->mattype);CHKERRQ(ierr);
6235   ierr = MatSetUp(*interpolation);CHKERRQ(ierr);
6236   ierr = MatSetFromOptions(*interpolation);CHKERRQ(ierr);
6237   ierr = MatSetOption(*interpolation, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_FALSE);CHKERRQ(ierr);
6238   ierr = DMGetApplicationContext(dmFine, &ctx);CHKERRQ(ierr);
6239   ierr = DMPlexComputeInterpolatorFEM(dmCoarse, dmFine, *interpolation, ctx);CHKERRQ(ierr);
6240   /* Use naive scaling */
6241   ierr = DMCreateInterpolationScale(dmCoarse, dmFine, *interpolation, scaling);CHKERRQ(ierr);
6242   PetscFunctionReturn(0);
6243 }
6244 
6245 #undef __FUNCT__
6246 #define __FUNCT__ "DMCreateInjection_Plex"
6247 PetscErrorCode DMCreateInjection_Plex(DM dmCoarse, DM dmFine, VecScatter *ctx)
6248 {
6249   Vec             cv,  fv;
6250   IS              cis, fis, fpointIS;
6251   PetscSection    sc, gsc, gsf;
6252   const PetscInt *fpoints;
6253   PetscInt       *cindices, *findices;
6254   PetscInt        cpStart, cpEnd, m, off, cp;
6255   PetscErrorCode  ierr;
6256 
6257   PetscFunctionBegin;
6258   ierr = DMGetDefaultGlobalSection(dmFine, &gsf);CHKERRQ(ierr);
6259   ierr = DMGetGlobalVector(dmFine, &fv);CHKERRQ(ierr);
6260   ierr = DMGetDefaultSection(dmCoarse, &sc);CHKERRQ(ierr);
6261   ierr = DMGetDefaultGlobalSection(dmCoarse, &gsc);CHKERRQ(ierr);
6262   ierr = DMGetGlobalVector(dmCoarse, &cv);CHKERRQ(ierr);
6263   ierr = DMPlexCreateCoarsePointIS(dmCoarse, &fpointIS);CHKERRQ(ierr);
6264   ierr = PetscSectionGetConstrainedStorageSize(gsc, &m);CHKERRQ(ierr);
6265   ierr = PetscMalloc2(m,&cindices,m,&findices);CHKERRQ(ierr);
6266   ierr = PetscSectionGetChart(gsc, &cpStart, &cpEnd);CHKERRQ(ierr);
6267   ierr = ISGetIndices(fpointIS, &fpoints);CHKERRQ(ierr);
6268   for (cp = cpStart, off = 0; cp < cpEnd; ++cp) {
6269     const PetscInt *cdofsC = NULL;
6270     PetscInt        fp     = fpoints[cp-cpStart], dofC, cdofC, dofF, offC, offF, d, e;
6271 
6272     ierr = PetscSectionGetDof(gsc, cp, &dofC);CHKERRQ(ierr);
6273     if (dofC <= 0) continue;
6274     ierr = PetscSectionGetConstraintDof(sc, cp, &cdofC);CHKERRQ(ierr);
6275     ierr = PetscSectionGetDof(gsf, fp, &dofF);CHKERRQ(ierr);
6276     ierr = PetscSectionGetOffset(gsc, cp, &offC);CHKERRQ(ierr);
6277     ierr = PetscSectionGetOffset(gsf, fp, &offF);CHKERRQ(ierr);
6278     if (cdofC) {ierr = PetscSectionGetConstraintIndices(sc, cp, &cdofsC);CHKERRQ(ierr);}
6279     if (dofC != dofF) SETERRQ4(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Point %d (%d) has %d coarse dofs != %d fine dofs", cp, fp, dofC, dofF);
6280     if (offC < 0 || offF < 0) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Coarse point %d has invalid offset %d (%d)", cp, offC, offF);
6281     for (d = 0, e = 0; d < dofC; ++d) {
6282       if (cdofsC && cdofsC[e] == d) {++e; continue;}
6283       cindices[off+d-e] = offC+d; findices[off+d-e] = offF+d;
6284     }
6285     if (e != cdofC) SETERRQ4(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Point %d (%d) has invalid number of constraints %d != %d", cp, fp, e, cdofC);
6286     off += dofC-cdofC;
6287   }
6288   ierr = ISRestoreIndices(fpointIS, &fpoints);CHKERRQ(ierr);
6289   if (off != m) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of coarse dofs %d != %d", off, m);
6290   ierr = ISCreateGeneral(PETSC_COMM_SELF, m, cindices, PETSC_OWN_POINTER, &cis);CHKERRQ(ierr);
6291   ierr = ISCreateGeneral(PETSC_COMM_SELF, m, findices, PETSC_OWN_POINTER, &fis);CHKERRQ(ierr);
6292   ierr = VecScatterCreate(cv, cis, fv, fis, ctx);CHKERRQ(ierr);
6293   ierr = ISDestroy(&cis);CHKERRQ(ierr);
6294   ierr = ISDestroy(&fis);CHKERRQ(ierr);
6295   ierr = DMRestoreGlobalVector(dmFine, &fv);CHKERRQ(ierr);
6296   ierr = DMRestoreGlobalVector(dmCoarse, &cv);CHKERRQ(ierr);
6297   ierr = ISDestroy(&fpointIS);CHKERRQ(ierr);
6298   PetscFunctionReturn(0);
6299 }
6300 
6301 #undef __FUNCT__
6302 #define __FUNCT__ "DMCreateDefaultSection_Plex"
6303 /* Pointwise interpolation
6304      Just code FEM for now
6305      u^f = I u^c
6306      sum_k u^f_k phi^f_k = I sum_l u^c_l phi^c_l
6307      u^f_i = sum_l int psi^f_i I phi^c_l u^c_l
6308      I_{ij} = int psi^f_i phi^c_j
6309 */
6310 PetscErrorCode DMCreateDefaultSection_Plex(DM dm)
6311 {
6312   PetscSection   section;
6313   IS            *bcPoints;
6314   PetscInt      *bcFields, *numComp, *numDof;
6315   PetscInt       depth, dim, numBd, numBC = 0, numFields, bd, bc, f;
6316   PetscErrorCode ierr;
6317 
6318   PetscFunctionBegin;
6319   /* Handle boundary conditions */
6320   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
6321   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
6322   ierr = DMPlexGetNumBoundary(dm, &numBd);CHKERRQ(ierr);
6323   for (bd = 0; bd < numBd; ++bd) {
6324     PetscBool isEssential;
6325     ierr = DMPlexGetBoundary(dm, bd, &isEssential, NULL, NULL, NULL, NULL, NULL, NULL);CHKERRQ(ierr);
6326     if (isEssential) ++numBC;
6327   }
6328   ierr = PetscMalloc2(numBC,&bcFields,numBC,&bcPoints);CHKERRQ(ierr);
6329   for (bd = 0, bc = 0; bd < numBd; ++bd) {
6330     const char     *bdLabel;
6331     DMLabel         label;
6332     const PetscInt *values;
6333     PetscInt        field, numValues;
6334     PetscBool       isEssential, has;
6335 
6336     ierr = DMPlexGetBoundary(dm, bd, &isEssential, &bdLabel, &field, NULL, &numValues, &values, NULL);CHKERRQ(ierr);
6337     if (numValues != 1) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Bug me and I will fix this");
6338     ierr = DMPlexHasLabel(dm, bdLabel, &has);CHKERRQ(ierr);
6339     if (!has) {
6340       ierr = DMPlexCreateLabel(dm, bdLabel);CHKERRQ(ierr);
6341       ierr = DMPlexGetLabel(dm, bdLabel, &label);CHKERRQ(ierr);
6342       ierr = DMPlexMarkBoundaryFaces(dm, label);CHKERRQ(ierr);
6343     }
6344     ierr = DMPlexGetLabel(dm, bdLabel, &label);CHKERRQ(ierr);
6345     ierr = DMPlexLabelComplete(dm, label);CHKERRQ(ierr);
6346     if (isEssential) {
6347       bcFields[bc] = field;
6348       ierr = DMPlexGetStratumIS(dm, bdLabel, values[0], &bcPoints[bc++]);CHKERRQ(ierr);
6349     }
6350   }
6351   /* Handle discretization */
6352   ierr = DMGetNumFields(dm, &numFields);CHKERRQ(ierr);
6353   ierr = PetscMalloc2(numFields,&numComp,numFields*(dim+1),&numDof);CHKERRQ(ierr);
6354   for (f = 0; f < numFields; ++f) {
6355     PetscFE         fe;
6356     const PetscInt *numFieldDof;
6357     PetscInt        d;
6358 
6359     ierr = DMGetField(dm, f, (PetscObject *) &fe);CHKERRQ(ierr);
6360     ierr = PetscFEGetNumComponents(fe, &numComp[f]);CHKERRQ(ierr);
6361     ierr = PetscFEGetNumDof(fe, &numFieldDof);CHKERRQ(ierr);
6362     for (d = 0; d < dim+1; ++d) numDof[f*(dim+1)+d] = numFieldDof[d];
6363   }
6364   for (f = 0; f < numFields; ++f) {
6365     PetscInt d;
6366     for (d = 1; d < dim; ++d) {
6367       if ((numDof[f*(dim+1)+d] > 0) && (depth < dim)) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Mesh must be interpolated when unknowns are specified on edges or faces.");
6368     }
6369   }
6370   ierr = DMPlexCreateSection(dm, dim, numFields, numComp, numDof, numBC, bcFields, bcPoints, &section);CHKERRQ(ierr);
6371   for (f = 0; f < numFields; ++f) {
6372     PetscFE     fe;
6373     const char *name;
6374 
6375     ierr = DMGetField(dm, f, (PetscObject *) &fe);CHKERRQ(ierr);
6376     ierr = PetscObjectGetName((PetscObject) fe, &name);CHKERRQ(ierr);
6377     ierr = PetscSectionSetFieldName(section, f, name);CHKERRQ(ierr);
6378   }
6379   ierr = DMSetDefaultSection(dm, section);CHKERRQ(ierr);
6380   ierr = PetscSectionDestroy(&section);CHKERRQ(ierr);
6381   for (bc = 0; bc < numBC; ++bc) {ierr = ISDestroy(&bcPoints[bc]);CHKERRQ(ierr);}
6382   ierr = PetscFree2(bcFields,bcPoints);CHKERRQ(ierr);
6383   ierr = PetscFree2(numComp,numDof);CHKERRQ(ierr);
6384   PetscFunctionReturn(0);
6385 }
6386 
6387 #undef __FUNCT__
6388 #define __FUNCT__ "DMPlexGetCoarseDM"
6389 /*@
6390   DMPlexGetCoarseDM - Get the coarse mesh from which this was obtained by refinement
6391 
6392   Input Parameter:
6393 . dm - The DMPlex object
6394 
6395   Output Parameter:
6396 . cdm - The coarse DM
6397 
6398   Level: intermediate
6399 
6400 .seealso: DMPlexSetCoarseDM()
6401 @*/
6402 PetscErrorCode DMPlexGetCoarseDM(DM dm, DM *cdm)
6403 {
6404   PetscFunctionBegin;
6405   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6406   PetscValidPointer(cdm, 2);
6407   *cdm = ((DM_Plex *) dm->data)->coarseMesh;
6408   PetscFunctionReturn(0);
6409 }
6410 
6411 #undef __FUNCT__
6412 #define __FUNCT__ "DMPlexSetCoarseDM"
6413 /*@
6414   DMPlexSetCoarseDM - Set the coarse mesh from which this was obtained by refinement
6415 
6416   Input Parameters:
6417 + dm - The DMPlex object
6418 - cdm - The coarse DM
6419 
6420   Level: intermediate
6421 
6422 .seealso: DMPlexGetCoarseDM()
6423 @*/
6424 PetscErrorCode DMPlexSetCoarseDM(DM dm, DM cdm)
6425 {
6426   DM_Plex       *mesh;
6427   PetscErrorCode ierr;
6428 
6429   PetscFunctionBegin;
6430   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6431   if (cdm) PetscValidHeaderSpecific(cdm, DM_CLASSID, 2);
6432   mesh = (DM_Plex *) dm->data;
6433   ierr = DMDestroy(&mesh->coarseMesh);CHKERRQ(ierr);
6434   mesh->coarseMesh = cdm;
6435   ierr = PetscObjectReference((PetscObject) mesh->coarseMesh);CHKERRQ(ierr);
6436   PetscFunctionReturn(0);
6437 }
6438