xref: /petsc/src/dm/impls/plex/plex.c (revision 2fb8446c541684a316151fd601c81003e751a50f)
1 #include <petsc-private/dmpleximpl.h>   /*I      "petscdmplex.h"   I*/
2 #include <../src/sys/utils/hash.h>
3 #include <petsc-private/isimpl.h>
4 #include <petscsf.h>
5 #include <petscviewerhdf5.h>
6 
7 /* Logging support */
8 PetscLogEvent DMPLEX_Interpolate, DMPLEX_Partition, DMPLEX_Distribute, DMPLEX_DistributeCones, DMPLEX_DistributeLabels, DMPLEX_DistributeSF, DMPLEX_DistributeField, DMPLEX_DistributeData, DMPLEX_Stratify, DMPLEX_Preallocate, DMPLEX_ResidualFEM, DMPLEX_JacobianFEM;
9 
10 PETSC_EXTERN PetscErrorCode VecView_Seq(Vec, PetscViewer);
11 PETSC_EXTERN PetscErrorCode VecView_MPI(Vec, PetscViewer);
12 PETSC_EXTERN PetscErrorCode VecLoad_Default(Vec, PetscViewer);
13 PETSC_EXTERN PetscErrorCode DMTSGetTimeStepNumber(DM,PetscInt*);
14 
15 #undef __FUNCT__
16 #define __FUNCT__ "GetField_Static"
17 static PetscErrorCode GetField_Static(DM dm, PetscSection section, PetscSection sectionGlobal, Vec v, PetscInt field, PetscInt pStart, PetscInt pEnd, IS *is, Vec *subv)
18 {
19   PetscInt      *subIndices;
20   PetscInt       Nc, subSize = 0, subOff = 0, p;
21   PetscErrorCode ierr;
22 
23   PetscFunctionBegin;
24   ierr = PetscSectionGetFieldComponents(section, field, &Nc);CHKERRQ(ierr);
25   for (p = pStart; p < pEnd; ++p) {
26     PetscInt gdof, fdof = 0;
27 
28     ierr = PetscSectionGetDof(sectionGlobal, p, &gdof);CHKERRQ(ierr);
29     if (gdof > 0) {ierr = PetscSectionGetFieldDof(section, p, field, &fdof);CHKERRQ(ierr);}
30     subSize += fdof;
31   }
32   ierr = PetscMalloc1(subSize, &subIndices);CHKERRQ(ierr);
33   for (p = pStart; p < pEnd; ++p) {
34     PetscInt gdof, goff;
35 
36     ierr = PetscSectionGetDof(sectionGlobal, p, &gdof);CHKERRQ(ierr);
37     if (gdof > 0) {
38       PetscInt fdof, fc, f2, poff = 0;
39 
40       ierr = PetscSectionGetOffset(sectionGlobal, p, &goff);CHKERRQ(ierr);
41       /* Can get rid of this loop by storing field information in the global section */
42       for (f2 = 0; f2 < field; ++f2) {
43         ierr  = PetscSectionGetFieldDof(section, p, f2, &fdof);CHKERRQ(ierr);
44         poff += fdof;
45       }
46       ierr = PetscSectionGetFieldDof(section, p, field, &fdof);CHKERRQ(ierr);
47       for (fc = 0; fc < fdof; ++fc, ++subOff) subIndices[subOff] = goff+poff+fc;
48     }
49   }
50   ierr = ISCreateGeneral(PetscObjectComm((PetscObject) dm), subSize, subIndices, PETSC_OWN_POINTER, is);CHKERRQ(ierr);
51   ierr = VecGetSubVector(v, *is, subv);CHKERRQ(ierr);
52   ierr = VecSetBlockSize(*subv, Nc);CHKERRQ(ierr);
53   PetscFunctionReturn(0);
54 }
55 
56 #undef __FUNCT__
57 #define __FUNCT__ "RestoreField_Static"
58 static PetscErrorCode RestoreField_Static(DM dm, PetscSection section, PetscSection sectionGlobal, Vec v, PetscInt field, PetscInt pStart, PetscInt pEnd, IS *is, Vec *subv)
59 {
60   PetscErrorCode ierr;
61 
62   PetscFunctionBegin;
63   ierr = VecRestoreSubVector(v, *is, subv);CHKERRQ(ierr);
64   ierr = ISDestroy(is);CHKERRQ(ierr);
65   PetscFunctionReturn(0);
66 }
67 
68 #undef __FUNCT__
69 #define __FUNCT__ "GetFieldType_Static"
70 static PetscErrorCode GetFieldType_Static(DM dm, PetscSection section, PetscInt field, PetscInt *sStart, PetscInt *sEnd, PetscViewerVTKFieldType *ft)
71 {
72   PetscInt       dim, pStart, pEnd, vStart, vEnd, cStart, cEnd, vdof = 0, cdof = 0;
73   PetscErrorCode ierr;
74 
75   PetscFunctionBegin;
76   *ft  = PETSC_VTK_POINT_FIELD;
77   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
78   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
79   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
80   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
81   if (field >= 0) {
82     if ((vStart >= pStart) && (vStart < pEnd)) {ierr = PetscSectionGetFieldDof(section, vStart, field, &vdof);CHKERRQ(ierr);}
83     if ((cStart >= pStart) && (cStart < pEnd)) {ierr = PetscSectionGetFieldDof(section, cStart, field, &cdof);CHKERRQ(ierr);}
84   } else {
85     if ((vStart >= pStart) && (vStart < pEnd)) {ierr = PetscSectionGetDof(section, vStart, &vdof);CHKERRQ(ierr);}
86     if ((cStart >= pStart) && (cStart < pEnd)) {ierr = PetscSectionGetDof(section, cStart, &cdof);CHKERRQ(ierr);}
87   }
88   if (vdof) {
89     *sStart = vStart;
90     *sEnd   = vEnd;
91     if (vdof == dim) *ft = PETSC_VTK_POINT_VECTOR_FIELD;
92     else             *ft = PETSC_VTK_POINT_FIELD;
93   } else if (cdof) {
94     *sStart = cStart;
95     *sEnd   = cEnd;
96     if (cdof == dim) *ft = PETSC_VTK_CELL_VECTOR_FIELD;
97     else             *ft = PETSC_VTK_CELL_FIELD;
98   } else SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Could not classify input Vec for VTK");
99   PetscFunctionReturn(0);
100 }
101 
102 #if defined(PETSC_HAVE_HDF5)
103 #undef __FUNCT__
104 #define __FUNCT__ "DMSequenceView_HDF5"
105 static PetscErrorCode DMSequenceView_HDF5(DM dm, const char *seqname, PetscInt seqnum, PetscScalar value, PetscViewer viewer)
106 {
107   Vec            stamp;
108   PetscMPIInt    rank;
109   PetscErrorCode ierr;
110 
111   PetscFunctionBegin;
112   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) viewer), &rank);CHKERRQ(ierr);
113   ierr = VecCreateMPI(PetscObjectComm((PetscObject) viewer), rank ? 0 : 1, 1, &stamp);CHKERRQ(ierr);
114   ierr = VecSetBlockSize(stamp, 1);CHKERRQ(ierr);
115   ierr = PetscObjectSetName((PetscObject) stamp, seqname);CHKERRQ(ierr);
116   if (!rank) {
117     PetscReal timeScale;
118     PetscBool istime;
119 
120     ierr = PetscStrncmp(seqname, "time", 5, &istime);CHKERRQ(ierr);
121     if (istime) {ierr = DMPlexGetScale(dm, PETSC_UNIT_TIME, &timeScale);CHKERRQ(ierr); value *= timeScale;}
122     ierr = VecSetValue(stamp, 0, value, INSERT_VALUES);CHKERRQ(ierr);
123   }
124   ierr = VecAssemblyBegin(stamp);CHKERRQ(ierr);
125   ierr = VecAssemblyEnd(stamp);CHKERRQ(ierr);
126   ierr = PetscViewerHDF5PushGroup(viewer, "/");CHKERRQ(ierr);
127   ierr = PetscViewerHDF5SetTimestep(viewer, seqnum);CHKERRQ(ierr);
128   ierr = VecView(stamp, viewer);CHKERRQ(ierr);
129   ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
130   ierr = VecDestroy(&stamp);CHKERRQ(ierr);
131   PetscFunctionReturn(0);
132 }
133 
134 #undef __FUNCT__
135 #define __FUNCT__ "VecView_Plex_Local_HDF5"
136 static PetscErrorCode VecView_Plex_Local_HDF5(Vec v, PetscViewer viewer)
137 {
138   DM                      dm;
139   DM                      dmBC;
140   PetscSection            section, sectionGlobal;
141   Vec                     gv;
142   const char             *name;
143   PetscViewerVTKFieldType ft;
144   PetscViewerFormat       format;
145   PetscInt                seqnum;
146   PetscBool               isseq;
147   PetscErrorCode          ierr;
148 
149   PetscFunctionBegin;
150   ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
151   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
152   ierr = DMGetDefaultSection(dm, &section);CHKERRQ(ierr);
153   ierr = DMGetOutputSequenceNumber(dm, &seqnum);CHKERRQ(ierr);
154   ierr = PetscViewerHDF5SetTimestep(viewer, seqnum);CHKERRQ(ierr);
155   ierr = DMSequenceView_HDF5(dm, "time", seqnum, (PetscScalar) seqnum, viewer);CHKERRQ(ierr);
156   ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
157   ierr = DMGetOutputDM(dm, &dmBC);CHKERRQ(ierr);
158   ierr = DMGetDefaultGlobalSection(dmBC, &sectionGlobal);CHKERRQ(ierr);
159   ierr = DMGetGlobalVector(dmBC, &gv);CHKERRQ(ierr);
160   ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
161   ierr = PetscObjectSetName((PetscObject) gv, name);CHKERRQ(ierr);
162   ierr = DMLocalToGlobalBegin(dmBC, v, INSERT_VALUES, gv);CHKERRQ(ierr);
163   ierr = DMLocalToGlobalEnd(dmBC, v, INSERT_VALUES, gv);CHKERRQ(ierr);
164   ierr = PetscObjectTypeCompare((PetscObject) gv, VECSEQ, &isseq);CHKERRQ(ierr);
165   if (isseq) {ierr = VecView_Seq(gv, viewer);CHKERRQ(ierr);}
166   else       {ierr = VecView_MPI(gv, viewer);CHKERRQ(ierr);}
167   if (format == PETSC_VIEWER_HDF5_VIZ) {
168     /* Output visualization representation */
169     PetscInt numFields, f;
170 
171     ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
172     for (f = 0; f < numFields; ++f) {
173       Vec         subv;
174       IS          is;
175       const char *fname, *fgroup;
176       char        group[PETSC_MAX_PATH_LEN];
177       PetscInt    pStart, pEnd;
178       PetscBool   flag;
179 
180       ierr = GetFieldType_Static(dm, section, f, &pStart, &pEnd, &ft);CHKERRQ(ierr);
181       fgroup = (ft == PETSC_VTK_POINT_VECTOR_FIELD) || (ft == PETSC_VTK_POINT_FIELD) ? "/vertex_fields" : "/cell_fields";
182       ierr = PetscSectionGetFieldName(section, f, &fname);CHKERRQ(ierr);
183       ierr = PetscViewerHDF5PushGroup(viewer, fgroup);CHKERRQ(ierr);
184       ierr = GetField_Static(dmBC, section, sectionGlobal, gv, f, pStart, pEnd, &is, &subv);CHKERRQ(ierr);
185       ierr = PetscObjectSetName((PetscObject) subv, fname);CHKERRQ(ierr);
186       if (isseq) {ierr = VecView_Seq(subv, viewer);CHKERRQ(ierr);}
187       else       {ierr = VecView_MPI(subv, viewer);CHKERRQ(ierr);}
188       ierr = RestoreField_Static(dmBC, section, sectionGlobal, gv, f, pStart, pEnd, &is, &subv);CHKERRQ(ierr);
189       ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
190       ierr = PetscSNPrintf(group, PETSC_MAX_PATH_LEN, "%s/%s", fgroup, fname);CHKERRQ(ierr);
191       ierr = PetscViewerHDF5HasAttribute(viewer, group, "vector_field_type", &flag);CHKERRQ(ierr);
192       if (!flag) {
193         if ((ft == PETSC_VTK_POINT_VECTOR_FIELD) || (ft == PETSC_VTK_CELL_VECTOR_FIELD)) {
194           ierr = PetscViewerHDF5WriteAttribute(viewer, group, "vector_field_type", PETSC_STRING, "vector");CHKERRQ(ierr);
195         } else {
196           ierr = PetscViewerHDF5WriteAttribute(viewer, group, "vector_field_type", PETSC_STRING, "scalar");CHKERRQ(ierr);
197         }
198       }
199     }
200   }
201   ierr = DMRestoreGlobalVector(dmBC, &gv);CHKERRQ(ierr);
202   PetscFunctionReturn(0);
203 }
204 #endif
205 
206 #undef __FUNCT__
207 #define __FUNCT__ "VecView_Plex_Local"
208 PetscErrorCode VecView_Plex_Local(Vec v, PetscViewer viewer)
209 {
210   DM             dm;
211   PetscBool      isvtk, ishdf5, isseq;
212   PetscErrorCode ierr;
213 
214   PetscFunctionBegin;
215   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
216   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
217   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,  &isvtk);CHKERRQ(ierr);
218   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
219   ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
220   if (isvtk || ishdf5) {ierr = DMPlexInsertBoundaryValues(dm, v);CHKERRQ(ierr);}
221   if (isvtk) {
222     PetscSection            section;
223     PetscViewerVTKFieldType ft;
224     PetscInt                pStart, pEnd;
225 
226     ierr = DMGetDefaultSection(dm, &section);CHKERRQ(ierr);
227     ierr = GetFieldType_Static(dm, section, PETSC_DETERMINE, &pStart, &pEnd, &ft);CHKERRQ(ierr);
228     ierr = PetscObjectReference((PetscObject) dm);CHKERRQ(ierr); /* viewer drops reference */
229     ierr = PetscObjectReference((PetscObject) v);CHKERRQ(ierr);  /* viewer drops reference */
230     ierr = PetscViewerVTKAddField(viewer, (PetscObject) dm, DMPlexVTKWriteAll, ft, (PetscObject) v);CHKERRQ(ierr);
231   } else if (ishdf5) {
232 #if defined(PETSC_HAVE_HDF5)
233     ierr = VecView_Plex_Local_HDF5(v, viewer);CHKERRQ(ierr);
234 #else
235     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
236 #endif
237   } else {
238     if (isseq) {ierr = VecView_Seq(v, viewer);CHKERRQ(ierr);}
239     else       {ierr = VecView_MPI(v, viewer);CHKERRQ(ierr);}
240   }
241   PetscFunctionReturn(0);
242 }
243 
244 #if defined(PETSC_HAVE_HDF5)
245 #undef __FUNCT__
246 #define __FUNCT__ "VecView_Plex_HDF5"
247 static PetscErrorCode VecView_Plex_HDF5(Vec v, PetscViewer viewer)
248 {
249   DM             dm;
250   Vec            locv;
251   const char    *name;
252   PetscErrorCode ierr;
253 
254   PetscFunctionBegin;
255   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
256   ierr = DMGetLocalVector(dm, &locv);CHKERRQ(ierr);
257   ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
258   ierr = PetscObjectSetName((PetscObject) locv, name);CHKERRQ(ierr);
259   ierr = DMGlobalToLocalBegin(dm, v, INSERT_VALUES, locv);CHKERRQ(ierr);
260   ierr = DMGlobalToLocalEnd(dm, v, INSERT_VALUES, locv);CHKERRQ(ierr);
261   ierr = PetscViewerHDF5PushGroup(viewer, "/fields");CHKERRQ(ierr);
262   ierr = PetscViewerPushFormat(viewer, PETSC_VIEWER_HDF5_VIZ);CHKERRQ(ierr);
263   ierr = VecView_Plex_Local(locv, viewer);CHKERRQ(ierr);
264   ierr = PetscViewerPopFormat(viewer);CHKERRQ(ierr);
265   ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
266   ierr = DMRestoreLocalVector(dm, &locv);CHKERRQ(ierr);
267   PetscFunctionReturn(0);
268 }
269 #endif
270 
271 #undef __FUNCT__
272 #define __FUNCT__ "VecView_Plex"
273 PetscErrorCode VecView_Plex(Vec v, PetscViewer viewer)
274 {
275   DM             dm;
276   PetscBool      isvtk, ishdf5, isseq;
277   PetscErrorCode ierr;
278 
279   PetscFunctionBegin;
280   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
281   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
282   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,  &isvtk);CHKERRQ(ierr);
283   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
284   ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
285   if (isvtk) {
286     Vec         locv;
287     const char *name;
288 
289     ierr = DMGetLocalVector(dm, &locv);CHKERRQ(ierr);
290     ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
291     ierr = PetscObjectSetName((PetscObject) locv, name);CHKERRQ(ierr);
292     ierr = DMGlobalToLocalBegin(dm, v, INSERT_VALUES, locv);CHKERRQ(ierr);
293     ierr = DMGlobalToLocalEnd(dm, v, INSERT_VALUES, locv);CHKERRQ(ierr);
294     ierr = VecView_Plex_Local(locv, viewer);CHKERRQ(ierr);
295     ierr = DMRestoreLocalVector(dm, &locv);CHKERRQ(ierr);
296   } else if (ishdf5) {
297 #if defined(PETSC_HAVE_HDF5)
298     ierr = VecView_Plex_HDF5(v, viewer);CHKERRQ(ierr);
299 #else
300     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
301 #endif
302   } else {
303     if (isseq) {ierr = VecView_Seq(v, viewer);CHKERRQ(ierr);}
304     else       {ierr = VecView_MPI(v, viewer);CHKERRQ(ierr);}
305   }
306   PetscFunctionReturn(0);
307 }
308 
309 #undef __FUNCT__
310 #define __FUNCT__ "VecLoad_Plex_Local"
311 PetscErrorCode VecLoad_Plex_Local(Vec v, PetscViewer viewer)
312 {
313   DM             dm;
314   PetscBool      ishdf5;
315   PetscErrorCode ierr;
316 
317   PetscFunctionBegin;
318   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
319   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
320   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
321   if (ishdf5) {
322     DM          dmBC;
323     Vec         gv;
324     const char *name;
325 
326     ierr = DMGetOutputDM(dm, &dmBC);CHKERRQ(ierr);
327     ierr = DMGetGlobalVector(dmBC, &gv);CHKERRQ(ierr);
328     ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
329     ierr = PetscObjectSetName((PetscObject) gv, name);CHKERRQ(ierr);
330     ierr = VecLoad_Default(gv, viewer);CHKERRQ(ierr);
331     ierr = DMGlobalToLocalBegin(dmBC, gv, INSERT_VALUES, v);CHKERRQ(ierr);
332     ierr = DMGlobalToLocalEnd(dmBC, gv, INSERT_VALUES, v);CHKERRQ(ierr);
333     ierr = DMRestoreGlobalVector(dmBC, &gv);CHKERRQ(ierr);
334   } else {
335     ierr = VecLoad_Default(v, viewer);CHKERRQ(ierr);
336   }
337   PetscFunctionReturn(0);
338 }
339 
340 #if defined(PETSC_HAVE_HDF5)
341 #undef __FUNCT__
342 #define __FUNCT__ "VecLoad_Plex_HDF5"
343 static PetscErrorCode VecLoad_Plex_HDF5(Vec v, PetscViewer viewer)
344 {
345   DM             dm;
346   Vec            locv;
347   const char    *name;
348   PetscInt       seqnum;
349   PetscErrorCode ierr;
350 
351   PetscFunctionBegin;
352   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
353   ierr = DMGetLocalVector(dm, &locv);CHKERRQ(ierr);
354   ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
355   ierr = PetscObjectSetName((PetscObject) locv, name);CHKERRQ(ierr);
356   ierr = DMGetOutputSequenceNumber(dm, &seqnum);CHKERRQ(ierr);
357   ierr = PetscViewerHDF5SetTimestep(viewer, seqnum);CHKERRQ(ierr);
358   ierr = PetscViewerHDF5PushGroup(viewer, "/fields");CHKERRQ(ierr);
359   ierr = VecLoad_Plex_Local(locv, viewer);CHKERRQ(ierr);
360   ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
361   ierr = DMLocalToGlobalBegin(dm, locv, INSERT_VALUES, v);CHKERRQ(ierr);
362   ierr = DMLocalToGlobalEnd(dm, locv, INSERT_VALUES, v);CHKERRQ(ierr);
363   ierr = DMRestoreLocalVector(dm, &locv);CHKERRQ(ierr);
364   PetscFunctionReturn(0);
365 }
366 #endif
367 
368 #undef __FUNCT__
369 #define __FUNCT__ "VecLoad_Plex"
370 PetscErrorCode VecLoad_Plex(Vec v, PetscViewer viewer)
371 {
372   DM             dm;
373   PetscBool      ishdf5;
374   PetscErrorCode ierr;
375 
376   PetscFunctionBegin;
377   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
378   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
379   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
380   if (ishdf5) {
381 #if defined(PETSC_HAVE_HDF5)
382     ierr = VecLoad_Plex_HDF5(v, viewer);CHKERRQ(ierr);
383 #else
384     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
385 #endif
386   } else {
387     ierr = VecLoad_Default(v, viewer);CHKERRQ(ierr);
388   }
389   PetscFunctionReturn(0);
390 }
391 
392 #undef __FUNCT__
393 #define __FUNCT__ "DMPlexView_Ascii"
394 PetscErrorCode DMPlexView_Ascii(DM dm, PetscViewer viewer)
395 {
396   DM_Plex          *mesh = (DM_Plex*) dm->data;
397   DM                cdm;
398   DMLabel           markers;
399   PetscSection      coordSection;
400   Vec               coordinates;
401   PetscViewerFormat format;
402   PetscErrorCode    ierr;
403 
404   PetscFunctionBegin;
405   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
406   ierr = DMGetDefaultSection(cdm, &coordSection);CHKERRQ(ierr);
407   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
408   ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
409   if (format == PETSC_VIEWER_ASCII_INFO_DETAIL) {
410     const char *name;
411     PetscInt    maxConeSize, maxSupportSize;
412     PetscInt    pStart, pEnd, p;
413     PetscMPIInt rank, size;
414 
415     ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRQ(ierr);
416     ierr = MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size);CHKERRQ(ierr);
417     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
418     ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
419     ierr = DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize);CHKERRQ(ierr);
420     ierr = PetscViewerASCIISynchronizedAllow(viewer, PETSC_TRUE);CHKERRQ(ierr);
421     ierr = PetscViewerASCIIPrintf(viewer, "Mesh '%s':\n", name);CHKERRQ(ierr);
422     ierr = PetscViewerASCIISynchronizedPrintf(viewer, "Max sizes cone: %D support: %D\n", maxConeSize, maxSupportSize);CHKERRQ(ierr);
423     ierr = PetscViewerASCIIPrintf(viewer, "orientation is missing\n", name);CHKERRQ(ierr);
424     ierr = PetscViewerASCIIPrintf(viewer, "cap --> base:\n", name);CHKERRQ(ierr);
425     for (p = pStart; p < pEnd; ++p) {
426       PetscInt dof, off, s;
427 
428       ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
429       ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
430       for (s = off; s < off+dof; ++s) {
431         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%D]: %D ----> %D\n", rank, p, mesh->supports[s]);CHKERRQ(ierr);
432       }
433     }
434     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
435     ierr = PetscViewerASCIIPrintf(viewer, "base <-- cap:\n", name);CHKERRQ(ierr);
436     for (p = pStart; p < pEnd; ++p) {
437       PetscInt dof, off, c;
438 
439       ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
440       ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
441       for (c = off; c < off+dof; ++c) {
442         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%D]: %D <---- %D (%D)\n", rank, p, mesh->cones[c], mesh->coneOrientations[c]);CHKERRQ(ierr);
443       }
444     }
445     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
446     ierr = PetscSectionGetChart(coordSection, &pStart, NULL);CHKERRQ(ierr);
447     if (pStart >= 0) {ierr = PetscSectionVecView(coordSection, coordinates, viewer);CHKERRQ(ierr);}
448     ierr = DMPlexGetLabel(dm, "marker", &markers);CHKERRQ(ierr);
449     ierr = DMLabelView(markers,viewer);CHKERRQ(ierr);
450     if (size > 1) {
451       PetscSF sf;
452 
453       ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr);
454       ierr = PetscSFView(sf, viewer);CHKERRQ(ierr);
455     }
456     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
457   } else if (format == PETSC_VIEWER_ASCII_LATEX) {
458     const char  *name;
459     const char  *colors[3] = {"red", "blue", "green"};
460     const int    numColors  = 3;
461     PetscReal    scale      = 2.0;
462     PetscScalar *coords;
463     PetscInt     depth, cStart, cEnd, c, vStart, vEnd, v, eStart = 0, eEnd = 0, e, p;
464     PetscMPIInt  rank, size;
465 
466     ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
467     ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRQ(ierr);
468     ierr = MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size);CHKERRQ(ierr);
469     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
470     ierr = PetscViewerASCIISynchronizedAllow(viewer, PETSC_TRUE);CHKERRQ(ierr);
471     ierr = PetscViewerASCIIPrintf(viewer, "\
472 \\documentclass[crop,multi=false]{standalone}\n\n\
473 \\usepackage{tikz}\n\
474 \\usepackage{pgflibraryshapes}\n\
475 \\usetikzlibrary{backgrounds}\n\
476 \\usetikzlibrary{arrows}\n\
477 \\begin{document}\n\
478 \\section{%s}\n\
479 \\begin{center}\n", name, 8.0/scale);CHKERRQ(ierr);
480     ierr = PetscViewerASCIIPrintf(viewer, "Mesh for process ");CHKERRQ(ierr);
481     for (p = 0; p < size; ++p) {
482       if (p > 0 && p == size-1) {
483         ierr = PetscViewerASCIIPrintf(viewer, ", and ", colors[p%numColors], p);CHKERRQ(ierr);
484       } else if (p > 0) {
485         ierr = PetscViewerASCIIPrintf(viewer, ", ", colors[p%numColors], p);CHKERRQ(ierr);
486       }
487       ierr = PetscViewerASCIIPrintf(viewer, "{\\textcolor{%s}%D}", colors[p%numColors], p);CHKERRQ(ierr);
488     }
489     ierr = PetscViewerASCIIPrintf(viewer, ".\n\n\n\
490 \\begin{tikzpicture}[scale = %g,font=\\fontsize{8}{8}\\selectfont]\n");CHKERRQ(ierr);
491     /* Plot vertices */
492     ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
493     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
494     ierr = PetscViewerASCIIPrintf(viewer, "\\path\n");CHKERRQ(ierr);
495     for (v = vStart; v < vEnd; ++v) {
496       PetscInt off, dof, d;
497 
498       ierr = PetscSectionGetDof(coordSection, v, &dof);CHKERRQ(ierr);
499       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
500       ierr = PetscViewerASCIISynchronizedPrintf(viewer, "(");CHKERRQ(ierr);
501       for (d = 0; d < dof; ++d) {
502         if (d > 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, ",");CHKERRQ(ierr);}
503         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "%g", (double)(scale*PetscRealPart(coords[off+d])));CHKERRQ(ierr);
504       }
505       ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%D) [draw,shape=circle,color=%s] {%D} --\n", v, rank, colors[rank%numColors], v);CHKERRQ(ierr);
506     }
507     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
508     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
509     ierr = PetscViewerASCIIPrintf(viewer, "(0,0);\n");CHKERRQ(ierr);
510     /* Plot edges */
511     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
512     ierr = PetscViewerASCIIPrintf(viewer, "\\path\n");CHKERRQ(ierr);
513     if (depth > 1) {ierr = DMPlexGetDepthStratum(dm, 1, &eStart, &eEnd);CHKERRQ(ierr);}
514     for (e = eStart; e < eEnd; ++e) {
515       const PetscInt *cone;
516       PetscInt        coneSize, offA, offB, dof, d;
517 
518       ierr = DMPlexGetConeSize(dm, e, &coneSize);CHKERRQ(ierr);
519       if (coneSize != 2) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Edge %d cone should have two vertices, not %d", e, coneSize);
520       ierr = DMPlexGetCone(dm, e, &cone);CHKERRQ(ierr);
521       ierr = PetscSectionGetDof(coordSection, cone[0], &dof);CHKERRQ(ierr);
522       ierr = PetscSectionGetOffset(coordSection, cone[0], &offA);CHKERRQ(ierr);
523       ierr = PetscSectionGetOffset(coordSection, cone[1], &offB);CHKERRQ(ierr);
524       ierr = PetscViewerASCIISynchronizedPrintf(viewer, "(");CHKERRQ(ierr);
525       for (d = 0; d < dof; ++d) {
526         if (d > 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, ",");CHKERRQ(ierr);}
527         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "%g", (double)(scale*0.5*PetscRealPart(coords[offA+d]+coords[offB+d])));CHKERRQ(ierr);
528       }
529       ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%D) [draw,shape=circle,color=%s] {%D} --\n", e, rank, colors[rank%numColors], e);CHKERRQ(ierr);
530     }
531     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
532     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
533     ierr = PetscViewerASCIIPrintf(viewer, "(0,0);\n");CHKERRQ(ierr);
534     /* Plot cells */
535     ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
536     for (c = cStart; c < cEnd; ++c) {
537       PetscInt *closure = NULL;
538       PetscInt  closureSize, firstPoint = -1;
539 
540       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
541       ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\draw[color=%s] ", colors[rank%numColors]);CHKERRQ(ierr);
542       for (p = 0; p < closureSize*2; p += 2) {
543         const PetscInt point = closure[p];
544 
545         if ((point < vStart) || (point >= vEnd)) continue;
546         if (firstPoint >= 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- ");CHKERRQ(ierr);}
547         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "(%D_%D)", point, rank);CHKERRQ(ierr);
548         if (firstPoint < 0) firstPoint = point;
549       }
550       /* Why doesn't this work? ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- cycle;\n");CHKERRQ(ierr); */
551       ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- (%D_%D);\n", firstPoint, rank);CHKERRQ(ierr);
552       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
553     }
554     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
555     ierr = PetscViewerASCIIPrintf(viewer, "\\end{tikzpicture}\n\\end{center}\n");CHKERRQ(ierr);
556     ierr = PetscViewerASCIIPrintf(viewer, "\\end{document}\n", name);CHKERRQ(ierr);
557   } else {
558     MPI_Comm    comm;
559     PetscInt   *sizes, *hybsizes;
560     PetscInt    locDepth, depth, dim, d, pMax[4];
561     PetscInt    pStart, pEnd, p;
562     PetscInt    numLabels, l;
563     const char *name;
564     PetscMPIInt size;
565 
566     ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
567     ierr = MPI_Comm_size(comm, &size);CHKERRQ(ierr);
568     ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
569     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
570     if (name) {ierr = PetscViewerASCIIPrintf(viewer, "%s in %D dimensions:\n", name, dim);CHKERRQ(ierr);}
571     else      {ierr = PetscViewerASCIIPrintf(viewer, "Mesh in %D dimensions:\n", dim);CHKERRQ(ierr);}
572     ierr = DMPlexGetDepth(dm, &locDepth);CHKERRQ(ierr);
573     ierr = MPI_Allreduce(&locDepth, &depth, 1, MPIU_INT, MPI_MAX, comm);CHKERRQ(ierr);
574     ierr = DMPlexGetHybridBounds(dm, &pMax[depth], &pMax[depth-1], &pMax[1], &pMax[0]);CHKERRQ(ierr);
575     ierr = PetscMalloc2(size,&sizes,size,&hybsizes);CHKERRQ(ierr);
576     if (depth == 1) {
577       ierr = DMPlexGetDepthStratum(dm, 0, &pStart, &pEnd);CHKERRQ(ierr);
578       pEnd = pEnd - pStart;
579       ierr = MPI_Gather(&pEnd, 1, MPIU_INT, sizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
580       ierr = PetscViewerASCIIPrintf(viewer, "  %D-cells:", 0);CHKERRQ(ierr);
581       for (p = 0; p < size; ++p) {ierr = PetscViewerASCIIPrintf(viewer, " %D", sizes[p]);CHKERRQ(ierr);}
582       ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
583       ierr = DMPlexGetHeightStratum(dm, 0, &pStart, &pEnd);CHKERRQ(ierr);
584       pEnd = pEnd - pStart;
585       ierr = MPI_Gather(&pEnd, 1, MPIU_INT, sizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
586       ierr = PetscViewerASCIIPrintf(viewer, "  %D-cells:", dim);CHKERRQ(ierr);
587       for (p = 0; p < size; ++p) {ierr = PetscViewerASCIIPrintf(viewer, " %D", sizes[p]);CHKERRQ(ierr);}
588       ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
589     } else {
590       for (d = 0; d <= dim; d++) {
591         ierr = DMPlexGetDepthStratum(dm, d, &pStart, &pEnd);CHKERRQ(ierr);
592         pEnd    -= pStart;
593         pMax[d] -= pStart;
594         ierr = MPI_Gather(&pEnd, 1, MPIU_INT, sizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
595         ierr = MPI_Gather(&pMax[d], 1, MPIU_INT, hybsizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
596         ierr = PetscViewerASCIIPrintf(viewer, "  %D-cells:", d);CHKERRQ(ierr);
597         for (p = 0; p < size; ++p) {
598           if (hybsizes[p] >= 0) {ierr = PetscViewerASCIIPrintf(viewer, " %D (%D)", sizes[p], sizes[p] - hybsizes[p]);CHKERRQ(ierr);}
599           else                  {ierr = PetscViewerASCIIPrintf(viewer, " %D", sizes[p]);CHKERRQ(ierr);}
600         }
601         ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
602       }
603     }
604     ierr = PetscFree2(sizes,hybsizes);CHKERRQ(ierr);
605     ierr = DMPlexGetNumLabels(dm, &numLabels);CHKERRQ(ierr);
606     if (numLabels) {ierr = PetscViewerASCIIPrintf(viewer, "Labels:\n");CHKERRQ(ierr);}
607     for (l = 0; l < numLabels; ++l) {
608       DMLabel         label;
609       const char     *name;
610       IS              valueIS;
611       const PetscInt *values;
612       PetscInt        numValues, v;
613 
614       ierr = DMPlexGetLabelName(dm, l, &name);CHKERRQ(ierr);
615       ierr = DMPlexGetLabel(dm, name, &label);CHKERRQ(ierr);
616       ierr = DMLabelGetNumValues(label, &numValues);CHKERRQ(ierr);
617       ierr = PetscViewerASCIIPrintf(viewer, "  %s: %d strata of sizes (", name, numValues);CHKERRQ(ierr);
618       ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
619       ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
620       for (v = 0; v < numValues; ++v) {
621         PetscInt size;
622 
623         ierr = DMLabelGetStratumSize(label, values[v], &size);CHKERRQ(ierr);
624         if (v > 0) {ierr = PetscViewerASCIIPrintf(viewer, ", ");CHKERRQ(ierr);}
625         ierr = PetscViewerASCIIPrintf(viewer, "%d", size);CHKERRQ(ierr);
626       }
627       ierr = PetscViewerASCIIPrintf(viewer, ")\n");CHKERRQ(ierr);
628       ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
629       ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
630     }
631   }
632   PetscFunctionReturn(0);
633 }
634 
635 #if defined(PETSC_HAVE_HDF5)
636 #undef __FUNCT__
637 #define __FUNCT__ "DMPlexView_HDF5"
638 /* We only write cells and vertices. Does this screw up parallel reading? */
639 static PetscErrorCode DMPlexView_HDF5(DM dm, PetscViewer viewer)
640 {
641   DM              cdm;
642   Vec             coordinates, newcoords;
643   Vec             coneVec, cellVec;
644   IS              globalVertexNumbers;
645   const PetscInt *gvertex;
646   PetscScalar    *sizes, *vertices;
647   PetscReal       lengthScale;
648   const char     *label   = NULL;
649   PetscInt        labelId = 0, dim;
650   char            group[PETSC_MAX_PATH_LEN];
651   PetscInt        vStart, vEnd, v, cellHeight, cStart, cEnd, cMax, cell, conesSize = 0, numCornersLocal = 0, numCorners, numLabels, l;
652   hid_t           fileId, groupId;
653   herr_t          status;
654   PetscErrorCode  ierr;
655 
656   PetscFunctionBegin;
657   /* Write coordinates */
658   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
659   ierr = DMPlexGetScale(dm, PETSC_UNIT_LENGTH, &lengthScale);CHKERRQ(ierr);
660   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
661   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
662   ierr = VecDuplicate(coordinates, &newcoords);CHKERRQ(ierr);
663   ierr = PetscObjectSetName((PetscObject) newcoords, "vertices");CHKERRQ(ierr);
664   ierr = VecCopy(coordinates, newcoords);CHKERRQ(ierr);
665   ierr = VecScale(newcoords, lengthScale);CHKERRQ(ierr);
666   /* Use the local version to bypass the default group setting */
667   ierr = PetscViewerHDF5PushGroup(viewer, "/geometry");CHKERRQ(ierr);
668   ierr = VecView(newcoords, viewer);CHKERRQ(ierr);
669   ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
670   ierr = VecDestroy(&newcoords);CHKERRQ(ierr);
671   /* Write toplogy */
672   ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
673   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
674   ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
675   ierr = DMPlexGetHybridBounds(dm, &cMax, PETSC_NULL, PETSC_NULL, PETSC_NULL);CHKERRQ(ierr);
676   if (cMax >= 0) cEnd = PetscMin(cEnd, cMax);
677 
678   ierr = VecCreate(PetscObjectComm((PetscObject) dm), &coneVec);CHKERRQ(ierr);
679   ierr = VecSetSizes(coneVec, cEnd-cStart, PETSC_DETERMINE);CHKERRQ(ierr);
680   ierr = VecSetBlockSize(coneVec, 1);CHKERRQ(ierr);
681   ierr = VecSetFromOptions(coneVec);CHKERRQ(ierr);
682   ierr = PetscObjectSetName((PetscObject) coneVec, "coneSize");CHKERRQ(ierr);
683   ierr = VecGetArray(coneVec, &sizes);CHKERRQ(ierr);
684   for (cell = cStart; cell < cEnd; ++cell) {
685     PetscInt *closure = NULL;
686     PetscInt  closureSize, v, Nc = 0;
687 
688     if (label) {
689       PetscInt value;
690       ierr = DMPlexGetLabelValue(dm, label, cell, &value);CHKERRQ(ierr);
691       if (value == labelId) continue;
692     }
693     ierr = DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
694     for (v = 0; v < closureSize*2; v += 2) {
695       if ((closure[v] >= vStart) && (closure[v] < vEnd)) ++Nc;
696     }
697     ierr = DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
698     conesSize += Nc;
699     if (!numCornersLocal)           numCornersLocal = Nc;
700     else if (numCornersLocal != Nc) numCornersLocal = 1;
701   }
702   ierr = VecRestoreArray(coneVec, &sizes);CHKERRQ(ierr);
703   ierr = MPI_Allreduce(&numCornersLocal, &numCorners, 1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject) dm));CHKERRQ(ierr);
704   if (numCornersLocal && numCornersLocal != numCorners) numCorners = 1;
705 
706   ierr = DMPlexGetVertexNumbering(dm, &globalVertexNumbers);CHKERRQ(ierr);
707   ierr = ISGetIndices(globalVertexNumbers, &gvertex);CHKERRQ(ierr);
708   ierr = VecCreate(PetscObjectComm((PetscObject) dm), &cellVec);CHKERRQ(ierr);
709   ierr = VecSetSizes(cellVec, conesSize, PETSC_DETERMINE);CHKERRQ(ierr);
710   ierr = VecSetBlockSize(cellVec, numCorners);CHKERRQ(ierr);
711   ierr = VecSetFromOptions(cellVec);CHKERRQ(ierr);
712   ierr = PetscObjectSetName((PetscObject) cellVec, "cells");CHKERRQ(ierr);
713   ierr = VecGetArray(cellVec, &vertices);CHKERRQ(ierr);
714   for (cell = cStart, v = 0; cell < cEnd; ++cell) {
715     PetscInt *closure = NULL;
716     PetscInt  closureSize, Nc = 0, p;
717 
718     if (label) {
719       PetscInt value;
720       ierr = DMPlexGetLabelValue(dm, label, cell, &value);CHKERRQ(ierr);
721       if (value == labelId) continue;
722     }
723     ierr = DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
724     for (p = 0; p < closureSize*2; p += 2) {
725       if ((closure[p] >= vStart) && (closure[p] < vEnd)) {
726         closure[Nc++] = closure[p];
727         }
728     }
729     ierr = DMPlexInvertCell(dim, Nc, closure);CHKERRQ(ierr);
730     for (p = 0; p < Nc; ++p) {
731       const PetscInt gv = gvertex[closure[p] - vStart];
732       vertices[v++] = gv < 0 ? -(gv+1) : gv;
733     }
734     ierr = DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
735   }
736   if (v != conesSize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_LIB, "Total number of cell vertices %d != %d", v, conesSize);
737   ierr = VecRestoreArray(cellVec, &vertices);CHKERRQ(ierr);
738   ierr = PetscViewerHDF5PushGroup(viewer, "/topology");CHKERRQ(ierr);
739   ierr = VecView(cellVec, viewer);CHKERRQ(ierr);
740   if (numCorners == 1) {
741     ierr = VecView(coneVec, viewer);CHKERRQ(ierr);
742   } else {
743     ierr = PetscViewerHDF5WriteAttribute(viewer, "/topology/cells", "cell_corners", PETSC_INT, (void *) &numCorners);CHKERRQ(ierr);
744   }
745   ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
746   ierr = VecDestroy(&cellVec);CHKERRQ(ierr);
747   ierr = VecDestroy(&coneVec);CHKERRQ(ierr);
748   ierr = ISRestoreIndices(globalVertexNumbers, &gvertex);CHKERRQ(ierr);
749 
750   ierr = PetscViewerHDF5WriteAttribute(viewer, "/topology/cells", "cell_dim", PETSC_INT, (void *) &dim);CHKERRQ(ierr);
751   /* Write Labels*/
752   ierr = PetscViewerHDF5PushGroup(viewer, "/labels");CHKERRQ(ierr);
753   ierr = PetscViewerHDF5OpenGroup(viewer, &fileId, &groupId);CHKERRQ(ierr);
754   if (groupId != fileId) {status = H5Gclose(groupId);CHKERRQ(status);}
755   ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
756   ierr = DMPlexGetNumLabels(dm, &numLabels);CHKERRQ(ierr);
757   for (l = 0; l < numLabels; ++l) {
758     DMLabel         label;
759     const char     *name;
760     IS              valueIS;
761     const PetscInt *values;
762     PetscInt        numValues, v;
763     PetscBool       isDepth;
764 
765     ierr = DMPlexGetLabelName(dm, l, &name);CHKERRQ(ierr);
766     ierr = PetscStrncmp(name, "depth", 10, &isDepth);CHKERRQ(ierr);
767     if (isDepth) continue;
768     ierr = DMPlexGetLabel(dm, name, &label);CHKERRQ(ierr);
769     ierr = DMLabelGetNumValues(label, &numValues);CHKERRQ(ierr);
770     ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
771     ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
772     ierr = PetscSNPrintf(group, PETSC_MAX_PATH_LEN, "/labels/%s", name);CHKERRQ(ierr);
773     ierr = PetscViewerHDF5PushGroup(viewer, group);CHKERRQ(ierr);
774     ierr = PetscViewerHDF5OpenGroup(viewer, &fileId, &groupId);CHKERRQ(ierr);
775     if (groupId != fileId) {status = H5Gclose(groupId);CHKERRQ(status);}
776     ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
777     /* TODO: Need to actually loop over the union of label values, ISAllGather() */
778     for (v = 0; v < numValues; ++v) {
779       IS   stratumIS;
780 
781       ierr = PetscSNPrintf(group, PETSC_MAX_PATH_LEN, "/labels/%s/%d", name, values[v]);CHKERRQ(ierr);
782       ierr = DMLabelGetStratumIS(label, values[v], &stratumIS);CHKERRQ(ierr);
783       /* TODO: Need to globalize point names and remove unowned points */
784       ierr = PetscViewerHDF5PushGroup(viewer, group);CHKERRQ(ierr);
785       ierr = ISView(stratumIS, viewer);CHKERRQ(ierr);
786       ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
787       ierr = ISDestroy(&stratumIS);CHKERRQ(ierr);
788     }
789     ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
790     ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
791   }
792   PetscFunctionReturn(0);
793 }
794 #endif
795 
796 #undef __FUNCT__
797 #define __FUNCT__ "DMView_Plex"
798 PetscErrorCode DMView_Plex(DM dm, PetscViewer viewer)
799 {
800   PetscBool      iascii, ishdf5;
801   PetscErrorCode ierr;
802 
803   PetscFunctionBegin;
804   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
805   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
806   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERASCII, &iascii);CHKERRQ(ierr);
807   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,  &ishdf5);CHKERRQ(ierr);
808   if (iascii) {
809     ierr = DMPlexView_Ascii(dm, viewer);CHKERRQ(ierr);
810   } else if (ishdf5) {
811 #if defined(PETSC_HAVE_HDF5)
812     ierr = DMPlexView_HDF5(dm, viewer);CHKERRQ(ierr);
813 #else
814     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
815 #endif
816   }
817   PetscFunctionReturn(0);
818 }
819 
820 #if defined(PETSC_HAVE_HDF5)
821 typedef struct {
822   DM          dm;
823   PetscViewer viewer;
824   DMLabel     label;
825 } LabelCtx;
826 
827 static herr_t ReadLabelStratumHDF5_Static(hid_t g_id, const char *name, const H5L_info_t *info, void *op_data)
828 {
829   PetscViewer     viewer = ((LabelCtx *) op_data)->viewer;
830   DMLabel         label  = ((LabelCtx *) op_data)->label;
831   IS              stratumIS;
832   const PetscInt *ind;
833   PetscInt        value, N, i;
834   const char     *lname;
835   char            group[PETSC_MAX_PATH_LEN];
836   PetscErrorCode  ierr;
837 
838   ierr = PetscOptionsStringToInt(name, &value);
839   ierr = ISCreate(PetscObjectComm((PetscObject) viewer), &stratumIS);
840   ierr = PetscObjectSetName((PetscObject) stratumIS, "indices");
841   ierr = DMLabelGetName(label, &lname);
842   ierr = PetscSNPrintf(group, PETSC_MAX_PATH_LEN, "/labels/%s/%s", lname, name);CHKERRQ(ierr);
843   ierr = PetscViewerHDF5PushGroup(viewer, group);CHKERRQ(ierr);
844   ierr = ISLoad(stratumIS, viewer);
845   ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
846   ierr = ISGetSize(stratumIS, &N);
847   ierr = ISGetIndices(stratumIS, &ind);
848   for (i = 0; i < N; ++i) {ierr = DMLabelSetValue(label, ind[i], value);}
849   ierr = ISRestoreIndices(stratumIS, &ind);
850   ierr = ISDestroy(&stratumIS);
851   return 0;
852 }
853 
854 static herr_t ReadLabelHDF5_Static(hid_t g_id, const char *name, const H5L_info_t *info, void *op_data)
855 {
856   DM             dm = ((LabelCtx *) op_data)->dm;
857   hsize_t        idx;
858   herr_t         status;
859   PetscErrorCode ierr;
860 
861   ierr = DMPlexCreateLabel(dm, name); if (ierr) return (herr_t) ierr;
862   ierr = DMPlexGetLabel(dm, name, &((LabelCtx *) op_data)->label); if (ierr) return (herr_t) ierr;
863   status = H5Literate_by_name(g_id, name, H5_INDEX_NAME, H5_ITER_NATIVE, &idx, ReadLabelStratumHDF5_Static, op_data, 0);
864   return status;
865 }
866 
867 #undef __FUNCT__
868 #define __FUNCT__ "DMPlexLoad_HDF5"
869 /* The first version will read everything onto proc 0, letting the user distribute
870    The next will create a naive partition, and then rebalance after reading
871 */
872 static PetscErrorCode DMPlexLoad_HDF5(DM dm, PetscViewer viewer)
873 {
874   LabelCtx        ctx;
875   PetscSection    coordSection;
876   Vec             coordinates;
877   Vec             cellVec;
878   PetscScalar    *cells;
879   PetscReal       lengthScale;
880   PetscInt       *cone;
881   PetscInt        dim, spatialDim, numVertices, v, numCorners, numCells, cell, c;
882   hid_t           fileId, groupId;
883   hsize_t         idx;
884   herr_t          status;
885   PetscErrorCode  ierr;
886 
887   PetscFunctionBegin;
888   /* Read geometry */
889   ierr = PetscViewerHDF5PushGroup(viewer, "/geometry");CHKERRQ(ierr);
890   ierr = VecCreate(PetscObjectComm((PetscObject) dm), &coordinates);CHKERRQ(ierr);
891   ierr = PetscObjectSetName((PetscObject) coordinates, "vertices");CHKERRQ(ierr);
892   ierr = VecLoad(coordinates, viewer);CHKERRQ(ierr);
893   ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
894   ierr = DMPlexGetScale(dm, PETSC_UNIT_LENGTH, &lengthScale);CHKERRQ(ierr);
895   ierr = VecScale(coordinates, 1.0/lengthScale);CHKERRQ(ierr);
896   ierr = VecGetSize(coordinates, &numVertices);CHKERRQ(ierr);
897   ierr = VecGetBlockSize(coordinates, &spatialDim);CHKERRQ(ierr);
898   numVertices /= spatialDim;
899   /* Read toplogy */
900   ierr = PetscViewerHDF5ReadAttribute(viewer, "/topology/cells", "cell_dim", PETSC_INT, (void *) &dim);CHKERRQ(ierr);
901   ierr = DMPlexSetDimension(dm, dim);CHKERRQ(ierr);
902   /*   TODO Check for coneSize vector rather than this attribute */
903   ierr = PetscViewerHDF5ReadAttribute(viewer, "/topology/cells", "cell_corners", PETSC_INT, (void *) &numCorners);CHKERRQ(ierr);
904   ierr = PetscViewerHDF5PushGroup(viewer, "/topology");CHKERRQ(ierr);
905   ierr = VecCreate(PetscObjectComm((PetscObject) dm), &cellVec);CHKERRQ(ierr);
906   ierr = VecSetBlockSize(cellVec, numCorners);CHKERRQ(ierr);
907   ierr = PetscObjectSetName((PetscObject) cellVec, "cells");CHKERRQ(ierr);
908   ierr = VecLoad(cellVec, viewer);CHKERRQ(ierr);
909   ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
910   ierr = VecGetSize(cellVec, &numCells);CHKERRQ(ierr);
911   numCells /= numCorners;
912   /* Create Plex */
913   ierr = DMPlexSetChart(dm, 0, numCells+numVertices);CHKERRQ(ierr);
914   for (cell = 0; cell < numCells; ++cell) {ierr = DMPlexSetConeSize(dm, cell, numCorners);CHKERRQ(ierr);}
915   ierr = DMSetUp(dm);CHKERRQ(ierr);
916   ierr = PetscMalloc1(numCorners,&cone);CHKERRQ(ierr);
917   ierr = VecGetArray(cellVec, &cells);CHKERRQ(ierr);
918   for (cell = 0; cell < numCells; ++cell) {
919     for (c = 0; c < numCorners; ++c) {cone[c] = numCells + cells[cell*numCorners+c];}
920     ierr = DMPlexSetCone(dm, cell, cone);CHKERRQ(ierr);
921   }
922   ierr = VecRestoreArray(cellVec, &cells);CHKERRQ(ierr);
923   ierr = PetscFree(cone);CHKERRQ(ierr);
924   ierr = VecDestroy(&cellVec);CHKERRQ(ierr);
925   ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
926   ierr = DMPlexStratify(dm);CHKERRQ(ierr);
927   /* Create coordinates */
928   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
929   ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
930   ierr = PetscSectionSetFieldComponents(coordSection, 0, spatialDim);CHKERRQ(ierr);
931   ierr = PetscSectionSetChart(coordSection, numCells, numCells+numVertices);CHKERRQ(ierr);
932   for (v = numCells; v < numCells+numVertices; ++v) {
933     ierr = PetscSectionSetDof(coordSection, v, spatialDim);CHKERRQ(ierr);
934     ierr = PetscSectionSetFieldDof(coordSection, v, 0, spatialDim);CHKERRQ(ierr);
935   }
936   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
937   ierr = DMSetCoordinates(dm, coordinates);CHKERRQ(ierr);
938   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
939   /* Read Labels*/
940   ctx.dm     = dm;
941   ctx.viewer = viewer;
942   ierr = PetscViewerHDF5PushGroup(viewer, "/labels");CHKERRQ(ierr);
943   ierr = PetscViewerHDF5OpenGroup(viewer, &fileId, &groupId);CHKERRQ(ierr);
944   status = H5Literate(groupId, H5_INDEX_NAME, H5_ITER_NATIVE, &idx, ReadLabelHDF5_Static, &ctx);CHKERRQ(status);
945   status = H5Gclose(groupId);CHKERRQ(status);
946   ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
947   PetscFunctionReturn(0);
948 }
949 #endif
950 
951 #undef __FUNCT__
952 #define __FUNCT__ "DMLoad_Plex"
953 PetscErrorCode DMLoad_Plex(DM dm, PetscViewer viewer)
954 {
955   PetscBool      isbinary, ishdf5;
956   PetscErrorCode ierr;
957 
958   PetscFunctionBegin;
959   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
960   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
961   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERBINARY, &isbinary);CHKERRQ(ierr);
962   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,   &ishdf5);CHKERRQ(ierr);
963   if (isbinary) {SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Do not yet support binary viewers");}
964   else if (ishdf5) {
965 #if defined(PETSC_HAVE_HDF5)
966     DM odm;
967     ierr = DMCreate(PetscObjectComm((PetscObject) dm), &odm);CHKERRQ(ierr);
968     ierr = DMSetType(odm, DMPLEX);CHKERRQ(ierr);
969     ierr = DMPlexLoad_HDF5(odm, viewer);CHKERRQ(ierr);
970     ierr = DMPlexInterpolate(odm, &dm);CHKERRQ(ierr);
971     ierr = DMPlexCopyCoordinates(odm, dm);CHKERRQ(ierr);
972     ierr = DMPlexCopyLabels(odm, dm);CHKERRQ(ierr);
973     ierr = DMDestroy(&odm);CHKERRQ(ierr);
974 #else
975     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
976 #endif
977   }
978   PetscFunctionReturn(0);
979 }
980 
981 #undef __FUNCT__
982 #define __FUNCT__ "BoundaryDestroy"
983 static PetscErrorCode BoundaryDestroy(DMBoundary *boundary)
984 {
985   DMBoundary     b, next;
986   PetscErrorCode ierr;
987 
988   PetscFunctionBeginUser;
989   if (!boundary) PetscFunctionReturn(0);
990   b = *boundary;
991   *boundary = NULL;
992   for (; b; b = next) {
993     next = b->next;
994     ierr = PetscFree(b->ids);CHKERRQ(ierr);
995     ierr = PetscFree(b->name);CHKERRQ(ierr);
996     ierr = PetscFree(b);CHKERRQ(ierr);
997   }
998   PetscFunctionReturn(0);
999 }
1000 
1001 #undef __FUNCT__
1002 #define __FUNCT__ "DMDestroy_Plex"
1003 PetscErrorCode DMDestroy_Plex(DM dm)
1004 {
1005   DM_Plex       *mesh = (DM_Plex*) dm->data;
1006   DMLabel        next  = mesh->labels;
1007   PetscErrorCode ierr;
1008 
1009   PetscFunctionBegin;
1010   if (--mesh->refct > 0) PetscFunctionReturn(0);
1011   ierr = PetscSectionDestroy(&mesh->coneSection);CHKERRQ(ierr);
1012   ierr = PetscFree(mesh->cones);CHKERRQ(ierr);
1013   ierr = PetscFree(mesh->coneOrientations);CHKERRQ(ierr);
1014   ierr = PetscSectionDestroy(&mesh->supportSection);CHKERRQ(ierr);
1015   ierr = PetscFree(mesh->supports);CHKERRQ(ierr);
1016   ierr = PetscFree(mesh->facesTmp);CHKERRQ(ierr);
1017   while (next) {
1018     DMLabel tmp = next->next;
1019 
1020     ierr = DMLabelDestroy(&next);CHKERRQ(ierr);
1021     next = tmp;
1022   }
1023   ierr = DMDestroy(&mesh->coarseMesh);CHKERRQ(ierr);
1024   ierr = DMLabelDestroy(&mesh->subpointMap);CHKERRQ(ierr);
1025   ierr = ISDestroy(&mesh->globalVertexNumbers);CHKERRQ(ierr);
1026   ierr = ISDestroy(&mesh->globalCellNumbers);CHKERRQ(ierr);
1027   ierr = BoundaryDestroy(&mesh->boundary);CHKERRQ(ierr);
1028   /* This was originally freed in DMDestroy(), but that prevents reference counting of backend objects */
1029   ierr = PetscFree(mesh);CHKERRQ(ierr);
1030   PetscFunctionReturn(0);
1031 }
1032 
1033 #undef __FUNCT__
1034 #define __FUNCT__ "DMCreateMatrix_Plex"
1035 PetscErrorCode DMCreateMatrix_Plex(DM dm, Mat *J)
1036 {
1037   PetscSection   section, sectionGlobal;
1038   PetscInt       bs = -1;
1039   PetscInt       localSize;
1040   PetscBool      isShell, isBlock, isSeqBlock, isMPIBlock, isSymBlock, isSymSeqBlock, isSymMPIBlock;
1041   PetscErrorCode ierr;
1042   MatType        mtype;
1043 
1044   PetscFunctionBegin;
1045   ierr = MatInitializePackage();CHKERRQ(ierr);
1046   mtype = dm->mattype;
1047   ierr = DMGetDefaultSection(dm, &section);CHKERRQ(ierr);
1048   ierr = DMGetDefaultGlobalSection(dm, &sectionGlobal);CHKERRQ(ierr);
1049   /* ierr = PetscSectionGetStorageSize(sectionGlobal, &localSize);CHKERRQ(ierr); */
1050   ierr = PetscSectionGetConstrainedStorageSize(sectionGlobal, &localSize);CHKERRQ(ierr);
1051   ierr = MatCreate(PetscObjectComm((PetscObject)dm), J);CHKERRQ(ierr);
1052   ierr = MatSetSizes(*J, localSize, localSize, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
1053   ierr = MatSetType(*J, mtype);CHKERRQ(ierr);
1054   ierr = MatSetFromOptions(*J);CHKERRQ(ierr);
1055   ierr = PetscStrcmp(mtype, MATSHELL, &isShell);CHKERRQ(ierr);
1056   ierr = PetscStrcmp(mtype, MATBAIJ, &isBlock);CHKERRQ(ierr);
1057   ierr = PetscStrcmp(mtype, MATSEQBAIJ, &isSeqBlock);CHKERRQ(ierr);
1058   ierr = PetscStrcmp(mtype, MATMPIBAIJ, &isMPIBlock);CHKERRQ(ierr);
1059   ierr = PetscStrcmp(mtype, MATSBAIJ, &isSymBlock);CHKERRQ(ierr);
1060   ierr = PetscStrcmp(mtype, MATSEQSBAIJ, &isSymSeqBlock);CHKERRQ(ierr);
1061   ierr = PetscStrcmp(mtype, MATMPISBAIJ, &isSymMPIBlock);CHKERRQ(ierr);
1062   if (!isShell) {
1063     PetscBool fillMatrix = (PetscBool) !dm->prealloc_only;
1064     PetscInt *dnz, *onz, *dnzu, *onzu, bsLocal, bsMax, bsMin;
1065 
1066     if (bs < 0) {
1067       if (isBlock || isSeqBlock || isMPIBlock || isSymBlock || isSymSeqBlock || isSymMPIBlock) {
1068         PetscInt pStart, pEnd, p, dof, cdof;
1069 
1070         ierr = PetscSectionGetChart(sectionGlobal, &pStart, &pEnd);CHKERRQ(ierr);
1071         for (p = pStart; p < pEnd; ++p) {
1072           ierr = PetscSectionGetDof(sectionGlobal, p, &dof);CHKERRQ(ierr);
1073           ierr = PetscSectionGetConstraintDof(sectionGlobal, p, &cdof);CHKERRQ(ierr);
1074           if (dof-cdof) {
1075             if (bs < 0) {
1076               bs = dof-cdof;
1077             } else if (bs != dof-cdof) {
1078               /* Layout does not admit a pointwise block size */
1079               bs = 1;
1080               break;
1081             }
1082           }
1083         }
1084         /* Must have same blocksize on all procs (some might have no points) */
1085         bsLocal = bs;
1086         ierr = MPI_Allreduce(&bsLocal, &bsMax, 1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
1087         bsLocal = bs < 0 ? bsMax : bs;
1088         ierr = MPI_Allreduce(&bsLocal, &bsMin, 1, MPIU_INT, MPI_MIN, PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
1089         if (bsMin != bsMax) {
1090           bs = 1;
1091         } else {
1092           bs = bsMax;
1093         }
1094       } else {
1095         bs = 1;
1096       }
1097     }
1098     ierr = PetscCalloc4(localSize/bs, &dnz, localSize/bs, &onz, localSize/bs, &dnzu, localSize/bs, &onzu);CHKERRQ(ierr);
1099     ierr = DMPlexPreallocateOperator(dm, bs, section, sectionGlobal, dnz, onz, dnzu, onzu, *J, fillMatrix);CHKERRQ(ierr);
1100     ierr = PetscFree4(dnz, onz, dnzu, onzu);CHKERRQ(ierr);
1101   }
1102   PetscFunctionReturn(0);
1103 }
1104 
1105 #undef __FUNCT__
1106 #define __FUNCT__ "DMPlexGetDimension"
1107 /*@
1108   DMPlexGetDimension - Return the topological mesh dimension
1109 
1110   Not collective
1111 
1112   Input Parameter:
1113 . mesh - The DMPlex
1114 
1115   Output Parameter:
1116 . dim - The topological mesh dimension
1117 
1118   Level: beginner
1119 
1120 .seealso: DMPlexCreate()
1121 @*/
1122 PetscErrorCode DMPlexGetDimension(DM dm, PetscInt *dim)
1123 {
1124   DM_Plex *mesh = (DM_Plex*) dm->data;
1125 
1126   PetscFunctionBegin;
1127   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1128   PetscValidPointer(dim, 2);
1129   *dim = mesh->dim;
1130   PetscFunctionReturn(0);
1131 }
1132 
1133 #undef __FUNCT__
1134 #define __FUNCT__ "DMPlexSetDimension"
1135 /*@
1136   DMPlexSetDimension - Set the topological mesh dimension
1137 
1138   Collective on mesh
1139 
1140   Input Parameters:
1141 + mesh - The DMPlex
1142 - dim - The topological mesh dimension
1143 
1144   Level: beginner
1145 
1146 .seealso: DMPlexCreate()
1147 @*/
1148 PetscErrorCode DMPlexSetDimension(DM dm, PetscInt dim)
1149 {
1150   DM_Plex *mesh = (DM_Plex*) dm->data;
1151 
1152   PetscFunctionBegin;
1153   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1154   PetscValidLogicalCollectiveInt(dm, dim, 2);
1155   mesh->dim = dim;
1156   PetscFunctionReturn(0);
1157 }
1158 
1159 #undef __FUNCT__
1160 #define __FUNCT__ "DMPlexGetChart"
1161 /*@
1162   DMPlexGetChart - Return the interval for all mesh points [pStart, pEnd)
1163 
1164   Not collective
1165 
1166   Input Parameter:
1167 . mesh - The DMPlex
1168 
1169   Output Parameters:
1170 + pStart - The first mesh point
1171 - pEnd   - The upper bound for mesh points
1172 
1173   Level: beginner
1174 
1175 .seealso: DMPlexCreate(), DMPlexSetChart()
1176 @*/
1177 PetscErrorCode DMPlexGetChart(DM dm, PetscInt *pStart, PetscInt *pEnd)
1178 {
1179   DM_Plex       *mesh = (DM_Plex*) dm->data;
1180   PetscErrorCode ierr;
1181 
1182   PetscFunctionBegin;
1183   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1184   ierr = PetscSectionGetChart(mesh->coneSection, pStart, pEnd);CHKERRQ(ierr);
1185   PetscFunctionReturn(0);
1186 }
1187 
1188 #undef __FUNCT__
1189 #define __FUNCT__ "DMPlexSetChart"
1190 /*@
1191   DMPlexSetChart - Set the interval for all mesh points [pStart, pEnd)
1192 
1193   Not collective
1194 
1195   Input Parameters:
1196 + mesh - The DMPlex
1197 . pStart - The first mesh point
1198 - pEnd   - The upper bound for mesh points
1199 
1200   Output Parameters:
1201 
1202   Level: beginner
1203 
1204 .seealso: DMPlexCreate(), DMPlexGetChart()
1205 @*/
1206 PetscErrorCode DMPlexSetChart(DM dm, PetscInt pStart, PetscInt pEnd)
1207 {
1208   DM_Plex       *mesh = (DM_Plex*) dm->data;
1209   PetscErrorCode ierr;
1210 
1211   PetscFunctionBegin;
1212   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1213   ierr = PetscSectionSetChart(mesh->coneSection, pStart, pEnd);CHKERRQ(ierr);
1214   ierr = PetscSectionSetChart(mesh->supportSection, pStart, pEnd);CHKERRQ(ierr);
1215   PetscFunctionReturn(0);
1216 }
1217 
1218 #undef __FUNCT__
1219 #define __FUNCT__ "DMPlexGetConeSize"
1220 /*@
1221   DMPlexGetConeSize - Return the number of in-edges for this point in the Sieve DAG
1222 
1223   Not collective
1224 
1225   Input Parameters:
1226 + mesh - The DMPlex
1227 - p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1228 
1229   Output Parameter:
1230 . size - The cone size for point p
1231 
1232   Level: beginner
1233 
1234 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart()
1235 @*/
1236 PetscErrorCode DMPlexGetConeSize(DM dm, PetscInt p, PetscInt *size)
1237 {
1238   DM_Plex       *mesh = (DM_Plex*) dm->data;
1239   PetscErrorCode ierr;
1240 
1241   PetscFunctionBegin;
1242   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1243   PetscValidPointer(size, 3);
1244   ierr = PetscSectionGetDof(mesh->coneSection, p, size);CHKERRQ(ierr);
1245   PetscFunctionReturn(0);
1246 }
1247 
1248 #undef __FUNCT__
1249 #define __FUNCT__ "DMPlexSetConeSize"
1250 /*@
1251   DMPlexSetConeSize - Set the number of in-edges for this point in the Sieve DAG
1252 
1253   Not collective
1254 
1255   Input Parameters:
1256 + mesh - The DMPlex
1257 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1258 - size - The cone size for point p
1259 
1260   Output Parameter:
1261 
1262   Note:
1263   This should be called after DMPlexSetChart().
1264 
1265   Level: beginner
1266 
1267 .seealso: DMPlexCreate(), DMPlexGetConeSize(), DMPlexSetChart()
1268 @*/
1269 PetscErrorCode DMPlexSetConeSize(DM dm, PetscInt p, PetscInt size)
1270 {
1271   DM_Plex       *mesh = (DM_Plex*) dm->data;
1272   PetscErrorCode ierr;
1273 
1274   PetscFunctionBegin;
1275   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1276   ierr = PetscSectionSetDof(mesh->coneSection, p, size);CHKERRQ(ierr);
1277 
1278   mesh->maxConeSize = PetscMax(mesh->maxConeSize, size);
1279   PetscFunctionReturn(0);
1280 }
1281 
1282 #undef __FUNCT__
1283 #define __FUNCT__ "DMPlexGetCone"
1284 /*@C
1285   DMPlexGetCone - Return the points on the in-edges for this point in the Sieve DAG
1286 
1287   Not collective
1288 
1289   Input Parameters:
1290 + mesh - The DMPlex
1291 - p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1292 
1293   Output Parameter:
1294 . cone - An array of points which are on the in-edges for point p
1295 
1296   Level: beginner
1297 
1298   Fortran Notes:
1299   Since it returns an array, this routine is only available in Fortran 90, and you must
1300   include petsc.h90 in your code.
1301 
1302   You must also call DMPlexRestoreCone() after you finish using the returned array.
1303 
1304 .seealso: DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart()
1305 @*/
1306 PetscErrorCode DMPlexGetCone(DM dm, PetscInt p, const PetscInt *cone[])
1307 {
1308   DM_Plex       *mesh = (DM_Plex*) dm->data;
1309   PetscInt       off;
1310   PetscErrorCode ierr;
1311 
1312   PetscFunctionBegin;
1313   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1314   PetscValidPointer(cone, 3);
1315   ierr  = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1316   *cone = &mesh->cones[off];
1317   PetscFunctionReturn(0);
1318 }
1319 
1320 #undef __FUNCT__
1321 #define __FUNCT__ "DMPlexSetCone"
1322 /*@
1323   DMPlexSetCone - Set the points on the in-edges for this point in the Sieve DAG
1324 
1325   Not collective
1326 
1327   Input Parameters:
1328 + mesh - The DMPlex
1329 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1330 - cone - An array of points which are on the in-edges for point p
1331 
1332   Output Parameter:
1333 
1334   Note:
1335   This should be called after all calls to DMPlexSetConeSize() and DMSetUp().
1336 
1337   Level: beginner
1338 
1339 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
1340 @*/
1341 PetscErrorCode DMPlexSetCone(DM dm, PetscInt p, const PetscInt cone[])
1342 {
1343   DM_Plex       *mesh = (DM_Plex*) dm->data;
1344   PetscInt       pStart, pEnd;
1345   PetscInt       dof, off, c;
1346   PetscErrorCode ierr;
1347 
1348   PetscFunctionBegin;
1349   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1350   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
1351   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
1352   if (dof) PetscValidPointer(cone, 3);
1353   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1354   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
1355   for (c = 0; c < dof; ++c) {
1356     if ((cone[c] < pStart) || (cone[c] >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone point %D is not in the valid range [%D, %D)", cone[c], pStart, pEnd);
1357     mesh->cones[off+c] = cone[c];
1358   }
1359   PetscFunctionReturn(0);
1360 }
1361 
1362 #undef __FUNCT__
1363 #define __FUNCT__ "DMPlexGetConeOrientation"
1364 /*@C
1365   DMPlexGetConeOrientation - Return the orientations on the in-edges for this point in the Sieve DAG
1366 
1367   Not collective
1368 
1369   Input Parameters:
1370 + mesh - The DMPlex
1371 - p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1372 
1373   Output Parameter:
1374 . coneOrientation - An array of orientations which are on the in-edges for point p. An orientation is an
1375                     integer giving the prescription for cone traversal. If it is negative, the cone is
1376                     traversed in the opposite direction. Its value 'o', or if negative '-(o+1)', gives
1377                     the index of the cone point on which to start.
1378 
1379   Level: beginner
1380 
1381   Fortran Notes:
1382   Since it returns an array, this routine is only available in Fortran 90, and you must
1383   include petsc.h90 in your code.
1384 
1385   You must also call DMPlexRestoreConeOrientation() after you finish using the returned array.
1386 
1387 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetCone(), DMPlexSetChart()
1388 @*/
1389 PetscErrorCode DMPlexGetConeOrientation(DM dm, PetscInt p, const PetscInt *coneOrientation[])
1390 {
1391   DM_Plex       *mesh = (DM_Plex*) dm->data;
1392   PetscInt       off;
1393   PetscErrorCode ierr;
1394 
1395   PetscFunctionBegin;
1396   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1397 #if defined(PETSC_USE_DEBUG)
1398   {
1399     PetscInt dof;
1400     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
1401     if (dof) PetscValidPointer(coneOrientation, 3);
1402   }
1403 #endif
1404   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1405 
1406   *coneOrientation = &mesh->coneOrientations[off];
1407   PetscFunctionReturn(0);
1408 }
1409 
1410 #undef __FUNCT__
1411 #define __FUNCT__ "DMPlexSetConeOrientation"
1412 /*@
1413   DMPlexSetConeOrientation - Set the orientations on the in-edges for this point in the Sieve DAG
1414 
1415   Not collective
1416 
1417   Input Parameters:
1418 + mesh - The DMPlex
1419 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1420 - coneOrientation - An array of orientations which are on the in-edges for point p. An orientation is an
1421                     integer giving the prescription for cone traversal. If it is negative, the cone is
1422                     traversed in the opposite direction. Its value 'o', or if negative '-(o+1)', gives
1423                     the index of the cone point on which to start.
1424 
1425   Output Parameter:
1426 
1427   Note:
1428   This should be called after all calls to DMPlexSetConeSize() and DMSetUp().
1429 
1430   Level: beginner
1431 
1432 .seealso: DMPlexCreate(), DMPlexGetConeOrientation(), DMPlexSetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
1433 @*/
1434 PetscErrorCode DMPlexSetConeOrientation(DM dm, PetscInt p, const PetscInt coneOrientation[])
1435 {
1436   DM_Plex       *mesh = (DM_Plex*) dm->data;
1437   PetscInt       pStart, pEnd;
1438   PetscInt       dof, off, c;
1439   PetscErrorCode ierr;
1440 
1441   PetscFunctionBegin;
1442   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1443   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
1444   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
1445   if (dof) PetscValidPointer(coneOrientation, 3);
1446   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1447   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
1448   for (c = 0; c < dof; ++c) {
1449     PetscInt cdof, o = coneOrientation[c];
1450 
1451     ierr = PetscSectionGetDof(mesh->coneSection, mesh->cones[off+c], &cdof);CHKERRQ(ierr);
1452     if (o && ((o < -(cdof+1)) || (o >= cdof))) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone orientation %D is not in the valid range [%D. %D)", o, -(cdof+1), cdof);
1453     mesh->coneOrientations[off+c] = o;
1454   }
1455   PetscFunctionReturn(0);
1456 }
1457 
1458 #undef __FUNCT__
1459 #define __FUNCT__ "DMPlexInsertCone"
1460 PetscErrorCode DMPlexInsertCone(DM dm, PetscInt p, PetscInt conePos, PetscInt conePoint)
1461 {
1462   DM_Plex       *mesh = (DM_Plex*) dm->data;
1463   PetscInt       pStart, pEnd;
1464   PetscInt       dof, off;
1465   PetscErrorCode ierr;
1466 
1467   PetscFunctionBegin;
1468   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1469   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
1470   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
1471   if ((conePoint < pStart) || (conePoint >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone point %D is not in the valid range [%D, %D)", conePoint, pStart, pEnd);
1472   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
1473   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1474   if ((conePos < 0) || (conePos >= dof)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone position %D of point %D is not in the valid range [0, %D)", conePos, p, dof);
1475   mesh->cones[off+conePos] = conePoint;
1476   PetscFunctionReturn(0);
1477 }
1478 
1479 #undef __FUNCT__
1480 #define __FUNCT__ "DMPlexInsertConeOrientation"
1481 PetscErrorCode DMPlexInsertConeOrientation(DM dm, PetscInt p, PetscInt conePos, PetscInt coneOrientation)
1482 {
1483   DM_Plex       *mesh = (DM_Plex*) dm->data;
1484   PetscInt       pStart, pEnd;
1485   PetscInt       dof, off;
1486   PetscErrorCode ierr;
1487 
1488   PetscFunctionBegin;
1489   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1490   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
1491   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
1492   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
1493   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1494   if ((conePos < 0) || (conePos >= dof)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone position %D of point %D is not in the valid range [0, %D)", conePos, p, dof);
1495   mesh->coneOrientations[off+conePos] = coneOrientation;
1496   PetscFunctionReturn(0);
1497 }
1498 
1499 #undef __FUNCT__
1500 #define __FUNCT__ "DMPlexGetSupportSize"
1501 /*@
1502   DMPlexGetSupportSize - Return the number of out-edges for this point in the Sieve DAG
1503 
1504   Not collective
1505 
1506   Input Parameters:
1507 + mesh - The DMPlex
1508 - p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1509 
1510   Output Parameter:
1511 . size - The support size for point p
1512 
1513   Level: beginner
1514 
1515 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart(), DMPlexGetConeSize()
1516 @*/
1517 PetscErrorCode DMPlexGetSupportSize(DM dm, PetscInt p, PetscInt *size)
1518 {
1519   DM_Plex       *mesh = (DM_Plex*) dm->data;
1520   PetscErrorCode ierr;
1521 
1522   PetscFunctionBegin;
1523   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1524   PetscValidPointer(size, 3);
1525   ierr = PetscSectionGetDof(mesh->supportSection, p, size);CHKERRQ(ierr);
1526   PetscFunctionReturn(0);
1527 }
1528 
1529 #undef __FUNCT__
1530 #define __FUNCT__ "DMPlexSetSupportSize"
1531 /*@
1532   DMPlexSetSupportSize - Set the number of out-edges for this point in the Sieve DAG
1533 
1534   Not collective
1535 
1536   Input Parameters:
1537 + mesh - The DMPlex
1538 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1539 - size - The support size for point p
1540 
1541   Output Parameter:
1542 
1543   Note:
1544   This should be called after DMPlexSetChart().
1545 
1546   Level: beginner
1547 
1548 .seealso: DMPlexCreate(), DMPlexGetSupportSize(), DMPlexSetChart()
1549 @*/
1550 PetscErrorCode DMPlexSetSupportSize(DM dm, PetscInt p, PetscInt size)
1551 {
1552   DM_Plex       *mesh = (DM_Plex*) dm->data;
1553   PetscErrorCode ierr;
1554 
1555   PetscFunctionBegin;
1556   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1557   ierr = PetscSectionSetDof(mesh->supportSection, p, size);CHKERRQ(ierr);
1558 
1559   mesh->maxSupportSize = PetscMax(mesh->maxSupportSize, size);
1560   PetscFunctionReturn(0);
1561 }
1562 
1563 #undef __FUNCT__
1564 #define __FUNCT__ "DMPlexGetSupport"
1565 /*@C
1566   DMPlexGetSupport - Return the points on the out-edges for this point in the Sieve DAG
1567 
1568   Not collective
1569 
1570   Input Parameters:
1571 + mesh - The DMPlex
1572 - p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1573 
1574   Output Parameter:
1575 . support - An array of points which are on the out-edges for point p
1576 
1577   Level: beginner
1578 
1579   Fortran Notes:
1580   Since it returns an array, this routine is only available in Fortran 90, and you must
1581   include petsc.h90 in your code.
1582 
1583   You must also call DMPlexRestoreSupport() after you finish using the returned array.
1584 
1585 .seealso: DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
1586 @*/
1587 PetscErrorCode DMPlexGetSupport(DM dm, PetscInt p, const PetscInt *support[])
1588 {
1589   DM_Plex       *mesh = (DM_Plex*) dm->data;
1590   PetscInt       off;
1591   PetscErrorCode ierr;
1592 
1593   PetscFunctionBegin;
1594   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1595   PetscValidPointer(support, 3);
1596   ierr     = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
1597   *support = &mesh->supports[off];
1598   PetscFunctionReturn(0);
1599 }
1600 
1601 #undef __FUNCT__
1602 #define __FUNCT__ "DMPlexSetSupport"
1603 /*@
1604   DMPlexSetSupport - Set the points on the out-edges for this point in the Sieve DAG
1605 
1606   Not collective
1607 
1608   Input Parameters:
1609 + mesh - The DMPlex
1610 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1611 - support - An array of points which are on the in-edges for point p
1612 
1613   Output Parameter:
1614 
1615   Note:
1616   This should be called after all calls to DMPlexSetSupportSize() and DMSetUp().
1617 
1618   Level: beginner
1619 
1620 .seealso: DMPlexCreate(), DMPlexGetSupport(), DMPlexSetChart(), DMPlexSetSupportSize(), DMSetUp()
1621 @*/
1622 PetscErrorCode DMPlexSetSupport(DM dm, PetscInt p, const PetscInt support[])
1623 {
1624   DM_Plex       *mesh = (DM_Plex*) dm->data;
1625   PetscInt       pStart, pEnd;
1626   PetscInt       dof, off, c;
1627   PetscErrorCode ierr;
1628 
1629   PetscFunctionBegin;
1630   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1631   ierr = PetscSectionGetChart(mesh->supportSection, &pStart, &pEnd);CHKERRQ(ierr);
1632   ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
1633   if (dof) PetscValidPointer(support, 3);
1634   ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
1635   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
1636   for (c = 0; c < dof; ++c) {
1637     if ((support[c] < pStart) || (support[c] >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support point %D is not in the valid range [%D, %D)", support[c], pStart, pEnd);
1638     mesh->supports[off+c] = support[c];
1639   }
1640   PetscFunctionReturn(0);
1641 }
1642 
1643 #undef __FUNCT__
1644 #define __FUNCT__ "DMPlexInsertSupport"
1645 PetscErrorCode DMPlexInsertSupport(DM dm, PetscInt p, PetscInt supportPos, PetscInt supportPoint)
1646 {
1647   DM_Plex       *mesh = (DM_Plex*) dm->data;
1648   PetscInt       pStart, pEnd;
1649   PetscInt       dof, off;
1650   PetscErrorCode ierr;
1651 
1652   PetscFunctionBegin;
1653   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1654   ierr = PetscSectionGetChart(mesh->supportSection, &pStart, &pEnd);CHKERRQ(ierr);
1655   ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
1656   ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
1657   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
1658   if ((supportPoint < pStart) || (supportPoint >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support point %D is not in the valid range [%D, %D)", supportPoint, pStart, pEnd);
1659   if (supportPos >= dof) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support position %D of point %D is not in the valid range [0, %D)", supportPos, p, dof);
1660   mesh->supports[off+supportPos] = supportPoint;
1661   PetscFunctionReturn(0);
1662 }
1663 
1664 #undef __FUNCT__
1665 #define __FUNCT__ "DMPlexGetTransitiveClosure"
1666 /*@C
1667   DMPlexGetTransitiveClosure - Return the points on the transitive closure of the in-edges or out-edges for this point in the Sieve DAG
1668 
1669   Not collective
1670 
1671   Input Parameters:
1672 + mesh - The DMPlex
1673 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1674 . useCone - PETSC_TRUE for in-edges,  otherwise use out-edges
1675 - points - If points is NULL on input, internal storage will be returned, otherwise the provided array is used
1676 
1677   Output Parameters:
1678 + numPoints - The number of points in the closure, so points[] is of size 2*numPoints
1679 - points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...]
1680 
1681   Note:
1682   If using internal storage (points is NULL on input), each call overwrites the last output.
1683 
1684   Fortran Notes:
1685   Since it returns an array, this routine is only available in Fortran 90, and you must
1686   include petsc.h90 in your code.
1687 
1688   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
1689 
1690   Level: beginner
1691 
1692 .seealso: DMPlexRestoreTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
1693 @*/
1694 PetscErrorCode DMPlexGetTransitiveClosure(DM dm, PetscInt p, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
1695 {
1696   DM_Plex        *mesh = (DM_Plex*) dm->data;
1697   PetscInt       *closure, *fifo;
1698   const PetscInt *tmp = NULL, *tmpO = NULL;
1699   PetscInt        tmpSize, t;
1700   PetscInt        depth       = 0, maxSize;
1701   PetscInt        closureSize = 2, fifoSize = 0, fifoStart = 0;
1702   PetscErrorCode  ierr;
1703 
1704   PetscFunctionBegin;
1705   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1706   ierr    = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
1707   /* This is only 1-level */
1708   if (useCone) {
1709     ierr = DMPlexGetConeSize(dm, p, &tmpSize);CHKERRQ(ierr);
1710     ierr = DMPlexGetCone(dm, p, &tmp);CHKERRQ(ierr);
1711     ierr = DMPlexGetConeOrientation(dm, p, &tmpO);CHKERRQ(ierr);
1712   } else {
1713     ierr = DMPlexGetSupportSize(dm, p, &tmpSize);CHKERRQ(ierr);
1714     ierr = DMPlexGetSupport(dm, p, &tmp);CHKERRQ(ierr);
1715   }
1716   if (depth == 1) {
1717     if (*points) {
1718       closure = *points;
1719     } else {
1720       maxSize = 2*(PetscMax(mesh->maxConeSize, mesh->maxSupportSize)+1);
1721       ierr = DMGetWorkArray(dm, maxSize, PETSC_INT, &closure);CHKERRQ(ierr);
1722     }
1723     closure[0] = p; closure[1] = 0;
1724     for (t = 0; t < tmpSize; ++t, closureSize += 2) {
1725       closure[closureSize]   = tmp[t];
1726       closure[closureSize+1] = tmpO ? tmpO[t] : 0;
1727     }
1728     if (numPoints) *numPoints = closureSize/2;
1729     if (points)    *points    = closure;
1730     PetscFunctionReturn(0);
1731   }
1732   maxSize = 2*PetscMax(PetscMax(PetscPowInt(mesh->maxConeSize,depth+1),PetscPowInt(mesh->maxSupportSize,depth+1)),depth+1);
1733   ierr    = DMGetWorkArray(dm, maxSize, PETSC_INT, &fifo);CHKERRQ(ierr);
1734   if (*points) {
1735     closure = *points;
1736   } else {
1737     ierr = DMGetWorkArray(dm, maxSize, PETSC_INT, &closure);CHKERRQ(ierr);
1738   }
1739   closure[0] = p; closure[1] = 0;
1740   for (t = 0; t < tmpSize; ++t, closureSize += 2, fifoSize += 2) {
1741     const PetscInt cp = tmp[t];
1742     const PetscInt co = tmpO ? tmpO[t] : 0;
1743 
1744     closure[closureSize]   = cp;
1745     closure[closureSize+1] = co;
1746     fifo[fifoSize]         = cp;
1747     fifo[fifoSize+1]       = co;
1748   }
1749   /* Should kick out early when depth is reached, rather than checking all vertices for empty cones */
1750   while (fifoSize - fifoStart) {
1751     const PetscInt q   = fifo[fifoStart];
1752     const PetscInt o   = fifo[fifoStart+1];
1753     const PetscInt rev = o >= 0 ? 0 : 1;
1754     const PetscInt off = rev ? -(o+1) : o;
1755 
1756     if (useCone) {
1757       ierr = DMPlexGetConeSize(dm, q, &tmpSize);CHKERRQ(ierr);
1758       ierr = DMPlexGetCone(dm, q, &tmp);CHKERRQ(ierr);
1759       ierr = DMPlexGetConeOrientation(dm, q, &tmpO);CHKERRQ(ierr);
1760     } else {
1761       ierr = DMPlexGetSupportSize(dm, q, &tmpSize);CHKERRQ(ierr);
1762       ierr = DMPlexGetSupport(dm, q, &tmp);CHKERRQ(ierr);
1763       tmpO = NULL;
1764     }
1765     for (t = 0; t < tmpSize; ++t) {
1766       const PetscInt i  = ((rev ? tmpSize-t : t) + off)%tmpSize;
1767       const PetscInt cp = tmp[i];
1768       /* Must propogate orientation: When we reverse orientation, we both reverse the direction of iteration and start at the other end of the chain. */
1769       /* HACK: It is worse to get the size here, than to change the interpretation of -(*+1)
1770        const PetscInt co = tmpO ? (rev ? -(tmpO[i]+1) : tmpO[i]) : 0; */
1771       PetscInt       co = tmpO ? tmpO[i] : 0;
1772       PetscInt       c;
1773 
1774       if (rev) {
1775         PetscInt childSize, coff;
1776         ierr = DMPlexGetConeSize(dm, cp, &childSize);CHKERRQ(ierr);
1777         coff = tmpO[i] < 0 ? -(tmpO[i]+1) : tmpO[i];
1778         co   = childSize ? -(((coff+childSize-1)%childSize)+1) : 0;
1779       }
1780       /* Check for duplicate */
1781       for (c = 0; c < closureSize; c += 2) {
1782         if (closure[c] == cp) break;
1783       }
1784       if (c == closureSize) {
1785         closure[closureSize]   = cp;
1786         closure[closureSize+1] = co;
1787         fifo[fifoSize]         = cp;
1788         fifo[fifoSize+1]       = co;
1789         closureSize           += 2;
1790         fifoSize              += 2;
1791       }
1792     }
1793     fifoStart += 2;
1794   }
1795   if (numPoints) *numPoints = closureSize/2;
1796   if (points)    *points    = closure;
1797   ierr = DMRestoreWorkArray(dm, maxSize, PETSC_INT, &fifo);CHKERRQ(ierr);
1798   PetscFunctionReturn(0);
1799 }
1800 
1801 #undef __FUNCT__
1802 #define __FUNCT__ "DMPlexGetTransitiveClosure_Internal"
1803 /*@C
1804   DMPlexGetTransitiveClosure_Internal - Return the points on the transitive closure of the in-edges or out-edges for this point in the Sieve DAG with a specified initial orientation
1805 
1806   Not collective
1807 
1808   Input Parameters:
1809 + mesh - The DMPlex
1810 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1811 . orientation - The orientation of the point
1812 . useCone - PETSC_TRUE for in-edges,  otherwise use out-edges
1813 - points - If points is NULL on input, internal storage will be returned, otherwise the provided array is used
1814 
1815   Output Parameters:
1816 + numPoints - The number of points in the closure, so points[] is of size 2*numPoints
1817 - points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...]
1818 
1819   Note:
1820   If using internal storage (points is NULL on input), each call overwrites the last output.
1821 
1822   Fortran Notes:
1823   Since it returns an array, this routine is only available in Fortran 90, and you must
1824   include petsc.h90 in your code.
1825 
1826   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
1827 
1828   Level: beginner
1829 
1830 .seealso: DMPlexRestoreTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
1831 @*/
1832 PetscErrorCode DMPlexGetTransitiveClosure_Internal(DM dm, PetscInt p, PetscInt ornt, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
1833 {
1834   DM_Plex        *mesh = (DM_Plex*) dm->data;
1835   PetscInt       *closure, *fifo;
1836   const PetscInt *tmp = NULL, *tmpO = NULL;
1837   PetscInt        tmpSize, t;
1838   PetscInt        depth       = 0, maxSize;
1839   PetscInt        closureSize = 2, fifoSize = 0, fifoStart = 0;
1840   PetscErrorCode  ierr;
1841 
1842   PetscFunctionBegin;
1843   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1844   ierr    = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
1845   /* This is only 1-level */
1846   if (useCone) {
1847     ierr = DMPlexGetConeSize(dm, p, &tmpSize);CHKERRQ(ierr);
1848     ierr = DMPlexGetCone(dm, p, &tmp);CHKERRQ(ierr);
1849     ierr = DMPlexGetConeOrientation(dm, p, &tmpO);CHKERRQ(ierr);
1850   } else {
1851     ierr = DMPlexGetSupportSize(dm, p, &tmpSize);CHKERRQ(ierr);
1852     ierr = DMPlexGetSupport(dm, p, &tmp);CHKERRQ(ierr);
1853   }
1854   if (depth == 1) {
1855     if (*points) {
1856       closure = *points;
1857     } else {
1858       maxSize = 2*(PetscMax(mesh->maxConeSize, mesh->maxSupportSize)+1);
1859       ierr = DMGetWorkArray(dm, maxSize, PETSC_INT, &closure);CHKERRQ(ierr);
1860     }
1861     closure[0] = p; closure[1] = ornt;
1862     for (t = 0; t < tmpSize; ++t, closureSize += 2) {
1863       const PetscInt i = ornt >= 0 ? (t+ornt)%tmpSize : (-(ornt+1) + tmpSize-t)%tmpSize;
1864       closure[closureSize]   = tmp[i];
1865       closure[closureSize+1] = tmpO ? tmpO[i] : 0;
1866     }
1867     if (numPoints) *numPoints = closureSize/2;
1868     if (points)    *points    = closure;
1869     PetscFunctionReturn(0);
1870   }
1871   maxSize = 2*PetscMax(PetscMax(PetscPowInt(mesh->maxConeSize,depth+1),PetscPowInt(mesh->maxSupportSize,depth+1)),depth+1);
1872   ierr    = DMGetWorkArray(dm, maxSize, PETSC_INT, &fifo);CHKERRQ(ierr);
1873   if (*points) {
1874     closure = *points;
1875   } else {
1876     ierr = DMGetWorkArray(dm, maxSize, PETSC_INT, &closure);CHKERRQ(ierr);
1877   }
1878   closure[0] = p; closure[1] = ornt;
1879   for (t = 0; t < tmpSize; ++t, closureSize += 2, fifoSize += 2) {
1880     const PetscInt i  = ornt >= 0 ? (t+ornt)%tmpSize : (-(ornt+1) + tmpSize-t)%tmpSize;
1881     const PetscInt cp = tmp[i];
1882     PetscInt       co = tmpO ? tmpO[i] : 0;
1883 
1884     if (ornt < 0) {
1885       PetscInt childSize, coff;
1886       ierr = DMPlexGetConeSize(dm, cp, &childSize);CHKERRQ(ierr);
1887       coff = tmpO[i] < 0 ? -(tmpO[i]+1) : tmpO[i];
1888       co   = childSize ? -(((coff+childSize-1)%childSize)+1) : 0;
1889     }
1890     closure[closureSize]   = cp;
1891     closure[closureSize+1] = co;
1892     fifo[fifoSize]         = cp;
1893     fifo[fifoSize+1]       = co;
1894   }
1895   /* Should kick out early when depth is reached, rather than checking all vertices for empty cones */
1896   while (fifoSize - fifoStart) {
1897     const PetscInt q   = fifo[fifoStart];
1898     const PetscInt o   = fifo[fifoStart+1];
1899     const PetscInt rev = o >= 0 ? 0 : 1;
1900     const PetscInt off = rev ? -(o+1) : o;
1901 
1902     if (useCone) {
1903       ierr = DMPlexGetConeSize(dm, q, &tmpSize);CHKERRQ(ierr);
1904       ierr = DMPlexGetCone(dm, q, &tmp);CHKERRQ(ierr);
1905       ierr = DMPlexGetConeOrientation(dm, q, &tmpO);CHKERRQ(ierr);
1906     } else {
1907       ierr = DMPlexGetSupportSize(dm, q, &tmpSize);CHKERRQ(ierr);
1908       ierr = DMPlexGetSupport(dm, q, &tmp);CHKERRQ(ierr);
1909       tmpO = NULL;
1910     }
1911     for (t = 0; t < tmpSize; ++t) {
1912       const PetscInt i  = ((rev ? tmpSize-t : t) + off)%tmpSize;
1913       const PetscInt cp = tmp[i];
1914       /* Must propogate orientation: When we reverse orientation, we both reverse the direction of iteration and start at the other end of the chain. */
1915       /* HACK: It is worse to get the size here, than to change the interpretation of -(*+1)
1916        const PetscInt co = tmpO ? (rev ? -(tmpO[i]+1) : tmpO[i]) : 0; */
1917       PetscInt       co = tmpO ? tmpO[i] : 0;
1918       PetscInt       c;
1919 
1920       if (rev) {
1921         PetscInt childSize, coff;
1922         ierr = DMPlexGetConeSize(dm, cp, &childSize);CHKERRQ(ierr);
1923         coff = tmpO[i] < 0 ? -(tmpO[i]+1) : tmpO[i];
1924         co   = childSize ? -(((coff+childSize-1)%childSize)+1) : 0;
1925       }
1926       /* Check for duplicate */
1927       for (c = 0; c < closureSize; c += 2) {
1928         if (closure[c] == cp) break;
1929       }
1930       if (c == closureSize) {
1931         closure[closureSize]   = cp;
1932         closure[closureSize+1] = co;
1933         fifo[fifoSize]         = cp;
1934         fifo[fifoSize+1]       = co;
1935         closureSize           += 2;
1936         fifoSize              += 2;
1937       }
1938     }
1939     fifoStart += 2;
1940   }
1941   if (numPoints) *numPoints = closureSize/2;
1942   if (points)    *points    = closure;
1943   ierr = DMRestoreWorkArray(dm, maxSize, PETSC_INT, &fifo);CHKERRQ(ierr);
1944   PetscFunctionReturn(0);
1945 }
1946 
1947 #undef __FUNCT__
1948 #define __FUNCT__ "DMPlexRestoreTransitiveClosure"
1949 /*@C
1950   DMPlexRestoreTransitiveClosure - Restore the array of points on the transitive closure of the in-edges or out-edges for this point in the Sieve DAG
1951 
1952   Not collective
1953 
1954   Input Parameters:
1955 + mesh - The DMPlex
1956 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1957 . useCone - PETSC_TRUE for in-edges,  otherwise use out-edges
1958 . numPoints - The number of points in the closure, so points[] is of size 2*numPoints, zeroed on exit
1959 - points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...], zeroed on exit
1960 
1961   Note:
1962   If not using internal storage (points is not NULL on input), this call is unnecessary
1963 
1964   Fortran Notes:
1965   Since it returns an array, this routine is only available in Fortran 90, and you must
1966   include petsc.h90 in your code.
1967 
1968   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
1969 
1970   Level: beginner
1971 
1972 .seealso: DMPlexGetTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
1973 @*/
1974 PetscErrorCode DMPlexRestoreTransitiveClosure(DM dm, PetscInt p, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
1975 {
1976   PetscErrorCode ierr;
1977 
1978   PetscFunctionBegin;
1979   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1980   if (numPoints) PetscValidIntPointer(numPoints,4);
1981   if (points) PetscValidPointer(points,5);
1982   ierr = DMRestoreWorkArray(dm, 0, PETSC_INT, points);CHKERRQ(ierr);
1983   if (numPoints) *numPoints = 0;
1984   PetscFunctionReturn(0);
1985 }
1986 
1987 #undef __FUNCT__
1988 #define __FUNCT__ "DMPlexGetMaxSizes"
1989 /*@
1990   DMPlexGetMaxSizes - Return the maximum number of in-edges (cone) and out-edges (support) for any point in the Sieve DAG
1991 
1992   Not collective
1993 
1994   Input Parameter:
1995 . mesh - The DMPlex
1996 
1997   Output Parameters:
1998 + maxConeSize - The maximum number of in-edges
1999 - maxSupportSize - The maximum number of out-edges
2000 
2001   Level: beginner
2002 
2003 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart()
2004 @*/
2005 PetscErrorCode DMPlexGetMaxSizes(DM dm, PetscInt *maxConeSize, PetscInt *maxSupportSize)
2006 {
2007   DM_Plex *mesh = (DM_Plex*) dm->data;
2008 
2009   PetscFunctionBegin;
2010   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2011   if (maxConeSize)    *maxConeSize    = mesh->maxConeSize;
2012   if (maxSupportSize) *maxSupportSize = mesh->maxSupportSize;
2013   PetscFunctionReturn(0);
2014 }
2015 
2016 #undef __FUNCT__
2017 #define __FUNCT__ "DMSetUp_Plex"
2018 PetscErrorCode DMSetUp_Plex(DM dm)
2019 {
2020   DM_Plex       *mesh = (DM_Plex*) dm->data;
2021   PetscInt       size;
2022   PetscErrorCode ierr;
2023 
2024   PetscFunctionBegin;
2025   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2026   ierr = PetscSectionSetUp(mesh->coneSection);CHKERRQ(ierr);
2027   ierr = PetscSectionGetStorageSize(mesh->coneSection, &size);CHKERRQ(ierr);
2028   ierr = PetscMalloc1(size, &mesh->cones);CHKERRQ(ierr);
2029   ierr = PetscCalloc1(size, &mesh->coneOrientations);CHKERRQ(ierr);
2030   if (mesh->maxSupportSize) {
2031     ierr = PetscSectionSetUp(mesh->supportSection);CHKERRQ(ierr);
2032     ierr = PetscSectionGetStorageSize(mesh->supportSection, &size);CHKERRQ(ierr);
2033     ierr = PetscMalloc1(size, &mesh->supports);CHKERRQ(ierr);
2034   }
2035   PetscFunctionReturn(0);
2036 }
2037 
2038 #undef __FUNCT__
2039 #define __FUNCT__ "DMCreateSubDM_Plex"
2040 PetscErrorCode DMCreateSubDM_Plex(DM dm, PetscInt numFields, PetscInt fields[], IS *is, DM *subdm)
2041 {
2042   PetscErrorCode ierr;
2043 
2044   PetscFunctionBegin;
2045   if (subdm) {ierr = DMClone(dm, subdm);CHKERRQ(ierr);}
2046   ierr = DMCreateSubDM_Section_Private(dm, numFields, fields, is, subdm);CHKERRQ(ierr);
2047   PetscFunctionReturn(0);
2048 }
2049 
2050 #undef __FUNCT__
2051 #define __FUNCT__ "DMPlexSymmetrize"
2052 /*@
2053   DMPlexSymmetrize - Creates support (out-edge) information from cone (in-edge) inoformation
2054 
2055   Not collective
2056 
2057   Input Parameter:
2058 . mesh - The DMPlex
2059 
2060   Output Parameter:
2061 
2062   Note:
2063   This should be called after all calls to DMPlexSetCone()
2064 
2065   Level: beginner
2066 
2067 .seealso: DMPlexCreate(), DMPlexSetChart(), DMPlexSetConeSize(), DMPlexSetCone()
2068 @*/
2069 PetscErrorCode DMPlexSymmetrize(DM dm)
2070 {
2071   DM_Plex       *mesh = (DM_Plex*) dm->data;
2072   PetscInt      *offsets;
2073   PetscInt       supportSize;
2074   PetscInt       pStart, pEnd, p;
2075   PetscErrorCode ierr;
2076 
2077   PetscFunctionBegin;
2078   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2079   if (mesh->supports) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONGSTATE, "Supports were already setup in this DMPlex");
2080   /* Calculate support sizes */
2081   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
2082   for (p = pStart; p < pEnd; ++p) {
2083     PetscInt dof, off, c;
2084 
2085     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2086     ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2087     for (c = off; c < off+dof; ++c) {
2088       ierr = PetscSectionAddDof(mesh->supportSection, mesh->cones[c], 1);CHKERRQ(ierr);
2089     }
2090   }
2091   for (p = pStart; p < pEnd; ++p) {
2092     PetscInt dof;
2093 
2094     ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
2095 
2096     mesh->maxSupportSize = PetscMax(mesh->maxSupportSize, dof);
2097   }
2098   ierr = PetscSectionSetUp(mesh->supportSection);CHKERRQ(ierr);
2099   /* Calculate supports */
2100   ierr = PetscSectionGetStorageSize(mesh->supportSection, &supportSize);CHKERRQ(ierr);
2101   ierr = PetscMalloc1(supportSize, &mesh->supports);CHKERRQ(ierr);
2102   ierr = PetscCalloc1(pEnd - pStart, &offsets);CHKERRQ(ierr);
2103   for (p = pStart; p < pEnd; ++p) {
2104     PetscInt dof, off, c;
2105 
2106     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2107     ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2108     for (c = off; c < off+dof; ++c) {
2109       const PetscInt q = mesh->cones[c];
2110       PetscInt       offS;
2111 
2112       ierr = PetscSectionGetOffset(mesh->supportSection, q, &offS);CHKERRQ(ierr);
2113 
2114       mesh->supports[offS+offsets[q]] = p;
2115       ++offsets[q];
2116     }
2117   }
2118   ierr = PetscFree(offsets);CHKERRQ(ierr);
2119   PetscFunctionReturn(0);
2120 }
2121 
2122 #undef __FUNCT__
2123 #define __FUNCT__ "DMPlexStratify"
2124 /*@
2125   DMPlexStratify - The Sieve DAG for most topologies is a graded poset (http://en.wikipedia.org/wiki/Graded_poset), and
2126   can be illustrated by Hasse Diagram (a http://en.wikipedia.org/wiki/Hasse_diagram). The strata group all points of the
2127   same grade, and this function calculates the strata. This grade can be seen as the height (or depth) of the point in
2128   the DAG.
2129 
2130   Not collective
2131 
2132   Input Parameter:
2133 . mesh - The DMPlex
2134 
2135   Output Parameter:
2136 
2137   Notes:
2138   The normal association for the point grade is element dimension (or co-dimension). For instance, all vertices would
2139   have depth 0, and all edges depth 1. Likewise, all cells heights would have height 0, and all faces height 1.
2140 
2141   This should be called after all calls to DMPlexSymmetrize()
2142 
2143   Level: beginner
2144 
2145 .seealso: DMPlexCreate(), DMPlexSymmetrize()
2146 @*/
2147 PetscErrorCode DMPlexStratify(DM dm)
2148 {
2149   DMLabel        label;
2150   PetscInt       pStart, pEnd, p;
2151   PetscInt       numRoots = 0, numLeaves = 0;
2152   PetscErrorCode ierr;
2153 
2154   PetscFunctionBegin;
2155   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2156   ierr = PetscLogEventBegin(DMPLEX_Stratify,dm,0,0,0);CHKERRQ(ierr);
2157   /* Calculate depth */
2158   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
2159   ierr = DMPlexCreateLabel(dm, "depth");CHKERRQ(ierr);
2160   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
2161   /* Initialize roots and count leaves */
2162   for (p = pStart; p < pEnd; ++p) {
2163     PetscInt coneSize, supportSize;
2164 
2165     ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
2166     ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
2167     if (!coneSize && supportSize) {
2168       ++numRoots;
2169       ierr = DMLabelSetValue(label, p, 0);CHKERRQ(ierr);
2170     } else if (!supportSize && coneSize) {
2171       ++numLeaves;
2172     } else if (!supportSize && !coneSize) {
2173       /* Isolated points */
2174       ierr = DMLabelSetValue(label, p, 0);CHKERRQ(ierr);
2175     }
2176   }
2177   if (numRoots + numLeaves == (pEnd - pStart)) {
2178     for (p = pStart; p < pEnd; ++p) {
2179       PetscInt coneSize, supportSize;
2180 
2181       ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
2182       ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
2183       if (!supportSize && coneSize) {
2184         ierr = DMLabelSetValue(label, p, 1);CHKERRQ(ierr);
2185       }
2186     }
2187   } else {
2188     IS       pointIS;
2189     PetscInt numPoints = 0, level = 0;
2190 
2191     ierr = DMLabelGetStratumIS(label, level, &pointIS);CHKERRQ(ierr);
2192     if (pointIS) {ierr = ISGetLocalSize(pointIS, &numPoints);CHKERRQ(ierr);}
2193     while (numPoints) {
2194       const PetscInt *points;
2195       const PetscInt  newLevel = level+1;
2196 
2197       ierr = ISGetIndices(pointIS, &points);CHKERRQ(ierr);
2198       for (p = 0; p < numPoints; ++p) {
2199         const PetscInt  point = points[p];
2200         const PetscInt *support;
2201         PetscInt        supportSize, s;
2202 
2203         ierr = DMPlexGetSupportSize(dm, point, &supportSize);CHKERRQ(ierr);
2204         ierr = DMPlexGetSupport(dm, point, &support);CHKERRQ(ierr);
2205         for (s = 0; s < supportSize; ++s) {
2206           ierr = DMLabelSetValue(label, support[s], newLevel);CHKERRQ(ierr);
2207         }
2208       }
2209       ierr = ISRestoreIndices(pointIS, &points);CHKERRQ(ierr);
2210       ++level;
2211       ierr = ISDestroy(&pointIS);CHKERRQ(ierr);
2212       ierr = DMLabelGetStratumIS(label, level, &pointIS);CHKERRQ(ierr);
2213       if (pointIS) {ierr = ISGetLocalSize(pointIS, &numPoints);CHKERRQ(ierr);}
2214       else         {numPoints = 0;}
2215     }
2216     ierr = ISDestroy(&pointIS);CHKERRQ(ierr);
2217   }
2218   ierr = PetscLogEventEnd(DMPLEX_Stratify,dm,0,0,0);CHKERRQ(ierr);
2219   PetscFunctionReturn(0);
2220 }
2221 
2222 #undef __FUNCT__
2223 #define __FUNCT__ "DMPlexGetJoin"
2224 /*@C
2225   DMPlexGetJoin - Get an array for the join of the set of points
2226 
2227   Not Collective
2228 
2229   Input Parameters:
2230 + dm - The DMPlex object
2231 . numPoints - The number of input points for the join
2232 - points - The input points
2233 
2234   Output Parameters:
2235 + numCoveredPoints - The number of points in the join
2236 - coveredPoints - The points in the join
2237 
2238   Level: intermediate
2239 
2240   Note: Currently, this is restricted to a single level join
2241 
2242   Fortran Notes:
2243   Since it returns an array, this routine is only available in Fortran 90, and you must
2244   include petsc.h90 in your code.
2245 
2246   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2247 
2248 .keywords: mesh
2249 .seealso: DMPlexRestoreJoin(), DMPlexGetMeet()
2250 @*/
2251 PetscErrorCode DMPlexGetJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
2252 {
2253   DM_Plex       *mesh = (DM_Plex*) dm->data;
2254   PetscInt      *join[2];
2255   PetscInt       joinSize, i = 0;
2256   PetscInt       dof, off, p, c, m;
2257   PetscErrorCode ierr;
2258 
2259   PetscFunctionBegin;
2260   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2261   PetscValidPointer(points, 2);
2262   PetscValidPointer(numCoveredPoints, 3);
2263   PetscValidPointer(coveredPoints, 4);
2264   ierr = DMGetWorkArray(dm, mesh->maxSupportSize, PETSC_INT, &join[0]);CHKERRQ(ierr);
2265   ierr = DMGetWorkArray(dm, mesh->maxSupportSize, PETSC_INT, &join[1]);CHKERRQ(ierr);
2266   /* Copy in support of first point */
2267   ierr = PetscSectionGetDof(mesh->supportSection, points[0], &dof);CHKERRQ(ierr);
2268   ierr = PetscSectionGetOffset(mesh->supportSection, points[0], &off);CHKERRQ(ierr);
2269   for (joinSize = 0; joinSize < dof; ++joinSize) {
2270     join[i][joinSize] = mesh->supports[off+joinSize];
2271   }
2272   /* Check each successive support */
2273   for (p = 1; p < numPoints; ++p) {
2274     PetscInt newJoinSize = 0;
2275 
2276     ierr = PetscSectionGetDof(mesh->supportSection, points[p], &dof);CHKERRQ(ierr);
2277     ierr = PetscSectionGetOffset(mesh->supportSection, points[p], &off);CHKERRQ(ierr);
2278     for (c = 0; c < dof; ++c) {
2279       const PetscInt point = mesh->supports[off+c];
2280 
2281       for (m = 0; m < joinSize; ++m) {
2282         if (point == join[i][m]) {
2283           join[1-i][newJoinSize++] = point;
2284           break;
2285         }
2286       }
2287     }
2288     joinSize = newJoinSize;
2289     i        = 1-i;
2290   }
2291   *numCoveredPoints = joinSize;
2292   *coveredPoints    = join[i];
2293   ierr              = DMRestoreWorkArray(dm, mesh->maxSupportSize, PETSC_INT, &join[1-i]);CHKERRQ(ierr);
2294   PetscFunctionReturn(0);
2295 }
2296 
2297 #undef __FUNCT__
2298 #define __FUNCT__ "DMPlexRestoreJoin"
2299 /*@C
2300   DMPlexRestoreJoin - Restore an array for the join of the set of points
2301 
2302   Not Collective
2303 
2304   Input Parameters:
2305 + dm - The DMPlex object
2306 . numPoints - The number of input points for the join
2307 - points - The input points
2308 
2309   Output Parameters:
2310 + numCoveredPoints - The number of points in the join
2311 - coveredPoints - The points in the join
2312 
2313   Fortran Notes:
2314   Since it returns an array, this routine is only available in Fortran 90, and you must
2315   include petsc.h90 in your code.
2316 
2317   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2318 
2319   Level: intermediate
2320 
2321 .keywords: mesh
2322 .seealso: DMPlexGetJoin(), DMPlexGetFullJoin(), DMPlexGetMeet()
2323 @*/
2324 PetscErrorCode DMPlexRestoreJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
2325 {
2326   PetscErrorCode ierr;
2327 
2328   PetscFunctionBegin;
2329   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2330   if (points) PetscValidIntPointer(points,3);
2331   if (numCoveredPoints) PetscValidIntPointer(numCoveredPoints,4);
2332   PetscValidPointer(coveredPoints, 5);
2333   ierr = DMRestoreWorkArray(dm, 0, PETSC_INT, (void*) coveredPoints);CHKERRQ(ierr);
2334   if (numCoveredPoints) *numCoveredPoints = 0;
2335   PetscFunctionReturn(0);
2336 }
2337 
2338 #undef __FUNCT__
2339 #define __FUNCT__ "DMPlexGetFullJoin"
2340 /*@C
2341   DMPlexGetFullJoin - Get an array for the join of the set of points
2342 
2343   Not Collective
2344 
2345   Input Parameters:
2346 + dm - The DMPlex object
2347 . numPoints - The number of input points for the join
2348 - points - The input points
2349 
2350   Output Parameters:
2351 + numCoveredPoints - The number of points in the join
2352 - coveredPoints - The points in the join
2353 
2354   Fortran Notes:
2355   Since it returns an array, this routine is only available in Fortran 90, and you must
2356   include petsc.h90 in your code.
2357 
2358   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2359 
2360   Level: intermediate
2361 
2362 .keywords: mesh
2363 .seealso: DMPlexGetJoin(), DMPlexRestoreJoin(), DMPlexGetMeet()
2364 @*/
2365 PetscErrorCode DMPlexGetFullJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
2366 {
2367   DM_Plex       *mesh = (DM_Plex*) dm->data;
2368   PetscInt      *offsets, **closures;
2369   PetscInt      *join[2];
2370   PetscInt       depth = 0, maxSize, joinSize = 0, i = 0;
2371   PetscInt       p, d, c, m;
2372   PetscErrorCode ierr;
2373 
2374   PetscFunctionBegin;
2375   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2376   PetscValidPointer(points, 2);
2377   PetscValidPointer(numCoveredPoints, 3);
2378   PetscValidPointer(coveredPoints, 4);
2379 
2380   ierr    = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
2381   ierr    = PetscCalloc1(numPoints, &closures);CHKERRQ(ierr);
2382   ierr    = DMGetWorkArray(dm, numPoints*(depth+2), PETSC_INT, &offsets);CHKERRQ(ierr);
2383   maxSize = PetscPowInt(mesh->maxSupportSize,depth+1);
2384   ierr    = DMGetWorkArray(dm, maxSize, PETSC_INT, &join[0]);CHKERRQ(ierr);
2385   ierr    = DMGetWorkArray(dm, maxSize, PETSC_INT, &join[1]);CHKERRQ(ierr);
2386 
2387   for (p = 0; p < numPoints; ++p) {
2388     PetscInt closureSize;
2389 
2390     ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_FALSE, &closureSize, &closures[p]);CHKERRQ(ierr);
2391 
2392     offsets[p*(depth+2)+0] = 0;
2393     for (d = 0; d < depth+1; ++d) {
2394       PetscInt pStart, pEnd, i;
2395 
2396       ierr = DMPlexGetDepthStratum(dm, d, &pStart, &pEnd);CHKERRQ(ierr);
2397       for (i = offsets[p*(depth+2)+d]; i < closureSize; ++i) {
2398         if ((pStart > closures[p][i*2]) || (pEnd <= closures[p][i*2])) {
2399           offsets[p*(depth+2)+d+1] = i;
2400           break;
2401         }
2402       }
2403       if (i == closureSize) offsets[p*(depth+2)+d+1] = i;
2404     }
2405     if (offsets[p*(depth+2)+depth+1] != closureSize) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Total size of closure %D should be %D", offsets[p*(depth+2)+depth+1], closureSize);
2406   }
2407   for (d = 0; d < depth+1; ++d) {
2408     PetscInt dof;
2409 
2410     /* Copy in support of first point */
2411     dof = offsets[d+1] - offsets[d];
2412     for (joinSize = 0; joinSize < dof; ++joinSize) {
2413       join[i][joinSize] = closures[0][(offsets[d]+joinSize)*2];
2414     }
2415     /* Check each successive cone */
2416     for (p = 1; p < numPoints && joinSize; ++p) {
2417       PetscInt newJoinSize = 0;
2418 
2419       dof = offsets[p*(depth+2)+d+1] - offsets[p*(depth+2)+d];
2420       for (c = 0; c < dof; ++c) {
2421         const PetscInt point = closures[p][(offsets[p*(depth+2)+d]+c)*2];
2422 
2423         for (m = 0; m < joinSize; ++m) {
2424           if (point == join[i][m]) {
2425             join[1-i][newJoinSize++] = point;
2426             break;
2427           }
2428         }
2429       }
2430       joinSize = newJoinSize;
2431       i        = 1-i;
2432     }
2433     if (joinSize) break;
2434   }
2435   *numCoveredPoints = joinSize;
2436   *coveredPoints    = join[i];
2437   for (p = 0; p < numPoints; ++p) {
2438     ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_FALSE, NULL, &closures[p]);CHKERRQ(ierr);
2439   }
2440   ierr = PetscFree(closures);CHKERRQ(ierr);
2441   ierr = DMRestoreWorkArray(dm, numPoints*(depth+2), PETSC_INT, &offsets);CHKERRQ(ierr);
2442   ierr = DMRestoreWorkArray(dm, mesh->maxSupportSize, PETSC_INT, &join[1-i]);CHKERRQ(ierr);
2443   PetscFunctionReturn(0);
2444 }
2445 
2446 #undef __FUNCT__
2447 #define __FUNCT__ "DMPlexGetMeet"
2448 /*@C
2449   DMPlexGetMeet - Get an array for the meet of the set of points
2450 
2451   Not Collective
2452 
2453   Input Parameters:
2454 + dm - The DMPlex object
2455 . numPoints - The number of input points for the meet
2456 - points - The input points
2457 
2458   Output Parameters:
2459 + numCoveredPoints - The number of points in the meet
2460 - coveredPoints - The points in the meet
2461 
2462   Level: intermediate
2463 
2464   Note: Currently, this is restricted to a single level meet
2465 
2466   Fortran Notes:
2467   Since it returns an array, this routine is only available in Fortran 90, and you must
2468   include petsc.h90 in your code.
2469 
2470   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2471 
2472 .keywords: mesh
2473 .seealso: DMPlexRestoreMeet(), DMPlexGetJoin()
2474 @*/
2475 PetscErrorCode DMPlexGetMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveringPoints, const PetscInt **coveringPoints)
2476 {
2477   DM_Plex       *mesh = (DM_Plex*) dm->data;
2478   PetscInt      *meet[2];
2479   PetscInt       meetSize, i = 0;
2480   PetscInt       dof, off, p, c, m;
2481   PetscErrorCode ierr;
2482 
2483   PetscFunctionBegin;
2484   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2485   PetscValidPointer(points, 2);
2486   PetscValidPointer(numCoveringPoints, 3);
2487   PetscValidPointer(coveringPoints, 4);
2488   ierr = DMGetWorkArray(dm, mesh->maxConeSize, PETSC_INT, &meet[0]);CHKERRQ(ierr);
2489   ierr = DMGetWorkArray(dm, mesh->maxConeSize, PETSC_INT, &meet[1]);CHKERRQ(ierr);
2490   /* Copy in cone of first point */
2491   ierr = PetscSectionGetDof(mesh->coneSection, points[0], &dof);CHKERRQ(ierr);
2492   ierr = PetscSectionGetOffset(mesh->coneSection, points[0], &off);CHKERRQ(ierr);
2493   for (meetSize = 0; meetSize < dof; ++meetSize) {
2494     meet[i][meetSize] = mesh->cones[off+meetSize];
2495   }
2496   /* Check each successive cone */
2497   for (p = 1; p < numPoints; ++p) {
2498     PetscInt newMeetSize = 0;
2499 
2500     ierr = PetscSectionGetDof(mesh->coneSection, points[p], &dof);CHKERRQ(ierr);
2501     ierr = PetscSectionGetOffset(mesh->coneSection, points[p], &off);CHKERRQ(ierr);
2502     for (c = 0; c < dof; ++c) {
2503       const PetscInt point = mesh->cones[off+c];
2504 
2505       for (m = 0; m < meetSize; ++m) {
2506         if (point == meet[i][m]) {
2507           meet[1-i][newMeetSize++] = point;
2508           break;
2509         }
2510       }
2511     }
2512     meetSize = newMeetSize;
2513     i        = 1-i;
2514   }
2515   *numCoveringPoints = meetSize;
2516   *coveringPoints    = meet[i];
2517   ierr               = DMRestoreWorkArray(dm, mesh->maxConeSize, PETSC_INT, &meet[1-i]);CHKERRQ(ierr);
2518   PetscFunctionReturn(0);
2519 }
2520 
2521 #undef __FUNCT__
2522 #define __FUNCT__ "DMPlexRestoreMeet"
2523 /*@C
2524   DMPlexRestoreMeet - Restore an array for the meet of the set of points
2525 
2526   Not Collective
2527 
2528   Input Parameters:
2529 + dm - The DMPlex object
2530 . numPoints - The number of input points for the meet
2531 - points - The input points
2532 
2533   Output Parameters:
2534 + numCoveredPoints - The number of points in the meet
2535 - coveredPoints - The points in the meet
2536 
2537   Level: intermediate
2538 
2539   Fortran Notes:
2540   Since it returns an array, this routine is only available in Fortran 90, and you must
2541   include petsc.h90 in your code.
2542 
2543   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2544 
2545 .keywords: mesh
2546 .seealso: DMPlexGetMeet(), DMPlexGetFullMeet(), DMPlexGetJoin()
2547 @*/
2548 PetscErrorCode DMPlexRestoreMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
2549 {
2550   PetscErrorCode ierr;
2551 
2552   PetscFunctionBegin;
2553   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2554   if (points) PetscValidIntPointer(points,3);
2555   if (numCoveredPoints) PetscValidIntPointer(numCoveredPoints,4);
2556   PetscValidPointer(coveredPoints,5);
2557   ierr = DMRestoreWorkArray(dm, 0, PETSC_INT, (void*) coveredPoints);CHKERRQ(ierr);
2558   if (numCoveredPoints) *numCoveredPoints = 0;
2559   PetscFunctionReturn(0);
2560 }
2561 
2562 #undef __FUNCT__
2563 #define __FUNCT__ "DMPlexGetFullMeet"
2564 /*@C
2565   DMPlexGetFullMeet - Get an array for the meet of the set of points
2566 
2567   Not Collective
2568 
2569   Input Parameters:
2570 + dm - The DMPlex object
2571 . numPoints - The number of input points for the meet
2572 - points - The input points
2573 
2574   Output Parameters:
2575 + numCoveredPoints - The number of points in the meet
2576 - coveredPoints - The points in the meet
2577 
2578   Level: intermediate
2579 
2580   Fortran Notes:
2581   Since it returns an array, this routine is only available in Fortran 90, and you must
2582   include petsc.h90 in your code.
2583 
2584   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2585 
2586 .keywords: mesh
2587 .seealso: DMPlexGetMeet(), DMPlexRestoreMeet(), DMPlexGetJoin()
2588 @*/
2589 PetscErrorCode DMPlexGetFullMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
2590 {
2591   DM_Plex       *mesh = (DM_Plex*) dm->data;
2592   PetscInt      *offsets, **closures;
2593   PetscInt      *meet[2];
2594   PetscInt       height = 0, maxSize, meetSize = 0, i = 0;
2595   PetscInt       p, h, c, m;
2596   PetscErrorCode ierr;
2597 
2598   PetscFunctionBegin;
2599   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2600   PetscValidPointer(points, 2);
2601   PetscValidPointer(numCoveredPoints, 3);
2602   PetscValidPointer(coveredPoints, 4);
2603 
2604   ierr    = DMPlexGetDepth(dm, &height);CHKERRQ(ierr);
2605   ierr    = PetscMalloc1(numPoints, &closures);CHKERRQ(ierr);
2606   ierr    = DMGetWorkArray(dm, numPoints*(height+2), PETSC_INT, &offsets);CHKERRQ(ierr);
2607   maxSize = PetscPowInt(mesh->maxConeSize,height+1);
2608   ierr    = DMGetWorkArray(dm, maxSize, PETSC_INT, &meet[0]);CHKERRQ(ierr);
2609   ierr    = DMGetWorkArray(dm, maxSize, PETSC_INT, &meet[1]);CHKERRQ(ierr);
2610 
2611   for (p = 0; p < numPoints; ++p) {
2612     PetscInt closureSize;
2613 
2614     ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_TRUE, &closureSize, &closures[p]);CHKERRQ(ierr);
2615 
2616     offsets[p*(height+2)+0] = 0;
2617     for (h = 0; h < height+1; ++h) {
2618       PetscInt pStart, pEnd, i;
2619 
2620       ierr = DMPlexGetHeightStratum(dm, h, &pStart, &pEnd);CHKERRQ(ierr);
2621       for (i = offsets[p*(height+2)+h]; i < closureSize; ++i) {
2622         if ((pStart > closures[p][i*2]) || (pEnd <= closures[p][i*2])) {
2623           offsets[p*(height+2)+h+1] = i;
2624           break;
2625         }
2626       }
2627       if (i == closureSize) offsets[p*(height+2)+h+1] = i;
2628     }
2629     if (offsets[p*(height+2)+height+1] != closureSize) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Total size of closure %D should be %D", offsets[p*(height+2)+height+1], closureSize);
2630   }
2631   for (h = 0; h < height+1; ++h) {
2632     PetscInt dof;
2633 
2634     /* Copy in cone of first point */
2635     dof = offsets[h+1] - offsets[h];
2636     for (meetSize = 0; meetSize < dof; ++meetSize) {
2637       meet[i][meetSize] = closures[0][(offsets[h]+meetSize)*2];
2638     }
2639     /* Check each successive cone */
2640     for (p = 1; p < numPoints && meetSize; ++p) {
2641       PetscInt newMeetSize = 0;
2642 
2643       dof = offsets[p*(height+2)+h+1] - offsets[p*(height+2)+h];
2644       for (c = 0; c < dof; ++c) {
2645         const PetscInt point = closures[p][(offsets[p*(height+2)+h]+c)*2];
2646 
2647         for (m = 0; m < meetSize; ++m) {
2648           if (point == meet[i][m]) {
2649             meet[1-i][newMeetSize++] = point;
2650             break;
2651           }
2652         }
2653       }
2654       meetSize = newMeetSize;
2655       i        = 1-i;
2656     }
2657     if (meetSize) break;
2658   }
2659   *numCoveredPoints = meetSize;
2660   *coveredPoints    = meet[i];
2661   for (p = 0; p < numPoints; ++p) {
2662     ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_TRUE, NULL, &closures[p]);CHKERRQ(ierr);
2663   }
2664   ierr = PetscFree(closures);CHKERRQ(ierr);
2665   ierr = DMRestoreWorkArray(dm, numPoints*(height+2), PETSC_INT, &offsets);CHKERRQ(ierr);
2666   ierr = DMRestoreWorkArray(dm, mesh->maxConeSize, PETSC_INT, &meet[1-i]);CHKERRQ(ierr);
2667   PetscFunctionReturn(0);
2668 }
2669 
2670 #undef __FUNCT__
2671 #define __FUNCT__ "DMPlexEqual"
2672 /*@C
2673   DMPlexEqual - Determine if two DMs have the same topology
2674 
2675   Not Collective
2676 
2677   Input Parameters:
2678 + dmA - A DMPlex object
2679 - dmB - A DMPlex object
2680 
2681   Output Parameters:
2682 . equal - PETSC_TRUE if the topologies are identical
2683 
2684   Level: intermediate
2685 
2686   Notes:
2687   We are not solving graph isomorphism, so we do not permutation.
2688 
2689 .keywords: mesh
2690 .seealso: DMPlexGetCone()
2691 @*/
2692 PetscErrorCode DMPlexEqual(DM dmA, DM dmB, PetscBool *equal)
2693 {
2694   PetscInt       depth, depthB, pStart, pEnd, pStartB, pEndB, p;
2695   PetscErrorCode ierr;
2696 
2697   PetscFunctionBegin;
2698   PetscValidHeaderSpecific(dmA, DM_CLASSID, 1);
2699   PetscValidHeaderSpecific(dmB, DM_CLASSID, 2);
2700   PetscValidPointer(equal, 3);
2701 
2702   *equal = PETSC_FALSE;
2703   ierr = DMPlexGetDepth(dmA, &depth);CHKERRQ(ierr);
2704   ierr = DMPlexGetDepth(dmB, &depthB);CHKERRQ(ierr);
2705   if (depth != depthB) PetscFunctionReturn(0);
2706   ierr = DMPlexGetChart(dmA, &pStart,  &pEnd);CHKERRQ(ierr);
2707   ierr = DMPlexGetChart(dmB, &pStartB, &pEndB);CHKERRQ(ierr);
2708   if ((pStart != pStartB) || (pEnd != pEndB)) PetscFunctionReturn(0);
2709   for (p = pStart; p < pEnd; ++p) {
2710     const PetscInt *cone, *coneB, *ornt, *orntB, *support, *supportB;
2711     PetscInt        coneSize, coneSizeB, c, supportSize, supportSizeB, s;
2712 
2713     ierr = DMPlexGetConeSize(dmA, p, &coneSize);CHKERRQ(ierr);
2714     ierr = DMPlexGetCone(dmA, p, &cone);CHKERRQ(ierr);
2715     ierr = DMPlexGetConeOrientation(dmA, p, &ornt);CHKERRQ(ierr);
2716     ierr = DMPlexGetConeSize(dmB, p, &coneSizeB);CHKERRQ(ierr);
2717     ierr = DMPlexGetCone(dmB, p, &coneB);CHKERRQ(ierr);
2718     ierr = DMPlexGetConeOrientation(dmB, p, &orntB);CHKERRQ(ierr);
2719     if (coneSize != coneSizeB) PetscFunctionReturn(0);
2720     for (c = 0; c < coneSize; ++c) {
2721       if (cone[c] != coneB[c]) PetscFunctionReturn(0);
2722       if (ornt[c] != orntB[c]) PetscFunctionReturn(0);
2723     }
2724     ierr = DMPlexGetSupportSize(dmA, p, &supportSize);CHKERRQ(ierr);
2725     ierr = DMPlexGetSupport(dmA, p, &support);CHKERRQ(ierr);
2726     ierr = DMPlexGetSupportSize(dmB, p, &supportSizeB);CHKERRQ(ierr);
2727     ierr = DMPlexGetSupport(dmB, p, &supportB);CHKERRQ(ierr);
2728     if (supportSize != supportSizeB) PetscFunctionReturn(0);
2729     for (s = 0; s < supportSize; ++s) {
2730       if (support[s] != supportB[s]) PetscFunctionReturn(0);
2731     }
2732   }
2733   *equal = PETSC_TRUE;
2734   PetscFunctionReturn(0);
2735 }
2736 
2737 #undef __FUNCT__
2738 #define __FUNCT__ "DMPlexGetNumFaceVertices"
2739 PetscErrorCode DMPlexGetNumFaceVertices(DM dm, PetscInt cellDim, PetscInt numCorners, PetscInt *numFaceVertices)
2740 {
2741   MPI_Comm       comm;
2742   PetscErrorCode ierr;
2743 
2744   PetscFunctionBegin;
2745   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
2746   PetscValidPointer(numFaceVertices,3);
2747   switch (cellDim) {
2748   case 0:
2749     *numFaceVertices = 0;
2750     break;
2751   case 1:
2752     *numFaceVertices = 1;
2753     break;
2754   case 2:
2755     switch (numCorners) {
2756     case 3: /* triangle */
2757       *numFaceVertices = 2; /* Edge has 2 vertices */
2758       break;
2759     case 4: /* quadrilateral */
2760       *numFaceVertices = 2; /* Edge has 2 vertices */
2761       break;
2762     case 6: /* quadratic triangle, tri and quad cohesive Lagrange cells */
2763       *numFaceVertices = 3; /* Edge has 3 vertices */
2764       break;
2765     case 9: /* quadratic quadrilateral, quadratic quad cohesive Lagrange cells */
2766       *numFaceVertices = 3; /* Edge has 3 vertices */
2767       break;
2768     default:
2769       SETERRQ2(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid number of face corners %d for dimension %d", numCorners, cellDim);
2770     }
2771     break;
2772   case 3:
2773     switch (numCorners) {
2774     case 4: /* tetradehdron */
2775       *numFaceVertices = 3; /* Face has 3 vertices */
2776       break;
2777     case 6: /* tet cohesive cells */
2778       *numFaceVertices = 4; /* Face has 4 vertices */
2779       break;
2780     case 8: /* hexahedron */
2781       *numFaceVertices = 4; /* Face has 4 vertices */
2782       break;
2783     case 9: /* tet cohesive Lagrange cells */
2784       *numFaceVertices = 6; /* Face has 6 vertices */
2785       break;
2786     case 10: /* quadratic tetrahedron */
2787       *numFaceVertices = 6; /* Face has 6 vertices */
2788       break;
2789     case 12: /* hex cohesive Lagrange cells */
2790       *numFaceVertices = 6; /* Face has 6 vertices */
2791       break;
2792     case 18: /* quadratic tet cohesive Lagrange cells */
2793       *numFaceVertices = 6; /* Face has 6 vertices */
2794       break;
2795     case 27: /* quadratic hexahedron, quadratic hex cohesive Lagrange cells */
2796       *numFaceVertices = 9; /* Face has 9 vertices */
2797       break;
2798     default:
2799       SETERRQ2(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid number of face corners %d for dimension %d", numCorners, cellDim);
2800     }
2801     break;
2802   default:
2803     SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid cell dimension %d", cellDim);
2804   }
2805   PetscFunctionReturn(0);
2806 }
2807 
2808 #undef __FUNCT__
2809 #define __FUNCT__ "DMPlexOrient"
2810 /* Trys to give the mesh a consistent orientation */
2811 PetscErrorCode DMPlexOrient(DM dm)
2812 {
2813   PetscBT        seenCells, flippedCells, seenFaces;
2814   PetscInt      *faceFIFO, fTop, fBottom;
2815   PetscInt       dim, h, cStart, cEnd, c, fStart, fEnd, face, maxConeSize, *revcone, *revconeO;
2816   PetscErrorCode ierr;
2817 
2818   PetscFunctionBegin;
2819   /* Truth Table
2820      mismatch    flips   do action   mismatch   flipA ^ flipB   action
2821          F       0 flips     no         F             F           F
2822          F       1 flip      yes        F             T           T
2823          F       2 flips     no         T             F           T
2824          T       0 flips     yes        T             T           F
2825          T       1 flip      no
2826          T       2 flips     yes
2827   */
2828   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
2829   ierr = DMPlexGetVTKCellHeight(dm, &h);CHKERRQ(ierr);
2830   ierr = DMPlexGetHeightStratum(dm, h,   &cStart, &cEnd);CHKERRQ(ierr);
2831   ierr = DMPlexGetHeightStratum(dm, h+1, &fStart, &fEnd);CHKERRQ(ierr);
2832   ierr = PetscBTCreate(cEnd - cStart, &seenCells);CHKERRQ(ierr);
2833   ierr = PetscBTMemzero(cEnd - cStart, seenCells);CHKERRQ(ierr);
2834   ierr = PetscBTCreate(cEnd - cStart, &flippedCells);CHKERRQ(ierr);
2835   ierr = PetscBTMemzero(cEnd - cStart, flippedCells);CHKERRQ(ierr);
2836   ierr = PetscBTCreate(fEnd - fStart, &seenFaces);CHKERRQ(ierr);
2837   ierr = PetscBTMemzero(fEnd - fStart, seenFaces);CHKERRQ(ierr);
2838   ierr = PetscMalloc1((fEnd - fStart), &faceFIFO);CHKERRQ(ierr);
2839   fTop = fBottom = 0;
2840   /* Initialize FIFO with first cell */
2841   if (cEnd > cStart) {
2842     const PetscInt *cone;
2843     PetscInt        coneSize;
2844 
2845     ierr = DMPlexGetConeSize(dm, cStart, &coneSize);CHKERRQ(ierr);
2846     ierr = DMPlexGetCone(dm, cStart, &cone);CHKERRQ(ierr);
2847     for (c = 0; c < coneSize; ++c) {
2848       faceFIFO[fBottom++] = cone[c];
2849       ierr = PetscBTSet(seenFaces, cone[c]-fStart);CHKERRQ(ierr);
2850     }
2851   }
2852   /* Consider each face in FIFO */
2853   while (fTop < fBottom) {
2854     const PetscInt *support, *coneA, *coneB, *coneOA, *coneOB;
2855     PetscInt        supportSize, coneSizeA, coneSizeB, posA = -1, posB = -1;
2856     PetscInt        seenA, flippedA, seenB, flippedB, mismatch;
2857 
2858     face = faceFIFO[fTop++];
2859     ierr = DMPlexGetSupportSize(dm, face, &supportSize);CHKERRQ(ierr);
2860     ierr = DMPlexGetSupport(dm, face, &support);CHKERRQ(ierr);
2861     if (supportSize < 2) continue;
2862     if (supportSize != 2) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Faces should separate only two cells, not %d", supportSize);
2863     seenA    = PetscBTLookup(seenCells,    support[0]-cStart);
2864     flippedA = PetscBTLookup(flippedCells, support[0]-cStart) ? 1 : 0;
2865     seenB    = PetscBTLookup(seenCells,    support[1]-cStart);
2866     flippedB = PetscBTLookup(flippedCells, support[1]-cStart) ? 1 : 0;
2867 
2868     ierr = DMPlexGetConeSize(dm, support[0], &coneSizeA);CHKERRQ(ierr);
2869     ierr = DMPlexGetConeSize(dm, support[1], &coneSizeB);CHKERRQ(ierr);
2870     ierr = DMPlexGetCone(dm, support[0], &coneA);CHKERRQ(ierr);
2871     ierr = DMPlexGetCone(dm, support[1], &coneB);CHKERRQ(ierr);
2872     ierr = DMPlexGetConeOrientation(dm, support[0], &coneOA);CHKERRQ(ierr);
2873     ierr = DMPlexGetConeOrientation(dm, support[1], &coneOB);CHKERRQ(ierr);
2874     for (c = 0; c < coneSizeA; ++c) {
2875       if (!PetscBTLookup(seenFaces, coneA[c]-fStart)) {
2876         faceFIFO[fBottom++] = coneA[c];
2877         ierr = PetscBTSet(seenFaces, coneA[c]-fStart);CHKERRQ(ierr);
2878       }
2879       if (coneA[c] == face) posA = c;
2880       if (fBottom > fEnd-fStart) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Face %d was pushed exceeding capacity %d > %d", coneA[c], fBottom, fEnd-fStart);
2881     }
2882     if (posA < 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %d could not be located in cell %d", face, support[0]);
2883     for (c = 0; c < coneSizeB; ++c) {
2884       if (!PetscBTLookup(seenFaces, coneB[c]-fStart)) {
2885         faceFIFO[fBottom++] = coneB[c];
2886         ierr = PetscBTSet(seenFaces, coneB[c]-fStart);CHKERRQ(ierr);
2887       }
2888       if (coneB[c] == face) posB = c;
2889       if (fBottom > fEnd-fStart) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Face %d was pushed exceeding capacity %d > %d", coneA[c], fBottom, fEnd-fStart);
2890     }
2891     if (posB < 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %d could not be located in cell %d", face, support[1]);
2892 
2893     if (dim == 1) {
2894       mismatch = posA == posB;
2895     } else {
2896       mismatch = coneOA[posA] == coneOB[posB];
2897     }
2898 
2899     if (mismatch ^ (flippedA ^ flippedB)) {
2900       if (seenA && seenB) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Previously seen cells %d and %d do not match: Fault mesh is non-orientable", support[0], support[1]);
2901       if (!seenA && !flippedA) {
2902         ierr = PetscBTSet(flippedCells, support[0]-cStart);CHKERRQ(ierr);
2903       } else if (!seenB && !flippedB) {
2904         ierr = PetscBTSet(flippedCells, support[1]-cStart);CHKERRQ(ierr);
2905       } else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Inconsistent mesh orientation: Fault mesh is non-orientable");
2906     } else if (mismatch && flippedA && flippedB) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Attempt to flip already flipped cell: Fault mesh is non-orientable");
2907     ierr = PetscBTSet(seenCells, support[0]-cStart);CHKERRQ(ierr);
2908     ierr = PetscBTSet(seenCells, support[1]-cStart);CHKERRQ(ierr);
2909   }
2910   /* Now all subdomains are oriented, but we need a consistent parallel orientation */
2911   {
2912     /* Find a representative face (edge) separating pairs of procs */
2913     PetscSF            sf;
2914     const PetscInt    *lpoints;
2915     const PetscSFNode *rpoints;
2916     PetscInt          *neighbors, *nranks;
2917     PetscInt           numLeaves, numRoots, numNeighbors = 0, l, n;
2918 
2919     ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr);
2920     ierr = PetscSFGetGraph(sf, &numRoots, &numLeaves, &lpoints, &rpoints);CHKERRQ(ierr);
2921     if (numLeaves >= 0) {
2922       const PetscInt *cone, *ornt, *support;
2923       PetscInt        coneSize, supportSize;
2924       int            *rornt, *lornt; /* PetscSF cannot handle smaller than int */
2925       PetscBool      *match, flipped = PETSC_FALSE;
2926 
2927       ierr = PetscMalloc1(numLeaves,&neighbors);CHKERRQ(ierr);
2928       /* I know this is p^2 time in general, but for bounded degree its alright */
2929       for (l = 0; l < numLeaves; ++l) {
2930         const PetscInt face = lpoints[l];
2931         if ((face >= fStart) && (face < fEnd)) {
2932           const PetscInt rank = rpoints[l].rank;
2933           for (n = 0; n < numNeighbors; ++n) if (rank == rpoints[neighbors[n]].rank) break;
2934           if (n >= numNeighbors) {
2935             PetscInt supportSize;
2936             ierr = DMPlexGetSupportSize(dm, face, &supportSize);CHKERRQ(ierr);
2937             if (supportSize != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Boundary faces should see one cell, not %d", supportSize);
2938             neighbors[numNeighbors++] = l;
2939           }
2940         }
2941       }
2942       ierr = PetscCalloc4(numNeighbors,&match,numNeighbors,&nranks,numRoots,&rornt,numRoots,&lornt);CHKERRQ(ierr);
2943       for (face = fStart; face < fEnd; ++face) {
2944         ierr = DMPlexGetSupportSize(dm, face, &supportSize);CHKERRQ(ierr);
2945         if (supportSize != 1) continue;
2946         ierr = DMPlexGetSupport(dm, face, &support);CHKERRQ(ierr);
2947 
2948         ierr = DMPlexGetCone(dm, support[0], &cone);CHKERRQ(ierr);
2949         ierr = DMPlexGetConeSize(dm, support[0], &coneSize);CHKERRQ(ierr);
2950         ierr = DMPlexGetConeOrientation(dm, support[0], &ornt);CHKERRQ(ierr);
2951         for (c = 0; c < coneSize; ++c) if (cone[c] == face) break;
2952         if (dim == 1) {
2953           /* Use cone position instead, shifted to -1 or 1 */
2954           rornt[face] = c*2-1;
2955         } else {
2956           if (PetscBTLookup(flippedCells, support[0]-cStart)) rornt[face] = ornt[c] < 0 ? -1 :  1;
2957           else                                                rornt[face] = ornt[c] < 0 ?  1 : -1;
2958         }
2959       }
2960       /* Mark each edge with match or nomatch */
2961       ierr = PetscSFBcastBegin(sf, MPI_INT, rornt, lornt);CHKERRQ(ierr);
2962       ierr = PetscSFBcastEnd(sf, MPI_INT, rornt, lornt);CHKERRQ(ierr);
2963       for (n = 0; n < numNeighbors; ++n) {
2964         const PetscInt face = lpoints[neighbors[n]];
2965 
2966         if (rornt[face]*lornt[face] < 0) match[n] = PETSC_TRUE;
2967         else                             match[n] = PETSC_FALSE;
2968         nranks[n] = rpoints[neighbors[n]].rank;
2969       }
2970       /* Collect the graph on 0 */
2971       {
2972         MPI_Comm     comm = PetscObjectComm((PetscObject) sf);
2973         PetscBT      seenProcs, flippedProcs;
2974         PetscInt    *procFIFO, pTop, pBottom;
2975         PetscInt    *adj = NULL;
2976         PetscBool   *val = NULL;
2977         PetscMPIInt *recvcounts = NULL, *displs = NULL, p;
2978         PetscMPIInt  N = numNeighbors, numProcs = 0, rank;
2979         PetscInt     debug = 0;
2980 
2981         ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
2982         if (!rank) {ierr = MPI_Comm_size(comm, &numProcs);CHKERRQ(ierr);}
2983         ierr = PetscCalloc2(numProcs,&recvcounts,numProcs+1,&displs);CHKERRQ(ierr);
2984         ierr = MPI_Gather(&N, 1, MPI_INT, recvcounts, 1, MPI_INT, 0, comm);CHKERRQ(ierr);
2985         for (p = 0; p < numProcs; ++p) {
2986           displs[p+1] = displs[p] + recvcounts[p];
2987         }
2988         if (!rank) {ierr = PetscMalloc2(displs[numProcs],&adj,displs[numProcs],&val);CHKERRQ(ierr);}
2989         ierr = MPI_Gatherv(nranks, numNeighbors, MPIU_INT, adj, recvcounts, displs, MPIU_INT, 0, comm);CHKERRQ(ierr);
2990         ierr = MPI_Gatherv(match, numNeighbors, MPIU_BOOL, val, recvcounts, displs, MPIU_BOOL, 0, comm);CHKERRQ(ierr);
2991         if (debug) {
2992           for (p = 0; p < numProcs; ++p) {
2993             ierr = PetscPrintf(comm, "Proc %d:\n", p);
2994             for (n = 0; n < recvcounts[p]; ++n) {
2995               ierr = PetscPrintf(comm, "  edge %d (%d):\n", adj[displs[p]+n], val[displs[p]+n]);
2996             }
2997           }
2998         }
2999         ierr = PetscBTCreate(numProcs, &seenProcs);CHKERRQ(ierr);
3000         ierr = PetscBTMemzero(numProcs, seenProcs);CHKERRQ(ierr);
3001         ierr = PetscBTCreate(numProcs, &flippedProcs);CHKERRQ(ierr);
3002         ierr = PetscBTMemzero(numProcs, flippedProcs);CHKERRQ(ierr);
3003         ierr = PetscMalloc1(numProcs,&procFIFO);CHKERRQ(ierr);
3004         pTop = pBottom = 0;
3005         for (p = 0; p < numProcs; ++p) {
3006           if (PetscBTLookup(seenProcs, p)) continue;
3007           /* Initialize FIFO with next proc */
3008           procFIFO[pBottom++] = p;
3009           ierr = PetscBTSet(seenProcs, p);CHKERRQ(ierr);
3010           /* Consider each proc in FIFO */
3011           while (pTop < pBottom) {
3012             PetscInt proc, nproc, seen, flippedA, flippedB, mismatch;
3013 
3014             proc     = procFIFO[pTop++];
3015             flippedA = PetscBTLookup(flippedProcs, proc) ? 1 : 0;
3016             /* Loop over neighboring procs */
3017             for (n = 0; n < recvcounts[proc]; ++n) {
3018               nproc    = adj[displs[proc]+n];
3019               mismatch = val[displs[proc]+n] ? 0 : 1;
3020               seen     = PetscBTLookup(seenProcs, nproc);
3021               flippedB = PetscBTLookup(flippedProcs, nproc) ? 1 : 0;
3022 
3023               if (mismatch ^ (flippedA ^ flippedB)) {
3024                 if (seen) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Previously seen procs %d and %d do not match: Fault mesh is non-orientable", proc, nproc);
3025                 if (!flippedB) {
3026                   ierr = PetscBTSet(flippedProcs, nproc);CHKERRQ(ierr);
3027               } else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Inconsistent mesh orientation: Fault mesh is non-orientable");
3028               } else if (mismatch && flippedA && flippedB) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Attempt to flip already flipped cell: Fault mesh is non-orientable");
3029               if (!seen) {
3030                 procFIFO[pBottom++] = nproc;
3031                 ierr = PetscBTSet(seenProcs, nproc);CHKERRQ(ierr);
3032               }
3033             }
3034           }
3035         }
3036         ierr = PetscFree(procFIFO);CHKERRQ(ierr);
3037 
3038         ierr = PetscFree2(recvcounts,displs);CHKERRQ(ierr);
3039         ierr = PetscFree2(adj,val);CHKERRQ(ierr);
3040         {
3041           PetscBool *flips;
3042 
3043           ierr = PetscMalloc1(numProcs,&flips);CHKERRQ(ierr);
3044           for (p = 0; p < numProcs; ++p) {
3045             flips[p] = PetscBTLookup(flippedProcs, p) ? PETSC_TRUE : PETSC_FALSE;
3046             if (debug && flips[p]) {ierr = PetscPrintf(comm, "Flipping Proc %d:\n", p);}
3047           }
3048           ierr = MPI_Scatter(flips, 1, MPIU_BOOL, &flipped, 1, MPIU_BOOL, 0, comm);CHKERRQ(ierr);
3049           ierr = PetscFree(flips);CHKERRQ(ierr);
3050         }
3051         ierr = PetscBTDestroy(&seenProcs);CHKERRQ(ierr);
3052         ierr = PetscBTDestroy(&flippedProcs);CHKERRQ(ierr);
3053       }
3054       ierr = PetscFree4(match,nranks,rornt,lornt);CHKERRQ(ierr);
3055       ierr = PetscFree(neighbors);CHKERRQ(ierr);
3056       if (flipped) {for (c = cStart; c < cEnd; ++c) {ierr = PetscBTNegate(flippedCells, c-cStart);CHKERRQ(ierr);}}
3057     }
3058   }
3059   /* Reverse flipped cells in the mesh */
3060   ierr = DMPlexGetMaxSizes(dm, &maxConeSize, NULL);CHKERRQ(ierr);
3061   ierr = DMGetWorkArray(dm, maxConeSize, PETSC_INT, &revcone);CHKERRQ(ierr);
3062   ierr = DMGetWorkArray(dm, maxConeSize, PETSC_INT, &revconeO);CHKERRQ(ierr);
3063   for (c = cStart; c < cEnd; ++c) {
3064     const PetscInt *cone, *coneO, *support;
3065     PetscInt        coneSize, supportSize, faceSize, cp, sp;
3066 
3067     if (!PetscBTLookup(flippedCells, c-cStart)) continue;
3068     ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
3069     ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
3070     ierr = DMPlexGetConeOrientation(dm, c, &coneO);CHKERRQ(ierr);
3071     for (cp = 0; cp < coneSize; ++cp) {
3072       const PetscInt rcp = coneSize-cp-1;
3073 
3074       ierr = DMPlexGetConeSize(dm, cone[rcp], &faceSize);CHKERRQ(ierr);
3075       revcone[cp]  = cone[rcp];
3076       revconeO[cp] = coneO[rcp] >= 0 ? -(faceSize-coneO[rcp]) : faceSize+coneO[rcp];
3077     }
3078     ierr = DMPlexSetCone(dm, c, revcone);CHKERRQ(ierr);
3079     ierr = DMPlexSetConeOrientation(dm, c, revconeO);CHKERRQ(ierr);
3080     /* Reverse orientations of support */
3081     faceSize = coneSize;
3082     ierr = DMPlexGetSupportSize(dm, c, &supportSize);CHKERRQ(ierr);
3083     ierr = DMPlexGetSupport(dm, c, &support);CHKERRQ(ierr);
3084     for (sp = 0; sp < supportSize; ++sp) {
3085       ierr = DMPlexGetConeSize(dm, support[sp], &coneSize);CHKERRQ(ierr);
3086       ierr = DMPlexGetCone(dm, support[sp], &cone);CHKERRQ(ierr);
3087       ierr = DMPlexGetConeOrientation(dm, support[sp], &coneO);CHKERRQ(ierr);
3088       for (cp = 0; cp < coneSize; ++cp) {
3089         if (cone[cp] != c) continue;
3090         ierr = DMPlexInsertConeOrientation(dm, support[sp], cp, coneO[cp] >= 0 ? -(faceSize-coneO[cp]) : faceSize+coneO[cp]);CHKERRQ(ierr);
3091       }
3092     }
3093   }
3094   ierr = DMRestoreWorkArray(dm, maxConeSize, PETSC_INT, &revcone);CHKERRQ(ierr);
3095   ierr = DMRestoreWorkArray(dm, maxConeSize, PETSC_INT, &revconeO);CHKERRQ(ierr);
3096   ierr = PetscBTDestroy(&seenCells);CHKERRQ(ierr);
3097   ierr = PetscBTDestroy(&flippedCells);CHKERRQ(ierr);
3098   ierr = PetscBTDestroy(&seenFaces);CHKERRQ(ierr);
3099   ierr = PetscFree(faceFIFO);CHKERRQ(ierr);
3100   PetscFunctionReturn(0);
3101 }
3102 
3103 #undef __FUNCT__
3104 #define __FUNCT__ "DMPlexInvertCell"
3105 /*@C
3106   DMPlexInvertCell - This flips tetrahedron and hexahedron orientation since Plex stores them internally with outward normals. Other cells are left untouched.
3107 
3108   Input Parameters:
3109 + numCorners - The number of vertices in a cell
3110 - cone - The incoming cone
3111 
3112   Output Parameter:
3113 . cone - The inverted cone (in-place)
3114 
3115   Level: developer
3116 
3117 .seealso: DMPlexGenerate()
3118 @*/
3119 PetscErrorCode DMPlexInvertCell(PetscInt dim, PetscInt numCorners, int cone[])
3120 {
3121   int tmpc;
3122 
3123   PetscFunctionBegin;
3124   if (dim != 3) PetscFunctionReturn(0);
3125   switch (numCorners) {
3126   case 4:
3127     tmpc    = cone[0];
3128     cone[0] = cone[1];
3129     cone[1] = tmpc;
3130     break;
3131   case 8:
3132     tmpc    = cone[1];
3133     cone[1] = cone[3];
3134     cone[3] = tmpc;
3135     break;
3136   default: break;
3137   }
3138   PetscFunctionReturn(0);
3139 }
3140 
3141 #undef __FUNCT__
3142 #define __FUNCT__ "DMPlexInvertCells_Internal"
3143 /* This is to fix the tetrahedron orientation from TetGen */
3144 PETSC_UNUSED static PetscErrorCode DMPlexInvertCells_Internal(PetscInt dim, PetscInt numCells, PetscInt numCorners, int cells[])
3145 {
3146   PetscInt       bound = numCells*numCorners, coff;
3147   PetscErrorCode ierr;
3148 
3149   PetscFunctionBegin;
3150   for (coff = 0; coff < bound; coff += numCorners) {
3151     ierr = DMPlexInvertCell(dim, numCorners, &cells[coff]);CHKERRQ(ierr);
3152   }
3153   PetscFunctionReturn(0);
3154 }
3155 
3156 #if defined(PETSC_HAVE_TRIANGLE)
3157 #include <triangle.h>
3158 
3159 #undef __FUNCT__
3160 #define __FUNCT__ "InitInput_Triangle"
3161 PetscErrorCode InitInput_Triangle(struct triangulateio *inputCtx)
3162 {
3163   PetscFunctionBegin;
3164   inputCtx->numberofpoints             = 0;
3165   inputCtx->numberofpointattributes    = 0;
3166   inputCtx->pointlist                  = NULL;
3167   inputCtx->pointattributelist         = NULL;
3168   inputCtx->pointmarkerlist            = NULL;
3169   inputCtx->numberofsegments           = 0;
3170   inputCtx->segmentlist                = NULL;
3171   inputCtx->segmentmarkerlist          = NULL;
3172   inputCtx->numberoftriangleattributes = 0;
3173   inputCtx->trianglelist               = NULL;
3174   inputCtx->numberofholes              = 0;
3175   inputCtx->holelist                   = NULL;
3176   inputCtx->numberofregions            = 0;
3177   inputCtx->regionlist                 = NULL;
3178   PetscFunctionReturn(0);
3179 }
3180 
3181 #undef __FUNCT__
3182 #define __FUNCT__ "InitOutput_Triangle"
3183 PetscErrorCode InitOutput_Triangle(struct triangulateio *outputCtx)
3184 {
3185   PetscFunctionBegin;
3186   outputCtx->numberofpoints        = 0;
3187   outputCtx->pointlist             = NULL;
3188   outputCtx->pointattributelist    = NULL;
3189   outputCtx->pointmarkerlist       = NULL;
3190   outputCtx->numberoftriangles     = 0;
3191   outputCtx->trianglelist          = NULL;
3192   outputCtx->triangleattributelist = NULL;
3193   outputCtx->neighborlist          = NULL;
3194   outputCtx->segmentlist           = NULL;
3195   outputCtx->segmentmarkerlist     = NULL;
3196   outputCtx->numberofedges         = 0;
3197   outputCtx->edgelist              = NULL;
3198   outputCtx->edgemarkerlist        = NULL;
3199   PetscFunctionReturn(0);
3200 }
3201 
3202 #undef __FUNCT__
3203 #define __FUNCT__ "FiniOutput_Triangle"
3204 PetscErrorCode FiniOutput_Triangle(struct triangulateio *outputCtx)
3205 {
3206   PetscFunctionBegin;
3207   free(outputCtx->pointlist);
3208   free(outputCtx->pointmarkerlist);
3209   free(outputCtx->segmentlist);
3210   free(outputCtx->segmentmarkerlist);
3211   free(outputCtx->edgelist);
3212   free(outputCtx->edgemarkerlist);
3213   free(outputCtx->trianglelist);
3214   free(outputCtx->neighborlist);
3215   PetscFunctionReturn(0);
3216 }
3217 
3218 #undef __FUNCT__
3219 #define __FUNCT__ "DMPlexGenerate_Triangle"
3220 PetscErrorCode DMPlexGenerate_Triangle(DM boundary, PetscBool interpolate, DM *dm)
3221 {
3222   MPI_Comm             comm;
3223   PetscInt             dim              = 2;
3224   const PetscBool      createConvexHull = PETSC_FALSE;
3225   const PetscBool      constrained      = PETSC_FALSE;
3226   struct triangulateio in;
3227   struct triangulateio out;
3228   PetscInt             vStart, vEnd, v, eStart, eEnd, e;
3229   PetscMPIInt          rank;
3230   PetscErrorCode       ierr;
3231 
3232   PetscFunctionBegin;
3233   ierr = PetscObjectGetComm((PetscObject)boundary,&comm);CHKERRQ(ierr);
3234   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
3235   ierr = InitInput_Triangle(&in);CHKERRQ(ierr);
3236   ierr = InitOutput_Triangle(&out);CHKERRQ(ierr);
3237   ierr = DMPlexGetDepthStratum(boundary, 0, &vStart, &vEnd);CHKERRQ(ierr);
3238 
3239   in.numberofpoints = vEnd - vStart;
3240   if (in.numberofpoints > 0) {
3241     PetscSection coordSection;
3242     Vec          coordinates;
3243     PetscScalar *array;
3244 
3245     ierr = PetscMalloc1(in.numberofpoints*dim, &in.pointlist);CHKERRQ(ierr);
3246     ierr = PetscMalloc1(in.numberofpoints, &in.pointmarkerlist);CHKERRQ(ierr);
3247     ierr = DMGetCoordinatesLocal(boundary, &coordinates);CHKERRQ(ierr);
3248     ierr = DMGetCoordinateSection(boundary, &coordSection);CHKERRQ(ierr);
3249     ierr = VecGetArray(coordinates, &array);CHKERRQ(ierr);
3250     for (v = vStart; v < vEnd; ++v) {
3251       const PetscInt idx = v - vStart;
3252       PetscInt       off, d;
3253 
3254       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
3255       for (d = 0; d < dim; ++d) {
3256         in.pointlist[idx*dim + d] = PetscRealPart(array[off+d]);
3257       }
3258       ierr = DMPlexGetLabelValue(boundary, "marker", v, &in.pointmarkerlist[idx]);CHKERRQ(ierr);
3259     }
3260     ierr = VecRestoreArray(coordinates, &array);CHKERRQ(ierr);
3261   }
3262   ierr  = DMPlexGetHeightStratum(boundary, 0, &eStart, &eEnd);CHKERRQ(ierr);
3263   in.numberofsegments = eEnd - eStart;
3264   if (in.numberofsegments > 0) {
3265     ierr = PetscMalloc1(in.numberofsegments*2, &in.segmentlist);CHKERRQ(ierr);
3266     ierr = PetscMalloc1(in.numberofsegments, &in.segmentmarkerlist);CHKERRQ(ierr);
3267     for (e = eStart; e < eEnd; ++e) {
3268       const PetscInt  idx = e - eStart;
3269       const PetscInt *cone;
3270 
3271       ierr = DMPlexGetCone(boundary, e, &cone);CHKERRQ(ierr);
3272 
3273       in.segmentlist[idx*2+0] = cone[0] - vStart;
3274       in.segmentlist[idx*2+1] = cone[1] - vStart;
3275 
3276       ierr = DMPlexGetLabelValue(boundary, "marker", e, &in.segmentmarkerlist[idx]);CHKERRQ(ierr);
3277     }
3278   }
3279 #if 0 /* Do not currently support holes */
3280   PetscReal *holeCoords;
3281   PetscInt   h, d;
3282 
3283   ierr = DMPlexGetHoles(boundary, &in.numberofholes, &holeCords);CHKERRQ(ierr);
3284   if (in.numberofholes > 0) {
3285     ierr = PetscMalloc1(in.numberofholes*dim, &in.holelist);CHKERRQ(ierr);
3286     for (h = 0; h < in.numberofholes; ++h) {
3287       for (d = 0; d < dim; ++d) {
3288         in.holelist[h*dim+d] = holeCoords[h*dim+d];
3289       }
3290     }
3291   }
3292 #endif
3293   if (!rank) {
3294     char args[32];
3295 
3296     /* Take away 'Q' for verbose output */
3297     ierr = PetscStrcpy(args, "pqezQ");CHKERRQ(ierr);
3298     if (createConvexHull) {
3299       ierr = PetscStrcat(args, "c");CHKERRQ(ierr);
3300     }
3301     if (constrained) {
3302       ierr = PetscStrcpy(args, "zepDQ");CHKERRQ(ierr);
3303     }
3304     triangulate(args, &in, &out, NULL);
3305   }
3306   ierr = PetscFree(in.pointlist);CHKERRQ(ierr);
3307   ierr = PetscFree(in.pointmarkerlist);CHKERRQ(ierr);
3308   ierr = PetscFree(in.segmentlist);CHKERRQ(ierr);
3309   ierr = PetscFree(in.segmentmarkerlist);CHKERRQ(ierr);
3310   ierr = PetscFree(in.holelist);CHKERRQ(ierr);
3311 
3312   {
3313     const PetscInt numCorners  = 3;
3314     const PetscInt numCells    = out.numberoftriangles;
3315     const PetscInt numVertices = out.numberofpoints;
3316     const int     *cells      = out.trianglelist;
3317     const double  *meshCoords = out.pointlist;
3318 
3319     ierr = DMPlexCreateFromCellList(comm, dim, numCells, numVertices, numCorners, interpolate, cells, dim, meshCoords, dm);CHKERRQ(ierr);
3320     /* Set labels */
3321     for (v = 0; v < numVertices; ++v) {
3322       if (out.pointmarkerlist[v]) {
3323         ierr = DMPlexSetLabelValue(*dm, "marker", v+numCells, out.pointmarkerlist[v]);CHKERRQ(ierr);
3324       }
3325     }
3326     if (interpolate) {
3327       for (e = 0; e < out.numberofedges; e++) {
3328         if (out.edgemarkerlist[e]) {
3329           const PetscInt  vertices[2] = {out.edgelist[e*2+0]+numCells, out.edgelist[e*2+1]+numCells};
3330           const PetscInt *edges;
3331           PetscInt        numEdges;
3332 
3333           ierr = DMPlexGetJoin(*dm, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3334           if (numEdges != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Two vertices must cover only one edge, not %D", numEdges);
3335           ierr = DMPlexSetLabelValue(*dm, "marker", edges[0], out.edgemarkerlist[e]);CHKERRQ(ierr);
3336           ierr = DMPlexRestoreJoin(*dm, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3337         }
3338       }
3339     }
3340     ierr = DMPlexSetRefinementUniform(*dm, PETSC_FALSE);CHKERRQ(ierr);
3341   }
3342 #if 0 /* Do not currently support holes */
3343   ierr = DMPlexCopyHoles(*dm, boundary);CHKERRQ(ierr);
3344 #endif
3345   ierr = FiniOutput_Triangle(&out);CHKERRQ(ierr);
3346   PetscFunctionReturn(0);
3347 }
3348 
3349 #undef __FUNCT__
3350 #define __FUNCT__ "DMPlexRefine_Triangle"
3351 PetscErrorCode DMPlexRefine_Triangle(DM dm, double *maxVolumes, DM *dmRefined)
3352 {
3353   MPI_Comm             comm;
3354   PetscInt             dim  = 2;
3355   struct triangulateio in;
3356   struct triangulateio out;
3357   PetscInt             vStart, vEnd, v, cStart, cEnd, c, depth, depthGlobal;
3358   PetscMPIInt          rank;
3359   PetscErrorCode       ierr;
3360 
3361   PetscFunctionBegin;
3362   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
3363   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
3364   ierr = InitInput_Triangle(&in);CHKERRQ(ierr);
3365   ierr = InitOutput_Triangle(&out);CHKERRQ(ierr);
3366   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3367   ierr = MPI_Allreduce(&depth, &depthGlobal, 1, MPIU_INT, MPI_MAX, comm);CHKERRQ(ierr);
3368   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
3369 
3370   in.numberofpoints = vEnd - vStart;
3371   if (in.numberofpoints > 0) {
3372     PetscSection coordSection;
3373     Vec          coordinates;
3374     PetscScalar *array;
3375 
3376     ierr = PetscMalloc1(in.numberofpoints*dim, &in.pointlist);CHKERRQ(ierr);
3377     ierr = PetscMalloc1(in.numberofpoints, &in.pointmarkerlist);CHKERRQ(ierr);
3378     ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
3379     ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
3380     ierr = VecGetArray(coordinates, &array);CHKERRQ(ierr);
3381     for (v = vStart; v < vEnd; ++v) {
3382       const PetscInt idx = v - vStart;
3383       PetscInt       off, d;
3384 
3385       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
3386       for (d = 0; d < dim; ++d) {
3387         in.pointlist[idx*dim + d] = PetscRealPart(array[off+d]);
3388       }
3389       ierr = DMPlexGetLabelValue(dm, "marker", v, &in.pointmarkerlist[idx]);CHKERRQ(ierr);
3390     }
3391     ierr = VecRestoreArray(coordinates, &array);CHKERRQ(ierr);
3392   }
3393   ierr  = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
3394 
3395   in.numberofcorners   = 3;
3396   in.numberoftriangles = cEnd - cStart;
3397 
3398   in.trianglearealist  = (double*) maxVolumes;
3399   if (in.numberoftriangles > 0) {
3400     ierr = PetscMalloc1(in.numberoftriangles*in.numberofcorners, &in.trianglelist);CHKERRQ(ierr);
3401     for (c = cStart; c < cEnd; ++c) {
3402       const PetscInt idx      = c - cStart;
3403       PetscInt      *closure = NULL;
3404       PetscInt       closureSize;
3405 
3406       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
3407       if ((closureSize != 4) && (closureSize != 7)) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Mesh has cell which is not a triangle, %D vertices in closure", closureSize);
3408       for (v = 0; v < 3; ++v) {
3409         in.trianglelist[idx*in.numberofcorners + v] = closure[(v+closureSize-3)*2] - vStart;
3410       }
3411       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
3412     }
3413   }
3414   /* TODO: Segment markers are missing on input */
3415 #if 0 /* Do not currently support holes */
3416   PetscReal *holeCoords;
3417   PetscInt   h, d;
3418 
3419   ierr = DMPlexGetHoles(boundary, &in.numberofholes, &holeCords);CHKERRQ(ierr);
3420   if (in.numberofholes > 0) {
3421     ierr = PetscMalloc1(in.numberofholes*dim, &in.holelist);CHKERRQ(ierr);
3422     for (h = 0; h < in.numberofholes; ++h) {
3423       for (d = 0; d < dim; ++d) {
3424         in.holelist[h*dim+d] = holeCoords[h*dim+d];
3425       }
3426     }
3427   }
3428 #endif
3429   if (!rank) {
3430     char args[32];
3431 
3432     /* Take away 'Q' for verbose output */
3433     ierr = PetscStrcpy(args, "pqezQra");CHKERRQ(ierr);
3434     triangulate(args, &in, &out, NULL);
3435   }
3436   ierr = PetscFree(in.pointlist);CHKERRQ(ierr);
3437   ierr = PetscFree(in.pointmarkerlist);CHKERRQ(ierr);
3438   ierr = PetscFree(in.segmentlist);CHKERRQ(ierr);
3439   ierr = PetscFree(in.segmentmarkerlist);CHKERRQ(ierr);
3440   ierr = PetscFree(in.trianglelist);CHKERRQ(ierr);
3441 
3442   {
3443     const PetscInt numCorners  = 3;
3444     const PetscInt numCells    = out.numberoftriangles;
3445     const PetscInt numVertices = out.numberofpoints;
3446     const int     *cells      = out.trianglelist;
3447     const double  *meshCoords = out.pointlist;
3448     PetscBool      interpolate = depthGlobal > 1 ? PETSC_TRUE : PETSC_FALSE;
3449 
3450     ierr = DMPlexCreateFromCellList(comm, dim, numCells, numVertices, numCorners, interpolate, cells, dim, meshCoords, dmRefined);CHKERRQ(ierr);
3451     /* Set labels */
3452     for (v = 0; v < numVertices; ++v) {
3453       if (out.pointmarkerlist[v]) {
3454         ierr = DMPlexSetLabelValue(*dmRefined, "marker", v+numCells, out.pointmarkerlist[v]);CHKERRQ(ierr);
3455       }
3456     }
3457     if (interpolate) {
3458       PetscInt e;
3459 
3460       for (e = 0; e < out.numberofedges; e++) {
3461         if (out.edgemarkerlist[e]) {
3462           const PetscInt  vertices[2] = {out.edgelist[e*2+0]+numCells, out.edgelist[e*2+1]+numCells};
3463           const PetscInt *edges;
3464           PetscInt        numEdges;
3465 
3466           ierr = DMPlexGetJoin(*dmRefined, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3467           if (numEdges != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Two vertices must cover only one edge, not %D", numEdges);
3468           ierr = DMPlexSetLabelValue(*dmRefined, "marker", edges[0], out.edgemarkerlist[e]);CHKERRQ(ierr);
3469           ierr = DMPlexRestoreJoin(*dmRefined, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3470         }
3471       }
3472     }
3473     ierr = DMPlexSetRefinementUniform(*dmRefined, PETSC_FALSE);CHKERRQ(ierr);
3474   }
3475 #if 0 /* Do not currently support holes */
3476   ierr = DMPlexCopyHoles(*dm, boundary);CHKERRQ(ierr);
3477 #endif
3478   ierr = FiniOutput_Triangle(&out);CHKERRQ(ierr);
3479   PetscFunctionReturn(0);
3480 }
3481 #endif
3482 
3483 #if defined(PETSC_HAVE_TETGEN)
3484 #include <tetgen.h>
3485 #undef __FUNCT__
3486 #define __FUNCT__ "DMPlexGenerate_Tetgen"
3487 PetscErrorCode DMPlexGenerate_Tetgen(DM boundary, PetscBool interpolate, DM *dm)
3488 {
3489   MPI_Comm       comm;
3490   const PetscInt dim  = 3;
3491   ::tetgenio     in;
3492   ::tetgenio     out;
3493   PetscInt       vStart, vEnd, v, fStart, fEnd, f;
3494   PetscMPIInt    rank;
3495   PetscErrorCode ierr;
3496 
3497   PetscFunctionBegin;
3498   ierr              = PetscObjectGetComm((PetscObject)boundary,&comm);CHKERRQ(ierr);
3499   ierr              = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
3500   ierr              = DMPlexGetDepthStratum(boundary, 0, &vStart, &vEnd);CHKERRQ(ierr);
3501   in.numberofpoints = vEnd - vStart;
3502   if (in.numberofpoints > 0) {
3503     PetscSection coordSection;
3504     Vec          coordinates;
3505     PetscScalar *array;
3506 
3507     in.pointlist       = new double[in.numberofpoints*dim];
3508     in.pointmarkerlist = new int[in.numberofpoints];
3509 
3510     ierr = DMGetCoordinatesLocal(boundary, &coordinates);CHKERRQ(ierr);
3511     ierr = DMGetCoordinateSection(boundary, &coordSection);CHKERRQ(ierr);
3512     ierr = VecGetArray(coordinates, &array);CHKERRQ(ierr);
3513     for (v = vStart; v < vEnd; ++v) {
3514       const PetscInt idx = v - vStart;
3515       PetscInt       off, d;
3516 
3517       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
3518       for (d = 0; d < dim; ++d) in.pointlist[idx*dim + d] = array[off+d];
3519       ierr = DMPlexGetLabelValue(boundary, "marker", v, &in.pointmarkerlist[idx]);CHKERRQ(ierr);
3520     }
3521     ierr = VecRestoreArray(coordinates, &array);CHKERRQ(ierr);
3522   }
3523   ierr  = DMPlexGetHeightStratum(boundary, 0, &fStart, &fEnd);CHKERRQ(ierr);
3524 
3525   in.numberoffacets = fEnd - fStart;
3526   if (in.numberoffacets > 0) {
3527     in.facetlist       = new tetgenio::facet[in.numberoffacets];
3528     in.facetmarkerlist = new int[in.numberoffacets];
3529     for (f = fStart; f < fEnd; ++f) {
3530       const PetscInt idx     = f - fStart;
3531       PetscInt      *points = NULL, numPoints, p, numVertices = 0, v;
3532 
3533       in.facetlist[idx].numberofpolygons = 1;
3534       in.facetlist[idx].polygonlist      = new tetgenio::polygon[in.facetlist[idx].numberofpolygons];
3535       in.facetlist[idx].numberofholes    = 0;
3536       in.facetlist[idx].holelist         = NULL;
3537 
3538       ierr = DMPlexGetTransitiveClosure(boundary, f, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
3539       for (p = 0; p < numPoints*2; p += 2) {
3540         const PetscInt point = points[p];
3541         if ((point >= vStart) && (point < vEnd)) points[numVertices++] = point;
3542       }
3543 
3544       tetgenio::polygon *poly = in.facetlist[idx].polygonlist;
3545       poly->numberofvertices = numVertices;
3546       poly->vertexlist       = new int[poly->numberofvertices];
3547       for (v = 0; v < numVertices; ++v) {
3548         const PetscInt vIdx = points[v] - vStart;
3549         poly->vertexlist[v] = vIdx;
3550       }
3551       ierr = DMPlexGetLabelValue(boundary, "marker", f, &in.facetmarkerlist[idx]);CHKERRQ(ierr);
3552       ierr = DMPlexRestoreTransitiveClosure(boundary, f, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
3553     }
3554   }
3555   if (!rank) {
3556     char args[32];
3557 
3558     /* Take away 'Q' for verbose output */
3559     ierr = PetscStrcpy(args, "pqezQ");CHKERRQ(ierr);
3560     ::tetrahedralize(args, &in, &out);
3561   }
3562   {
3563     const PetscInt numCorners  = 4;
3564     const PetscInt numCells    = out.numberoftetrahedra;
3565     const PetscInt numVertices = out.numberofpoints;
3566     const double   *meshCoords = out.pointlist;
3567     int            *cells      = out.tetrahedronlist;
3568 
3569     ierr = DMPlexInvertCells_Internal(dim, numCells, numCorners, cells);CHKERRQ(ierr);
3570     ierr = DMPlexCreateFromCellList(comm, dim, numCells, numVertices, numCorners, interpolate, cells, dim, meshCoords, dm);CHKERRQ(ierr);
3571     /* Set labels */
3572     for (v = 0; v < numVertices; ++v) {
3573       if (out.pointmarkerlist[v]) {
3574         ierr = DMPlexSetLabelValue(*dm, "marker", v+numCells, out.pointmarkerlist[v]);CHKERRQ(ierr);
3575       }
3576     }
3577     if (interpolate) {
3578       PetscInt e;
3579 
3580       for (e = 0; e < out.numberofedges; e++) {
3581         if (out.edgemarkerlist[e]) {
3582           const PetscInt  vertices[2] = {out.edgelist[e*2+0]+numCells, out.edgelist[e*2+1]+numCells};
3583           const PetscInt *edges;
3584           PetscInt        numEdges;
3585 
3586           ierr = DMPlexGetJoin(*dm, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3587           if (numEdges != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Two vertices must cover only one edge, not %D", numEdges);
3588           ierr = DMPlexSetLabelValue(*dm, "marker", edges[0], out.edgemarkerlist[e]);CHKERRQ(ierr);
3589           ierr = DMPlexRestoreJoin(*dm, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3590         }
3591       }
3592       for (f = 0; f < out.numberoftrifaces; f++) {
3593         if (out.trifacemarkerlist[f]) {
3594           const PetscInt  vertices[3] = {out.trifacelist[f*3+0]+numCells, out.trifacelist[f*3+1]+numCells, out.trifacelist[f*3+2]+numCells};
3595           const PetscInt *faces;
3596           PetscInt        numFaces;
3597 
3598           ierr = DMPlexGetJoin(*dm, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
3599           if (numFaces != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Three vertices must cover only one face, not %D", numFaces);
3600           ierr = DMPlexSetLabelValue(*dm, "marker", faces[0], out.trifacemarkerlist[f]);CHKERRQ(ierr);
3601           ierr = DMPlexRestoreJoin(*dm, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
3602         }
3603       }
3604     }
3605     ierr = DMPlexSetRefinementUniform(*dm, PETSC_FALSE);CHKERRQ(ierr);
3606   }
3607   PetscFunctionReturn(0);
3608 }
3609 
3610 #undef __FUNCT__
3611 #define __FUNCT__ "DMPlexRefine_Tetgen"
3612 PetscErrorCode DMPlexRefine_Tetgen(DM dm, double *maxVolumes, DM *dmRefined)
3613 {
3614   MPI_Comm       comm;
3615   const PetscInt dim  = 3;
3616   ::tetgenio     in;
3617   ::tetgenio     out;
3618   PetscInt       vStart, vEnd, v, cStart, cEnd, c, depth, depthGlobal;
3619   PetscMPIInt    rank;
3620   PetscErrorCode ierr;
3621 
3622   PetscFunctionBegin;
3623   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
3624   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
3625   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3626   ierr = MPI_Allreduce(&depth, &depthGlobal, 1, MPIU_INT, MPI_MAX, comm);CHKERRQ(ierr);
3627   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
3628 
3629   in.numberofpoints = vEnd - vStart;
3630   if (in.numberofpoints > 0) {
3631     PetscSection coordSection;
3632     Vec          coordinates;
3633     PetscScalar *array;
3634 
3635     in.pointlist       = new double[in.numberofpoints*dim];
3636     in.pointmarkerlist = new int[in.numberofpoints];
3637 
3638     ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
3639     ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
3640     ierr = VecGetArray(coordinates, &array);CHKERRQ(ierr);
3641     for (v = vStart; v < vEnd; ++v) {
3642       const PetscInt idx = v - vStart;
3643       PetscInt       off, d;
3644 
3645       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
3646       for (d = 0; d < dim; ++d) in.pointlist[idx*dim + d] = array[off+d];
3647       ierr = DMPlexGetLabelValue(dm, "marker", v, &in.pointmarkerlist[idx]);CHKERRQ(ierr);
3648     }
3649     ierr = VecRestoreArray(coordinates, &array);CHKERRQ(ierr);
3650   }
3651   ierr  = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
3652 
3653   in.numberofcorners       = 4;
3654   in.numberoftetrahedra    = cEnd - cStart;
3655   in.tetrahedronvolumelist = (double*) maxVolumes;
3656   if (in.numberoftetrahedra > 0) {
3657     in.tetrahedronlist = new int[in.numberoftetrahedra*in.numberofcorners];
3658     for (c = cStart; c < cEnd; ++c) {
3659       const PetscInt idx      = c - cStart;
3660       PetscInt      *closure = NULL;
3661       PetscInt       closureSize;
3662 
3663       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
3664       if ((closureSize != 5) && (closureSize != 15)) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Mesh has cell which is not a tetrahedron, %D vertices in closure", closureSize);
3665       for (v = 0; v < 4; ++v) {
3666         in.tetrahedronlist[idx*in.numberofcorners + v] = closure[(v+closureSize-4)*2] - vStart;
3667       }
3668       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
3669     }
3670   }
3671   /* TODO: Put in boundary faces with markers */
3672   if (!rank) {
3673     char args[32];
3674 
3675     /* Take away 'Q' for verbose output */
3676     /*ierr = PetscStrcpy(args, "qezQra");CHKERRQ(ierr); */
3677     ierr = PetscStrcpy(args, "qezraVVVV");CHKERRQ(ierr);
3678     ::tetrahedralize(args, &in, &out);
3679   }
3680   in.tetrahedronvolumelist = NULL;
3681 
3682   {
3683     const PetscInt numCorners  = 4;
3684     const PetscInt numCells    = out.numberoftetrahedra;
3685     const PetscInt numVertices = out.numberofpoints;
3686     const double   *meshCoords = out.pointlist;
3687     int            *cells      = out.tetrahedronlist;
3688 
3689     PetscBool      interpolate = depthGlobal > 1 ? PETSC_TRUE : PETSC_FALSE;
3690 
3691     ierr = DMPlexInvertCells_Internal(dim, numCells, numCorners, cells);CHKERRQ(ierr);
3692     ierr = DMPlexCreateFromCellList(comm, dim, numCells, numVertices, numCorners, interpolate, cells, dim, meshCoords, dmRefined);CHKERRQ(ierr);
3693     /* Set labels */
3694     for (v = 0; v < numVertices; ++v) {
3695       if (out.pointmarkerlist[v]) {
3696         ierr = DMPlexSetLabelValue(*dmRefined, "marker", v+numCells, out.pointmarkerlist[v]);CHKERRQ(ierr);
3697       }
3698     }
3699     if (interpolate) {
3700       PetscInt e, f;
3701 
3702       for (e = 0; e < out.numberofedges; e++) {
3703         if (out.edgemarkerlist[e]) {
3704           const PetscInt  vertices[2] = {out.edgelist[e*2+0]+numCells, out.edgelist[e*2+1]+numCells};
3705           const PetscInt *edges;
3706           PetscInt        numEdges;
3707 
3708           ierr = DMPlexGetJoin(*dmRefined, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3709           if (numEdges != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Two vertices must cover only one edge, not %D", numEdges);
3710           ierr = DMPlexSetLabelValue(*dmRefined, "marker", edges[0], out.edgemarkerlist[e]);CHKERRQ(ierr);
3711           ierr = DMPlexRestoreJoin(*dmRefined, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3712         }
3713       }
3714       for (f = 0; f < out.numberoftrifaces; f++) {
3715         if (out.trifacemarkerlist[f]) {
3716           const PetscInt  vertices[3] = {out.trifacelist[f*3+0]+numCells, out.trifacelist[f*3+1]+numCells, out.trifacelist[f*3+2]+numCells};
3717           const PetscInt *faces;
3718           PetscInt        numFaces;
3719 
3720           ierr = DMPlexGetJoin(*dmRefined, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
3721           if (numFaces != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Three vertices must cover only one face, not %D", numFaces);
3722           ierr = DMPlexSetLabelValue(*dmRefined, "marker", faces[0], out.trifacemarkerlist[f]);CHKERRQ(ierr);
3723           ierr = DMPlexRestoreJoin(*dmRefined, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
3724         }
3725       }
3726     }
3727     ierr = DMPlexSetRefinementUniform(*dmRefined, PETSC_FALSE);CHKERRQ(ierr);
3728   }
3729   PetscFunctionReturn(0);
3730 }
3731 #endif
3732 
3733 #if defined(PETSC_HAVE_CTETGEN)
3734 #include <ctetgen.h>
3735 
3736 #undef __FUNCT__
3737 #define __FUNCT__ "DMPlexGenerate_CTetgen"
3738 PetscErrorCode DMPlexGenerate_CTetgen(DM boundary, PetscBool interpolate, DM *dm)
3739 {
3740   MPI_Comm       comm;
3741   const PetscInt dim  = 3;
3742   PLC           *in, *out;
3743   PetscInt       verbose = 0, vStart, vEnd, v, fStart, fEnd, f;
3744   PetscMPIInt    rank;
3745   PetscErrorCode ierr;
3746 
3747   PetscFunctionBegin;
3748   ierr = PetscObjectGetComm((PetscObject)boundary,&comm);CHKERRQ(ierr);
3749   ierr = PetscOptionsGetInt(((PetscObject) boundary)->prefix, "-ctetgen_verbose", &verbose, NULL);CHKERRQ(ierr);
3750   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
3751   ierr = DMPlexGetDepthStratum(boundary, 0, &vStart, &vEnd);CHKERRQ(ierr);
3752   ierr = PLCCreate(&in);CHKERRQ(ierr);
3753   ierr = PLCCreate(&out);CHKERRQ(ierr);
3754 
3755   in->numberofpoints = vEnd - vStart;
3756   if (in->numberofpoints > 0) {
3757     PetscSection coordSection;
3758     Vec          coordinates;
3759     PetscScalar *array;
3760 
3761     ierr = PetscMalloc1(in->numberofpoints*dim, &in->pointlist);CHKERRQ(ierr);
3762     ierr = PetscMalloc1(in->numberofpoints,       &in->pointmarkerlist);CHKERRQ(ierr);
3763     ierr = DMGetCoordinatesLocal(boundary, &coordinates);CHKERRQ(ierr);
3764     ierr = DMGetCoordinateSection(boundary, &coordSection);CHKERRQ(ierr);
3765     ierr = VecGetArray(coordinates, &array);CHKERRQ(ierr);
3766     for (v = vStart; v < vEnd; ++v) {
3767       const PetscInt idx = v - vStart;
3768       PetscInt       off, d, m;
3769 
3770       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
3771       for (d = 0; d < dim; ++d) {
3772         in->pointlist[idx*dim + d] = PetscRealPart(array[off+d]);
3773       }
3774       ierr = DMPlexGetLabelValue(boundary, "marker", v, &m);CHKERRQ(ierr);
3775 
3776       in->pointmarkerlist[idx] = (int) m;
3777     }
3778     ierr = VecRestoreArray(coordinates, &array);CHKERRQ(ierr);
3779   }
3780   ierr  = DMPlexGetHeightStratum(boundary, 0, &fStart, &fEnd);CHKERRQ(ierr);
3781 
3782   in->numberoffacets = fEnd - fStart;
3783   if (in->numberoffacets > 0) {
3784     ierr = PetscMalloc1(in->numberoffacets, &in->facetlist);CHKERRQ(ierr);
3785     ierr = PetscMalloc1(in->numberoffacets,   &in->facetmarkerlist);CHKERRQ(ierr);
3786     for (f = fStart; f < fEnd; ++f) {
3787       const PetscInt idx     = f - fStart;
3788       PetscInt      *points = NULL, numPoints, p, numVertices = 0, v, m;
3789       polygon       *poly;
3790 
3791       in->facetlist[idx].numberofpolygons = 1;
3792 
3793       ierr = PetscMalloc1(in->facetlist[idx].numberofpolygons, &in->facetlist[idx].polygonlist);CHKERRQ(ierr);
3794 
3795       in->facetlist[idx].numberofholes    = 0;
3796       in->facetlist[idx].holelist         = NULL;
3797 
3798       ierr = DMPlexGetTransitiveClosure(boundary, f, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
3799       for (p = 0; p < numPoints*2; p += 2) {
3800         const PetscInt point = points[p];
3801         if ((point >= vStart) && (point < vEnd)) points[numVertices++] = point;
3802       }
3803 
3804       poly                   = in->facetlist[idx].polygonlist;
3805       poly->numberofvertices = numVertices;
3806       ierr                   = PetscMalloc1(poly->numberofvertices, &poly->vertexlist);CHKERRQ(ierr);
3807       for (v = 0; v < numVertices; ++v) {
3808         const PetscInt vIdx = points[v] - vStart;
3809         poly->vertexlist[v] = vIdx;
3810       }
3811       ierr                     = DMPlexGetLabelValue(boundary, "marker", f, &m);CHKERRQ(ierr);
3812       in->facetmarkerlist[idx] = (int) m;
3813       ierr                     = DMPlexRestoreTransitiveClosure(boundary, f, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
3814     }
3815   }
3816   if (!rank) {
3817     TetGenOpts t;
3818 
3819     ierr        = TetGenOptsInitialize(&t);CHKERRQ(ierr);
3820     t.in        = boundary; /* Should go away */
3821     t.plc       = 1;
3822     t.quality   = 1;
3823     t.edgesout  = 1;
3824     t.zeroindex = 1;
3825     t.quiet     = 1;
3826     t.verbose   = verbose;
3827     ierr        = TetGenCheckOpts(&t);CHKERRQ(ierr);
3828     ierr        = TetGenTetrahedralize(&t, in, out);CHKERRQ(ierr);
3829   }
3830   {
3831     const PetscInt numCorners  = 4;
3832     const PetscInt numCells    = out->numberoftetrahedra;
3833     const PetscInt numVertices = out->numberofpoints;
3834     const double   *meshCoords = out->pointlist;
3835     int            *cells      = out->tetrahedronlist;
3836 
3837     ierr = DMPlexInvertCells_Internal(dim, numCells, numCorners, cells);CHKERRQ(ierr);
3838     ierr = DMPlexCreateFromCellList(comm, dim, numCells, numVertices, numCorners, interpolate, cells, dim, meshCoords, dm);CHKERRQ(ierr);
3839     /* Set labels */
3840     for (v = 0; v < numVertices; ++v) {
3841       if (out->pointmarkerlist[v]) {
3842         ierr = DMPlexSetLabelValue(*dm, "marker", v+numCells, out->pointmarkerlist[v]);CHKERRQ(ierr);
3843       }
3844     }
3845     if (interpolate) {
3846       PetscInt e;
3847 
3848       for (e = 0; e < out->numberofedges; e++) {
3849         if (out->edgemarkerlist[e]) {
3850           const PetscInt  vertices[2] = {out->edgelist[e*2+0]+numCells, out->edgelist[e*2+1]+numCells};
3851           const PetscInt *edges;
3852           PetscInt        numEdges;
3853 
3854           ierr = DMPlexGetJoin(*dm, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3855           if (numEdges != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Two vertices must cover only one edge, not %D", numEdges);
3856           ierr = DMPlexSetLabelValue(*dm, "marker", edges[0], out->edgemarkerlist[e]);CHKERRQ(ierr);
3857           ierr = DMPlexRestoreJoin(*dm, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3858         }
3859       }
3860       for (f = 0; f < out->numberoftrifaces; f++) {
3861         if (out->trifacemarkerlist[f]) {
3862           const PetscInt  vertices[3] = {out->trifacelist[f*3+0]+numCells, out->trifacelist[f*3+1]+numCells, out->trifacelist[f*3+2]+numCells};
3863           const PetscInt *faces;
3864           PetscInt        numFaces;
3865 
3866           ierr = DMPlexGetFullJoin(*dm, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
3867           if (numFaces != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Three vertices must cover only one face, not %D", numFaces);
3868           ierr = DMPlexSetLabelValue(*dm, "marker", faces[0], out->trifacemarkerlist[f]);CHKERRQ(ierr);
3869           ierr = DMPlexRestoreJoin(*dm, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
3870         }
3871       }
3872     }
3873     ierr = DMPlexSetRefinementUniform(*dm, PETSC_FALSE);CHKERRQ(ierr);
3874   }
3875 
3876   ierr = PLCDestroy(&in);CHKERRQ(ierr);
3877   ierr = PLCDestroy(&out);CHKERRQ(ierr);
3878   PetscFunctionReturn(0);
3879 }
3880 
3881 #undef __FUNCT__
3882 #define __FUNCT__ "DMPlexRefine_CTetgen"
3883 PetscErrorCode DMPlexRefine_CTetgen(DM dm, PetscReal *maxVolumes, DM *dmRefined)
3884 {
3885   MPI_Comm       comm;
3886   const PetscInt dim  = 3;
3887   PLC           *in, *out;
3888   PetscInt       verbose = 0, vStart, vEnd, v, cStart, cEnd, c, depth, depthGlobal;
3889   PetscMPIInt    rank;
3890   PetscErrorCode ierr;
3891 
3892   PetscFunctionBegin;
3893   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
3894   ierr = PetscOptionsGetInt(((PetscObject) dm)->prefix, "-ctetgen_verbose", &verbose, NULL);CHKERRQ(ierr);
3895   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
3896   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3897   ierr = MPI_Allreduce(&depth, &depthGlobal, 1, MPIU_INT, MPI_MAX, comm);CHKERRQ(ierr);
3898   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
3899   ierr = PLCCreate(&in);CHKERRQ(ierr);
3900   ierr = PLCCreate(&out);CHKERRQ(ierr);
3901 
3902   in->numberofpoints = vEnd - vStart;
3903   if (in->numberofpoints > 0) {
3904     PetscSection coordSection;
3905     Vec          coordinates;
3906     PetscScalar *array;
3907 
3908     ierr = PetscMalloc1(in->numberofpoints*dim, &in->pointlist);CHKERRQ(ierr);
3909     ierr = PetscMalloc1(in->numberofpoints,       &in->pointmarkerlist);CHKERRQ(ierr);
3910     ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
3911     ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
3912     ierr = VecGetArray(coordinates, &array);CHKERRQ(ierr);
3913     for (v = vStart; v < vEnd; ++v) {
3914       const PetscInt idx = v - vStart;
3915       PetscInt       off, d, m;
3916 
3917       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
3918       for (d = 0; d < dim; ++d) {
3919         in->pointlist[idx*dim + d] = PetscRealPart(array[off+d]);
3920       }
3921       ierr = DMPlexGetLabelValue(dm, "marker", v, &m);CHKERRQ(ierr);
3922 
3923       in->pointmarkerlist[idx] = (int) m;
3924     }
3925     ierr = VecRestoreArray(coordinates, &array);CHKERRQ(ierr);
3926   }
3927   ierr  = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
3928 
3929   in->numberofcorners       = 4;
3930   in->numberoftetrahedra    = cEnd - cStart;
3931   in->tetrahedronvolumelist = maxVolumes;
3932   if (in->numberoftetrahedra > 0) {
3933     ierr = PetscMalloc1(in->numberoftetrahedra*in->numberofcorners, &in->tetrahedronlist);CHKERRQ(ierr);
3934     for (c = cStart; c < cEnd; ++c) {
3935       const PetscInt idx      = c - cStart;
3936       PetscInt      *closure = NULL;
3937       PetscInt       closureSize;
3938 
3939       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
3940       if ((closureSize != 5) && (closureSize != 15)) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Mesh has cell which is not a tetrahedron, %D vertices in closure", closureSize);
3941       for (v = 0; v < 4; ++v) {
3942         in->tetrahedronlist[idx*in->numberofcorners + v] = closure[(v+closureSize-4)*2] - vStart;
3943       }
3944       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
3945     }
3946   }
3947   if (!rank) {
3948     TetGenOpts t;
3949 
3950     ierr = TetGenOptsInitialize(&t);CHKERRQ(ierr);
3951 
3952     t.in        = dm; /* Should go away */
3953     t.refine    = 1;
3954     t.varvolume = 1;
3955     t.quality   = 1;
3956     t.edgesout  = 1;
3957     t.zeroindex = 1;
3958     t.quiet     = 1;
3959     t.verbose   = verbose; /* Change this */
3960 
3961     ierr = TetGenCheckOpts(&t);CHKERRQ(ierr);
3962     ierr = TetGenTetrahedralize(&t, in, out);CHKERRQ(ierr);
3963   }
3964   {
3965     const PetscInt numCorners  = 4;
3966     const PetscInt numCells    = out->numberoftetrahedra;
3967     const PetscInt numVertices = out->numberofpoints;
3968     const double   *meshCoords = out->pointlist;
3969     int            *cells      = out->tetrahedronlist;
3970     PetscBool      interpolate = depthGlobal > 1 ? PETSC_TRUE : PETSC_FALSE;
3971 
3972     ierr = DMPlexInvertCells_Internal(dim, numCells, numCorners, cells);CHKERRQ(ierr);
3973     ierr = DMPlexCreateFromCellList(comm, dim, numCells, numVertices, numCorners, interpolate, cells, dim, meshCoords, dmRefined);CHKERRQ(ierr);
3974     /* Set labels */
3975     for (v = 0; v < numVertices; ++v) {
3976       if (out->pointmarkerlist[v]) {
3977         ierr = DMPlexSetLabelValue(*dmRefined, "marker", v+numCells, out->pointmarkerlist[v]);CHKERRQ(ierr);
3978       }
3979     }
3980     if (interpolate) {
3981       PetscInt e, f;
3982 
3983       for (e = 0; e < out->numberofedges; e++) {
3984         if (out->edgemarkerlist[e]) {
3985           const PetscInt  vertices[2] = {out->edgelist[e*2+0]+numCells, out->edgelist[e*2+1]+numCells};
3986           const PetscInt *edges;
3987           PetscInt        numEdges;
3988 
3989           ierr = DMPlexGetJoin(*dmRefined, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3990           if (numEdges != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Two vertices must cover only one edge, not %D", numEdges);
3991           ierr = DMPlexSetLabelValue(*dmRefined, "marker", edges[0], out->edgemarkerlist[e]);CHKERRQ(ierr);
3992           ierr = DMPlexRestoreJoin(*dmRefined, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3993         }
3994       }
3995       for (f = 0; f < out->numberoftrifaces; f++) {
3996         if (out->trifacemarkerlist[f]) {
3997           const PetscInt  vertices[3] = {out->trifacelist[f*3+0]+numCells, out->trifacelist[f*3+1]+numCells, out->trifacelist[f*3+2]+numCells};
3998           const PetscInt *faces;
3999           PetscInt        numFaces;
4000 
4001           ierr = DMPlexGetFullJoin(*dmRefined, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
4002           if (numFaces != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Three vertices must cover only one face, not %D", numFaces);
4003           ierr = DMPlexSetLabelValue(*dmRefined, "marker", faces[0], out->trifacemarkerlist[f]);CHKERRQ(ierr);
4004           ierr = DMPlexRestoreJoin(*dmRefined, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
4005         }
4006       }
4007     }
4008     ierr = DMPlexSetRefinementUniform(*dmRefined, PETSC_FALSE);CHKERRQ(ierr);
4009   }
4010   ierr = PLCDestroy(&in);CHKERRQ(ierr);
4011   ierr = PLCDestroy(&out);CHKERRQ(ierr);
4012   PetscFunctionReturn(0);
4013 }
4014 #endif
4015 
4016 #undef __FUNCT__
4017 #define __FUNCT__ "DMPlexGenerate"
4018 /*@C
4019   DMPlexGenerate - Generates a mesh.
4020 
4021   Not Collective
4022 
4023   Input Parameters:
4024 + boundary - The DMPlex boundary object
4025 . name - The mesh generation package name
4026 - interpolate - Flag to create intermediate mesh elements
4027 
4028   Output Parameter:
4029 . mesh - The DMPlex object
4030 
4031   Level: intermediate
4032 
4033 .keywords: mesh, elements
4034 .seealso: DMPlexCreate(), DMRefine()
4035 @*/
4036 PetscErrorCode DMPlexGenerate(DM boundary, const char name[], PetscBool interpolate, DM *mesh)
4037 {
4038   PetscInt       dim;
4039   char           genname[1024];
4040   PetscBool      isTriangle = PETSC_FALSE, isTetgen = PETSC_FALSE, isCTetgen = PETSC_FALSE, flg;
4041   PetscErrorCode ierr;
4042 
4043   PetscFunctionBegin;
4044   PetscValidHeaderSpecific(boundary, DM_CLASSID, 1);
4045   PetscValidLogicalCollectiveBool(boundary, interpolate, 2);
4046   ierr = DMPlexGetDimension(boundary, &dim);CHKERRQ(ierr);
4047   ierr = PetscOptionsGetString(((PetscObject) boundary)->prefix, "-dm_plex_generator", genname, 1024, &flg);CHKERRQ(ierr);
4048   if (flg) name = genname;
4049   if (name) {
4050     ierr = PetscStrcmp(name, "triangle", &isTriangle);CHKERRQ(ierr);
4051     ierr = PetscStrcmp(name, "tetgen",   &isTetgen);CHKERRQ(ierr);
4052     ierr = PetscStrcmp(name, "ctetgen",  &isCTetgen);CHKERRQ(ierr);
4053   }
4054   switch (dim) {
4055   case 1:
4056     if (!name || isTriangle) {
4057 #if defined(PETSC_HAVE_TRIANGLE)
4058       ierr = DMPlexGenerate_Triangle(boundary, interpolate, mesh);CHKERRQ(ierr);
4059 #else
4060       SETERRQ(PetscObjectComm((PetscObject)boundary), PETSC_ERR_SUP, "Mesh generation needs external package support.\nPlease reconfigure with --download-triangle.");
4061 #endif
4062     } else SETERRQ1(PetscObjectComm((PetscObject)boundary), PETSC_ERR_SUP, "Unknown 2D mesh generation package %s", name);
4063     break;
4064   case 2:
4065     if (!name || isCTetgen) {
4066 #if defined(PETSC_HAVE_CTETGEN)
4067       ierr = DMPlexGenerate_CTetgen(boundary, interpolate, mesh);CHKERRQ(ierr);
4068 #else
4069       SETERRQ(PetscObjectComm((PetscObject)boundary), PETSC_ERR_SUP, "CTetgen needs external package support.\nPlease reconfigure with --download-ctetgen.");
4070 #endif
4071     } else if (isTetgen) {
4072 #if defined(PETSC_HAVE_TETGEN)
4073       ierr = DMPlexGenerate_Tetgen(boundary, interpolate, mesh);CHKERRQ(ierr);
4074 #else
4075       SETERRQ(PetscObjectComm((PetscObject)boundary), PETSC_ERR_SUP, "Tetgen needs external package support.\nPlease reconfigure with --with-c-language=cxx --download-tetgen.");
4076 #endif
4077     } else SETERRQ1(PetscObjectComm((PetscObject)boundary), PETSC_ERR_SUP, "Unknown 3D mesh generation package %s", name);
4078     break;
4079   default:
4080     SETERRQ1(PetscObjectComm((PetscObject)boundary), PETSC_ERR_SUP, "Mesh generation for a dimension %d boundary is not supported.", dim);
4081   }
4082   PetscFunctionReturn(0);
4083 }
4084 
4085 #undef __FUNCT__
4086 #define __FUNCT__ "DMRefine_Plex"
4087 PetscErrorCode DMRefine_Plex(DM dm, MPI_Comm comm, DM *dmRefined)
4088 {
4089   PetscReal      refinementLimit;
4090   PetscInt       dim, cStart, cEnd;
4091   char           genname[1024], *name = NULL;
4092   PetscBool      isUniform, isTriangle = PETSC_FALSE, isTetgen = PETSC_FALSE, isCTetgen = PETSC_FALSE, flg;
4093   PetscErrorCode ierr;
4094 
4095   PetscFunctionBegin;
4096   ierr = DMPlexGetRefinementUniform(dm, &isUniform);CHKERRQ(ierr);
4097   if (isUniform) {
4098     CellRefiner cellRefiner;
4099 
4100     ierr = DMPlexGetCellRefiner_Internal(dm, &cellRefiner);CHKERRQ(ierr);
4101     ierr = DMPlexRefineUniform_Internal(dm, cellRefiner, dmRefined);CHKERRQ(ierr);
4102     PetscFunctionReturn(0);
4103   }
4104   ierr = DMPlexGetRefinementLimit(dm, &refinementLimit);CHKERRQ(ierr);
4105   if (refinementLimit == 0.0) PetscFunctionReturn(0);
4106   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
4107   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
4108   ierr = PetscOptionsGetString(((PetscObject) dm)->prefix, "-dm_plex_generator", genname, 1024, &flg);CHKERRQ(ierr);
4109   if (flg) name = genname;
4110   if (name) {
4111     ierr = PetscStrcmp(name, "triangle", &isTriangle);CHKERRQ(ierr);
4112     ierr = PetscStrcmp(name, "tetgen",   &isTetgen);CHKERRQ(ierr);
4113     ierr = PetscStrcmp(name, "ctetgen",  &isCTetgen);CHKERRQ(ierr);
4114   }
4115   switch (dim) {
4116   case 2:
4117     if (!name || isTriangle) {
4118 #if defined(PETSC_HAVE_TRIANGLE)
4119       double  *maxVolumes;
4120       PetscInt c;
4121 
4122       ierr = PetscMalloc1((cEnd - cStart), &maxVolumes);CHKERRQ(ierr);
4123       for (c = 0; c < cEnd-cStart; ++c) maxVolumes[c] = refinementLimit;
4124       ierr = DMPlexRefine_Triangle(dm, maxVolumes, dmRefined);CHKERRQ(ierr);
4125       ierr = PetscFree(maxVolumes);CHKERRQ(ierr);
4126 #else
4127       SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Mesh refinement needs external package support.\nPlease reconfigure with --download-triangle.");
4128 #endif
4129     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Unknown 2D mesh generation package %s", name);
4130     break;
4131   case 3:
4132     if (!name || isCTetgen) {
4133 #if defined(PETSC_HAVE_CTETGEN)
4134       PetscReal *maxVolumes;
4135       PetscInt   c;
4136 
4137       ierr = PetscMalloc1((cEnd - cStart), &maxVolumes);CHKERRQ(ierr);
4138       for (c = 0; c < cEnd-cStart; ++c) maxVolumes[c] = refinementLimit;
4139       ierr = DMPlexRefine_CTetgen(dm, maxVolumes, dmRefined);CHKERRQ(ierr);
4140 #else
4141       SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "CTetgen needs external package support.\nPlease reconfigure with --download-ctetgen.");
4142 #endif
4143     } else if (isTetgen) {
4144 #if defined(PETSC_HAVE_TETGEN)
4145       double  *maxVolumes;
4146       PetscInt c;
4147 
4148       ierr = PetscMalloc1((cEnd - cStart), &maxVolumes);CHKERRQ(ierr);
4149       for (c = 0; c < cEnd-cStart; ++c) maxVolumes[c] = refinementLimit;
4150       ierr = DMPlexRefine_Tetgen(dm, maxVolumes, dmRefined);CHKERRQ(ierr);
4151 #else
4152       SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Tetgen needs external package support.\nPlease reconfigure with --with-c-language=cxx --download-tetgen.");
4153 #endif
4154     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Unknown 3D mesh generation package %s", name);
4155     break;
4156   default:
4157     SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Mesh refinement in dimension %d is not supported.", dim);
4158   }
4159   PetscFunctionReturn(0);
4160 }
4161 
4162 #undef __FUNCT__
4163 #define __FUNCT__ "DMRefineHierarchy_Plex"
4164 PetscErrorCode DMRefineHierarchy_Plex(DM dm, PetscInt nlevels, DM dmRefined[])
4165 {
4166   DM             cdm = dm;
4167   PetscInt       r;
4168   PetscBool      isUniform;
4169   PetscErrorCode ierr;
4170 
4171   PetscFunctionBegin;
4172   ierr = DMPlexGetRefinementUniform(dm, &isUniform);CHKERRQ(ierr);
4173   if (!isUniform) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Non-uniform refinement is incompatible with the hierarchy");
4174   for (r = 0; r < nlevels; ++r) {
4175     CellRefiner cellRefiner;
4176 
4177     ierr = DMPlexGetCellRefiner_Internal(cdm, &cellRefiner);CHKERRQ(ierr);
4178     ierr = DMPlexRefineUniform_Internal(cdm, cellRefiner, &dmRefined[r]);CHKERRQ(ierr);
4179     ierr = DMPlexSetCoarseDM(dmRefined[r], cdm);CHKERRQ(ierr);
4180     cdm  = dmRefined[r];
4181   }
4182   PetscFunctionReturn(0);
4183 }
4184 
4185 #undef __FUNCT__
4186 #define __FUNCT__ "DMCoarsen_Plex"
4187 PetscErrorCode DMCoarsen_Plex(DM dm, MPI_Comm comm, DM *dmCoarsened)
4188 {
4189   DM_Plex       *mesh = (DM_Plex*) dm->data;
4190   PetscErrorCode ierr;
4191 
4192   PetscFunctionBegin;
4193   ierr = PetscObjectReference((PetscObject) mesh->coarseMesh);CHKERRQ(ierr);
4194   *dmCoarsened = mesh->coarseMesh;
4195   PetscFunctionReturn(0);
4196 }
4197 
4198 #undef __FUNCT__
4199 #define __FUNCT__ "DMPlexGetDepthLabel"
4200 /*@
4201   DMPlexGetDepthLabel - Get the DMLabel recording the depth of each point
4202 
4203   Not Collective
4204 
4205   Input Parameter:
4206 . dm    - The DMPlex object
4207 
4208   Output Parameter:
4209 . depthLabel - The DMLabel recording point depth
4210 
4211   Level: developer
4212 
4213 .keywords: mesh, points
4214 .seealso: DMPlexGetDepth(), DMPlexGetHeightStratum(), DMPlexGetDepthStratum()
4215 @*/
4216 PetscErrorCode DMPlexGetDepthLabel(DM dm, DMLabel *depthLabel)
4217 {
4218   DM_Plex       *mesh = (DM_Plex*) dm->data;
4219   PetscErrorCode ierr;
4220 
4221   PetscFunctionBegin;
4222   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4223   PetscValidPointer(depthLabel, 2);
4224   if (!mesh->depthLabel) {ierr = DMPlexGetLabel(dm, "depth", &mesh->depthLabel);CHKERRQ(ierr);}
4225   *depthLabel = mesh->depthLabel;
4226   PetscFunctionReturn(0);
4227 }
4228 
4229 #undef __FUNCT__
4230 #define __FUNCT__ "DMPlexGetDepth"
4231 /*@
4232   DMPlexGetDepth - Get the depth of the DAG representing this mesh
4233 
4234   Not Collective
4235 
4236   Input Parameter:
4237 . dm    - The DMPlex object
4238 
4239   Output Parameter:
4240 . depth - The number of strata (breadth first levels) in the DAG
4241 
4242   Level: developer
4243 
4244 .keywords: mesh, points
4245 .seealso: DMPlexGetDepthLabel(), DMPlexGetHeightStratum(), DMPlexGetDepthStratum()
4246 @*/
4247 PetscErrorCode DMPlexGetDepth(DM dm, PetscInt *depth)
4248 {
4249   DMLabel        label;
4250   PetscInt       d = 0;
4251   PetscErrorCode ierr;
4252 
4253   PetscFunctionBegin;
4254   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4255   PetscValidPointer(depth, 2);
4256   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
4257   if (label) {ierr = DMLabelGetNumValues(label, &d);CHKERRQ(ierr);}
4258   *depth = d-1;
4259   PetscFunctionReturn(0);
4260 }
4261 
4262 #undef __FUNCT__
4263 #define __FUNCT__ "DMPlexGetDepthStratum"
4264 /*@
4265   DMPlexGetDepthStratum - Get the bounds [start, end) for all points at a certain depth.
4266 
4267   Not Collective
4268 
4269   Input Parameters:
4270 + dm           - The DMPlex object
4271 - stratumValue - The requested depth
4272 
4273   Output Parameters:
4274 + start - The first point at this depth
4275 - end   - One beyond the last point at this depth
4276 
4277   Level: developer
4278 
4279 .keywords: mesh, points
4280 .seealso: DMPlexGetHeightStratum(), DMPlexGetDepth()
4281 @*/
4282 PetscErrorCode DMPlexGetDepthStratum(DM dm, PetscInt stratumValue, PetscInt *start, PetscInt *end)
4283 {
4284   DMLabel        label;
4285   PetscInt       pStart, pEnd;
4286   PetscErrorCode ierr;
4287 
4288   PetscFunctionBegin;
4289   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4290   if (start) {PetscValidPointer(start, 3); *start = 0;}
4291   if (end)   {PetscValidPointer(end,   4); *end   = 0;}
4292   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
4293   if (pStart == pEnd) PetscFunctionReturn(0);
4294   if (stratumValue < 0) {
4295     if (start) *start = pStart;
4296     if (end)   *end   = pEnd;
4297     PetscFunctionReturn(0);
4298   }
4299   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
4300   if (!label) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "No label named depth was found");
4301   ierr = DMLabelGetStratumBounds(label, stratumValue, start, end);CHKERRQ(ierr);
4302   PetscFunctionReturn(0);
4303 }
4304 
4305 #undef __FUNCT__
4306 #define __FUNCT__ "DMPlexGetHeightStratum"
4307 /*@
4308   DMPlexGetHeightStratum - Get the bounds [start, end) for all points at a certain height.
4309 
4310   Not Collective
4311 
4312   Input Parameters:
4313 + dm           - The DMPlex object
4314 - stratumValue - The requested height
4315 
4316   Output Parameters:
4317 + start - The first point at this height
4318 - end   - One beyond the last point at this height
4319 
4320   Level: developer
4321 
4322 .keywords: mesh, points
4323 .seealso: DMPlexGetDepthStratum(), DMPlexGetDepth()
4324 @*/
4325 PetscErrorCode DMPlexGetHeightStratum(DM dm, PetscInt stratumValue, PetscInt *start, PetscInt *end)
4326 {
4327   DMLabel        label;
4328   PetscInt       depth, pStart, pEnd;
4329   PetscErrorCode ierr;
4330 
4331   PetscFunctionBegin;
4332   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4333   if (start) {PetscValidPointer(start, 3); *start = 0;}
4334   if (end)   {PetscValidPointer(end,   4); *end   = 0;}
4335   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
4336   if (pStart == pEnd) PetscFunctionReturn(0);
4337   if (stratumValue < 0) {
4338     if (start) *start = pStart;
4339     if (end)   *end   = pEnd;
4340     PetscFunctionReturn(0);
4341   }
4342   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
4343   if (!label) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "No label named depth was found");CHKERRQ(ierr);
4344   ierr = DMLabelGetNumValues(label, &depth);CHKERRQ(ierr);
4345   ierr = DMLabelGetStratumBounds(label, depth-1-stratumValue, start, end);CHKERRQ(ierr);
4346   PetscFunctionReturn(0);
4347 }
4348 
4349 #undef __FUNCT__
4350 #define __FUNCT__ "DMPlexCreateSectionInitial"
4351 /* Set the number of dof on each point and separate by fields */
4352 PetscErrorCode DMPlexCreateSectionInitial(DM dm, PetscInt dim, PetscInt numFields,const PetscInt numComp[],const PetscInt numDof[], PetscSection *section)
4353 {
4354   PetscInt      *numDofTot;
4355   PetscInt       depth, pStart = 0, pEnd = 0;
4356   PetscInt       p, d, dep, f;
4357   PetscErrorCode ierr;
4358 
4359   PetscFunctionBegin;
4360   ierr = PetscMalloc1((dim+1), &numDofTot);CHKERRQ(ierr);
4361   for (d = 0; d <= dim; ++d) {
4362     numDofTot[d] = 0;
4363     for (f = 0; f < numFields; ++f) numDofTot[d] += numDof[f*(dim+1)+d];
4364   }
4365   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), section);CHKERRQ(ierr);
4366   if (numFields > 0) {
4367     ierr = PetscSectionSetNumFields(*section, numFields);CHKERRQ(ierr);
4368     if (numComp) {
4369       for (f = 0; f < numFields; ++f) {
4370         ierr = PetscSectionSetFieldComponents(*section, f, numComp[f]);CHKERRQ(ierr);
4371       }
4372     }
4373   }
4374   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
4375   ierr = PetscSectionSetChart(*section, pStart, pEnd);CHKERRQ(ierr);
4376   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
4377   for (dep = 0; dep <= depth; ++dep) {
4378     d    = dim == depth ? dep : (!dep ? 0 : dim);
4379     ierr = DMPlexGetDepthStratum(dm, dep, &pStart, &pEnd);CHKERRQ(ierr);
4380     for (p = pStart; p < pEnd; ++p) {
4381       for (f = 0; f < numFields; ++f) {
4382         ierr = PetscSectionSetFieldDof(*section, p, f, numDof[f*(dim+1)+d]);CHKERRQ(ierr);
4383       }
4384       ierr = PetscSectionSetDof(*section, p, numDofTot[d]);CHKERRQ(ierr);
4385     }
4386   }
4387   ierr = PetscFree(numDofTot);CHKERRQ(ierr);
4388   PetscFunctionReturn(0);
4389 }
4390 
4391 #undef __FUNCT__
4392 #define __FUNCT__ "DMPlexCreateSectionBCDof"
4393 /* Set the number of dof on each point and separate by fields
4394    If constDof is PETSC_DETERMINE, constrain every dof on the point
4395 */
4396 PetscErrorCode DMPlexCreateSectionBCDof(DM dm, PetscInt numBC,const PetscInt bcField[],const IS bcPoints[], PetscInt constDof, PetscSection section)
4397 {
4398   PetscInt       numFields;
4399   PetscInt       bc;
4400   PetscErrorCode ierr;
4401 
4402   PetscFunctionBegin;
4403   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
4404   for (bc = 0; bc < numBC; ++bc) {
4405     PetscInt        field = 0;
4406     const PetscInt *idx;
4407     PetscInt        n, i;
4408 
4409     if (numFields) field = bcField[bc];
4410     ierr = ISGetLocalSize(bcPoints[bc], &n);CHKERRQ(ierr);
4411     ierr = ISGetIndices(bcPoints[bc], &idx);CHKERRQ(ierr);
4412     for (i = 0; i < n; ++i) {
4413       const PetscInt p        = idx[i];
4414       PetscInt       numConst = constDof;
4415 
4416       /* Constrain every dof on the point */
4417       if (numConst < 0) {
4418         if (numFields) {
4419           ierr = PetscSectionGetFieldDof(section, p, field, &numConst);CHKERRQ(ierr);
4420         } else {
4421           ierr = PetscSectionGetDof(section, p, &numConst);CHKERRQ(ierr);
4422         }
4423       }
4424       if (numFields) {
4425         ierr = PetscSectionAddFieldConstraintDof(section, p, field, numConst);CHKERRQ(ierr);
4426       }
4427       ierr = PetscSectionAddConstraintDof(section, p, numConst);CHKERRQ(ierr);
4428     }
4429     ierr = ISRestoreIndices(bcPoints[bc], &idx);CHKERRQ(ierr);
4430   }
4431   PetscFunctionReturn(0);
4432 }
4433 
4434 #undef __FUNCT__
4435 #define __FUNCT__ "DMPlexCreateSectionBCIndicesAll"
4436 /* Set the constrained indices on each point and separate by fields */
4437 PetscErrorCode DMPlexCreateSectionBCIndicesAll(DM dm, PetscSection section)
4438 {
4439   PetscInt      *maxConstraints;
4440   PetscInt       numFields, f, pStart = 0, pEnd = 0, p;
4441   PetscErrorCode ierr;
4442 
4443   PetscFunctionBegin;
4444   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
4445   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
4446   ierr = PetscMalloc1((numFields+1), &maxConstraints);CHKERRQ(ierr);
4447   for (f = 0; f <= numFields; ++f) maxConstraints[f] = 0;
4448   for (p = pStart; p < pEnd; ++p) {
4449     PetscInt cdof;
4450 
4451     if (numFields) {
4452       for (f = 0; f < numFields; ++f) {
4453         ierr              = PetscSectionGetFieldConstraintDof(section, p, f, &cdof);CHKERRQ(ierr);
4454         maxConstraints[f] = PetscMax(maxConstraints[f], cdof);
4455       }
4456     } else {
4457       ierr              = PetscSectionGetConstraintDof(section, p, &cdof);CHKERRQ(ierr);
4458       maxConstraints[0] = PetscMax(maxConstraints[0], cdof);
4459     }
4460   }
4461   for (f = 0; f < numFields; ++f) {
4462     maxConstraints[numFields] += maxConstraints[f];
4463   }
4464   if (maxConstraints[numFields]) {
4465     PetscInt *indices;
4466 
4467     ierr = PetscMalloc1(maxConstraints[numFields], &indices);CHKERRQ(ierr);
4468     for (p = pStart; p < pEnd; ++p) {
4469       PetscInt cdof, d;
4470 
4471       ierr = PetscSectionGetConstraintDof(section, p, &cdof);CHKERRQ(ierr);
4472       if (cdof) {
4473         if (cdof > maxConstraints[numFields]) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_LIB, "Likely memory corruption, point %D cDof %D > maxConstraints %D", p, cdof, maxConstraints[numFields]);
4474         if (numFields) {
4475           PetscInt numConst = 0, foff = 0;
4476 
4477           for (f = 0; f < numFields; ++f) {
4478             PetscInt cfdof, fdof;
4479 
4480             ierr = PetscSectionGetFieldDof(section, p, f, &fdof);CHKERRQ(ierr);
4481             ierr = PetscSectionGetFieldConstraintDof(section, p, f, &cfdof);CHKERRQ(ierr);
4482             /* Change constraint numbering from absolute local dof number to field relative local dof number */
4483             for (d = 0; d < cfdof; ++d) indices[numConst+d] = d;
4484             ierr = PetscSectionSetFieldConstraintIndices(section, p, f, &indices[numConst]);CHKERRQ(ierr);
4485             for (d = 0; d < cfdof; ++d) indices[numConst+d] += foff;
4486             numConst += cfdof;
4487             foff     += fdof;
4488           }
4489           if (cdof != numConst) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_LIB, "Total number of field constraints %D should be %D", numConst, cdof);
4490         } else {
4491           for (d = 0; d < cdof; ++d) indices[d] = d;
4492         }
4493         ierr = PetscSectionSetConstraintIndices(section, p, indices);CHKERRQ(ierr);
4494       }
4495     }
4496     ierr = PetscFree(indices);CHKERRQ(ierr);
4497   }
4498   ierr = PetscFree(maxConstraints);CHKERRQ(ierr);
4499   PetscFunctionReturn(0);
4500 }
4501 
4502 #undef __FUNCT__
4503 #define __FUNCT__ "DMPlexCreateSectionBCIndicesField"
4504 /* Set the constrained field indices on each point */
4505 PetscErrorCode DMPlexCreateSectionBCIndicesField(DM dm, PetscInt field, IS bcPoints, IS constraintIndices, PetscSection section)
4506 {
4507   const PetscInt *points, *indices;
4508   PetscInt        numFields, maxDof, numPoints, p, numConstraints;
4509   PetscErrorCode  ierr;
4510 
4511   PetscFunctionBegin;
4512   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
4513   if ((field < 0) || (field >= numFields)) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Section field %d should be in [%d, %d)", field, 0, numFields);
4514 
4515   ierr = ISGetLocalSize(bcPoints, &numPoints);CHKERRQ(ierr);
4516   ierr = ISGetIndices(bcPoints, &points);CHKERRQ(ierr);
4517   if (!constraintIndices) {
4518     PetscInt *idx, i;
4519 
4520     ierr = PetscSectionGetMaxDof(section, &maxDof);CHKERRQ(ierr);
4521     ierr = PetscMalloc1(maxDof, &idx);CHKERRQ(ierr);
4522     for (i = 0; i < maxDof; ++i) idx[i] = i;
4523     for (p = 0; p < numPoints; ++p) {
4524       ierr = PetscSectionSetFieldConstraintIndices(section, points[p], field, idx);CHKERRQ(ierr);
4525     }
4526     ierr = PetscFree(idx);CHKERRQ(ierr);
4527   } else {
4528     ierr = ISGetLocalSize(constraintIndices, &numConstraints);CHKERRQ(ierr);
4529     ierr = ISGetIndices(constraintIndices, &indices);CHKERRQ(ierr);
4530     for (p = 0; p < numPoints; ++p) {
4531       PetscInt fcdof;
4532 
4533       ierr = PetscSectionGetFieldConstraintDof(section, points[p], field, &fcdof);CHKERRQ(ierr);
4534       if (fcdof != numConstraints) SETERRQ4(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Section point %d field %d has %d constraints, but yo ugave %d indices", p, field, fcdof, numConstraints);
4535       ierr = PetscSectionSetFieldConstraintIndices(section, points[p], field, indices);CHKERRQ(ierr);
4536     }
4537     ierr = ISRestoreIndices(constraintIndices, &indices);CHKERRQ(ierr);
4538   }
4539   ierr = ISRestoreIndices(bcPoints, &points);CHKERRQ(ierr);
4540   PetscFunctionReturn(0);
4541 }
4542 
4543 #undef __FUNCT__
4544 #define __FUNCT__ "DMPlexCreateSectionBCIndices"
4545 /* Set the constrained indices on each point and separate by fields */
4546 PetscErrorCode DMPlexCreateSectionBCIndices(DM dm, PetscSection section)
4547 {
4548   PetscInt      *indices;
4549   PetscInt       numFields, maxDof, f, pStart = 0, pEnd = 0, p;
4550   PetscErrorCode ierr;
4551 
4552   PetscFunctionBegin;
4553   ierr = PetscSectionGetMaxDof(section, &maxDof);CHKERRQ(ierr);
4554   ierr = PetscMalloc1(maxDof, &indices);CHKERRQ(ierr);
4555   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
4556   if (!numFields) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "This function only works after users have set field constraint indices.");
4557   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
4558   for (p = pStart; p < pEnd; ++p) {
4559     PetscInt cdof, d;
4560 
4561     ierr = PetscSectionGetConstraintDof(section, p, &cdof);CHKERRQ(ierr);
4562     if (cdof) {
4563       PetscInt numConst = 0, foff = 0;
4564 
4565       for (f = 0; f < numFields; ++f) {
4566         const PetscInt *fcind;
4567         PetscInt        fdof, fcdof;
4568 
4569         ierr = PetscSectionGetFieldDof(section, p, f, &fdof);CHKERRQ(ierr);
4570         ierr = PetscSectionGetFieldConstraintDof(section, p, f, &fcdof);CHKERRQ(ierr);
4571         if (fcdof) {ierr = PetscSectionGetFieldConstraintIndices(section, p, f, &fcind);CHKERRQ(ierr);}
4572         /* Change constraint numbering from field relative local dof number to absolute local dof number */
4573         for (d = 0; d < fcdof; ++d) indices[numConst+d] = fcind[d]+foff;
4574         foff     += fdof;
4575         numConst += fcdof;
4576       }
4577       if (cdof != numConst) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_LIB, "Total number of field constraints %D should be %D", numConst, cdof);
4578       ierr = PetscSectionSetConstraintIndices(section, p, indices);CHKERRQ(ierr);
4579     }
4580   }
4581   ierr = PetscFree(indices);CHKERRQ(ierr);
4582   PetscFunctionReturn(0);
4583 }
4584 
4585 #undef __FUNCT__
4586 #define __FUNCT__ "DMPlexCreateSection"
4587 /*@C
4588   DMPlexCreateSection - Create a PetscSection based upon the dof layout specification provided.
4589 
4590   Not Collective
4591 
4592   Input Parameters:
4593 + dm        - The DMPlex object
4594 . dim       - The spatial dimension of the problem
4595 . numFields - The number of fields in the problem
4596 . numComp   - An array of size numFields that holds the number of components for each field
4597 . numDof    - An array of size numFields*(dim+1) which holds the number of dof for each field on a mesh piece of dimension d
4598 . numBC     - The number of boundary conditions
4599 . bcField   - An array of size numBC giving the field number for each boundry condition
4600 - bcPoints  - An array of size numBC giving an IS holding the sieve points to which each boundary condition applies
4601 
4602   Output Parameter:
4603 . section - The PetscSection object
4604 
4605   Notes: numDof[f*(dim+1)+d] gives the number of dof for field f on sieve points of dimension d. For instance, numDof[1] is the
4606   nubmer of dof for field 0 on each edge.
4607 
4608   Level: developer
4609 
4610   Fortran Notes:
4611   A Fortran 90 version is available as DMPlexCreateSectionF90()
4612 
4613 .keywords: mesh, elements
4614 .seealso: DMPlexCreate(), PetscSectionCreate()
4615 @*/
4616 PetscErrorCode DMPlexCreateSection(DM dm, PetscInt dim, PetscInt numFields,const PetscInt numComp[],const PetscInt numDof[], PetscInt numBC,const PetscInt bcField[],const IS bcPoints[], PetscSection *section)
4617 {
4618   PetscErrorCode ierr;
4619 
4620   PetscFunctionBegin;
4621   ierr = DMPlexCreateSectionInitial(dm, dim, numFields, numComp, numDof, section);CHKERRQ(ierr);
4622   ierr = DMPlexCreateSectionBCDof(dm, numBC, bcField, bcPoints, PETSC_DETERMINE, *section);CHKERRQ(ierr);
4623   ierr = PetscSectionSetUp(*section);CHKERRQ(ierr);
4624   if (numBC) {ierr = DMPlexCreateSectionBCIndicesAll(dm, *section);CHKERRQ(ierr);}
4625   ierr = PetscSectionViewFromOptions(*section,NULL,"-section_view");CHKERRQ(ierr);
4626   PetscFunctionReturn(0);
4627 }
4628 
4629 #undef __FUNCT__
4630 #define __FUNCT__ "DMCreateCoordinateDM_Plex"
4631 PetscErrorCode DMCreateCoordinateDM_Plex(DM dm, DM *cdm)
4632 {
4633   PetscSection   section;
4634   PetscErrorCode ierr;
4635 
4636   PetscFunctionBegin;
4637   ierr = DMClone(dm, cdm);CHKERRQ(ierr);
4638   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &section);CHKERRQ(ierr);
4639   ierr = DMSetDefaultSection(*cdm, section);CHKERRQ(ierr);
4640   ierr = PetscSectionDestroy(&section);CHKERRQ(ierr);
4641   PetscFunctionReturn(0);
4642 }
4643 
4644 #undef __FUNCT__
4645 #define __FUNCT__ "DMPlexGetConeSection"
4646 PetscErrorCode DMPlexGetConeSection(DM dm, PetscSection *section)
4647 {
4648   DM_Plex *mesh = (DM_Plex*) dm->data;
4649 
4650   PetscFunctionBegin;
4651   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4652   if (section) *section = mesh->coneSection;
4653   PetscFunctionReturn(0);
4654 }
4655 
4656 #undef __FUNCT__
4657 #define __FUNCT__ "DMPlexGetSupportSection"
4658 PetscErrorCode DMPlexGetSupportSection(DM dm, PetscSection *section)
4659 {
4660   DM_Plex *mesh = (DM_Plex*) dm->data;
4661 
4662   PetscFunctionBegin;
4663   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4664   if (section) *section = mesh->supportSection;
4665   PetscFunctionReturn(0);
4666 }
4667 
4668 #undef __FUNCT__
4669 #define __FUNCT__ "DMPlexGetCones"
4670 PetscErrorCode DMPlexGetCones(DM dm, PetscInt *cones[])
4671 {
4672   DM_Plex *mesh = (DM_Plex*) dm->data;
4673 
4674   PetscFunctionBegin;
4675   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4676   if (cones) *cones = mesh->cones;
4677   PetscFunctionReturn(0);
4678 }
4679 
4680 #undef __FUNCT__
4681 #define __FUNCT__ "DMPlexGetConeOrientations"
4682 PetscErrorCode DMPlexGetConeOrientations(DM dm, PetscInt *coneOrientations[])
4683 {
4684   DM_Plex *mesh = (DM_Plex*) dm->data;
4685 
4686   PetscFunctionBegin;
4687   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4688   if (coneOrientations) *coneOrientations = mesh->coneOrientations;
4689   PetscFunctionReturn(0);
4690 }
4691 
4692 /******************************** FEM Support **********************************/
4693 
4694 #undef __FUNCT__
4695 #define __FUNCT__ "DMPlexVecGetClosure_Depth1_Static"
4696 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecGetClosure_Depth1_Static(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
4697 {
4698   PetscScalar    *array, *vArray;
4699   const PetscInt *cone, *coneO;
4700   PetscInt        pStart, pEnd, p, numPoints, size = 0, offset = 0;
4701   PetscErrorCode  ierr;
4702 
4703   PetscFunctionBeginHot;
4704   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
4705   ierr = DMPlexGetConeSize(dm, point, &numPoints);CHKERRQ(ierr);
4706   ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
4707   ierr = DMPlexGetConeOrientation(dm, point, &coneO);CHKERRQ(ierr);
4708   if (!values || !*values) {
4709     if ((point >= pStart) && (point < pEnd)) {
4710       PetscInt dof;
4711 
4712       ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4713       size += dof;
4714     }
4715     for (p = 0; p < numPoints; ++p) {
4716       const PetscInt cp = cone[p];
4717       PetscInt       dof;
4718 
4719       if ((cp < pStart) || (cp >= pEnd)) continue;
4720       ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
4721       size += dof;
4722     }
4723     if (!values) {
4724       if (csize) *csize = size;
4725       PetscFunctionReturn(0);
4726     }
4727     ierr = DMGetWorkArray(dm, size, PETSC_SCALAR, &array);CHKERRQ(ierr);
4728   } else {
4729     array = *values;
4730   }
4731   size = 0;
4732   ierr = VecGetArray(v, &vArray);CHKERRQ(ierr);
4733   if ((point >= pStart) && (point < pEnd)) {
4734     PetscInt     dof, off, d;
4735     PetscScalar *varr;
4736 
4737     ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4738     ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
4739     varr = &vArray[off];
4740     for (d = 0; d < dof; ++d, ++offset) {
4741       array[offset] = varr[d];
4742     }
4743     size += dof;
4744   }
4745   for (p = 0; p < numPoints; ++p) {
4746     const PetscInt cp = cone[p];
4747     PetscInt       o  = coneO[p];
4748     PetscInt       dof, off, d;
4749     PetscScalar   *varr;
4750 
4751     if ((cp < pStart) || (cp >= pEnd)) continue;
4752     ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
4753     ierr = PetscSectionGetOffset(section, cp, &off);CHKERRQ(ierr);
4754     varr = &vArray[off];
4755     if (o >= 0) {
4756       for (d = 0; d < dof; ++d, ++offset) {
4757         array[offset] = varr[d];
4758       }
4759     } else {
4760       for (d = dof-1; d >= 0; --d, ++offset) {
4761         array[offset] = varr[d];
4762       }
4763     }
4764     size += dof;
4765   }
4766   ierr = VecRestoreArray(v, &vArray);CHKERRQ(ierr);
4767   if (!*values) {
4768     if (csize) *csize = size;
4769     *values = array;
4770   } else {
4771     if (size > *csize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Size of input array %d < actual size %d", *csize, size);
4772     *csize = size;
4773   }
4774   PetscFunctionReturn(0);
4775 }
4776 
4777 #undef __FUNCT__
4778 #define __FUNCT__ "DMPlexVecGetClosure_Static"
4779 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecGetClosure_Static(PetscSection section, PetscInt numPoints, const PetscInt points[], const PetscScalar vArray[], PetscInt *size, PetscScalar array[])
4780 {
4781   PetscInt       offset = 0, p;
4782   PetscErrorCode ierr;
4783 
4784   PetscFunctionBeginHot;
4785   *size = 0;
4786   for (p = 0; p < numPoints*2; p += 2) {
4787     const PetscInt point = points[p];
4788     const PetscInt o     = points[p+1];
4789     PetscInt       dof, off, d;
4790     const PetscScalar *varr;
4791 
4792     ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4793     ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
4794     varr = &vArray[off];
4795     if (o >= 0) {
4796       for (d = 0; d < dof; ++d, ++offset)    array[offset] = varr[d];
4797     } else {
4798       for (d = dof-1; d >= 0; --d, ++offset) array[offset] = varr[d];
4799     }
4800   }
4801   *size = offset;
4802   PetscFunctionReturn(0);
4803 }
4804 
4805 #undef __FUNCT__
4806 #define __FUNCT__ "DMPlexVecGetClosure_Fields_Static"
4807 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecGetClosure_Fields_Static(PetscSection section, PetscInt numPoints, const PetscInt points[], PetscInt numFields, const PetscScalar vArray[], PetscInt *size, PetscScalar array[])
4808 {
4809   PetscInt       offset = 0, f;
4810   PetscErrorCode ierr;
4811 
4812   PetscFunctionBeginHot;
4813   *size = 0;
4814   for (f = 0; f < numFields; ++f) {
4815     PetscInt fcomp, p;
4816 
4817     ierr = PetscSectionGetFieldComponents(section, f, &fcomp);CHKERRQ(ierr);
4818     for (p = 0; p < numPoints*2; p += 2) {
4819       const PetscInt point = points[p];
4820       const PetscInt o     = points[p+1];
4821       PetscInt       fdof, foff, d, c;
4822       const PetscScalar *varr;
4823 
4824       ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
4825       ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
4826       varr = &vArray[foff];
4827       if (o >= 0) {
4828         for (d = 0; d < fdof; ++d, ++offset) array[offset] = varr[d];
4829       } else {
4830         for (d = fdof/fcomp-1; d >= 0; --d) {
4831           for (c = 0; c < fcomp; ++c, ++offset) {
4832             array[offset] = varr[d*fcomp+c];
4833           }
4834         }
4835       }
4836     }
4837   }
4838   *size = offset;
4839   PetscFunctionReturn(0);
4840 }
4841 
4842 #undef __FUNCT__
4843 #define __FUNCT__ "DMPlexVecGetClosure"
4844 /*@C
4845   DMPlexVecGetClosure - Get an array of the values on the closure of 'point'
4846 
4847   Not collective
4848 
4849   Input Parameters:
4850 + dm - The DM
4851 . section - The section describing the layout in v, or NULL to use the default section
4852 . v - The local vector
4853 - point - The sieve point in the DM
4854 
4855   Output Parameters:
4856 + csize - The number of values in the closure, or NULL
4857 - values - The array of values, which is a borrowed array and should not be freed
4858 
4859   Fortran Notes:
4860   Since it returns an array, this routine is only available in Fortran 90, and you must
4861   include petsc.h90 in your code.
4862 
4863   The csize argument is not present in the Fortran 90 binding since it is internal to the array.
4864 
4865   Level: intermediate
4866 
4867 .seealso DMPlexVecRestoreClosure(), DMPlexVecSetClosure(), DMPlexMatSetClosure()
4868 @*/
4869 PetscErrorCode DMPlexVecGetClosure(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
4870 {
4871   PetscSection    clSection;
4872   IS              clPoints;
4873   PetscScalar    *array, *vArray;
4874   PetscInt       *points = NULL;
4875   const PetscInt *clp;
4876   PetscInt        depth, numFields, numPoints, size;
4877   PetscErrorCode  ierr;
4878 
4879   PetscFunctionBeginHot;
4880   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4881   if (!section) {ierr = DMGetDefaultSection(dm, &section);CHKERRQ(ierr);}
4882   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
4883   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
4884   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
4885   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
4886   if (depth == 1 && numFields < 2) {
4887     ierr = DMPlexVecGetClosure_Depth1_Static(dm, section, v, point, csize, values);CHKERRQ(ierr);
4888     PetscFunctionReturn(0);
4889   }
4890   /* Get points */
4891   ierr = PetscSectionGetClosureIndex(section, (PetscObject) dm, &clSection, &clPoints);CHKERRQ(ierr);
4892   if (!clPoints) {
4893     PetscInt pStart, pEnd, p, q;
4894 
4895     ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
4896     ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
4897     /* Compress out points not in the section */
4898     for (p = 0, q = 0; p < numPoints*2; p += 2) {
4899       if ((points[p] >= pStart) && (points[p] < pEnd)) {
4900         points[q*2]   = points[p];
4901         points[q*2+1] = points[p+1];
4902         ++q;
4903       }
4904     }
4905     numPoints = q;
4906   } else {
4907     PetscInt dof, off;
4908 
4909     ierr = PetscSectionGetDof(clSection, point, &dof);CHKERRQ(ierr);
4910     ierr = PetscSectionGetOffset(clSection, point, &off);CHKERRQ(ierr);
4911     ierr = ISGetIndices(clPoints, &clp);CHKERRQ(ierr);
4912     numPoints = dof/2;
4913     points    = (PetscInt *) &clp[off];
4914   }
4915   /* Get array */
4916   if (!values || !*values) {
4917     PetscInt asize = 0, dof, p;
4918 
4919     for (p = 0; p < numPoints*2; p += 2) {
4920       ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
4921       asize += dof;
4922     }
4923     if (!values) {
4924       if (!clPoints) {ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);}
4925       else           {ierr = ISRestoreIndices(clPoints, &clp);CHKERRQ(ierr);}
4926       if (csize) *csize = asize;
4927       PetscFunctionReturn(0);
4928     }
4929     ierr = DMGetWorkArray(dm, asize, PETSC_SCALAR, &array);CHKERRQ(ierr);
4930   } else {
4931     array = *values;
4932   }
4933   ierr = VecGetArray(v, &vArray);CHKERRQ(ierr);
4934   /* Get values */
4935   if (numFields > 0) {ierr = DMPlexVecGetClosure_Fields_Static(section, numPoints, points, numFields, vArray, &size, array);CHKERRQ(ierr);}
4936   else               {ierr = DMPlexVecGetClosure_Static(section, numPoints, points, vArray, &size, array);CHKERRQ(ierr);}
4937   /* Cleanup points */
4938   if (!clPoints) {ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);}
4939   else           {ierr = ISRestoreIndices(clPoints, &clp);CHKERRQ(ierr);}
4940   /* Cleanup array */
4941   ierr = VecRestoreArray(v, &vArray);CHKERRQ(ierr);
4942   if (!*values) {
4943     if (csize) *csize = size;
4944     *values = array;
4945   } else {
4946     if (size > *csize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Size of input array %d < actual size %d", *csize, size);
4947     *csize = size;
4948   }
4949   PetscFunctionReturn(0);
4950 }
4951 
4952 #undef __FUNCT__
4953 #define __FUNCT__ "DMPlexVecRestoreClosure"
4954 /*@C
4955   DMPlexVecRestoreClosure - Restore the array of the values on the closure of 'point'
4956 
4957   Not collective
4958 
4959   Input Parameters:
4960 + dm - The DM
4961 . section - The section describing the layout in v, or NULL to use the default section
4962 . v - The local vector
4963 . point - The sieve point in the DM
4964 . csize - The number of values in the closure, or NULL
4965 - values - The array of values, which is a borrowed array and should not be freed
4966 
4967   Fortran Notes:
4968   Since it returns an array, this routine is only available in Fortran 90, and you must
4969   include petsc.h90 in your code.
4970 
4971   The csize argument is not present in the Fortran 90 binding since it is internal to the array.
4972 
4973   Level: intermediate
4974 
4975 .seealso DMPlexVecGetClosure(), DMPlexVecSetClosure(), DMPlexMatSetClosure()
4976 @*/
4977 PetscErrorCode DMPlexVecRestoreClosure(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
4978 {
4979   PetscInt       size = 0;
4980   PetscErrorCode ierr;
4981 
4982   PetscFunctionBegin;
4983   /* Should work without recalculating size */
4984   ierr = DMRestoreWorkArray(dm, size, PETSC_SCALAR, (void*) values);CHKERRQ(ierr);
4985   PetscFunctionReturn(0);
4986 }
4987 
4988 PETSC_STATIC_INLINE void add   (PetscScalar *x, PetscScalar y) {*x += y;}
4989 PETSC_STATIC_INLINE void insert(PetscScalar *x, PetscScalar y) {*x  = y;}
4990 
4991 #undef __FUNCT__
4992 #define __FUNCT__ "updatePoint_private"
4993 PETSC_STATIC_INLINE PetscErrorCode updatePoint_private(PetscSection section, PetscInt point, PetscInt dof, void (*fuse)(PetscScalar*, PetscScalar), PetscBool setBC, PetscInt orientation, const PetscScalar values[], PetscScalar array[])
4994 {
4995   PetscInt        cdof;   /* The number of constraints on this point */
4996   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
4997   PetscScalar    *a;
4998   PetscInt        off, cind = 0, k;
4999   PetscErrorCode  ierr;
5000 
5001   PetscFunctionBegin;
5002   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
5003   ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
5004   a    = &array[off];
5005   if (!cdof || setBC) {
5006     if (orientation >= 0) {
5007       for (k = 0; k < dof; ++k) {
5008         fuse(&a[k], values[k]);
5009       }
5010     } else {
5011       for (k = 0; k < dof; ++k) {
5012         fuse(&a[k], values[dof-k-1]);
5013       }
5014     }
5015   } else {
5016     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
5017     if (orientation >= 0) {
5018       for (k = 0; k < dof; ++k) {
5019         if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
5020         fuse(&a[k], values[k]);
5021       }
5022     } else {
5023       for (k = 0; k < dof; ++k) {
5024         if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
5025         fuse(&a[k], values[dof-k-1]);
5026       }
5027     }
5028   }
5029   PetscFunctionReturn(0);
5030 }
5031 
5032 #undef __FUNCT__
5033 #define __FUNCT__ "updatePointBC_private"
5034 PETSC_STATIC_INLINE PetscErrorCode updatePointBC_private(PetscSection section, PetscInt point, PetscInt dof, void (*fuse)(PetscScalar*, PetscScalar), PetscInt orientation, const PetscScalar values[], PetscScalar array[])
5035 {
5036   PetscInt        cdof;   /* The number of constraints on this point */
5037   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
5038   PetscScalar    *a;
5039   PetscInt        off, cind = 0, k;
5040   PetscErrorCode  ierr;
5041 
5042   PetscFunctionBegin;
5043   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
5044   ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
5045   a    = &array[off];
5046   if (cdof) {
5047     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
5048     if (orientation >= 0) {
5049       for (k = 0; k < dof; ++k) {
5050         if ((cind < cdof) && (k == cdofs[cind])) {
5051           fuse(&a[k], values[k]);
5052           ++cind;
5053         }
5054       }
5055     } else {
5056       for (k = 0; k < dof; ++k) {
5057         if ((cind < cdof) && (k == cdofs[cind])) {
5058           fuse(&a[k], values[dof-k-1]);
5059           ++cind;
5060         }
5061       }
5062     }
5063   }
5064   PetscFunctionReturn(0);
5065 }
5066 
5067 #undef __FUNCT__
5068 #define __FUNCT__ "updatePointFields_private"
5069 PETSC_STATIC_INLINE PetscErrorCode updatePointFields_private(PetscSection section, PetscInt point, PetscInt o, PetscInt f, PetscInt fcomp, void (*fuse)(PetscScalar*, PetscScalar), PetscBool setBC, const PetscScalar values[], PetscInt *offset, PetscScalar array[])
5070 {
5071   PetscScalar    *a;
5072   PetscInt        fdof, foff, fcdof, foffset = *offset;
5073   const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
5074   PetscInt        cind = 0, k, c;
5075   PetscErrorCode  ierr;
5076 
5077   PetscFunctionBegin;
5078   ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
5079   ierr = PetscSectionGetFieldConstraintDof(section, point, f, &fcdof);CHKERRQ(ierr);
5080   ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
5081   a    = &array[foff];
5082   if (!fcdof || setBC) {
5083     if (o >= 0) {
5084       for (k = 0; k < fdof; ++k) fuse(&a[k], values[foffset+k]);
5085     } else {
5086       for (k = fdof/fcomp-1; k >= 0; --k) {
5087         for (c = 0; c < fcomp; ++c) {
5088           fuse(&a[(fdof/fcomp-1-k)*fcomp+c], values[foffset+k*fcomp+c]);
5089         }
5090       }
5091     }
5092   } else {
5093     ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
5094     if (o >= 0) {
5095       for (k = 0; k < fdof; ++k) {
5096         if ((cind < fcdof) && (k == fcdofs[cind])) {++cind; continue;}
5097         fuse(&a[k], values[foffset+k]);
5098       }
5099     } else {
5100       for (k = fdof/fcomp-1; k >= 0; --k) {
5101         for (c = 0; c < fcomp; ++c) {
5102           if ((cind < fcdof) && (k*fcomp+c == fcdofs[cind])) {++cind; continue;}
5103           fuse(&a[(fdof/fcomp-1-k)*fcomp+c], values[foffset+k*fcomp+c]);
5104         }
5105       }
5106     }
5107   }
5108   *offset += fdof;
5109   PetscFunctionReturn(0);
5110 }
5111 
5112 #undef __FUNCT__
5113 #define __FUNCT__ "updatePointFieldsBC_private"
5114 PETSC_STATIC_INLINE PetscErrorCode updatePointFieldsBC_private(PetscSection section, PetscInt point, PetscInt o, PetscInt f, PetscInt fcomp, void (*fuse)(PetscScalar*, PetscScalar), const PetscScalar values[], PetscInt *offset, PetscScalar array[])
5115 {
5116   PetscScalar    *a;
5117   PetscInt        fdof, foff, fcdof, foffset = *offset;
5118   const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
5119   PetscInt        cind = 0, k, c;
5120   PetscErrorCode  ierr;
5121 
5122   PetscFunctionBegin;
5123   ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
5124   ierr = PetscSectionGetFieldConstraintDof(section, point, f, &fcdof);CHKERRQ(ierr);
5125   ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
5126   a    = &array[foff];
5127   if (fcdof) {
5128     ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
5129     if (o >= 0) {
5130       for (k = 0; k < fdof; ++k) {
5131         if ((cind < fcdof) && (k == fcdofs[cind])) {
5132           fuse(&a[k], values[foffset+k]);
5133           ++cind;
5134         }
5135       }
5136     } else {
5137       for (k = fdof/fcomp-1; k >= 0; --k) {
5138         for (c = 0; c < fcomp; ++c) {
5139           if ((cind < fcdof) && (k*fcomp+c == fcdofs[cind])) {
5140             fuse(&a[(fdof/fcomp-1-k)*fcomp+c], values[foffset+k*fcomp+c]);
5141             ++cind;
5142           }
5143         }
5144       }
5145     }
5146   }
5147   *offset += fdof;
5148   PetscFunctionReturn(0);
5149 }
5150 
5151 #undef __FUNCT__
5152 #define __FUNCT__ "DMPlexVecSetClosure_Static"
5153 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecSetClosure_Static(DM dm, PetscSection section, Vec v, PetscInt point, const PetscScalar values[], InsertMode mode)
5154 {
5155   PetscScalar    *array;
5156   const PetscInt *cone, *coneO;
5157   PetscInt        pStart, pEnd, p, numPoints, off, dof;
5158   PetscErrorCode  ierr;
5159 
5160   PetscFunctionBeginHot;
5161   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
5162   ierr = DMPlexGetConeSize(dm, point, &numPoints);CHKERRQ(ierr);
5163   ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
5164   ierr = DMPlexGetConeOrientation(dm, point, &coneO);CHKERRQ(ierr);
5165   ierr = VecGetArray(v, &array);CHKERRQ(ierr);
5166   for (p = 0, off = 0; p <= numPoints; ++p, off += dof) {
5167     const PetscInt cp = !p ? point : cone[p-1];
5168     const PetscInt o  = !p ? 0     : coneO[p-1];
5169 
5170     if ((cp < pStart) || (cp >= pEnd)) {dof = 0; continue;}
5171     ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
5172     /* ADD_VALUES */
5173     {
5174       const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
5175       PetscScalar    *a;
5176       PetscInt        cdof, coff, cind = 0, k;
5177 
5178       ierr = PetscSectionGetConstraintDof(section, cp, &cdof);CHKERRQ(ierr);
5179       ierr = PetscSectionGetOffset(section, cp, &coff);CHKERRQ(ierr);
5180       a    = &array[coff];
5181       if (!cdof) {
5182         if (o >= 0) {
5183           for (k = 0; k < dof; ++k) {
5184             a[k] += values[off+k];
5185           }
5186         } else {
5187           for (k = 0; k < dof; ++k) {
5188             a[k] += values[off+dof-k-1];
5189           }
5190         }
5191       } else {
5192         ierr = PetscSectionGetConstraintIndices(section, cp, &cdofs);CHKERRQ(ierr);
5193         if (o >= 0) {
5194           for (k = 0; k < dof; ++k) {
5195             if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
5196             a[k] += values[off+k];
5197           }
5198         } else {
5199           for (k = 0; k < dof; ++k) {
5200             if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
5201             a[k] += values[off+dof-k-1];
5202           }
5203         }
5204       }
5205     }
5206   }
5207   ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
5208   PetscFunctionReturn(0);
5209 }
5210 
5211 #undef __FUNCT__
5212 #define __FUNCT__ "DMPlexVecSetClosure"
5213 /*@C
5214   DMPlexVecSetClosure - Set an array of the values on the closure of 'point'
5215 
5216   Not collective
5217 
5218   Input Parameters:
5219 + dm - The DM
5220 . section - The section describing the layout in v, or NULL to use the default section
5221 . v - The local vector
5222 . point - The sieve point in the DM
5223 . values - The array of values
5224 - mode - The insert mode, where INSERT_ALL_VALUES and ADD_ALL_VALUES also overwrite boundary conditions
5225 
5226   Fortran Notes:
5227   This routine is only available in Fortran 90, and you must include petsc.h90 in your code.
5228 
5229   Level: intermediate
5230 
5231 .seealso DMPlexVecGetClosure(), DMPlexMatSetClosure()
5232 @*/
5233 PetscErrorCode DMPlexVecSetClosure(DM dm, PetscSection section, Vec v, PetscInt point, const PetscScalar values[], InsertMode mode)
5234 {
5235   PetscSection    clSection;
5236   IS              clPoints;
5237   PetscScalar    *array;
5238   PetscInt       *points = NULL;
5239   const PetscInt *clp;
5240   PetscInt        depth, numFields, numPoints, p;
5241   PetscErrorCode  ierr;
5242 
5243   PetscFunctionBeginHot;
5244   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5245   if (!section) {ierr = DMGetDefaultSection(dm, &section);CHKERRQ(ierr);}
5246   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
5247   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
5248   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
5249   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5250   if (depth == 1 && numFields < 2 && mode == ADD_VALUES) {
5251     ierr = DMPlexVecSetClosure_Static(dm, section, v, point, values, mode);CHKERRQ(ierr);
5252     PetscFunctionReturn(0);
5253   }
5254   /* Get points */
5255   ierr = PetscSectionGetClosureIndex(section, (PetscObject) dm, &clSection, &clPoints);CHKERRQ(ierr);
5256   if (!clPoints) {
5257     PetscInt pStart, pEnd, q;
5258 
5259     ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
5260     ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
5261     /* Compress out points not in the section */
5262     for (p = 0, q = 0; p < numPoints*2; p += 2) {
5263       if ((points[p] >= pStart) && (points[p] < pEnd)) {
5264         points[q*2]   = points[p];
5265         points[q*2+1] = points[p+1];
5266         ++q;
5267       }
5268     }
5269     numPoints = q;
5270   } else {
5271     PetscInt dof, off;
5272 
5273     ierr = PetscSectionGetDof(clSection, point, &dof);CHKERRQ(ierr);
5274     ierr = PetscSectionGetOffset(clSection, point, &off);CHKERRQ(ierr);
5275     ierr = ISGetIndices(clPoints, &clp);CHKERRQ(ierr);
5276     numPoints = dof/2;
5277     points    = (PetscInt *) &clp[off];
5278   }
5279   /* Get array */
5280   ierr = VecGetArray(v, &array);CHKERRQ(ierr);
5281   /* Get values */
5282   if (numFields > 0) {
5283     PetscInt offset = 0, fcomp, f;
5284     for (f = 0; f < numFields; ++f) {
5285       ierr = PetscSectionGetFieldComponents(section, f, &fcomp);CHKERRQ(ierr);
5286       switch (mode) {
5287       case INSERT_VALUES:
5288         for (p = 0; p < numPoints*2; p += 2) {
5289           const PetscInt point = points[p];
5290           const PetscInt o     = points[p+1];
5291           updatePointFields_private(section, point, o, f, fcomp, insert, PETSC_FALSE, values, &offset, array);
5292         } break;
5293       case INSERT_ALL_VALUES:
5294         for (p = 0; p < numPoints*2; p += 2) {
5295           const PetscInt point = points[p];
5296           const PetscInt o     = points[p+1];
5297           updatePointFields_private(section, point, o, f, fcomp, insert, PETSC_TRUE, values, &offset, array);
5298         } break;
5299       case INSERT_BC_VALUES:
5300         for (p = 0; p < numPoints*2; p += 2) {
5301           const PetscInt point = points[p];
5302           const PetscInt o     = points[p+1];
5303           updatePointFieldsBC_private(section, point, o, f, fcomp, insert, values, &offset, array);
5304         } break;
5305       case ADD_VALUES:
5306         for (p = 0; p < numPoints*2; p += 2) {
5307           const PetscInt point = points[p];
5308           const PetscInt o     = points[p+1];
5309           updatePointFields_private(section, point, o, f, fcomp, add, PETSC_FALSE, values, &offset, array);
5310         } break;
5311       case ADD_ALL_VALUES:
5312         for (p = 0; p < numPoints*2; p += 2) {
5313           const PetscInt point = points[p];
5314           const PetscInt o     = points[p+1];
5315           updatePointFields_private(section, point, o, f, fcomp, add, PETSC_TRUE, values, &offset, array);
5316         } break;
5317       default:
5318         SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %D", mode);
5319       }
5320     }
5321   } else {
5322     PetscInt dof, off;
5323 
5324     switch (mode) {
5325     case INSERT_VALUES:
5326       for (p = 0, off = 0; p < numPoints*2; p += 2, off += dof) {
5327         PetscInt o = points[p+1];
5328         ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5329         updatePoint_private(section, points[p], dof, insert, PETSC_FALSE, o, &values[off], array);
5330       } break;
5331     case INSERT_ALL_VALUES:
5332       for (p = 0, off = 0; p < numPoints*2; p += 2, off += dof) {
5333         PetscInt o = points[p+1];
5334         ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5335         updatePoint_private(section, points[p], dof, insert, PETSC_TRUE,  o, &values[off], array);
5336       } break;
5337     case INSERT_BC_VALUES:
5338       for (p = 0, off = 0; p < numPoints*2; p += 2, off += dof) {
5339         PetscInt o = points[p+1];
5340         ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5341         updatePointBC_private(section, points[p], dof, insert,  o, &values[off], array);
5342       } break;
5343     case ADD_VALUES:
5344       for (p = 0, off = 0; p < numPoints*2; p += 2, off += dof) {
5345         PetscInt o = points[p+1];
5346         ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5347         updatePoint_private(section, points[p], dof, add,    PETSC_FALSE, o, &values[off], array);
5348       } break;
5349     case ADD_ALL_VALUES:
5350       for (p = 0, off = 0; p < numPoints*2; p += 2, off += dof) {
5351         PetscInt o = points[p+1];
5352         ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5353         updatePoint_private(section, points[p], dof, add,    PETSC_TRUE,  o, &values[off], array);
5354       } break;
5355     default:
5356       SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %D", mode);
5357     }
5358   }
5359   /* Cleanup points */
5360   if (!clPoints) {ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);}
5361   else           {ierr = ISRestoreIndices(clPoints, &clp);CHKERRQ(ierr);}
5362   /* Cleanup array */
5363   ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
5364   PetscFunctionReturn(0);
5365 }
5366 
5367 #undef __FUNCT__
5368 #define __FUNCT__ "DMPlexPrintMatSetValues"
5369 PetscErrorCode DMPlexPrintMatSetValues(PetscViewer viewer, Mat A, PetscInt point, PetscInt numRIndices, const PetscInt rindices[], PetscInt numCIndices, const PetscInt cindices[], const PetscScalar values[])
5370 {
5371   PetscMPIInt    rank;
5372   PetscInt       i, j;
5373   PetscErrorCode ierr;
5374 
5375   PetscFunctionBegin;
5376   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRQ(ierr);
5377   ierr = PetscViewerASCIIPrintf(viewer, "[%D]mat for sieve point %D\n", rank, point);CHKERRQ(ierr);
5378   for (i = 0; i < numRIndices; i++) {ierr = PetscViewerASCIIPrintf(viewer, "[%D]mat row indices[%D] = %D\n", rank, i, rindices[i]);CHKERRQ(ierr);}
5379   for (i = 0; i < numCIndices; i++) {ierr = PetscViewerASCIIPrintf(viewer, "[%D]mat col indices[%D] = %D\n", rank, i, cindices[i]);CHKERRQ(ierr);}
5380   numCIndices = numCIndices ? numCIndices : numRIndices;
5381   for (i = 0; i < numRIndices; i++) {
5382     ierr = PetscViewerASCIIPrintf(viewer, "[%D]", rank);CHKERRQ(ierr);
5383     for (j = 0; j < numCIndices; j++) {
5384 #if defined(PETSC_USE_COMPLEX)
5385       ierr = PetscViewerASCIIPrintf(viewer, " (%g,%g)", (double)PetscRealPart(values[i*numCIndices+j]), (double)PetscImaginaryPart(values[i*numCIndices+j]));CHKERRQ(ierr);
5386 #else
5387       ierr = PetscViewerASCIIPrintf(viewer, " %g", (double)values[i*numCIndices+j]);CHKERRQ(ierr);
5388 #endif
5389     }
5390     ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
5391   }
5392   PetscFunctionReturn(0);
5393 }
5394 
5395 #undef __FUNCT__
5396 #define __FUNCT__ "indicesPoint_private"
5397 /* . off - The global offset of this point */
5398 PetscErrorCode indicesPoint_private(PetscSection section, PetscInt point, PetscInt off, PetscInt *loff, PetscBool setBC, PetscInt orientation, PetscInt indices[])
5399 {
5400   PetscInt        dof;    /* The number of unknowns on this point */
5401   PetscInt        cdof;   /* The number of constraints on this point */
5402   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
5403   PetscInt        cind = 0, k;
5404   PetscErrorCode  ierr;
5405 
5406   PetscFunctionBegin;
5407   ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
5408   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
5409   if (!cdof || setBC) {
5410     if (orientation >= 0) {
5411       for (k = 0; k < dof; ++k) indices[*loff+k] = off+k;
5412     } else {
5413       for (k = 0; k < dof; ++k) indices[*loff+dof-k-1] = off+k;
5414     }
5415   } else {
5416     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
5417     if (orientation >= 0) {
5418       for (k = 0; k < dof; ++k) {
5419         if ((cind < cdof) && (k == cdofs[cind])) {
5420           /* Insert check for returning constrained indices */
5421           indices[*loff+k] = -(off+k+1);
5422           ++cind;
5423         } else {
5424           indices[*loff+k] = off+k-cind;
5425         }
5426       }
5427     } else {
5428       for (k = 0; k < dof; ++k) {
5429         if ((cind < cdof) && (k == cdofs[cind])) {
5430           /* Insert check for returning constrained indices */
5431           indices[*loff+dof-k-1] = -(off+k+1);
5432           ++cind;
5433         } else {
5434           indices[*loff+dof-k-1] = off+k-cind;
5435         }
5436       }
5437     }
5438   }
5439   *loff += dof;
5440   PetscFunctionReturn(0);
5441 }
5442 
5443 #undef __FUNCT__
5444 #define __FUNCT__ "indicesPointFields_private"
5445 /* . off - The global offset of this point */
5446 PetscErrorCode indicesPointFields_private(PetscSection section, PetscInt point, PetscInt off, PetscInt foffs[], PetscBool setBC, PetscInt orientation, PetscInt indices[])
5447 {
5448   PetscInt       numFields, foff, f;
5449   PetscErrorCode ierr;
5450 
5451   PetscFunctionBegin;
5452   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5453   for (f = 0, foff = 0; f < numFields; ++f) {
5454     PetscInt        fdof, fcomp, cfdof;
5455     const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
5456     PetscInt        cind = 0, k, c;
5457 
5458     ierr = PetscSectionGetFieldComponents(section, f, &fcomp);CHKERRQ(ierr);
5459     ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
5460     ierr = PetscSectionGetFieldConstraintDof(section, point, f, &cfdof);CHKERRQ(ierr);
5461     if (!cfdof || setBC) {
5462       if (orientation >= 0) {
5463         for (k = 0; k < fdof; ++k) indices[foffs[f]+k] = off+foff+k;
5464       } else {
5465         for (k = fdof/fcomp-1; k >= 0; --k) {
5466           for (c = 0; c < fcomp; ++c) {
5467             indices[foffs[f]+k*fcomp+c] = off+foff+(fdof/fcomp-1-k)*fcomp+c;
5468           }
5469         }
5470       }
5471     } else {
5472       ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
5473       if (orientation >= 0) {
5474         for (k = 0; k < fdof; ++k) {
5475           if ((cind < cfdof) && (k == fcdofs[cind])) {
5476             indices[foffs[f]+k] = -(off+foff+k+1);
5477             ++cind;
5478           } else {
5479             indices[foffs[f]+k] = off+foff+k-cind;
5480           }
5481         }
5482       } else {
5483         for (k = fdof/fcomp-1; k >= 0; --k) {
5484           for (c = 0; c < fcomp; ++c) {
5485             if ((cind < cfdof) && ((fdof/fcomp-1-k)*fcomp+c == fcdofs[cind])) {
5486               indices[foffs[f]+k*fcomp+c] = -(off+foff+(fdof/fcomp-1-k)*fcomp+c+1);
5487               ++cind;
5488             } else {
5489               indices[foffs[f]+k*fcomp+c] = off+foff+(fdof/fcomp-1-k)*fcomp+c-cind;
5490             }
5491           }
5492         }
5493       }
5494     }
5495     foff     += fdof - cfdof;
5496     foffs[f] += fdof;
5497   }
5498   PetscFunctionReturn(0);
5499 }
5500 
5501 #undef __FUNCT__
5502 #define __FUNCT__ "DMPlexMatSetClosure"
5503 /*@C
5504   DMPlexMatSetClosure - Set an array of the values on the closure of 'point'
5505 
5506   Not collective
5507 
5508   Input Parameters:
5509 + dm - The DM
5510 . section - The section describing the layout in v, or NULL to use the default section
5511 . globalSection - The section describing the layout in v, or NULL to use the default global section
5512 . A - The matrix
5513 . point - The sieve point in the DM
5514 . values - The array of values
5515 - mode - The insert mode, where INSERT_ALL_VALUES and ADD_ALL_VALUES also overwrite boundary conditions
5516 
5517   Fortran Notes:
5518   This routine is only available in Fortran 90, and you must include petsc.h90 in your code.
5519 
5520   Level: intermediate
5521 
5522 .seealso DMPlexVecGetClosure(), DMPlexVecSetClosure()
5523 @*/
5524 PetscErrorCode DMPlexMatSetClosure(DM dm, PetscSection section, PetscSection globalSection, Mat A, PetscInt point, const PetscScalar values[], InsertMode mode)
5525 {
5526   DM_Plex        *mesh   = (DM_Plex*) dm->data;
5527   PetscSection    clSection;
5528   IS              clPoints;
5529   PetscInt       *points = NULL;
5530   const PetscInt *clp;
5531   PetscInt       *indices;
5532   PetscInt        offsets[32];
5533   PetscInt        numFields, numPoints, numIndices, dof, off, globalOff, pStart, pEnd, p, q, f;
5534   PetscErrorCode  ierr;
5535 
5536   PetscFunctionBegin;
5537   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5538   if (!section) {ierr = DMGetDefaultSection(dm, &section);CHKERRQ(ierr);}
5539   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
5540   if (!globalSection) {ierr = DMGetDefaultGlobalSection(dm, &globalSection);CHKERRQ(ierr);}
5541   PetscValidHeaderSpecific(globalSection, PETSC_SECTION_CLASSID, 3);
5542   PetscValidHeaderSpecific(A, MAT_CLASSID, 4);
5543   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5544   if (numFields > 31) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", numFields);
5545   ierr = PetscMemzero(offsets, 32 * sizeof(PetscInt));CHKERRQ(ierr);
5546   ierr = PetscSectionGetClosureIndex(section, (PetscObject) dm, &clSection, &clPoints);CHKERRQ(ierr);
5547   if (!clPoints) {
5548     ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
5549     /* Compress out points not in the section */
5550     ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
5551     for (p = 0, q = 0; p < numPoints*2; p += 2) {
5552       if ((points[p] >= pStart) && (points[p] < pEnd)) {
5553         points[q*2]   = points[p];
5554         points[q*2+1] = points[p+1];
5555         ++q;
5556       }
5557     }
5558     numPoints = q;
5559   } else {
5560     PetscInt dof, off;
5561 
5562     ierr = PetscSectionGetDof(clSection, point, &dof);CHKERRQ(ierr);
5563     numPoints = dof/2;
5564     ierr = PetscSectionGetOffset(clSection, point, &off);CHKERRQ(ierr);
5565     ierr = ISGetIndices(clPoints, &clp);CHKERRQ(ierr);
5566     points = (PetscInt *) &clp[off];
5567   }
5568   for (p = 0, numIndices = 0; p < numPoints*2; p += 2) {
5569     PetscInt fdof;
5570 
5571     ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5572     for (f = 0; f < numFields; ++f) {
5573       ierr          = PetscSectionGetFieldDof(section, points[p], f, &fdof);CHKERRQ(ierr);
5574       offsets[f+1] += fdof;
5575     }
5576     numIndices += dof;
5577   }
5578   for (f = 1; f < numFields; ++f) offsets[f+1] += offsets[f];
5579 
5580   if (numFields && offsets[numFields] != numIndices) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid size for closure %d should be %d", offsets[numFields], numIndices);
5581   ierr = DMGetWorkArray(dm, numIndices, PETSC_INT, &indices);CHKERRQ(ierr);
5582   if (numFields) {
5583     for (p = 0; p < numPoints*2; p += 2) {
5584       PetscInt o = points[p+1];
5585       ierr = PetscSectionGetOffset(globalSection, points[p], &globalOff);CHKERRQ(ierr);
5586       indicesPointFields_private(section, points[p], globalOff < 0 ? -(globalOff+1) : globalOff, offsets, PETSC_FALSE, o, indices);
5587     }
5588   } else {
5589     for (p = 0, off = 0; p < numPoints*2; p += 2) {
5590       PetscInt o = points[p+1];
5591       ierr = PetscSectionGetOffset(globalSection, points[p], &globalOff);CHKERRQ(ierr);
5592       indicesPoint_private(section, points[p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, o, indices);
5593     }
5594   }
5595   if (mesh->printSetValues) {ierr = DMPlexPrintMatSetValues(PETSC_VIEWER_STDOUT_SELF, A, point, numIndices, indices, 0, NULL, values);CHKERRQ(ierr);}
5596   ierr = MatSetValues(A, numIndices, indices, numIndices, indices, values, mode);
5597   if (ierr) {
5598     PetscMPIInt    rank;
5599     PetscErrorCode ierr2;
5600 
5601     ierr2 = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRQ(ierr2);
5602     ierr2 = (*PetscErrorPrintf)("[%D]ERROR in DMPlexMatSetClosure\n", rank);CHKERRQ(ierr2);
5603     ierr2 = DMPlexPrintMatSetValues(PETSC_VIEWER_STDERR_SELF, A, point, numIndices, indices, 0, NULL, values);CHKERRQ(ierr2);
5604     ierr2 = DMRestoreWorkArray(dm, numIndices, PETSC_INT, &indices);CHKERRQ(ierr2);
5605     CHKERRQ(ierr);
5606   }
5607   if (!clPoints) {
5608     ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
5609   } else {
5610     ierr = ISRestoreIndices(clPoints, &clp);CHKERRQ(ierr);
5611   }
5612   ierr = DMRestoreWorkArray(dm, numIndices, PETSC_INT, &indices);CHKERRQ(ierr);
5613   PetscFunctionReturn(0);
5614 }
5615 
5616 #undef __FUNCT__
5617 #define __FUNCT__ "DMPlexMatSetClosureRefined"
5618 PetscErrorCode DMPlexMatSetClosureRefined(DM dmf, PetscSection fsection, PetscSection globalFSection, DM dmc, PetscSection csection, PetscSection globalCSection, Mat A, PetscInt point, const PetscScalar values[], InsertMode mode)
5619 {
5620   DM_Plex        *mesh   = (DM_Plex*) dmf->data;
5621   PetscInt       *fpoints = NULL, *ftotpoints = NULL;
5622   PetscInt       *cpoints = NULL;
5623   PetscInt       *findices, *cindices;
5624   PetscInt        foffsets[32], coffsets[32];
5625   CellRefiner     cellRefiner;
5626   PetscInt        numFields, numSubcells, maxFPoints, numFPoints, numCPoints, numFIndices, numCIndices, dof, off, globalOff, pStart, pEnd, p, q, r, s, f;
5627   PetscErrorCode  ierr;
5628 
5629   PetscFunctionBegin;
5630   PetscValidHeaderSpecific(dmf, DM_CLASSID, 1);
5631   PetscValidHeaderSpecific(dmc, DM_CLASSID, 4);
5632   if (!fsection) {ierr = DMGetDefaultSection(dmf, &fsection);CHKERRQ(ierr);}
5633   PetscValidHeaderSpecific(fsection, PETSC_SECTION_CLASSID, 2);
5634   if (!csection) {ierr = DMGetDefaultSection(dmc, &csection);CHKERRQ(ierr);}
5635   PetscValidHeaderSpecific(csection, PETSC_SECTION_CLASSID, 5);
5636   if (!globalFSection) {ierr = DMGetDefaultGlobalSection(dmf, &globalFSection);CHKERRQ(ierr);}
5637   PetscValidHeaderSpecific(globalFSection, PETSC_SECTION_CLASSID, 3);
5638   if (!globalCSection) {ierr = DMGetDefaultGlobalSection(dmc, &globalCSection);CHKERRQ(ierr);}
5639   PetscValidHeaderSpecific(globalCSection, PETSC_SECTION_CLASSID, 6);
5640   PetscValidHeaderSpecific(A, MAT_CLASSID, 7);
5641   ierr = PetscSectionGetNumFields(fsection, &numFields);CHKERRQ(ierr);
5642   if (numFields > 31) SETERRQ1(PetscObjectComm((PetscObject)dmf), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", numFields);
5643   ierr = PetscMemzero(foffsets, 32 * sizeof(PetscInt));CHKERRQ(ierr);
5644   ierr = PetscMemzero(coffsets, 32 * sizeof(PetscInt));CHKERRQ(ierr);
5645   /* Column indices */
5646   ierr = DMPlexGetTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
5647   maxFPoints = numCPoints;
5648   /* Compress out points not in the section */
5649   /*   TODO: Squeeze out points with 0 dof as well */
5650   ierr = PetscSectionGetChart(csection, &pStart, &pEnd);CHKERRQ(ierr);
5651   for (p = 0, q = 0; p < numCPoints*2; p += 2) {
5652     if ((cpoints[p] >= pStart) && (cpoints[p] < pEnd)) {
5653       cpoints[q*2]   = cpoints[p];
5654       cpoints[q*2+1] = cpoints[p+1];
5655       ++q;
5656     }
5657   }
5658   numCPoints = q;
5659   for (p = 0, numCIndices = 0; p < numCPoints*2; p += 2) {
5660     PetscInt fdof;
5661 
5662     ierr = PetscSectionGetDof(csection, cpoints[p], &dof);CHKERRQ(ierr);
5663     if (!dof) continue;
5664     for (f = 0; f < numFields; ++f) {
5665       ierr           = PetscSectionGetFieldDof(csection, cpoints[p], f, &fdof);CHKERRQ(ierr);
5666       coffsets[f+1] += fdof;
5667     }
5668     numCIndices += dof;
5669   }
5670   for (f = 1; f < numFields; ++f) coffsets[f+1] += coffsets[f];
5671   /* Row indices */
5672   ierr = DMPlexGetCellRefiner_Internal(dmc, &cellRefiner);CHKERRQ(ierr);
5673   ierr = CellRefinerGetAffineTransforms_Internal(cellRefiner, &numSubcells, NULL, NULL, NULL);CHKERRQ(ierr);
5674   ierr = DMGetWorkArray(dmf, maxFPoints*2*numSubcells, PETSC_INT, &ftotpoints);CHKERRQ(ierr);
5675   for (r = 0, q = 0; r < numSubcells; ++r) {
5676     /* TODO Map from coarse to fine cells */
5677     ierr = DMPlexGetTransitiveClosure(dmf, point*numSubcells + r, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
5678     /* Compress out points not in the section */
5679     ierr = PetscSectionGetChart(fsection, &pStart, &pEnd);CHKERRQ(ierr);
5680     for (p = 0; p < numFPoints*2; p += 2) {
5681       if ((fpoints[p] >= pStart) && (fpoints[p] < pEnd)) {
5682         ierr = PetscSectionGetDof(fsection, fpoints[p], &dof);CHKERRQ(ierr);
5683         if (!dof) continue;
5684         for (s = 0; s < q; ++s) if (fpoints[p] == ftotpoints[s*2]) break;
5685         if (s < q) continue;
5686         ftotpoints[q*2]   = fpoints[p];
5687         ftotpoints[q*2+1] = fpoints[p+1];
5688         ++q;
5689       }
5690     }
5691     ierr = DMPlexRestoreTransitiveClosure(dmf, point, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
5692   }
5693   numFPoints = q;
5694   for (p = 0, numFIndices = 0; p < numFPoints*2; p += 2) {
5695     PetscInt fdof;
5696 
5697     ierr = PetscSectionGetDof(fsection, ftotpoints[p], &dof);CHKERRQ(ierr);
5698     if (!dof) continue;
5699     for (f = 0; f < numFields; ++f) {
5700       ierr           = PetscSectionGetFieldDof(fsection, ftotpoints[p], f, &fdof);CHKERRQ(ierr);
5701       foffsets[f+1] += fdof;
5702     }
5703     numFIndices += dof;
5704   }
5705   for (f = 1; f < numFields; ++f) foffsets[f+1] += foffsets[f];
5706 
5707   if (numFields && foffsets[numFields] != numFIndices) SETERRQ2(PetscObjectComm((PetscObject)dmf), PETSC_ERR_PLIB, "Invalid size for closure %d should be %d", foffsets[numFields], numFIndices);
5708   if (numFields && coffsets[numFields] != numCIndices) SETERRQ2(PetscObjectComm((PetscObject)dmc), PETSC_ERR_PLIB, "Invalid size for closure %d should be %d", coffsets[numFields], numCIndices);
5709   ierr = DMGetWorkArray(dmf, numFIndices, PETSC_INT, &findices);CHKERRQ(ierr);
5710   ierr = DMGetWorkArray(dmc, numCIndices, PETSC_INT, &cindices);CHKERRQ(ierr);
5711   if (numFields) {
5712     for (p = 0; p < numFPoints*2; p += 2) {
5713       PetscInt o = ftotpoints[p+1];
5714       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[p], &globalOff);CHKERRQ(ierr);
5715       indicesPointFields_private(fsection, ftotpoints[p], globalOff < 0 ? -(globalOff+1) : globalOff, foffsets, PETSC_FALSE, o, findices);
5716     }
5717     for (p = 0; p < numCPoints*2; p += 2) {
5718       PetscInt o = cpoints[p+1];
5719       ierr = PetscSectionGetOffset(globalCSection, cpoints[p], &globalOff);CHKERRQ(ierr);
5720       indicesPointFields_private(csection, cpoints[p], globalOff < 0 ? -(globalOff+1) : globalOff, coffsets, PETSC_FALSE, o, cindices);
5721     }
5722   } else {
5723     for (p = 0, off = 0; p < numFPoints*2; p += 2) {
5724       PetscInt o = ftotpoints[p+1];
5725       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[p], &globalOff);CHKERRQ(ierr);
5726       indicesPoint_private(fsection, ftotpoints[p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, o, findices);
5727     }
5728     for (p = 0, off = 0; p < numCPoints*2; p += 2) {
5729       PetscInt o = cpoints[p+1];
5730       ierr = PetscSectionGetOffset(globalCSection, cpoints[p], &globalOff);CHKERRQ(ierr);
5731       indicesPoint_private(csection, cpoints[p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, o, cindices);
5732     }
5733   }
5734   if (mesh->printSetValues) {ierr = DMPlexPrintMatSetValues(PETSC_VIEWER_STDOUT_SELF, A, point, numFIndices, findices, numCIndices, cindices, values);CHKERRQ(ierr);}
5735   ierr = MatSetValues(A, numFIndices, findices, numCIndices, cindices, values, mode);
5736   if (ierr) {
5737     PetscMPIInt    rank;
5738     PetscErrorCode ierr2;
5739 
5740     ierr2 = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRQ(ierr2);
5741     ierr2 = (*PetscErrorPrintf)("[%D]ERROR in DMPlexMatSetClosure\n", rank);CHKERRQ(ierr2);
5742     ierr2 = DMPlexPrintMatSetValues(PETSC_VIEWER_STDERR_SELF, A, point, numFIndices, findices, numCIndices, cindices, values);CHKERRQ(ierr2);
5743     ierr2 = DMRestoreWorkArray(dmf, numFIndices, PETSC_INT, &findices);CHKERRQ(ierr2);
5744     ierr2 = DMRestoreWorkArray(dmc, numCIndices, PETSC_INT, &cindices);CHKERRQ(ierr2);
5745     CHKERRQ(ierr);
5746   }
5747   ierr = DMRestoreWorkArray(dmf, numCPoints*2*4, PETSC_INT, &ftotpoints);CHKERRQ(ierr);
5748   ierr = DMPlexRestoreTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
5749   ierr = DMRestoreWorkArray(dmf, numFIndices, PETSC_INT, &findices);CHKERRQ(ierr);
5750   ierr = DMRestoreWorkArray(dmc, numCIndices, PETSC_INT, &cindices);CHKERRQ(ierr);
5751   PetscFunctionReturn(0);
5752 }
5753 
5754 #undef __FUNCT__
5755 #define __FUNCT__ "DMPlexGetHybridBounds"
5756 /*@
5757   DMPlexGetHybridBounds - Get the first mesh point of each dimension which is a hybrid
5758 
5759   Input Parameter:
5760 . dm - The DMPlex object
5761 
5762   Output Parameters:
5763 + cMax - The first hybrid cell
5764 . cMax - The first hybrid face
5765 . cMax - The first hybrid edge
5766 - cMax - The first hybrid vertex
5767 
5768   Level: developer
5769 
5770 .seealso DMPlexCreateHybridMesh()
5771 @*/
5772 PetscErrorCode DMPlexGetHybridBounds(DM dm, PetscInt *cMax, PetscInt *fMax, PetscInt *eMax, PetscInt *vMax)
5773 {
5774   DM_Plex       *mesh = (DM_Plex*) dm->data;
5775   PetscInt       dim;
5776   PetscErrorCode ierr;
5777 
5778   PetscFunctionBegin;
5779   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5780   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
5781   if (cMax) *cMax = mesh->hybridPointMax[dim];
5782   if (fMax) *fMax = mesh->hybridPointMax[dim-1];
5783   if (eMax) *eMax = mesh->hybridPointMax[1];
5784   if (vMax) *vMax = mesh->hybridPointMax[0];
5785   PetscFunctionReturn(0);
5786 }
5787 
5788 #undef __FUNCT__
5789 #define __FUNCT__ "DMPlexSetHybridBounds"
5790 PetscErrorCode DMPlexSetHybridBounds(DM dm, PetscInt cMax, PetscInt fMax, PetscInt eMax, PetscInt vMax)
5791 {
5792   DM_Plex       *mesh = (DM_Plex*) dm->data;
5793   PetscInt       dim;
5794   PetscErrorCode ierr;
5795 
5796   PetscFunctionBegin;
5797   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5798   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
5799   if (cMax >= 0) mesh->hybridPointMax[dim]   = cMax;
5800   if (fMax >= 0) mesh->hybridPointMax[dim-1] = fMax;
5801   if (eMax >= 0) mesh->hybridPointMax[1]     = eMax;
5802   if (vMax >= 0) mesh->hybridPointMax[0]     = vMax;
5803   PetscFunctionReturn(0);
5804 }
5805 
5806 #undef __FUNCT__
5807 #define __FUNCT__ "DMPlexGetVTKCellHeight"
5808 PetscErrorCode DMPlexGetVTKCellHeight(DM dm, PetscInt *cellHeight)
5809 {
5810   DM_Plex *mesh = (DM_Plex*) dm->data;
5811 
5812   PetscFunctionBegin;
5813   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5814   PetscValidPointer(cellHeight, 2);
5815   *cellHeight = mesh->vtkCellHeight;
5816   PetscFunctionReturn(0);
5817 }
5818 
5819 #undef __FUNCT__
5820 #define __FUNCT__ "DMPlexSetVTKCellHeight"
5821 PetscErrorCode DMPlexSetVTKCellHeight(DM dm, PetscInt cellHeight)
5822 {
5823   DM_Plex *mesh = (DM_Plex*) dm->data;
5824 
5825   PetscFunctionBegin;
5826   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5827   mesh->vtkCellHeight = cellHeight;
5828   PetscFunctionReturn(0);
5829 }
5830 
5831 #undef __FUNCT__
5832 #define __FUNCT__ "DMPlexCreateNumbering_Private"
5833 /* We can easily have a form that takes an IS instead */
5834 PetscErrorCode DMPlexCreateNumbering_Private(DM dm, PetscInt pStart, PetscInt pEnd, PetscSF sf, IS *numbering)
5835 {
5836   PetscSection   section, globalSection;
5837   PetscInt      *numbers, p;
5838   PetscErrorCode ierr;
5839 
5840   PetscFunctionBegin;
5841   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &section);CHKERRQ(ierr);
5842   ierr = PetscSectionSetChart(section, pStart, pEnd);CHKERRQ(ierr);
5843   for (p = pStart; p < pEnd; ++p) {
5844     ierr = PetscSectionSetDof(section, p, 1);CHKERRQ(ierr);
5845   }
5846   ierr = PetscSectionSetUp(section);CHKERRQ(ierr);
5847   ierr = PetscSectionCreateGlobalSection(section, sf, PETSC_FALSE, &globalSection);CHKERRQ(ierr);
5848   ierr = PetscMalloc1((pEnd - pStart), &numbers);CHKERRQ(ierr);
5849   for (p = pStart; p < pEnd; ++p) {
5850     ierr = PetscSectionGetOffset(globalSection, p, &numbers[p-pStart]);CHKERRQ(ierr);
5851   }
5852   ierr = ISCreateGeneral(PetscObjectComm((PetscObject)dm), pEnd - pStart, numbers, PETSC_OWN_POINTER, numbering);CHKERRQ(ierr);
5853   ierr = PetscSectionDestroy(&section);CHKERRQ(ierr);
5854   ierr = PetscSectionDestroy(&globalSection);CHKERRQ(ierr);
5855   PetscFunctionReturn(0);
5856 }
5857 
5858 #undef __FUNCT__
5859 #define __FUNCT__ "DMPlexGetCellNumbering"
5860 PetscErrorCode DMPlexGetCellNumbering(DM dm, IS *globalCellNumbers)
5861 {
5862   DM_Plex       *mesh = (DM_Plex*) dm->data;
5863   PetscInt       cellHeight, cStart, cEnd, cMax;
5864   PetscErrorCode ierr;
5865 
5866   PetscFunctionBegin;
5867   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5868   if (!mesh->globalCellNumbers) {
5869     ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
5870     ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
5871     ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
5872     if (cMax >= 0) cEnd = PetscMin(cEnd, cMax);
5873     ierr = DMPlexCreateNumbering_Private(dm, cStart, cEnd, dm->sf, &mesh->globalCellNumbers);CHKERRQ(ierr);
5874   }
5875   *globalCellNumbers = mesh->globalCellNumbers;
5876   PetscFunctionReturn(0);
5877 }
5878 
5879 #undef __FUNCT__
5880 #define __FUNCT__ "DMPlexGetVertexNumbering"
5881 PetscErrorCode DMPlexGetVertexNumbering(DM dm, IS *globalVertexNumbers)
5882 {
5883   DM_Plex       *mesh = (DM_Plex*) dm->data;
5884   PetscInt       vStart, vEnd, vMax;
5885   PetscErrorCode ierr;
5886 
5887   PetscFunctionBegin;
5888   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5889   if (!mesh->globalVertexNumbers) {
5890     ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
5891     ierr = DMPlexGetHybridBounds(dm, NULL, NULL, NULL, &vMax);CHKERRQ(ierr);
5892     if (vMax >= 0) vEnd = PetscMin(vEnd, vMax);
5893     ierr = DMPlexCreateNumbering_Private(dm, vStart, vEnd, dm->sf, &mesh->globalVertexNumbers);CHKERRQ(ierr);
5894   }
5895   *globalVertexNumbers = mesh->globalVertexNumbers;
5896   PetscFunctionReturn(0);
5897 }
5898 
5899 
5900 #undef __FUNCT__
5901 #define __FUNCT__ "PetscSectionCreateGlobalSectionLabel"
5902 /*@C
5903   PetscSectionCreateGlobalSectionLabel - Create a section describing the global field layout using
5904   the local section and an SF describing the section point overlap.
5905 
5906   Input Parameters:
5907   + s - The PetscSection for the local field layout
5908   . sf - The SF describing parallel layout of the section points
5909   . includeConstraints - By default this is PETSC_FALSE, meaning that the global field vector will not possess constrained dofs
5910   . label - The label specifying the points
5911   - labelValue - The label stratum specifying the points
5912 
5913   Output Parameter:
5914   . gsection - The PetscSection for the global field layout
5915 
5916   Note: This gives negative sizes and offsets to points not owned by this process
5917 
5918   Level: developer
5919 
5920 .seealso: PetscSectionCreate()
5921 @*/
5922 PetscErrorCode PetscSectionCreateGlobalSectionLabel(PetscSection s, PetscSF sf, PetscBool includeConstraints, DMLabel label, PetscInt labelValue, PetscSection *gsection)
5923 {
5924   PetscInt      *neg = NULL, *tmpOff = NULL;
5925   PetscInt       pStart, pEnd, p, dof, cdof, off, globalOff = 0, nroots;
5926   PetscErrorCode ierr;
5927 
5928   PetscFunctionBegin;
5929   ierr = PetscSectionCreate(s->atlasLayout.comm, gsection);CHKERRQ(ierr);
5930   ierr = PetscSectionGetChart(s, &pStart, &pEnd);CHKERRQ(ierr);
5931   ierr = PetscSectionSetChart(*gsection, pStart, pEnd);CHKERRQ(ierr);
5932   ierr = PetscSFGetGraph(sf, &nroots, NULL, NULL, NULL);CHKERRQ(ierr);
5933   if (nroots >= 0) {
5934     if (nroots < pEnd-pStart) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "PetscSF nroots %d < %d section size", nroots, pEnd-pStart);
5935     ierr = PetscCalloc1(nroots, &neg);CHKERRQ(ierr);
5936     if (nroots > pEnd-pStart) {
5937       ierr = PetscCalloc1(nroots, &tmpOff);CHKERRQ(ierr);
5938     } else {
5939       tmpOff = &(*gsection)->atlasDof[-pStart];
5940     }
5941   }
5942   /* Mark ghost points with negative dof */
5943   for (p = pStart; p < pEnd; ++p) {
5944     PetscInt value;
5945 
5946     ierr = DMLabelGetValue(label, p, &value);CHKERRQ(ierr);
5947     if (value != labelValue) continue;
5948     ierr = PetscSectionGetDof(s, p, &dof);CHKERRQ(ierr);
5949     ierr = PetscSectionSetDof(*gsection, p, dof);CHKERRQ(ierr);
5950     ierr = PetscSectionGetConstraintDof(s, p, &cdof);CHKERRQ(ierr);
5951     if (!includeConstraints && cdof > 0) {ierr = PetscSectionSetConstraintDof(*gsection, p, cdof);CHKERRQ(ierr);}
5952     if (neg) neg[p] = -(dof+1);
5953   }
5954   ierr = PetscSectionSetUpBC(*gsection);CHKERRQ(ierr);
5955   if (nroots >= 0) {
5956     ierr = PetscSFBcastBegin(sf, MPIU_INT, neg, tmpOff);CHKERRQ(ierr);
5957     ierr = PetscSFBcastEnd(sf, MPIU_INT, neg, tmpOff);CHKERRQ(ierr);
5958     if (nroots > pEnd-pStart) {
5959       for (p = pStart; p < pEnd; ++p) {if (tmpOff[p] < 0) (*gsection)->atlasDof[p-pStart] = tmpOff[p];}
5960     }
5961   }
5962   /* Calculate new sizes, get proccess offset, and calculate point offsets */
5963   for (p = 0, off = 0; p < pEnd-pStart; ++p) {
5964     cdof = (!includeConstraints && s->bc) ? s->bc->atlasDof[p] : 0;
5965     (*gsection)->atlasOff[p] = off;
5966     off += (*gsection)->atlasDof[p] > 0 ? (*gsection)->atlasDof[p]-cdof : 0;
5967   }
5968   ierr       = MPI_Scan(&off, &globalOff, 1, MPIU_INT, MPI_SUM, s->atlasLayout.comm);CHKERRQ(ierr);
5969   globalOff -= off;
5970   for (p = 0, off = 0; p < pEnd-pStart; ++p) {
5971     (*gsection)->atlasOff[p] += globalOff;
5972     if (neg) neg[p] = -((*gsection)->atlasOff[p]+1);
5973   }
5974   /* Put in negative offsets for ghost points */
5975   if (nroots >= 0) {
5976     ierr = PetscSFBcastBegin(sf, MPIU_INT, neg, tmpOff);CHKERRQ(ierr);
5977     ierr = PetscSFBcastEnd(sf, MPIU_INT, neg, tmpOff);CHKERRQ(ierr);
5978     if (nroots > pEnd-pStart) {
5979       for (p = pStart; p < pEnd; ++p) {if (tmpOff[p] < 0) (*gsection)->atlasOff[p-pStart] = tmpOff[p];}
5980     }
5981   }
5982   if (nroots >= 0 && nroots > pEnd-pStart) {ierr = PetscFree(tmpOff);CHKERRQ(ierr);}
5983   ierr = PetscFree(neg);CHKERRQ(ierr);
5984   PetscFunctionReturn(0);
5985 }
5986 
5987 #undef __FUNCT__
5988 #define __FUNCT__ "DMPlexCheckSymmetry"
5989 /*@
5990   DMPlexCheckSymmetry - Check that the adjacency information in the mesh is symmetric.
5991 
5992   Input Parameters:
5993   + dm - The DMPlex object
5994 
5995   Note: This is a useful diagnostic when creating meshes programmatically.
5996 
5997   Level: developer
5998 
5999 .seealso: DMCreate(), DMCheckSkeleton(), DMCheckFaces()
6000 @*/
6001 PetscErrorCode DMPlexCheckSymmetry(DM dm)
6002 {
6003   PetscSection    coneSection, supportSection;
6004   const PetscInt *cone, *support;
6005   PetscInt        coneSize, c, supportSize, s;
6006   PetscInt        pStart, pEnd, p, csize, ssize;
6007   PetscErrorCode  ierr;
6008 
6009   PetscFunctionBegin;
6010   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6011   ierr = DMPlexGetConeSection(dm, &coneSection);CHKERRQ(ierr);
6012   ierr = DMPlexGetSupportSection(dm, &supportSection);CHKERRQ(ierr);
6013   /* Check that point p is found in the support of its cone points, and vice versa */
6014   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
6015   for (p = pStart; p < pEnd; ++p) {
6016     ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
6017     ierr = DMPlexGetCone(dm, p, &cone);CHKERRQ(ierr);
6018     for (c = 0; c < coneSize; ++c) {
6019       PetscBool dup = PETSC_FALSE;
6020       PetscInt  d;
6021       for (d = c-1; d >= 0; --d) {
6022         if (cone[c] == cone[d]) {dup = PETSC_TRUE; break;}
6023       }
6024       ierr = DMPlexGetSupportSize(dm, cone[c], &supportSize);CHKERRQ(ierr);
6025       ierr = DMPlexGetSupport(dm, cone[c], &support);CHKERRQ(ierr);
6026       for (s = 0; s < supportSize; ++s) {
6027         if (support[s] == p) break;
6028       }
6029       if ((s >= supportSize) || (dup && (support[s+1] != p))) {
6030         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %d cone: ", p);
6031         for (s = 0; s < coneSize; ++s) {
6032           ierr = PetscPrintf(PETSC_COMM_SELF, "%d, ", cone[s]);
6033         }
6034         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");
6035         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %d support: ", cone[c]);
6036         for (s = 0; s < supportSize; ++s) {
6037           ierr = PetscPrintf(PETSC_COMM_SELF, "%d, ", support[s]);
6038         }
6039         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");
6040         if (dup) {
6041           SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %d not repeatedly found in support of repeated cone point %d", p, cone[c]);
6042         } else {
6043           SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %d not found in support of cone point %d", p, cone[c]);
6044         }
6045       }
6046     }
6047     ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
6048     ierr = DMPlexGetSupport(dm, p, &support);CHKERRQ(ierr);
6049     for (s = 0; s < supportSize; ++s) {
6050       ierr = DMPlexGetConeSize(dm, support[s], &coneSize);CHKERRQ(ierr);
6051       ierr = DMPlexGetCone(dm, support[s], &cone);CHKERRQ(ierr);
6052       for (c = 0; c < coneSize; ++c) {
6053         if (cone[c] == p) break;
6054       }
6055       if (c >= coneSize) {
6056         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %d support: ", p);
6057         for (c = 0; c < supportSize; ++c) {
6058           ierr = PetscPrintf(PETSC_COMM_SELF, "%d, ", support[c]);
6059         }
6060         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");
6061         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %d cone: ", support[s]);
6062         for (c = 0; c < coneSize; ++c) {
6063           ierr = PetscPrintf(PETSC_COMM_SELF, "%d, ", cone[c]);
6064         }
6065         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");
6066         SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %d not found in cone of support point %d", p, support[s]);
6067       }
6068     }
6069   }
6070   ierr = PetscSectionGetStorageSize(coneSection, &csize);CHKERRQ(ierr);
6071   ierr = PetscSectionGetStorageSize(supportSection, &ssize);CHKERRQ(ierr);
6072   if (csize != ssize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "Total cone size %d != Total support size %d", csize, ssize);
6073   PetscFunctionReturn(0);
6074 }
6075 
6076 #undef __FUNCT__
6077 #define __FUNCT__ "DMPlexCheckSkeleton"
6078 /*@
6079   DMPlexCheckSkeleton - Check that each cell has the correct number of vertices
6080 
6081   Input Parameters:
6082 + dm - The DMPlex object
6083 . isSimplex - Are the cells simplices or tensor products
6084 - cellHeight - Normally 0
6085 
6086   Note: This is a useful diagnostic when creating meshes programmatically.
6087 
6088   Level: developer
6089 
6090 .seealso: DMCreate(), DMCheckSymmetry(), DMCheckFaces()
6091 @*/
6092 PetscErrorCode DMPlexCheckSkeleton(DM dm, PetscBool isSimplex, PetscInt cellHeight)
6093 {
6094   PetscInt       dim, numCorners, numHybridCorners, vStart, vEnd, cStart, cEnd, cMax, c;
6095   PetscErrorCode ierr;
6096 
6097   PetscFunctionBegin;
6098   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6099   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
6100   switch (dim) {
6101   case 1: numCorners = isSimplex ? 2 : 2; numHybridCorners = isSimplex ? 2 : 2; break;
6102   case 2: numCorners = isSimplex ? 3 : 4; numHybridCorners = isSimplex ? 4 : 4; break;
6103   case 3: numCorners = isSimplex ? 4 : 8; numHybridCorners = isSimplex ? 6 : 8; break;
6104   default:
6105     SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_OUTOFRANGE, "Cannot handle meshes of dimension %d", dim);
6106   }
6107   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
6108   ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
6109   ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
6110   cMax = cMax >= 0 ? cMax : cEnd;
6111   for (c = cStart; c < cMax; ++c) {
6112     PetscInt *closure = NULL, closureSize, cl, coneSize = 0;
6113 
6114     ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6115     for (cl = 0; cl < closureSize*2; cl += 2) {
6116       const PetscInt p = closure[cl];
6117       if ((p >= vStart) && (p < vEnd)) ++coneSize;
6118     }
6119     ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6120     if (coneSize != numCorners) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %d has  %d vertices != %d", c, coneSize, numCorners);
6121   }
6122   for (c = cMax; c < cEnd; ++c) {
6123     PetscInt *closure = NULL, closureSize, cl, coneSize = 0;
6124 
6125     ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6126     for (cl = 0; cl < closureSize*2; cl += 2) {
6127       const PetscInt p = closure[cl];
6128       if ((p >= vStart) && (p < vEnd)) ++coneSize;
6129     }
6130     ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6131     if (coneSize > numHybridCorners) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Hybrid cell %d has  %d vertices > %d", c, coneSize, numHybridCorners);
6132   }
6133   PetscFunctionReturn(0);
6134 }
6135 
6136 #undef __FUNCT__
6137 #define __FUNCT__ "DMPlexCheckFaces"
6138 /*@
6139   DMPlexCheckFaces - Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type
6140 
6141   Input Parameters:
6142 + dm - The DMPlex object
6143 . isSimplex - Are the cells simplices or tensor products
6144 - cellHeight - Normally 0
6145 
6146   Note: This is a useful diagnostic when creating meshes programmatically.
6147 
6148   Level: developer
6149 
6150 .seealso: DMCreate(), DMCheckSymmetry(), DMCheckSkeleton()
6151 @*/
6152 PetscErrorCode DMPlexCheckFaces(DM dm, PetscBool isSimplex, PetscInt cellHeight)
6153 {
6154   PetscInt       pMax[4];
6155   PetscInt       dim, vStart, vEnd, cStart, cEnd, c, h;
6156   PetscErrorCode ierr;
6157 
6158   PetscFunctionBegin;
6159   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6160   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
6161   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
6162   ierr = DMPlexGetHybridBounds(dm, &pMax[dim], &pMax[dim-1], &pMax[1], &pMax[0]);CHKERRQ(ierr);
6163   for (h = cellHeight; h < dim; ++h) {
6164     ierr = DMPlexGetHeightStratum(dm, h, &cStart, &cEnd);CHKERRQ(ierr);
6165     for (c = cStart; c < cEnd; ++c) {
6166       const PetscInt *cone, *ornt, *faces;
6167       PetscInt        numFaces, faceSize, coneSize,f;
6168       PetscInt       *closure = NULL, closureSize, cl, numCorners = 0;
6169 
6170       if (pMax[dim-h] >= 0 && c >= pMax[dim-h]) continue;
6171       ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
6172       ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
6173       ierr = DMPlexGetConeOrientation(dm, c, &ornt);CHKERRQ(ierr);
6174       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6175       for (cl = 0; cl < closureSize*2; cl += 2) {
6176         const PetscInt p = closure[cl];
6177         if ((p >= vStart) && (p < vEnd)) closure[numCorners++] = p;
6178       }
6179       ierr = DMPlexGetRawFaces_Internal(dm, dim-h, numCorners, closure, &numFaces, &faceSize, &faces);CHKERRQ(ierr);
6180       if (coneSize != numFaces) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %d has %d faces but should have %d", c, coneSize, numFaces);
6181       for (f = 0; f < numFaces; ++f) {
6182         PetscInt *fclosure = NULL, fclosureSize, cl, fnumCorners = 0, v;
6183 
6184         ierr = DMPlexGetTransitiveClosure_Internal(dm, cone[f], ornt[f], PETSC_TRUE, &fclosureSize, &fclosure);CHKERRQ(ierr);
6185         for (cl = 0; cl < fclosureSize*2; cl += 2) {
6186           const PetscInt p = fclosure[cl];
6187           if ((p >= vStart) && (p < vEnd)) fclosure[fnumCorners++] = p;
6188         }
6189         if (fnumCorners != faceSize) SETERRQ5(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %d (%d) of cell %d has %d vertices but should have %d", cone[f], f, c, fnumCorners, faceSize);
6190         for (v = 0; v < fnumCorners; ++v) {
6191           if (fclosure[v] != faces[f*faceSize+v]) SETERRQ6(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %d (%d) of cell %d vertex %d, %d != %d", cone[f], f, c, v, fclosure[v], faces[f*faceSize+v]);
6192         }
6193         ierr = DMPlexRestoreTransitiveClosure(dm, cone[f], PETSC_TRUE, &fclosureSize, &fclosure);CHKERRQ(ierr);
6194       }
6195       ierr = DMPlexRestoreFaces_Internal(dm, dim, c, &numFaces, &faceSize, &faces);CHKERRQ(ierr);
6196       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6197     }
6198   }
6199   PetscFunctionReturn(0);
6200 }
6201 
6202 #undef __FUNCT__
6203 #define __FUNCT__ "DMCreateInterpolation_Plex"
6204 /* Pointwise interpolation
6205      Just code FEM for now
6206      u^f = I u^c
6207      sum_k u^f_k phi^f_k = I sum_j u^c_j phi^c_j
6208      u^f_i = sum_j psi^f_i I phi^c_j u^c_j
6209      I_{ij} = psi^f_i phi^c_j
6210 */
6211 PetscErrorCode DMCreateInterpolation_Plex(DM dmCoarse, DM dmFine, Mat *interpolation, Vec *scaling)
6212 {
6213   PetscSection   gsc, gsf;
6214   PetscInt       m, n;
6215   void          *ctx;
6216   PetscErrorCode ierr;
6217 
6218   PetscFunctionBegin;
6219   /*
6220   Loop over coarse cells
6221     Loop over coarse basis functions
6222       Loop over fine cells in coarse cell
6223         Loop over fine dual basis functions
6224           Evaluate coarse basis on fine dual basis quad points
6225           Sum
6226           Update local element matrix
6227     Accumulate to interpolation matrix
6228 
6229    Can extend PetscFEIntegrateJacobian_Basic() to do a specialized cell loop
6230   */
6231   ierr = DMGetDefaultGlobalSection(dmFine, &gsf);CHKERRQ(ierr);
6232   ierr = PetscSectionGetConstrainedStorageSize(gsf, &m);CHKERRQ(ierr);
6233   ierr = DMGetDefaultGlobalSection(dmCoarse, &gsc);CHKERRQ(ierr);
6234   ierr = PetscSectionGetConstrainedStorageSize(gsc, &n);CHKERRQ(ierr);
6235   /* We need to preallocate properly */
6236   ierr = MatCreate(PetscObjectComm((PetscObject) dmCoarse), interpolation);CHKERRQ(ierr);
6237   ierr = MatSetSizes(*interpolation, m, n, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
6238   ierr = MatSetType(*interpolation, dmCoarse->mattype);CHKERRQ(ierr);
6239   ierr = MatSetUp(*interpolation);CHKERRQ(ierr);
6240   ierr = MatSetFromOptions(*interpolation);CHKERRQ(ierr);
6241   ierr = MatSetOption(*interpolation, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_FALSE);CHKERRQ(ierr);
6242   ierr = DMGetApplicationContext(dmFine, &ctx);CHKERRQ(ierr);
6243   ierr = DMPlexComputeInterpolatorFEM(dmCoarse, dmFine, *interpolation, ctx);CHKERRQ(ierr);
6244   /* Use naive scaling */
6245   ierr = DMCreateInterpolationScale(dmCoarse, dmFine, *interpolation, scaling);CHKERRQ(ierr);
6246   PetscFunctionReturn(0);
6247 }
6248 
6249 #undef __FUNCT__
6250 #define __FUNCT__ "DMCreateInjection_Plex"
6251 PetscErrorCode DMCreateInjection_Plex(DM dmCoarse, DM dmFine, VecScatter *ctx)
6252 {
6253   Vec             cv,  fv;
6254   IS              cis, fis, fpointIS;
6255   PetscSection    sc, gsc, gsf;
6256   const PetscInt *fpoints;
6257   PetscInt       *cindices, *findices;
6258   PetscInt        cpStart, cpEnd, m, off, cp;
6259   PetscErrorCode  ierr;
6260 
6261   PetscFunctionBegin;
6262   ierr = DMGetDefaultGlobalSection(dmFine, &gsf);CHKERRQ(ierr);
6263   ierr = DMGetGlobalVector(dmFine, &fv);CHKERRQ(ierr);
6264   ierr = DMGetDefaultSection(dmCoarse, &sc);CHKERRQ(ierr);
6265   ierr = DMGetDefaultGlobalSection(dmCoarse, &gsc);CHKERRQ(ierr);
6266   ierr = DMGetGlobalVector(dmCoarse, &cv);CHKERRQ(ierr);
6267   ierr = DMPlexCreateCoarsePointIS(dmCoarse, &fpointIS);CHKERRQ(ierr);
6268   ierr = PetscSectionGetConstrainedStorageSize(gsc, &m);CHKERRQ(ierr);
6269   ierr = PetscMalloc2(m,&cindices,m,&findices);CHKERRQ(ierr);
6270   ierr = PetscSectionGetChart(gsc, &cpStart, &cpEnd);CHKERRQ(ierr);
6271   ierr = ISGetIndices(fpointIS, &fpoints);CHKERRQ(ierr);
6272   for (cp = cpStart, off = 0; cp < cpEnd; ++cp) {
6273     const PetscInt *cdofsC = NULL;
6274     PetscInt        fp     = fpoints[cp-cpStart], dofC, cdofC, dofF, offC, offF, d, e;
6275 
6276     ierr = PetscSectionGetDof(gsc, cp, &dofC);CHKERRQ(ierr);
6277     if (dofC <= 0) continue;
6278     ierr = PetscSectionGetConstraintDof(sc, cp, &cdofC);CHKERRQ(ierr);
6279     ierr = PetscSectionGetDof(gsf, fp, &dofF);CHKERRQ(ierr);
6280     ierr = PetscSectionGetOffset(gsc, cp, &offC);CHKERRQ(ierr);
6281     ierr = PetscSectionGetOffset(gsf, fp, &offF);CHKERRQ(ierr);
6282     if (cdofC) {ierr = PetscSectionGetConstraintIndices(sc, cp, &cdofsC);CHKERRQ(ierr);}
6283     if (dofC != dofF) SETERRQ4(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Point %d (%d) has %d coarse dofs != %d fine dofs", cp, fp, dofC, dofF);
6284     if (offC < 0 || offF < 0) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Coarse point %d has invalid offset %d (%d)", cp, offC, offF);
6285     for (d = 0, e = 0; d < dofC; ++d) {
6286       if (cdofsC && cdofsC[e] == d) {++e; continue;}
6287       cindices[off+d-e] = offC+d; findices[off+d-e] = offF+d;
6288     }
6289     if (e != cdofC) SETERRQ4(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Point %d (%d) has invalid number of constraints %d != %d", cp, fp, e, cdofC);
6290     off += dofC-cdofC;
6291   }
6292   ierr = ISRestoreIndices(fpointIS, &fpoints);CHKERRQ(ierr);
6293   if (off != m) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of coarse dofs %d != %d", off, m);
6294   ierr = ISCreateGeneral(PETSC_COMM_SELF, m, cindices, PETSC_OWN_POINTER, &cis);CHKERRQ(ierr);
6295   ierr = ISCreateGeneral(PETSC_COMM_SELF, m, findices, PETSC_OWN_POINTER, &fis);CHKERRQ(ierr);
6296   ierr = VecScatterCreate(cv, cis, fv, fis, ctx);CHKERRQ(ierr);
6297   ierr = ISDestroy(&cis);CHKERRQ(ierr);
6298   ierr = ISDestroy(&fis);CHKERRQ(ierr);
6299   ierr = DMRestoreGlobalVector(dmFine, &fv);CHKERRQ(ierr);
6300   ierr = DMRestoreGlobalVector(dmCoarse, &cv);CHKERRQ(ierr);
6301   ierr = ISDestroy(&fpointIS);CHKERRQ(ierr);
6302   PetscFunctionReturn(0);
6303 }
6304 
6305 #undef __FUNCT__
6306 #define __FUNCT__ "DMCreateDefaultSection_Plex"
6307 /* Pointwise interpolation
6308      Just code FEM for now
6309      u^f = I u^c
6310      sum_k u^f_k phi^f_k = I sum_l u^c_l phi^c_l
6311      u^f_i = sum_l int psi^f_i I phi^c_l u^c_l
6312      I_{ij} = int psi^f_i phi^c_j
6313 */
6314 PetscErrorCode DMCreateDefaultSection_Plex(DM dm)
6315 {
6316   PetscSection   section;
6317   IS            *bcPoints;
6318   PetscInt      *bcFields, *numComp, *numDof;
6319   PetscInt       depth, dim, numBd, numBC = 0, numFields, bd, bc, f;
6320   PetscErrorCode ierr;
6321 
6322   PetscFunctionBegin;
6323   /* Handle boundary conditions */
6324   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
6325   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
6326   ierr = DMPlexGetNumBoundary(dm, &numBd);CHKERRQ(ierr);
6327   for (bd = 0; bd < numBd; ++bd) {
6328     PetscBool isEssential;
6329     ierr = DMPlexGetBoundary(dm, bd, &isEssential, NULL, NULL, NULL, NULL, NULL, NULL);CHKERRQ(ierr);
6330     if (isEssential) ++numBC;
6331   }
6332   ierr = PetscMalloc2(numBC,&bcFields,numBC,&bcPoints);CHKERRQ(ierr);
6333   for (bd = 0, bc = 0; bd < numBd; ++bd) {
6334     const char     *bdLabel;
6335     DMLabel         label;
6336     const PetscInt *values;
6337     PetscInt        field, numValues;
6338     PetscBool       isEssential, has;
6339 
6340     ierr = DMPlexGetBoundary(dm, bd, &isEssential, &bdLabel, &field, NULL, &numValues, &values, NULL);CHKERRQ(ierr);
6341     if (numValues != 1) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Bug me and I will fix this");
6342     ierr = DMPlexHasLabel(dm, bdLabel, &has);CHKERRQ(ierr);
6343     if (!has) {
6344       ierr = DMPlexCreateLabel(dm, bdLabel);CHKERRQ(ierr);
6345       ierr = DMPlexGetLabel(dm, bdLabel, &label);CHKERRQ(ierr);
6346       ierr = DMPlexMarkBoundaryFaces(dm, label);CHKERRQ(ierr);
6347     }
6348     ierr = DMPlexGetLabel(dm, bdLabel, &label);CHKERRQ(ierr);
6349     ierr = DMPlexLabelComplete(dm, label);CHKERRQ(ierr);
6350     if (isEssential) {
6351       bcFields[bc] = field;
6352       ierr = DMPlexGetStratumIS(dm, bdLabel, values[0], &bcPoints[bc++]);CHKERRQ(ierr);
6353     }
6354   }
6355   /* Handle discretization */
6356   ierr = DMGetNumFields(dm, &numFields);CHKERRQ(ierr);
6357   ierr = PetscMalloc2(numFields,&numComp,numFields*(dim+1),&numDof);CHKERRQ(ierr);
6358   for (f = 0; f < numFields; ++f) {
6359     PetscFE         fe;
6360     const PetscInt *numFieldDof;
6361     PetscInt        d;
6362 
6363     ierr = DMGetField(dm, f, (PetscObject *) &fe);CHKERRQ(ierr);
6364     ierr = PetscFEGetNumComponents(fe, &numComp[f]);CHKERRQ(ierr);
6365     ierr = PetscFEGetNumDof(fe, &numFieldDof);CHKERRQ(ierr);
6366     for (d = 0; d < dim+1; ++d) numDof[f*(dim+1)+d] = numFieldDof[d];
6367   }
6368   for (f = 0; f < numFields; ++f) {
6369     PetscInt d;
6370     for (d = 1; d < dim; ++d) {
6371       if ((numDof[f*(dim+1)+d] > 0) && (depth < dim)) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Mesh must be interpolated when unknowns are specified on edges or faces.");
6372     }
6373   }
6374   ierr = DMPlexCreateSection(dm, dim, numFields, numComp, numDof, numBC, bcFields, bcPoints, &section);CHKERRQ(ierr);
6375   for (f = 0; f < numFields; ++f) {
6376     PetscFE     fe;
6377     const char *name;
6378 
6379     ierr = DMGetField(dm, f, (PetscObject *) &fe);CHKERRQ(ierr);
6380     ierr = PetscObjectGetName((PetscObject) fe, &name);CHKERRQ(ierr);
6381     ierr = PetscSectionSetFieldName(section, f, name);CHKERRQ(ierr);
6382   }
6383   ierr = DMSetDefaultSection(dm, section);CHKERRQ(ierr);
6384   ierr = PetscSectionDestroy(&section);CHKERRQ(ierr);
6385   for (bc = 0; bc < numBC; ++bc) {ierr = ISDestroy(&bcPoints[bc]);CHKERRQ(ierr);}
6386   ierr = PetscFree2(bcFields,bcPoints);CHKERRQ(ierr);
6387   ierr = PetscFree2(numComp,numDof);CHKERRQ(ierr);
6388   PetscFunctionReturn(0);
6389 }
6390 
6391 #undef __FUNCT__
6392 #define __FUNCT__ "DMPlexGetCoarseDM"
6393 /*@
6394   DMPlexGetCoarseDM - Get the coarse mesh from which this was obtained by refinement
6395 
6396   Input Parameter:
6397 . dm - The DMPlex object
6398 
6399   Output Parameter:
6400 . cdm - The coarse DM
6401 
6402   Level: intermediate
6403 
6404 .seealso: DMPlexSetCoarseDM()
6405 @*/
6406 PetscErrorCode DMPlexGetCoarseDM(DM dm, DM *cdm)
6407 {
6408   PetscFunctionBegin;
6409   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6410   PetscValidPointer(cdm, 2);
6411   *cdm = ((DM_Plex *) dm->data)->coarseMesh;
6412   PetscFunctionReturn(0);
6413 }
6414 
6415 #undef __FUNCT__
6416 #define __FUNCT__ "DMPlexSetCoarseDM"
6417 /*@
6418   DMPlexSetCoarseDM - Set the coarse mesh from which this was obtained by refinement
6419 
6420   Input Parameters:
6421 + dm - The DMPlex object
6422 - cdm - The coarse DM
6423 
6424   Level: intermediate
6425 
6426 .seealso: DMPlexGetCoarseDM()
6427 @*/
6428 PetscErrorCode DMPlexSetCoarseDM(DM dm, DM cdm)
6429 {
6430   DM_Plex       *mesh;
6431   PetscErrorCode ierr;
6432 
6433   PetscFunctionBegin;
6434   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6435   if (cdm) PetscValidHeaderSpecific(cdm, DM_CLASSID, 2);
6436   mesh = (DM_Plex *) dm->data;
6437   ierr = DMDestroy(&mesh->coarseMesh);CHKERRQ(ierr);
6438   mesh->coarseMesh = cdm;
6439   ierr = PetscObjectReference((PetscObject) mesh->coarseMesh);CHKERRQ(ierr);
6440   PetscFunctionReturn(0);
6441 }
6442