xref: /petsc/src/dm/impls/plex/plex.c (revision ec007e15ec5b946b9877d15ae12e42ed876a5dbd)
1 #include <petsc-private/dmpleximpl.h>   /*I      "petscdmplex.h"   I*/
2 #include <../src/sys/utils/hash.h>
3 #include <petsc-private/isimpl.h>
4 #include <petscsf.h>
5 #include <petscviewerhdf5.h>
6 
7 /* Logging support */
8 PetscLogEvent DMPLEX_Interpolate, DMPLEX_Partition, DMPLEX_Distribute, DMPLEX_DistributeCones, DMPLEX_DistributeLabels, DMPLEX_DistributeSF, DMPLEX_DistributeField, DMPLEX_DistributeData, DMPLEX_Stratify, DMPLEX_Preallocate, DMPLEX_ResidualFEM, DMPLEX_JacobianFEM;
9 
10 PETSC_EXTERN PetscErrorCode VecView_Seq(Vec, PetscViewer);
11 PETSC_EXTERN PetscErrorCode VecView_MPI(Vec, PetscViewer);
12 PETSC_EXTERN PetscErrorCode VecLoad_Default(Vec, PetscViewer);
13 PETSC_EXTERN PetscErrorCode DMTSGetTimeStepNumber(DM,PetscInt*);
14 
15 #undef __FUNCT__
16 #define __FUNCT__ "GetFieldType_Static"
17 static PetscErrorCode GetFieldType_Static(DM dm, PetscSection section, PetscInt field, PetscInt *sStart, PetscInt *sEnd, PetscViewerVTKFieldType *ft)
18 {
19   PetscInt       dim, pStart, pEnd, vStart, vEnd, cStart, cEnd, vdof = 0, cdof = 0;
20   PetscErrorCode ierr;
21 
22   PetscFunctionBegin;
23   *ft  = PETSC_VTK_POINT_FIELD;
24   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
25   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
26   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
27   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
28   if (field >= 0) {
29     if ((vStart >= pStart) && (vStart < pEnd)) {ierr = PetscSectionGetFieldDof(section, vStart, field, &vdof);CHKERRQ(ierr);}
30     if ((cStart >= pStart) && (cStart < pEnd)) {ierr = PetscSectionGetFieldDof(section, cStart, field, &cdof);CHKERRQ(ierr);}
31   } else {
32     if ((vStart >= pStart) && (vStart < pEnd)) {ierr = PetscSectionGetDof(section, vStart, &vdof);CHKERRQ(ierr);}
33     if ((cStart >= pStart) && (cStart < pEnd)) {ierr = PetscSectionGetDof(section, cStart, &cdof);CHKERRQ(ierr);}
34   }
35   if (vdof) {
36     *sStart = vStart;
37     *sEnd   = vEnd;
38     if (vdof == dim) *ft = PETSC_VTK_POINT_VECTOR_FIELD;
39     else             *ft = PETSC_VTK_POINT_FIELD;
40   } else if (cdof) {
41     *sStart = cStart;
42     *sEnd   = cEnd;
43     if (cdof == dim) *ft = PETSC_VTK_CELL_VECTOR_FIELD;
44     else             *ft = PETSC_VTK_CELL_FIELD;
45   } else SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Could not classify input Vec for VTK");
46   PetscFunctionReturn(0);
47 }
48 
49 #if defined(PETSC_HAVE_HDF5)
50 #undef __FUNCT__
51 #define __FUNCT__ "GetField_Static"
52 static PetscErrorCode GetField_Static(DM dm, PetscSection section, PetscSection sectionGlobal, Vec v, PetscInt field, PetscInt pStart, PetscInt pEnd, IS *is, Vec *subv)
53 {
54   PetscInt      *subIndices;
55   PetscInt       Nc, subSize = 0, subOff = 0, p;
56   PetscErrorCode ierr;
57 
58   PetscFunctionBegin;
59   ierr = PetscSectionGetFieldComponents(section, field, &Nc);CHKERRQ(ierr);
60   for (p = pStart; p < pEnd; ++p) {
61     PetscInt gdof, fdof = 0;
62 
63     ierr = PetscSectionGetDof(sectionGlobal, p, &gdof);CHKERRQ(ierr);
64     if (gdof > 0) {ierr = PetscSectionGetFieldDof(section, p, field, &fdof);CHKERRQ(ierr);}
65     subSize += fdof;
66   }
67   ierr = PetscMalloc1(subSize, &subIndices);CHKERRQ(ierr);
68   for (p = pStart; p < pEnd; ++p) {
69     PetscInt gdof, goff;
70 
71     ierr = PetscSectionGetDof(sectionGlobal, p, &gdof);CHKERRQ(ierr);
72     if (gdof > 0) {
73       PetscInt fdof, fc, f2, poff = 0;
74 
75       ierr = PetscSectionGetOffset(sectionGlobal, p, &goff);CHKERRQ(ierr);
76       /* Can get rid of this loop by storing field information in the global section */
77       for (f2 = 0; f2 < field; ++f2) {
78         ierr  = PetscSectionGetFieldDof(section, p, f2, &fdof);CHKERRQ(ierr);
79         poff += fdof;
80       }
81       ierr = PetscSectionGetFieldDof(section, p, field, &fdof);CHKERRQ(ierr);
82       for (fc = 0; fc < fdof; ++fc, ++subOff) subIndices[subOff] = goff+poff+fc;
83     }
84   }
85   ierr = ISCreateGeneral(PetscObjectComm((PetscObject) dm), subSize, subIndices, PETSC_OWN_POINTER, is);CHKERRQ(ierr);
86   ierr = VecGetSubVector(v, *is, subv);CHKERRQ(ierr);
87   ierr = VecSetBlockSize(*subv, Nc);CHKERRQ(ierr);
88   PetscFunctionReturn(0);
89 }
90 
91 #undef __FUNCT__
92 #define __FUNCT__ "RestoreField_Static"
93 static PetscErrorCode RestoreField_Static(DM dm, PetscSection section, PetscSection sectionGlobal, Vec v, PetscInt field, PetscInt pStart, PetscInt pEnd, IS *is, Vec *subv)
94 {
95   PetscErrorCode ierr;
96 
97   PetscFunctionBegin;
98   ierr = VecRestoreSubVector(v, *is, subv);CHKERRQ(ierr);
99   ierr = ISDestroy(is);CHKERRQ(ierr);
100   PetscFunctionReturn(0);
101 }
102 
103 #undef __FUNCT__
104 #define __FUNCT__ "DMSequenceView_HDF5"
105 static PetscErrorCode DMSequenceView_HDF5(DM dm, const char *seqname, PetscInt seqnum, PetscScalar value, PetscViewer viewer)
106 {
107   Vec            stamp;
108   PetscMPIInt    rank;
109   PetscErrorCode ierr;
110 
111   PetscFunctionBegin;
112   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) viewer), &rank);CHKERRQ(ierr);
113   ierr = VecCreateMPI(PetscObjectComm((PetscObject) viewer), rank ? 0 : 1, 1, &stamp);CHKERRQ(ierr);
114   ierr = VecSetBlockSize(stamp, 1);CHKERRQ(ierr);
115   ierr = PetscObjectSetName((PetscObject) stamp, seqname);CHKERRQ(ierr);
116   if (!rank) {
117     PetscReal timeScale;
118     PetscBool istime;
119 
120     ierr = PetscStrncmp(seqname, "time", 5, &istime);CHKERRQ(ierr);
121     if (istime) {ierr = DMPlexGetScale(dm, PETSC_UNIT_TIME, &timeScale);CHKERRQ(ierr); value *= timeScale;}
122     ierr = VecSetValue(stamp, 0, value, INSERT_VALUES);CHKERRQ(ierr);
123   }
124   ierr = VecAssemblyBegin(stamp);CHKERRQ(ierr);
125   ierr = VecAssemblyEnd(stamp);CHKERRQ(ierr);
126   ierr = PetscViewerHDF5PushGroup(viewer, "/");CHKERRQ(ierr);
127   ierr = PetscViewerHDF5SetTimestep(viewer, seqnum);CHKERRQ(ierr);
128   ierr = VecView(stamp, viewer);CHKERRQ(ierr);
129   ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
130   ierr = VecDestroy(&stamp);CHKERRQ(ierr);
131   PetscFunctionReturn(0);
132 }
133 
134 #undef __FUNCT__
135 #define __FUNCT__ "VecView_Plex_Local_HDF5"
136 static PetscErrorCode VecView_Plex_Local_HDF5(Vec v, PetscViewer viewer)
137 {
138   DM                      dm;
139   DM                      dmBC;
140   PetscSection            section, sectionGlobal;
141   Vec                     gv;
142   const char             *name;
143   PetscViewerVTKFieldType ft;
144   PetscViewerFormat       format;
145   PetscInt                seqnum;
146   PetscBool               isseq;
147   PetscErrorCode          ierr;
148 
149   PetscFunctionBegin;
150   ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
151   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
152   ierr = DMGetDefaultSection(dm, &section);CHKERRQ(ierr);
153   ierr = DMGetOutputSequenceNumber(dm, &seqnum);CHKERRQ(ierr);
154   ierr = PetscViewerHDF5SetTimestep(viewer, seqnum);CHKERRQ(ierr);
155   ierr = DMSequenceView_HDF5(dm, "time", seqnum, (PetscScalar) seqnum, viewer);CHKERRQ(ierr);
156   ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
157   ierr = DMGetOutputDM(dm, &dmBC);CHKERRQ(ierr);
158   ierr = DMGetDefaultGlobalSection(dmBC, &sectionGlobal);CHKERRQ(ierr);
159   ierr = DMGetGlobalVector(dmBC, &gv);CHKERRQ(ierr);
160   ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
161   ierr = PetscObjectSetName((PetscObject) gv, name);CHKERRQ(ierr);
162   ierr = DMLocalToGlobalBegin(dmBC, v, INSERT_VALUES, gv);CHKERRQ(ierr);
163   ierr = DMLocalToGlobalEnd(dmBC, v, INSERT_VALUES, gv);CHKERRQ(ierr);
164   ierr = PetscObjectTypeCompare((PetscObject) gv, VECSEQ, &isseq);CHKERRQ(ierr);
165   if (isseq) {ierr = VecView_Seq(gv, viewer);CHKERRQ(ierr);}
166   else       {ierr = VecView_MPI(gv, viewer);CHKERRQ(ierr);}
167   if (format == PETSC_VIEWER_HDF5_VIZ) {
168     /* Output visualization representation */
169     PetscInt numFields, f;
170 
171     ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
172     for (f = 0; f < numFields; ++f) {
173       Vec         subv;
174       IS          is;
175       const char *fname, *fgroup;
176       char        group[PETSC_MAX_PATH_LEN];
177       PetscInt    pStart, pEnd;
178       PetscBool   flag;
179 
180       ierr = GetFieldType_Static(dm, section, f, &pStart, &pEnd, &ft);CHKERRQ(ierr);
181       fgroup = (ft == PETSC_VTK_POINT_VECTOR_FIELD) || (ft == PETSC_VTK_POINT_FIELD) ? "/vertex_fields" : "/cell_fields";
182       ierr = PetscSectionGetFieldName(section, f, &fname);CHKERRQ(ierr);
183       ierr = PetscViewerHDF5PushGroup(viewer, fgroup);CHKERRQ(ierr);
184       ierr = GetField_Static(dmBC, section, sectionGlobal, gv, f, pStart, pEnd, &is, &subv);CHKERRQ(ierr);
185       ierr = PetscObjectSetName((PetscObject) subv, fname);CHKERRQ(ierr);
186       if (isseq) {ierr = VecView_Seq(subv, viewer);CHKERRQ(ierr);}
187       else       {ierr = VecView_MPI(subv, viewer);CHKERRQ(ierr);}
188       ierr = RestoreField_Static(dmBC, section, sectionGlobal, gv, f, pStart, pEnd, &is, &subv);CHKERRQ(ierr);
189       ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
190       ierr = PetscSNPrintf(group, PETSC_MAX_PATH_LEN, "%s/%s", fgroup, fname);CHKERRQ(ierr);
191       ierr = PetscViewerHDF5HasAttribute(viewer, group, "vector_field_type", &flag);CHKERRQ(ierr);
192       if (!flag) {
193         if ((ft == PETSC_VTK_POINT_VECTOR_FIELD) || (ft == PETSC_VTK_CELL_VECTOR_FIELD)) {
194           ierr = PetscViewerHDF5WriteAttribute(viewer, group, "vector_field_type", PETSC_STRING, "vector");CHKERRQ(ierr);
195         } else {
196           ierr = PetscViewerHDF5WriteAttribute(viewer, group, "vector_field_type", PETSC_STRING, "scalar");CHKERRQ(ierr);
197         }
198       }
199     }
200   }
201   ierr = DMRestoreGlobalVector(dmBC, &gv);CHKERRQ(ierr);
202   PetscFunctionReturn(0);
203 }
204 #endif
205 
206 #undef __FUNCT__
207 #define __FUNCT__ "VecView_Plex_Local"
208 PetscErrorCode VecView_Plex_Local(Vec v, PetscViewer viewer)
209 {
210   DM             dm;
211   PetscBool      isvtk, ishdf5, isseq;
212   PetscErrorCode ierr;
213 
214   PetscFunctionBegin;
215   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
216   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
217   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,  &isvtk);CHKERRQ(ierr);
218   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
219   ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
220   if (isvtk || ishdf5) {ierr = DMPlexInsertBoundaryValuesFEM(dm, v);CHKERRQ(ierr);}
221   if (isvtk) {
222     PetscSection            section;
223     PetscViewerVTKFieldType ft;
224     PetscInt                pStart, pEnd;
225 
226     ierr = DMGetDefaultSection(dm, &section);CHKERRQ(ierr);
227     ierr = GetFieldType_Static(dm, section, PETSC_DETERMINE, &pStart, &pEnd, &ft);CHKERRQ(ierr);
228     ierr = PetscObjectReference((PetscObject) dm);CHKERRQ(ierr); /* viewer drops reference */
229     ierr = PetscObjectReference((PetscObject) v);CHKERRQ(ierr);  /* viewer drops reference */
230     ierr = PetscViewerVTKAddField(viewer, (PetscObject) dm, DMPlexVTKWriteAll, ft, (PetscObject) v);CHKERRQ(ierr);
231   } else if (ishdf5) {
232 #if defined(PETSC_HAVE_HDF5)
233     ierr = VecView_Plex_Local_HDF5(v, viewer);CHKERRQ(ierr);
234 #else
235     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
236 #endif
237   } else {
238     if (isseq) {ierr = VecView_Seq(v, viewer);CHKERRQ(ierr);}
239     else       {ierr = VecView_MPI(v, viewer);CHKERRQ(ierr);}
240   }
241   PetscFunctionReturn(0);
242 }
243 
244 #if defined(PETSC_HAVE_HDF5)
245 #undef __FUNCT__
246 #define __FUNCT__ "VecView_Plex_HDF5"
247 static PetscErrorCode VecView_Plex_HDF5(Vec v, PetscViewer viewer)
248 {
249   DM             dm;
250   Vec            locv;
251   const char    *name;
252   PetscErrorCode ierr;
253 
254   PetscFunctionBegin;
255   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
256   ierr = DMGetLocalVector(dm, &locv);CHKERRQ(ierr);
257   ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
258   ierr = PetscObjectSetName((PetscObject) locv, name);CHKERRQ(ierr);
259   ierr = DMGlobalToLocalBegin(dm, v, INSERT_VALUES, locv);CHKERRQ(ierr);
260   ierr = DMGlobalToLocalEnd(dm, v, INSERT_VALUES, locv);CHKERRQ(ierr);
261   ierr = PetscViewerHDF5PushGroup(viewer, "/fields");CHKERRQ(ierr);
262   ierr = PetscViewerPushFormat(viewer, PETSC_VIEWER_HDF5_VIZ);CHKERRQ(ierr);
263   ierr = VecView_Plex_Local(locv, viewer);CHKERRQ(ierr);
264   ierr = PetscViewerPopFormat(viewer);CHKERRQ(ierr);
265   ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
266   ierr = DMRestoreLocalVector(dm, &locv);CHKERRQ(ierr);
267   PetscFunctionReturn(0);
268 }
269 #endif
270 
271 #undef __FUNCT__
272 #define __FUNCT__ "VecView_Plex"
273 PetscErrorCode VecView_Plex(Vec v, PetscViewer viewer)
274 {
275   DM             dm;
276   PetscBool      isvtk, ishdf5, isseq;
277   PetscErrorCode ierr;
278 
279   PetscFunctionBegin;
280   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
281   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
282   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,  &isvtk);CHKERRQ(ierr);
283   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
284   ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
285   if (isvtk) {
286     Vec         locv;
287     const char *name;
288 
289     ierr = DMGetLocalVector(dm, &locv);CHKERRQ(ierr);
290     ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
291     ierr = PetscObjectSetName((PetscObject) locv, name);CHKERRQ(ierr);
292     ierr = DMGlobalToLocalBegin(dm, v, INSERT_VALUES, locv);CHKERRQ(ierr);
293     ierr = DMGlobalToLocalEnd(dm, v, INSERT_VALUES, locv);CHKERRQ(ierr);
294     ierr = VecView_Plex_Local(locv, viewer);CHKERRQ(ierr);
295     ierr = DMRestoreLocalVector(dm, &locv);CHKERRQ(ierr);
296   } else if (ishdf5) {
297 #if defined(PETSC_HAVE_HDF5)
298     ierr = VecView_Plex_HDF5(v, viewer);CHKERRQ(ierr);
299 #else
300     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
301 #endif
302   } else {
303     if (isseq) {ierr = VecView_Seq(v, viewer);CHKERRQ(ierr);}
304     else       {ierr = VecView_MPI(v, viewer);CHKERRQ(ierr);}
305   }
306   PetscFunctionReturn(0);
307 }
308 
309 #undef __FUNCT__
310 #define __FUNCT__ "VecLoad_Plex_Local"
311 PetscErrorCode VecLoad_Plex_Local(Vec v, PetscViewer viewer)
312 {
313   DM             dm;
314   PetscBool      ishdf5;
315   PetscErrorCode ierr;
316 
317   PetscFunctionBegin;
318   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
319   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
320   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
321   if (ishdf5) {
322     DM          dmBC;
323     Vec         gv;
324     const char *name;
325 
326     ierr = DMGetOutputDM(dm, &dmBC);CHKERRQ(ierr);
327     ierr = DMGetGlobalVector(dmBC, &gv);CHKERRQ(ierr);
328     ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
329     ierr = PetscObjectSetName((PetscObject) gv, name);CHKERRQ(ierr);
330     ierr = VecLoad_Default(gv, viewer);CHKERRQ(ierr);
331     ierr = DMGlobalToLocalBegin(dmBC, gv, INSERT_VALUES, v);CHKERRQ(ierr);
332     ierr = DMGlobalToLocalEnd(dmBC, gv, INSERT_VALUES, v);CHKERRQ(ierr);
333     ierr = DMRestoreGlobalVector(dmBC, &gv);CHKERRQ(ierr);
334   } else {
335     ierr = VecLoad_Default(v, viewer);CHKERRQ(ierr);
336   }
337   PetscFunctionReturn(0);
338 }
339 
340 #if defined(PETSC_HAVE_HDF5)
341 #undef __FUNCT__
342 #define __FUNCT__ "VecLoad_Plex_HDF5"
343 static PetscErrorCode VecLoad_Plex_HDF5(Vec v, PetscViewer viewer)
344 {
345   DM             dm;
346   Vec            locv;
347   const char    *name;
348   PetscInt       seqnum;
349   PetscErrorCode ierr;
350 
351   PetscFunctionBegin;
352   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
353   ierr = DMGetLocalVector(dm, &locv);CHKERRQ(ierr);
354   ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
355   ierr = PetscObjectSetName((PetscObject) locv, name);CHKERRQ(ierr);
356   ierr = DMGetOutputSequenceNumber(dm, &seqnum);CHKERRQ(ierr);
357   ierr = PetscViewerHDF5SetTimestep(viewer, seqnum);CHKERRQ(ierr);
358   ierr = PetscViewerHDF5PushGroup(viewer, "/fields");CHKERRQ(ierr);
359   ierr = VecLoad_Plex_Local(locv, viewer);CHKERRQ(ierr);
360   ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
361   ierr = DMLocalToGlobalBegin(dm, locv, INSERT_VALUES, v);CHKERRQ(ierr);
362   ierr = DMLocalToGlobalEnd(dm, locv, INSERT_VALUES, v);CHKERRQ(ierr);
363   ierr = DMRestoreLocalVector(dm, &locv);CHKERRQ(ierr);
364   PetscFunctionReturn(0);
365 }
366 #endif
367 
368 #undef __FUNCT__
369 #define __FUNCT__ "VecLoad_Plex"
370 PetscErrorCode VecLoad_Plex(Vec v, PetscViewer viewer)
371 {
372   DM             dm;
373   PetscBool      ishdf5;
374   PetscErrorCode ierr;
375 
376   PetscFunctionBegin;
377   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
378   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
379   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
380   if (ishdf5) {
381 #if defined(PETSC_HAVE_HDF5)
382     ierr = VecLoad_Plex_HDF5(v, viewer);CHKERRQ(ierr);
383 #else
384     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
385 #endif
386   } else {
387     ierr = VecLoad_Default(v, viewer);CHKERRQ(ierr);
388   }
389   PetscFunctionReturn(0);
390 }
391 
392 #undef __FUNCT__
393 #define __FUNCT__ "DMPlexView_Ascii"
394 PetscErrorCode DMPlexView_Ascii(DM dm, PetscViewer viewer)
395 {
396   DM_Plex          *mesh = (DM_Plex*) dm->data;
397   DM                cdm;
398   DMLabel           markers;
399   PetscSection      coordSection;
400   Vec               coordinates;
401   PetscViewerFormat format;
402   PetscErrorCode    ierr;
403 
404   PetscFunctionBegin;
405   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
406   ierr = DMGetDefaultSection(cdm, &coordSection);CHKERRQ(ierr);
407   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
408   ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
409   if (format == PETSC_VIEWER_ASCII_INFO_DETAIL) {
410     const char *name;
411     PetscInt    maxConeSize, maxSupportSize;
412     PetscInt    pStart, pEnd, p;
413     PetscMPIInt rank, size;
414 
415     ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRQ(ierr);
416     ierr = MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size);CHKERRQ(ierr);
417     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
418     ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
419     ierr = DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize);CHKERRQ(ierr);
420     ierr = PetscViewerASCIISynchronizedAllow(viewer, PETSC_TRUE);CHKERRQ(ierr);
421     ierr = PetscViewerASCIIPrintf(viewer, "Mesh '%s':\n", name);CHKERRQ(ierr);
422     ierr = PetscViewerASCIISynchronizedPrintf(viewer, "Max sizes cone: %D support: %D\n", maxConeSize, maxSupportSize);CHKERRQ(ierr);
423     ierr = PetscViewerASCIIPrintf(viewer, "orientation is missing\n", name);CHKERRQ(ierr);
424     ierr = PetscViewerASCIIPrintf(viewer, "cap --> base:\n", name);CHKERRQ(ierr);
425     for (p = pStart; p < pEnd; ++p) {
426       PetscInt dof, off, s;
427 
428       ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
429       ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
430       for (s = off; s < off+dof; ++s) {
431         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%D]: %D ----> %D\n", rank, p, mesh->supports[s]);CHKERRQ(ierr);
432       }
433     }
434     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
435     ierr = PetscViewerASCIIPrintf(viewer, "base <-- cap:\n", name);CHKERRQ(ierr);
436     for (p = pStart; p < pEnd; ++p) {
437       PetscInt dof, off, c;
438 
439       ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
440       ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
441       for (c = off; c < off+dof; ++c) {
442         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%D]: %D <---- %D (%D)\n", rank, p, mesh->cones[c], mesh->coneOrientations[c]);CHKERRQ(ierr);
443       }
444     }
445     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
446     ierr = PetscSectionGetChart(coordSection, &pStart, NULL);CHKERRQ(ierr);
447     if (pStart >= 0) {ierr = PetscSectionVecView(coordSection, coordinates, viewer);CHKERRQ(ierr);}
448     ierr = DMPlexGetLabel(dm, "marker", &markers);CHKERRQ(ierr);
449     ierr = DMLabelView(markers,viewer);CHKERRQ(ierr);
450     if (size > 1) {
451       PetscSF sf;
452 
453       ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr);
454       ierr = PetscSFView(sf, viewer);CHKERRQ(ierr);
455     }
456     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
457   } else if (format == PETSC_VIEWER_ASCII_LATEX) {
458     const char  *name;
459     const char  *colors[3] = {"red", "blue", "green"};
460     const int    numColors  = 3;
461     PetscReal    scale      = 2.0;
462     PetscScalar *coords;
463     PetscInt     depth, cStart, cEnd, c, vStart, vEnd, v, eStart = 0, eEnd = 0, e, p;
464     PetscMPIInt  rank, size;
465 
466     ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
467     ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRQ(ierr);
468     ierr = MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size);CHKERRQ(ierr);
469     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
470     ierr = PetscViewerASCIISynchronizedAllow(viewer, PETSC_TRUE);CHKERRQ(ierr);
471     ierr = PetscViewerASCIIPrintf(viewer, "\
472 \\documentclass[crop,multi=false]{standalone}\n\n\
473 \\usepackage{tikz}\n\
474 \\usepackage{pgflibraryshapes}\n\
475 \\usetikzlibrary{backgrounds}\n\
476 \\usetikzlibrary{arrows}\n\
477 \\begin{document}\n\
478 \\section{%s}\n\
479 \\begin{center}\n", name, 8.0/scale);CHKERRQ(ierr);
480     ierr = PetscViewerASCIIPrintf(viewer, "Mesh for process ");CHKERRQ(ierr);
481     for (p = 0; p < size; ++p) {
482       if (p > 0 && p == size-1) {
483         ierr = PetscViewerASCIIPrintf(viewer, ", and ", colors[p%numColors], p);CHKERRQ(ierr);
484       } else if (p > 0) {
485         ierr = PetscViewerASCIIPrintf(viewer, ", ", colors[p%numColors], p);CHKERRQ(ierr);
486       }
487       ierr = PetscViewerASCIIPrintf(viewer, "{\\textcolor{%s}%D}", colors[p%numColors], p);CHKERRQ(ierr);
488     }
489     ierr = PetscViewerASCIIPrintf(viewer, ".\n\n\n\
490 \\begin{tikzpicture}[scale = %g,font=\\fontsize{8}{8}\\selectfont]\n");CHKERRQ(ierr);
491     /* Plot vertices */
492     ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
493     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
494     ierr = PetscViewerASCIIPrintf(viewer, "\\path\n");CHKERRQ(ierr);
495     for (v = vStart; v < vEnd; ++v) {
496       PetscInt off, dof, d;
497 
498       ierr = PetscSectionGetDof(coordSection, v, &dof);CHKERRQ(ierr);
499       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
500       ierr = PetscViewerASCIISynchronizedPrintf(viewer, "(");CHKERRQ(ierr);
501       for (d = 0; d < dof; ++d) {
502         if (d > 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, ",");CHKERRQ(ierr);}
503         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "%g", (double)(scale*PetscRealPart(coords[off+d])));CHKERRQ(ierr);
504       }
505       ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%D) [draw,shape=circle,color=%s] {%D} --\n", v, rank, colors[rank%numColors], v);CHKERRQ(ierr);
506     }
507     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
508     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
509     ierr = PetscViewerASCIIPrintf(viewer, "(0,0);\n");CHKERRQ(ierr);
510     /* Plot edges */
511     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
512     ierr = PetscViewerASCIIPrintf(viewer, "\\path\n");CHKERRQ(ierr);
513     if (depth > 1) {ierr = DMPlexGetDepthStratum(dm, 1, &eStart, &eEnd);CHKERRQ(ierr);}
514     for (e = eStart; e < eEnd; ++e) {
515       const PetscInt *cone;
516       PetscInt        coneSize, offA, offB, dof, d;
517 
518       ierr = DMPlexGetConeSize(dm, e, &coneSize);CHKERRQ(ierr);
519       if (coneSize != 2) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Edge %d cone should have two vertices, not %d", e, coneSize);
520       ierr = DMPlexGetCone(dm, e, &cone);CHKERRQ(ierr);
521       ierr = PetscSectionGetDof(coordSection, cone[0], &dof);CHKERRQ(ierr);
522       ierr = PetscSectionGetOffset(coordSection, cone[0], &offA);CHKERRQ(ierr);
523       ierr = PetscSectionGetOffset(coordSection, cone[1], &offB);CHKERRQ(ierr);
524       ierr = PetscViewerASCIISynchronizedPrintf(viewer, "(");CHKERRQ(ierr);
525       for (d = 0; d < dof; ++d) {
526         if (d > 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, ",");CHKERRQ(ierr);}
527         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "%g", (double)(scale*0.5*PetscRealPart(coords[offA+d]+coords[offB+d])));CHKERRQ(ierr);
528       }
529       ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%D) [draw,shape=circle,color=%s] {%D} --\n", e, rank, colors[rank%numColors], e);CHKERRQ(ierr);
530     }
531     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
532     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
533     ierr = PetscViewerASCIIPrintf(viewer, "(0,0);\n");CHKERRQ(ierr);
534     /* Plot cells */
535     ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
536     for (c = cStart; c < cEnd; ++c) {
537       PetscInt *closure = NULL;
538       PetscInt  closureSize, firstPoint = -1;
539 
540       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
541       ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\draw[color=%s] ", colors[rank%numColors]);CHKERRQ(ierr);
542       for (p = 0; p < closureSize*2; p += 2) {
543         const PetscInt point = closure[p];
544 
545         if ((point < vStart) || (point >= vEnd)) continue;
546         if (firstPoint >= 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- ");CHKERRQ(ierr);}
547         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "(%D_%D)", point, rank);CHKERRQ(ierr);
548         if (firstPoint < 0) firstPoint = point;
549       }
550       /* Why doesn't this work? ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- cycle;\n");CHKERRQ(ierr); */
551       ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- (%D_%D);\n", firstPoint, rank);CHKERRQ(ierr);
552       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
553     }
554     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
555     ierr = PetscViewerASCIIPrintf(viewer, "\\end{tikzpicture}\n\\end{center}\n");CHKERRQ(ierr);
556     ierr = PetscViewerASCIIPrintf(viewer, "\\end{document}\n", name);CHKERRQ(ierr);
557   } else {
558     MPI_Comm    comm;
559     PetscInt   *sizes, *hybsizes;
560     PetscInt    locDepth, depth, dim, d, pMax[4];
561     PetscInt    pStart, pEnd, p;
562     PetscInt    numLabels, l;
563     const char *name;
564     PetscMPIInt size;
565 
566     ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
567     ierr = MPI_Comm_size(comm, &size);CHKERRQ(ierr);
568     ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
569     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
570     if (name) {ierr = PetscViewerASCIIPrintf(viewer, "%s in %D dimensions:\n", name, dim);CHKERRQ(ierr);}
571     else      {ierr = PetscViewerASCIIPrintf(viewer, "Mesh in %D dimensions:\n", dim);CHKERRQ(ierr);}
572     ierr = DMPlexGetDepth(dm, &locDepth);CHKERRQ(ierr);
573     ierr = MPI_Allreduce(&locDepth, &depth, 1, MPIU_INT, MPI_MAX, comm);CHKERRQ(ierr);
574     ierr = DMPlexGetHybridBounds(dm, &pMax[depth], &pMax[depth-1], &pMax[1], &pMax[0]);CHKERRQ(ierr);
575     ierr = PetscMalloc2(size,&sizes,size,&hybsizes);CHKERRQ(ierr);
576     if (depth == 1) {
577       ierr = DMPlexGetDepthStratum(dm, 0, &pStart, &pEnd);CHKERRQ(ierr);
578       pEnd = pEnd - pStart;
579       ierr = MPI_Gather(&pEnd, 1, MPIU_INT, sizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
580       ierr = PetscViewerASCIIPrintf(viewer, "  %D-cells:", 0);CHKERRQ(ierr);
581       for (p = 0; p < size; ++p) {ierr = PetscViewerASCIIPrintf(viewer, " %D", sizes[p]);CHKERRQ(ierr);}
582       ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
583       ierr = DMPlexGetHeightStratum(dm, 0, &pStart, &pEnd);CHKERRQ(ierr);
584       pEnd = pEnd - pStart;
585       ierr = MPI_Gather(&pEnd, 1, MPIU_INT, sizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
586       ierr = PetscViewerASCIIPrintf(viewer, "  %D-cells:", dim);CHKERRQ(ierr);
587       for (p = 0; p < size; ++p) {ierr = PetscViewerASCIIPrintf(viewer, " %D", sizes[p]);CHKERRQ(ierr);}
588       ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
589     } else {
590       for (d = 0; d <= dim; d++) {
591         ierr = DMPlexGetDepthStratum(dm, d, &pStart, &pEnd);CHKERRQ(ierr);
592         pEnd    -= pStart;
593         pMax[d] -= pStart;
594         ierr = MPI_Gather(&pEnd, 1, MPIU_INT, sizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
595         ierr = MPI_Gather(&pMax[d], 1, MPIU_INT, hybsizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
596         ierr = PetscViewerASCIIPrintf(viewer, "  %D-cells:", d);CHKERRQ(ierr);
597         for (p = 0; p < size; ++p) {
598           if (hybsizes[p] >= 0) {ierr = PetscViewerASCIIPrintf(viewer, " %D (%D)", sizes[p], sizes[p] - hybsizes[p]);CHKERRQ(ierr);}
599           else                  {ierr = PetscViewerASCIIPrintf(viewer, " %D", sizes[p]);CHKERRQ(ierr);}
600         }
601         ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
602       }
603     }
604     ierr = PetscFree2(sizes,hybsizes);CHKERRQ(ierr);
605     ierr = DMPlexGetNumLabels(dm, &numLabels);CHKERRQ(ierr);
606     if (numLabels) {ierr = PetscViewerASCIIPrintf(viewer, "Labels:\n");CHKERRQ(ierr);}
607     for (l = 0; l < numLabels; ++l) {
608       DMLabel         label;
609       const char     *name;
610       IS              valueIS;
611       const PetscInt *values;
612       PetscInt        numValues, v;
613 
614       ierr = DMPlexGetLabelName(dm, l, &name);CHKERRQ(ierr);
615       ierr = DMPlexGetLabel(dm, name, &label);CHKERRQ(ierr);
616       ierr = DMLabelGetNumValues(label, &numValues);CHKERRQ(ierr);
617       ierr = PetscViewerASCIIPrintf(viewer, "  %s: %d strata of sizes (", name, numValues);CHKERRQ(ierr);
618       ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
619       ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
620       for (v = 0; v < numValues; ++v) {
621         PetscInt size;
622 
623         ierr = DMLabelGetStratumSize(label, values[v], &size);CHKERRQ(ierr);
624         if (v > 0) {ierr = PetscViewerASCIIPrintf(viewer, ", ");CHKERRQ(ierr);}
625         ierr = PetscViewerASCIIPrintf(viewer, "%d", size);CHKERRQ(ierr);
626       }
627       ierr = PetscViewerASCIIPrintf(viewer, ")\n");CHKERRQ(ierr);
628       ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
629       ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
630     }
631   }
632   PetscFunctionReturn(0);
633 }
634 
635 #if defined(PETSC_HAVE_HDF5)
636 #undef __FUNCT__
637 #define __FUNCT__ "DMPlexView_HDF5"
638 /* We only write cells and vertices. Does this screw up parallel reading? */
639 static PetscErrorCode DMPlexView_HDF5(DM dm, PetscViewer viewer)
640 {
641   DM              cdm;
642   Vec             coordinates, newcoords;
643   Vec             coneVec, cellVec;
644   IS              globalVertexNumbers;
645   const PetscInt *gvertex;
646   PetscScalar    *sizes, *vertices;
647   PetscReal       lengthScale;
648   const char     *label   = NULL;
649   PetscInt        labelId = 0, dim;
650   char            group[PETSC_MAX_PATH_LEN];
651   PetscInt        vStart, vEnd, v, cellHeight, cStart, cEnd, cMax, cell, conesSize = 0, numCornersLocal = 0, numCorners, numLabels, l;
652   hid_t           fileId, groupId;
653   herr_t          status;
654   PetscErrorCode  ierr;
655 
656   PetscFunctionBegin;
657   /* Write coordinates */
658   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
659   ierr = DMPlexGetScale(dm, PETSC_UNIT_LENGTH, &lengthScale);CHKERRQ(ierr);
660   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
661   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
662   ierr = VecDuplicate(coordinates, &newcoords);CHKERRQ(ierr);
663   ierr = PetscObjectSetName((PetscObject) newcoords, "vertices");CHKERRQ(ierr);
664   ierr = VecCopy(coordinates, newcoords);CHKERRQ(ierr);
665   ierr = VecScale(newcoords, lengthScale);CHKERRQ(ierr);
666   /* Use the local version to bypass the default group setting */
667   ierr = PetscViewerHDF5PushGroup(viewer, "/geometry");CHKERRQ(ierr);
668   ierr = VecView(newcoords, viewer);CHKERRQ(ierr);
669   ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
670   ierr = VecDestroy(&newcoords);CHKERRQ(ierr);
671   /* Write toplogy */
672   ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
673   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
674   ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
675   ierr = DMPlexGetHybridBounds(dm, &cMax, PETSC_NULL, PETSC_NULL, PETSC_NULL);CHKERRQ(ierr);
676   if (cMax >= 0) cEnd = PetscMin(cEnd, cMax);
677 
678   ierr = VecCreate(PetscObjectComm((PetscObject) dm), &coneVec);CHKERRQ(ierr);
679   ierr = VecSetSizes(coneVec, cEnd-cStart, PETSC_DETERMINE);CHKERRQ(ierr);
680   ierr = VecSetBlockSize(coneVec, 1);CHKERRQ(ierr);
681   ierr = VecSetFromOptions(coneVec);CHKERRQ(ierr);
682   ierr = PetscObjectSetName((PetscObject) coneVec, "coneSize");CHKERRQ(ierr);
683   ierr = VecGetArray(coneVec, &sizes);CHKERRQ(ierr);
684   for (cell = cStart; cell < cEnd; ++cell) {
685     PetscInt *closure = NULL;
686     PetscInt  closureSize, v, Nc = 0;
687 
688     if (label) {
689       PetscInt value;
690       ierr = DMPlexGetLabelValue(dm, label, cell, &value);CHKERRQ(ierr);
691       if (value == labelId) continue;
692     }
693     ierr = DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
694     for (v = 0; v < closureSize*2; v += 2) {
695       if ((closure[v] >= vStart) && (closure[v] < vEnd)) ++Nc;
696     }
697     ierr = DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
698     conesSize += Nc;
699     if (!numCornersLocal)           numCornersLocal = Nc;
700     else if (numCornersLocal != Nc) numCornersLocal = 1;
701   }
702   ierr = VecRestoreArray(coneVec, &sizes);CHKERRQ(ierr);
703   ierr = MPI_Allreduce(&numCornersLocal, &numCorners, 1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject) dm));CHKERRQ(ierr);
704   if (numCornersLocal && numCornersLocal != numCorners) numCorners = 1;
705 
706   ierr = DMPlexGetVertexNumbering(dm, &globalVertexNumbers);CHKERRQ(ierr);
707   ierr = ISGetIndices(globalVertexNumbers, &gvertex);CHKERRQ(ierr);
708   ierr = VecCreate(PetscObjectComm((PetscObject) dm), &cellVec);CHKERRQ(ierr);
709   ierr = VecSetSizes(cellVec, conesSize, PETSC_DETERMINE);CHKERRQ(ierr);
710   ierr = VecSetBlockSize(cellVec, numCorners);CHKERRQ(ierr);
711   ierr = VecSetFromOptions(cellVec);CHKERRQ(ierr);
712   ierr = PetscObjectSetName((PetscObject) cellVec, "cells");CHKERRQ(ierr);
713   ierr = VecGetArray(cellVec, &vertices);CHKERRQ(ierr);
714   for (cell = cStart, v = 0; cell < cEnd; ++cell) {
715     PetscInt *closure = NULL;
716     PetscInt  closureSize, Nc = 0, p;
717 
718     if (label) {
719       PetscInt value;
720       ierr = DMPlexGetLabelValue(dm, label, cell, &value);CHKERRQ(ierr);
721       if (value == labelId) continue;
722     }
723     ierr = DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
724     for (p = 0; p < closureSize*2; p += 2) {
725       if ((closure[p] >= vStart) && (closure[p] < vEnd)) {
726         closure[Nc++] = closure[p];
727         }
728     }
729     ierr = DMPlexInvertCell(dim, Nc, closure);CHKERRQ(ierr);
730     for (p = 0; p < Nc; ++p) {
731       const PetscInt gv = gvertex[closure[p] - vStart];
732       vertices[v++] = gv < 0 ? -(gv+1) : gv;
733     }
734     ierr = DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
735   }
736   if (v != conesSize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_LIB, "Total number of cell vertices %d != %d", v, conesSize);
737   ierr = VecRestoreArray(cellVec, &vertices);CHKERRQ(ierr);
738   ierr = PetscViewerHDF5PushGroup(viewer, "/topology");CHKERRQ(ierr);
739   ierr = VecView(cellVec, viewer);CHKERRQ(ierr);
740   if (numCorners == 1) {
741     ierr = VecView(coneVec, viewer);CHKERRQ(ierr);
742   } else {
743     ierr = PetscViewerHDF5WriteAttribute(viewer, "/topology/cells", "cell_corners", PETSC_INT, (void *) &numCorners);CHKERRQ(ierr);
744   }
745   ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
746   ierr = VecDestroy(&cellVec);CHKERRQ(ierr);
747   ierr = VecDestroy(&coneVec);CHKERRQ(ierr);
748   ierr = ISRestoreIndices(globalVertexNumbers, &gvertex);CHKERRQ(ierr);
749 
750   ierr = PetscViewerHDF5WriteAttribute(viewer, "/topology/cells", "cell_dim", PETSC_INT, (void *) &dim);CHKERRQ(ierr);
751   /* Write Labels*/
752   ierr = PetscViewerHDF5PushGroup(viewer, "/labels");CHKERRQ(ierr);
753   ierr = PetscViewerHDF5OpenGroup(viewer, &fileId, &groupId);CHKERRQ(ierr);
754   if (groupId != fileId) {status = H5Gclose(groupId);CHKERRQ(status);}
755   ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
756   ierr = DMPlexGetNumLabels(dm, &numLabels);CHKERRQ(ierr);
757   for (l = 0; l < numLabels; ++l) {
758     DMLabel         label;
759     const char     *name;
760     IS              valueIS;
761     const PetscInt *values;
762     PetscInt        numValues, v;
763     PetscBool       isDepth;
764 
765     ierr = DMPlexGetLabelName(dm, l, &name);CHKERRQ(ierr);
766     ierr = PetscStrncmp(name, "depth", 10, &isDepth);CHKERRQ(ierr);
767     if (isDepth) continue;
768     ierr = DMPlexGetLabel(dm, name, &label);CHKERRQ(ierr);
769     ierr = DMLabelGetNumValues(label, &numValues);CHKERRQ(ierr);
770     ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
771     ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
772     ierr = PetscSNPrintf(group, PETSC_MAX_PATH_LEN, "/labels/%s", name);CHKERRQ(ierr);
773     ierr = PetscViewerHDF5PushGroup(viewer, group);CHKERRQ(ierr);
774     ierr = PetscViewerHDF5OpenGroup(viewer, &fileId, &groupId);CHKERRQ(ierr);
775     if (groupId != fileId) {status = H5Gclose(groupId);CHKERRQ(status);}
776     ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
777     /* TODO: Need to actually loop over the union of label values, ISAllGather() */
778     for (v = 0; v < numValues; ++v) {
779       IS   stratumIS;
780 
781       ierr = PetscSNPrintf(group, PETSC_MAX_PATH_LEN, "/labels/%s/%d", name, values[v]);CHKERRQ(ierr);
782       ierr = DMLabelGetStratumIS(label, values[v], &stratumIS);CHKERRQ(ierr);
783       /* TODO: Need to globalize point names and remove unowned points */
784       ierr = PetscViewerHDF5PushGroup(viewer, group);CHKERRQ(ierr);
785       ierr = ISView(stratumIS, viewer);CHKERRQ(ierr);
786       ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
787       ierr = ISDestroy(&stratumIS);CHKERRQ(ierr);
788     }
789     ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
790     ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
791   }
792   PetscFunctionReturn(0);
793 }
794 #endif
795 
796 #undef __FUNCT__
797 #define __FUNCT__ "DMView_Plex"
798 PetscErrorCode DMView_Plex(DM dm, PetscViewer viewer)
799 {
800   PetscBool      iascii, ishdf5;
801   PetscErrorCode ierr;
802 
803   PetscFunctionBegin;
804   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
805   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
806   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERASCII, &iascii);CHKERRQ(ierr);
807   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,  &ishdf5);CHKERRQ(ierr);
808   if (iascii) {
809     ierr = DMPlexView_Ascii(dm, viewer);CHKERRQ(ierr);
810   } else if (ishdf5) {
811 #if defined(PETSC_HAVE_HDF5)
812     ierr = DMPlexView_HDF5(dm, viewer);CHKERRQ(ierr);
813 #else
814     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
815 #endif
816   }
817   PetscFunctionReturn(0);
818 }
819 
820 #if defined(PETSC_HAVE_HDF5)
821 typedef struct {
822   DM          dm;
823   PetscViewer viewer;
824   DMLabel     label;
825 } LabelCtx;
826 
827 static herr_t ReadLabelStratumHDF5_Static(hid_t g_id, const char *name, const H5L_info_t *info, void *op_data)
828 {
829   PetscViewer     viewer = ((LabelCtx *) op_data)->viewer;
830   DMLabel         label  = ((LabelCtx *) op_data)->label;
831   IS              stratumIS;
832   const PetscInt *ind;
833   PetscInt        value, N, i;
834   const char     *lname;
835   char            group[PETSC_MAX_PATH_LEN];
836   PetscErrorCode  ierr;
837 
838   ierr = PetscOptionsStringToInt(name, &value);
839   ierr = ISCreate(PetscObjectComm((PetscObject) viewer), &stratumIS);
840   ierr = PetscObjectSetName((PetscObject) stratumIS, "indices");
841   ierr = DMLabelGetName(label, &lname);
842   ierr = PetscSNPrintf(group, PETSC_MAX_PATH_LEN, "/labels/%s/%s", lname, name);CHKERRQ(ierr);
843   ierr = PetscViewerHDF5PushGroup(viewer, group);CHKERRQ(ierr);
844   ierr = ISLoad(stratumIS, viewer);
845   ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
846   ierr = ISGetSize(stratumIS, &N);
847   ierr = ISGetIndices(stratumIS, &ind);
848   for (i = 0; i < N; ++i) {ierr = DMLabelSetValue(label, ind[i], value);}
849   ierr = ISRestoreIndices(stratumIS, &ind);
850   ierr = ISDestroy(&stratumIS);
851   return 0;
852 }
853 
854 static herr_t ReadLabelHDF5_Static(hid_t g_id, const char *name, const H5L_info_t *info, void *op_data)
855 {
856   DM             dm = ((LabelCtx *) op_data)->dm;
857   hsize_t        idx;
858   herr_t         status;
859   PetscErrorCode ierr;
860 
861   ierr = DMPlexCreateLabel(dm, name); if (ierr) return (herr_t) ierr;
862   ierr = DMPlexGetLabel(dm, name, &((LabelCtx *) op_data)->label); if (ierr) return (herr_t) ierr;
863   status = H5Literate_by_name(g_id, name, H5_INDEX_NAME, H5_ITER_NATIVE, &idx, ReadLabelStratumHDF5_Static, op_data, 0);
864   return status;
865 }
866 
867 #undef __FUNCT__
868 #define __FUNCT__ "DMPlexLoad_HDF5"
869 /* The first version will read everything onto proc 0, letting the user distribute
870    The next will create a naive partition, and then rebalance after reading
871 */
872 static PetscErrorCode DMPlexLoad_HDF5(DM dm, PetscViewer viewer)
873 {
874   LabelCtx        ctx;
875   PetscSection    coordSection;
876   Vec             coordinates;
877   Vec             cellVec;
878   PetscScalar    *cells;
879   PetscReal       lengthScale;
880   PetscInt       *cone;
881   PetscInt        dim, spatialDim, numVertices, v, numCorners, numCells, cell, c;
882   hid_t           fileId, groupId;
883   hsize_t         idx = 0;
884   herr_t          status;
885   PetscErrorCode  ierr;
886 
887   PetscFunctionBegin;
888   /* Read geometry */
889   ierr = PetscViewerHDF5PushGroup(viewer, "/geometry");CHKERRQ(ierr);
890   ierr = VecCreate(PetscObjectComm((PetscObject) dm), &coordinates);CHKERRQ(ierr);
891   ierr = PetscObjectSetName((PetscObject) coordinates, "vertices");CHKERRQ(ierr);
892   ierr = VecLoad(coordinates, viewer);CHKERRQ(ierr);
893   ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
894   ierr = DMPlexGetScale(dm, PETSC_UNIT_LENGTH, &lengthScale);CHKERRQ(ierr);
895   ierr = VecScale(coordinates, 1.0/lengthScale);CHKERRQ(ierr);
896   ierr = VecGetSize(coordinates, &numVertices);CHKERRQ(ierr);
897   ierr = VecGetBlockSize(coordinates, &spatialDim);CHKERRQ(ierr);
898   numVertices /= spatialDim;
899   /* Read toplogy */
900   ierr = PetscViewerHDF5ReadAttribute(viewer, "/topology/cells", "cell_dim", PETSC_INT, (void *) &dim);CHKERRQ(ierr);
901   ierr = DMPlexSetDimension(dm, dim);CHKERRQ(ierr);
902   /*   TODO Check for coneSize vector rather than this attribute */
903   ierr = PetscViewerHDF5ReadAttribute(viewer, "/topology/cells", "cell_corners", PETSC_INT, (void *) &numCorners);CHKERRQ(ierr);
904   ierr = PetscViewerHDF5PushGroup(viewer, "/topology");CHKERRQ(ierr);
905   ierr = VecCreate(PetscObjectComm((PetscObject) dm), &cellVec);CHKERRQ(ierr);
906   ierr = VecSetBlockSize(cellVec, numCorners);CHKERRQ(ierr);
907   ierr = PetscObjectSetName((PetscObject) cellVec, "cells");CHKERRQ(ierr);
908   ierr = VecLoad(cellVec, viewer);CHKERRQ(ierr);
909   ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
910   ierr = VecGetSize(cellVec, &numCells);CHKERRQ(ierr);
911   numCells /= numCorners;
912   /* Create Plex */
913   ierr = DMPlexSetChart(dm, 0, numCells+numVertices);CHKERRQ(ierr);
914   for (cell = 0; cell < numCells; ++cell) {ierr = DMPlexSetConeSize(dm, cell, numCorners);CHKERRQ(ierr);}
915   ierr = DMSetUp(dm);CHKERRQ(ierr);
916   ierr = PetscMalloc1(numCorners,&cone);CHKERRQ(ierr);
917   ierr = VecGetArray(cellVec, &cells);CHKERRQ(ierr);
918   for (cell = 0; cell < numCells; ++cell) {
919     for (c = 0; c < numCorners; ++c) {cone[c] = numCells + cells[cell*numCorners+c];}
920     ierr = DMPlexSetCone(dm, cell, cone);CHKERRQ(ierr);
921   }
922   ierr = VecRestoreArray(cellVec, &cells);CHKERRQ(ierr);
923   ierr = PetscFree(cone);CHKERRQ(ierr);
924   ierr = VecDestroy(&cellVec);CHKERRQ(ierr);
925   ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
926   ierr = DMPlexStratify(dm);CHKERRQ(ierr);
927   /* Create coordinates */
928   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
929   ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
930   ierr = PetscSectionSetFieldComponents(coordSection, 0, spatialDim);CHKERRQ(ierr);
931   ierr = PetscSectionSetChart(coordSection, numCells, numCells+numVertices);CHKERRQ(ierr);
932   for (v = numCells; v < numCells+numVertices; ++v) {
933     ierr = PetscSectionSetDof(coordSection, v, spatialDim);CHKERRQ(ierr);
934     ierr = PetscSectionSetFieldDof(coordSection, v, 0, spatialDim);CHKERRQ(ierr);
935   }
936   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
937   ierr = DMSetCoordinates(dm, coordinates);CHKERRQ(ierr);
938   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
939   /* Read Labels*/
940   ctx.dm     = dm;
941   ctx.viewer = viewer;
942   ierr = PetscViewerHDF5PushGroup(viewer, "/labels");CHKERRQ(ierr);
943   ierr = PetscViewerHDF5OpenGroup(viewer, &fileId, &groupId);CHKERRQ(ierr);
944   status = H5Literate(groupId, H5_INDEX_NAME, H5_ITER_NATIVE, &idx, ReadLabelHDF5_Static, &ctx);CHKERRQ(status);
945   status = H5Gclose(groupId);CHKERRQ(status);
946   ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
947   PetscFunctionReturn(0);
948 }
949 #endif
950 
951 #undef __FUNCT__
952 #define __FUNCT__ "DMLoad_Plex"
953 PetscErrorCode DMLoad_Plex(DM dm, PetscViewer viewer)
954 {
955   PetscBool      isbinary, ishdf5;
956   PetscErrorCode ierr;
957 
958   PetscFunctionBegin;
959   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
960   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
961   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERBINARY, &isbinary);CHKERRQ(ierr);
962   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,   &ishdf5);CHKERRQ(ierr);
963   if (isbinary) {SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Do not yet support binary viewers");}
964   else if (ishdf5) {
965 #if defined(PETSC_HAVE_HDF5)
966     DM odm;
967     ierr = DMCreate(PetscObjectComm((PetscObject) dm), &odm);CHKERRQ(ierr);
968     ierr = DMSetType(odm, DMPLEX);CHKERRQ(ierr);
969     ierr = DMPlexLoad_HDF5(odm, viewer);CHKERRQ(ierr);
970     ierr = DMPlexInterpolate(odm, &dm);CHKERRQ(ierr);
971     ierr = DMPlexCopyCoordinates(odm, dm);CHKERRQ(ierr);
972     ierr = DMPlexCopyLabels(odm, dm);CHKERRQ(ierr);
973     ierr = DMDestroy(&odm);CHKERRQ(ierr);
974 #else
975     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
976 #endif
977   }
978   PetscFunctionReturn(0);
979 }
980 
981 #undef __FUNCT__
982 #define __FUNCT__ "BoundaryDestroy"
983 static PetscErrorCode BoundaryDestroy(DMBoundary *boundary)
984 {
985   DMBoundary     b, next;
986   PetscErrorCode ierr;
987 
988   PetscFunctionBeginUser;
989   if (!boundary) PetscFunctionReturn(0);
990   b = *boundary;
991   *boundary = NULL;
992   for (; b; b = next) {
993     next = b->next;
994     ierr = PetscFree(b->ids);CHKERRQ(ierr);
995     ierr = PetscFree(b->name);CHKERRQ(ierr);
996     ierr = PetscFree(b);CHKERRQ(ierr);
997   }
998   PetscFunctionReturn(0);
999 }
1000 
1001 #undef __FUNCT__
1002 #define __FUNCT__ "DMDestroy_Plex"
1003 PetscErrorCode DMDestroy_Plex(DM dm)
1004 {
1005   DM_Plex       *mesh = (DM_Plex*) dm->data;
1006   DMLabel        next  = mesh->labels;
1007   PetscErrorCode ierr;
1008 
1009   PetscFunctionBegin;
1010   if (--mesh->refct > 0) PetscFunctionReturn(0);
1011   ierr = PetscSectionDestroy(&mesh->coneSection);CHKERRQ(ierr);
1012   ierr = PetscFree(mesh->cones);CHKERRQ(ierr);
1013   ierr = PetscFree(mesh->coneOrientations);CHKERRQ(ierr);
1014   ierr = PetscSectionDestroy(&mesh->supportSection);CHKERRQ(ierr);
1015   ierr = PetscFree(mesh->supports);CHKERRQ(ierr);
1016   ierr = PetscFree(mesh->facesTmp);CHKERRQ(ierr);
1017   while (next) {
1018     DMLabel tmp = next->next;
1019 
1020     ierr = DMLabelDestroy(&next);CHKERRQ(ierr);
1021     next = tmp;
1022   }
1023   ierr = DMDestroy(&mesh->coarseMesh);CHKERRQ(ierr);
1024   ierr = DMLabelDestroy(&mesh->subpointMap);CHKERRQ(ierr);
1025   ierr = ISDestroy(&mesh->globalVertexNumbers);CHKERRQ(ierr);
1026   ierr = ISDestroy(&mesh->globalCellNumbers);CHKERRQ(ierr);
1027   ierr = BoundaryDestroy(&mesh->boundary);CHKERRQ(ierr);
1028   /* This was originally freed in DMDestroy(), but that prevents reference counting of backend objects */
1029   ierr = PetscFree(mesh);CHKERRQ(ierr);
1030   PetscFunctionReturn(0);
1031 }
1032 
1033 #undef __FUNCT__
1034 #define __FUNCT__ "DMCreateMatrix_Plex"
1035 PetscErrorCode DMCreateMatrix_Plex(DM dm, Mat *J)
1036 {
1037   PetscSection   section, sectionGlobal;
1038   PetscInt       bs = -1;
1039   PetscInt       localSize;
1040   PetscBool      isShell, isBlock, isSeqBlock, isMPIBlock, isSymBlock, isSymSeqBlock, isSymMPIBlock;
1041   PetscErrorCode ierr;
1042   MatType        mtype;
1043 
1044   PetscFunctionBegin;
1045   ierr = MatInitializePackage();CHKERRQ(ierr);
1046   mtype = dm->mattype;
1047   ierr = DMGetDefaultSection(dm, &section);CHKERRQ(ierr);
1048   ierr = DMGetDefaultGlobalSection(dm, &sectionGlobal);CHKERRQ(ierr);
1049   /* ierr = PetscSectionGetStorageSize(sectionGlobal, &localSize);CHKERRQ(ierr); */
1050   ierr = PetscSectionGetConstrainedStorageSize(sectionGlobal, &localSize);CHKERRQ(ierr);
1051   ierr = MatCreate(PetscObjectComm((PetscObject)dm), J);CHKERRQ(ierr);
1052   ierr = MatSetSizes(*J, localSize, localSize, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
1053   ierr = MatSetType(*J, mtype);CHKERRQ(ierr);
1054   ierr = MatSetFromOptions(*J);CHKERRQ(ierr);
1055   ierr = PetscStrcmp(mtype, MATSHELL, &isShell);CHKERRQ(ierr);
1056   ierr = PetscStrcmp(mtype, MATBAIJ, &isBlock);CHKERRQ(ierr);
1057   ierr = PetscStrcmp(mtype, MATSEQBAIJ, &isSeqBlock);CHKERRQ(ierr);
1058   ierr = PetscStrcmp(mtype, MATMPIBAIJ, &isMPIBlock);CHKERRQ(ierr);
1059   ierr = PetscStrcmp(mtype, MATSBAIJ, &isSymBlock);CHKERRQ(ierr);
1060   ierr = PetscStrcmp(mtype, MATSEQSBAIJ, &isSymSeqBlock);CHKERRQ(ierr);
1061   ierr = PetscStrcmp(mtype, MATMPISBAIJ, &isSymMPIBlock);CHKERRQ(ierr);
1062   if (!isShell) {
1063     PetscBool fillMatrix = (PetscBool) !dm->prealloc_only;
1064     PetscInt *dnz, *onz, *dnzu, *onzu, bsLocal, bsMax, bsMin;
1065 
1066     if (bs < 0) {
1067       if (isBlock || isSeqBlock || isMPIBlock || isSymBlock || isSymSeqBlock || isSymMPIBlock) {
1068         PetscInt pStart, pEnd, p, dof, cdof;
1069 
1070         ierr = PetscSectionGetChart(sectionGlobal, &pStart, &pEnd);CHKERRQ(ierr);
1071         for (p = pStart; p < pEnd; ++p) {
1072           ierr = PetscSectionGetDof(sectionGlobal, p, &dof);CHKERRQ(ierr);
1073           ierr = PetscSectionGetConstraintDof(sectionGlobal, p, &cdof);CHKERRQ(ierr);
1074           if (dof-cdof) {
1075             if (bs < 0) {
1076               bs = dof-cdof;
1077             } else if (bs != dof-cdof) {
1078               /* Layout does not admit a pointwise block size */
1079               bs = 1;
1080               break;
1081             }
1082           }
1083         }
1084         /* Must have same blocksize on all procs (some might have no points) */
1085         bsLocal = bs;
1086         ierr = MPI_Allreduce(&bsLocal, &bsMax, 1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
1087         bsLocal = bs < 0 ? bsMax : bs;
1088         ierr = MPI_Allreduce(&bsLocal, &bsMin, 1, MPIU_INT, MPI_MIN, PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
1089         if (bsMin != bsMax) {
1090           bs = 1;
1091         } else {
1092           bs = bsMax;
1093         }
1094       } else {
1095         bs = 1;
1096       }
1097     }
1098     ierr = PetscCalloc4(localSize/bs, &dnz, localSize/bs, &onz, localSize/bs, &dnzu, localSize/bs, &onzu);CHKERRQ(ierr);
1099     ierr = DMPlexPreallocateOperator(dm, bs, section, sectionGlobal, dnz, onz, dnzu, onzu, *J, fillMatrix);CHKERRQ(ierr);
1100     ierr = PetscFree4(dnz, onz, dnzu, onzu);CHKERRQ(ierr);
1101   }
1102   PetscFunctionReturn(0);
1103 }
1104 
1105 #undef __FUNCT__
1106 #define __FUNCT__ "DMPlexGetDimension"
1107 /*@
1108   DMPlexGetDimension - Return the topological mesh dimension
1109 
1110   Not collective
1111 
1112   Input Parameter:
1113 . mesh - The DMPlex
1114 
1115   Output Parameter:
1116 . dim - The topological mesh dimension
1117 
1118   Level: beginner
1119 
1120 .seealso: DMPlexCreate()
1121 @*/
1122 PetscErrorCode DMPlexGetDimension(DM dm, PetscInt *dim)
1123 {
1124   DM_Plex *mesh = (DM_Plex*) dm->data;
1125 
1126   PetscFunctionBegin;
1127   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1128   PetscValidPointer(dim, 2);
1129   *dim = mesh->dim;
1130   PetscFunctionReturn(0);
1131 }
1132 
1133 #undef __FUNCT__
1134 #define __FUNCT__ "DMPlexSetDimension"
1135 /*@
1136   DMPlexSetDimension - Set the topological mesh dimension
1137 
1138   Collective on mesh
1139 
1140   Input Parameters:
1141 + mesh - The DMPlex
1142 - dim - The topological mesh dimension
1143 
1144   Level: beginner
1145 
1146 .seealso: DMPlexCreate()
1147 @*/
1148 PetscErrorCode DMPlexSetDimension(DM dm, PetscInt dim)
1149 {
1150   DM_Plex *mesh = (DM_Plex*) dm->data;
1151 
1152   PetscFunctionBegin;
1153   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1154   PetscValidLogicalCollectiveInt(dm, dim, 2);
1155   mesh->dim = dim;
1156   PetscFunctionReturn(0);
1157 }
1158 
1159 #undef __FUNCT__
1160 #define __FUNCT__ "DMPlexGetChart"
1161 /*@
1162   DMPlexGetChart - Return the interval for all mesh points [pStart, pEnd)
1163 
1164   Not collective
1165 
1166   Input Parameter:
1167 . mesh - The DMPlex
1168 
1169   Output Parameters:
1170 + pStart - The first mesh point
1171 - pEnd   - The upper bound for mesh points
1172 
1173   Level: beginner
1174 
1175 .seealso: DMPlexCreate(), DMPlexSetChart()
1176 @*/
1177 PetscErrorCode DMPlexGetChart(DM dm, PetscInt *pStart, PetscInt *pEnd)
1178 {
1179   DM_Plex       *mesh = (DM_Plex*) dm->data;
1180   PetscErrorCode ierr;
1181 
1182   PetscFunctionBegin;
1183   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1184   ierr = PetscSectionGetChart(mesh->coneSection, pStart, pEnd);CHKERRQ(ierr);
1185   PetscFunctionReturn(0);
1186 }
1187 
1188 #undef __FUNCT__
1189 #define __FUNCT__ "DMPlexSetChart"
1190 /*@
1191   DMPlexSetChart - Set the interval for all mesh points [pStart, pEnd)
1192 
1193   Not collective
1194 
1195   Input Parameters:
1196 + mesh - The DMPlex
1197 . pStart - The first mesh point
1198 - pEnd   - The upper bound for mesh points
1199 
1200   Output Parameters:
1201 
1202   Level: beginner
1203 
1204 .seealso: DMPlexCreate(), DMPlexGetChart()
1205 @*/
1206 PetscErrorCode DMPlexSetChart(DM dm, PetscInt pStart, PetscInt pEnd)
1207 {
1208   DM_Plex       *mesh = (DM_Plex*) dm->data;
1209   PetscErrorCode ierr;
1210 
1211   PetscFunctionBegin;
1212   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1213   ierr = PetscSectionSetChart(mesh->coneSection, pStart, pEnd);CHKERRQ(ierr);
1214   ierr = PetscSectionSetChart(mesh->supportSection, pStart, pEnd);CHKERRQ(ierr);
1215   PetscFunctionReturn(0);
1216 }
1217 
1218 #undef __FUNCT__
1219 #define __FUNCT__ "DMPlexGetConeSize"
1220 /*@
1221   DMPlexGetConeSize - Return the number of in-edges for this point in the Sieve DAG
1222 
1223   Not collective
1224 
1225   Input Parameters:
1226 + mesh - The DMPlex
1227 - p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1228 
1229   Output Parameter:
1230 . size - The cone size for point p
1231 
1232   Level: beginner
1233 
1234 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart()
1235 @*/
1236 PetscErrorCode DMPlexGetConeSize(DM dm, PetscInt p, PetscInt *size)
1237 {
1238   DM_Plex       *mesh = (DM_Plex*) dm->data;
1239   PetscErrorCode ierr;
1240 
1241   PetscFunctionBegin;
1242   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1243   PetscValidPointer(size, 3);
1244   ierr = PetscSectionGetDof(mesh->coneSection, p, size);CHKERRQ(ierr);
1245   PetscFunctionReturn(0);
1246 }
1247 
1248 #undef __FUNCT__
1249 #define __FUNCT__ "DMPlexSetConeSize"
1250 /*@
1251   DMPlexSetConeSize - Set the number of in-edges for this point in the Sieve DAG
1252 
1253   Not collective
1254 
1255   Input Parameters:
1256 + mesh - The DMPlex
1257 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1258 - size - The cone size for point p
1259 
1260   Output Parameter:
1261 
1262   Note:
1263   This should be called after DMPlexSetChart().
1264 
1265   Level: beginner
1266 
1267 .seealso: DMPlexCreate(), DMPlexGetConeSize(), DMPlexSetChart()
1268 @*/
1269 PetscErrorCode DMPlexSetConeSize(DM dm, PetscInt p, PetscInt size)
1270 {
1271   DM_Plex       *mesh = (DM_Plex*) dm->data;
1272   PetscErrorCode ierr;
1273 
1274   PetscFunctionBegin;
1275   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1276   ierr = PetscSectionSetDof(mesh->coneSection, p, size);CHKERRQ(ierr);
1277 
1278   mesh->maxConeSize = PetscMax(mesh->maxConeSize, size);
1279   PetscFunctionReturn(0);
1280 }
1281 
1282 #undef __FUNCT__
1283 #define __FUNCT__ "DMPlexGetCone"
1284 /*@C
1285   DMPlexGetCone - Return the points on the in-edges for this point in the Sieve DAG
1286 
1287   Not collective
1288 
1289   Input Parameters:
1290 + mesh - The DMPlex
1291 - p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1292 
1293   Output Parameter:
1294 . cone - An array of points which are on the in-edges for point p
1295 
1296   Level: beginner
1297 
1298   Fortran Notes:
1299   Since it returns an array, this routine is only available in Fortran 90, and you must
1300   include petsc.h90 in your code.
1301 
1302   You must also call DMPlexRestoreCone() after you finish using the returned array.
1303 
1304 .seealso: DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart()
1305 @*/
1306 PetscErrorCode DMPlexGetCone(DM dm, PetscInt p, const PetscInt *cone[])
1307 {
1308   DM_Plex       *mesh = (DM_Plex*) dm->data;
1309   PetscInt       off;
1310   PetscErrorCode ierr;
1311 
1312   PetscFunctionBegin;
1313   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1314   PetscValidPointer(cone, 3);
1315   ierr  = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1316   *cone = &mesh->cones[off];
1317   PetscFunctionReturn(0);
1318 }
1319 
1320 #undef __FUNCT__
1321 #define __FUNCT__ "DMPlexSetCone"
1322 /*@
1323   DMPlexSetCone - Set the points on the in-edges for this point in the Sieve DAG
1324 
1325   Not collective
1326 
1327   Input Parameters:
1328 + mesh - The DMPlex
1329 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1330 - cone - An array of points which are on the in-edges for point p
1331 
1332   Output Parameter:
1333 
1334   Note:
1335   This should be called after all calls to DMPlexSetConeSize() and DMSetUp().
1336 
1337   Level: beginner
1338 
1339 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
1340 @*/
1341 PetscErrorCode DMPlexSetCone(DM dm, PetscInt p, const PetscInt cone[])
1342 {
1343   DM_Plex       *mesh = (DM_Plex*) dm->data;
1344   PetscInt       pStart, pEnd;
1345   PetscInt       dof, off, c;
1346   PetscErrorCode ierr;
1347 
1348   PetscFunctionBegin;
1349   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1350   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
1351   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
1352   if (dof) PetscValidPointer(cone, 3);
1353   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1354   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
1355   for (c = 0; c < dof; ++c) {
1356     if ((cone[c] < pStart) || (cone[c] >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone point %D is not in the valid range [%D, %D)", cone[c], pStart, pEnd);
1357     mesh->cones[off+c] = cone[c];
1358   }
1359   PetscFunctionReturn(0);
1360 }
1361 
1362 #undef __FUNCT__
1363 #define __FUNCT__ "DMPlexGetConeOrientation"
1364 /*@C
1365   DMPlexGetConeOrientation - Return the orientations on the in-edges for this point in the Sieve DAG
1366 
1367   Not collective
1368 
1369   Input Parameters:
1370 + mesh - The DMPlex
1371 - p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1372 
1373   Output Parameter:
1374 . coneOrientation - An array of orientations which are on the in-edges for point p. An orientation is an
1375                     integer giving the prescription for cone traversal. If it is negative, the cone is
1376                     traversed in the opposite direction. Its value 'o', or if negative '-(o+1)', gives
1377                     the index of the cone point on which to start.
1378 
1379   Level: beginner
1380 
1381   Fortran Notes:
1382   Since it returns an array, this routine is only available in Fortran 90, and you must
1383   include petsc.h90 in your code.
1384 
1385   You must also call DMPlexRestoreConeOrientation() after you finish using the returned array.
1386 
1387 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetCone(), DMPlexSetChart()
1388 @*/
1389 PetscErrorCode DMPlexGetConeOrientation(DM dm, PetscInt p, const PetscInt *coneOrientation[])
1390 {
1391   DM_Plex       *mesh = (DM_Plex*) dm->data;
1392   PetscInt       off;
1393   PetscErrorCode ierr;
1394 
1395   PetscFunctionBegin;
1396   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1397 #if defined(PETSC_USE_DEBUG)
1398   {
1399     PetscInt dof;
1400     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
1401     if (dof) PetscValidPointer(coneOrientation, 3);
1402   }
1403 #endif
1404   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1405 
1406   *coneOrientation = &mesh->coneOrientations[off];
1407   PetscFunctionReturn(0);
1408 }
1409 
1410 #undef __FUNCT__
1411 #define __FUNCT__ "DMPlexSetConeOrientation"
1412 /*@
1413   DMPlexSetConeOrientation - Set the orientations on the in-edges for this point in the Sieve DAG
1414 
1415   Not collective
1416 
1417   Input Parameters:
1418 + mesh - The DMPlex
1419 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1420 - coneOrientation - An array of orientations which are on the in-edges for point p. An orientation is an
1421                     integer giving the prescription for cone traversal. If it is negative, the cone is
1422                     traversed in the opposite direction. Its value 'o', or if negative '-(o+1)', gives
1423                     the index of the cone point on which to start.
1424 
1425   Output Parameter:
1426 
1427   Note:
1428   This should be called after all calls to DMPlexSetConeSize() and DMSetUp().
1429 
1430   Level: beginner
1431 
1432 .seealso: DMPlexCreate(), DMPlexGetConeOrientation(), DMPlexSetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
1433 @*/
1434 PetscErrorCode DMPlexSetConeOrientation(DM dm, PetscInt p, const PetscInt coneOrientation[])
1435 {
1436   DM_Plex       *mesh = (DM_Plex*) dm->data;
1437   PetscInt       pStart, pEnd;
1438   PetscInt       dof, off, c;
1439   PetscErrorCode ierr;
1440 
1441   PetscFunctionBegin;
1442   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1443   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
1444   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
1445   if (dof) PetscValidPointer(coneOrientation, 3);
1446   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1447   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
1448   for (c = 0; c < dof; ++c) {
1449     PetscInt cdof, o = coneOrientation[c];
1450 
1451     ierr = PetscSectionGetDof(mesh->coneSection, mesh->cones[off+c], &cdof);CHKERRQ(ierr);
1452     if (o && ((o < -(cdof+1)) || (o >= cdof))) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone orientation %D is not in the valid range [%D. %D)", o, -(cdof+1), cdof);
1453     mesh->coneOrientations[off+c] = o;
1454   }
1455   PetscFunctionReturn(0);
1456 }
1457 
1458 #undef __FUNCT__
1459 #define __FUNCT__ "DMPlexInsertCone"
1460 PetscErrorCode DMPlexInsertCone(DM dm, PetscInt p, PetscInt conePos, PetscInt conePoint)
1461 {
1462   DM_Plex       *mesh = (DM_Plex*) dm->data;
1463   PetscInt       pStart, pEnd;
1464   PetscInt       dof, off;
1465   PetscErrorCode ierr;
1466 
1467   PetscFunctionBegin;
1468   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1469   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
1470   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
1471   if ((conePoint < pStart) || (conePoint >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone point %D is not in the valid range [%D, %D)", conePoint, pStart, pEnd);
1472   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
1473   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1474   if ((conePos < 0) || (conePos >= dof)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone position %D of point %D is not in the valid range [0, %D)", conePos, p, dof);
1475   mesh->cones[off+conePos] = conePoint;
1476   PetscFunctionReturn(0);
1477 }
1478 
1479 #undef __FUNCT__
1480 #define __FUNCT__ "DMPlexInsertConeOrientation"
1481 PetscErrorCode DMPlexInsertConeOrientation(DM dm, PetscInt p, PetscInt conePos, PetscInt coneOrientation)
1482 {
1483   DM_Plex       *mesh = (DM_Plex*) dm->data;
1484   PetscInt       pStart, pEnd;
1485   PetscInt       dof, off;
1486   PetscErrorCode ierr;
1487 
1488   PetscFunctionBegin;
1489   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1490   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
1491   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
1492   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
1493   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1494   if ((conePos < 0) || (conePos >= dof)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone position %D of point %D is not in the valid range [0, %D)", conePos, p, dof);
1495   mesh->coneOrientations[off+conePos] = coneOrientation;
1496   PetscFunctionReturn(0);
1497 }
1498 
1499 #undef __FUNCT__
1500 #define __FUNCT__ "DMPlexGetSupportSize"
1501 /*@
1502   DMPlexGetSupportSize - Return the number of out-edges for this point in the Sieve DAG
1503 
1504   Not collective
1505 
1506   Input Parameters:
1507 + mesh - The DMPlex
1508 - p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1509 
1510   Output Parameter:
1511 . size - The support size for point p
1512 
1513   Level: beginner
1514 
1515 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart(), DMPlexGetConeSize()
1516 @*/
1517 PetscErrorCode DMPlexGetSupportSize(DM dm, PetscInt p, PetscInt *size)
1518 {
1519   DM_Plex       *mesh = (DM_Plex*) dm->data;
1520   PetscErrorCode ierr;
1521 
1522   PetscFunctionBegin;
1523   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1524   PetscValidPointer(size, 3);
1525   ierr = PetscSectionGetDof(mesh->supportSection, p, size);CHKERRQ(ierr);
1526   PetscFunctionReturn(0);
1527 }
1528 
1529 #undef __FUNCT__
1530 #define __FUNCT__ "DMPlexSetSupportSize"
1531 /*@
1532   DMPlexSetSupportSize - Set the number of out-edges for this point in the Sieve DAG
1533 
1534   Not collective
1535 
1536   Input Parameters:
1537 + mesh - The DMPlex
1538 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1539 - size - The support size for point p
1540 
1541   Output Parameter:
1542 
1543   Note:
1544   This should be called after DMPlexSetChart().
1545 
1546   Level: beginner
1547 
1548 .seealso: DMPlexCreate(), DMPlexGetSupportSize(), DMPlexSetChart()
1549 @*/
1550 PetscErrorCode DMPlexSetSupportSize(DM dm, PetscInt p, PetscInt size)
1551 {
1552   DM_Plex       *mesh = (DM_Plex*) dm->data;
1553   PetscErrorCode ierr;
1554 
1555   PetscFunctionBegin;
1556   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1557   ierr = PetscSectionSetDof(mesh->supportSection, p, size);CHKERRQ(ierr);
1558 
1559   mesh->maxSupportSize = PetscMax(mesh->maxSupportSize, size);
1560   PetscFunctionReturn(0);
1561 }
1562 
1563 #undef __FUNCT__
1564 #define __FUNCT__ "DMPlexGetSupport"
1565 /*@C
1566   DMPlexGetSupport - Return the points on the out-edges for this point in the Sieve DAG
1567 
1568   Not collective
1569 
1570   Input Parameters:
1571 + mesh - The DMPlex
1572 - p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1573 
1574   Output Parameter:
1575 . support - An array of points which are on the out-edges for point p
1576 
1577   Level: beginner
1578 
1579   Fortran Notes:
1580   Since it returns an array, this routine is only available in Fortran 90, and you must
1581   include petsc.h90 in your code.
1582 
1583   You must also call DMPlexRestoreSupport() after you finish using the returned array.
1584 
1585 .seealso: DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
1586 @*/
1587 PetscErrorCode DMPlexGetSupport(DM dm, PetscInt p, const PetscInt *support[])
1588 {
1589   DM_Plex       *mesh = (DM_Plex*) dm->data;
1590   PetscInt       off;
1591   PetscErrorCode ierr;
1592 
1593   PetscFunctionBegin;
1594   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1595   PetscValidPointer(support, 3);
1596   ierr     = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
1597   *support = &mesh->supports[off];
1598   PetscFunctionReturn(0);
1599 }
1600 
1601 #undef __FUNCT__
1602 #define __FUNCT__ "DMPlexSetSupport"
1603 /*@
1604   DMPlexSetSupport - Set the points on the out-edges for this point in the Sieve DAG
1605 
1606   Not collective
1607 
1608   Input Parameters:
1609 + mesh - The DMPlex
1610 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1611 - support - An array of points which are on the in-edges for point p
1612 
1613   Output Parameter:
1614 
1615   Note:
1616   This should be called after all calls to DMPlexSetSupportSize() and DMSetUp().
1617 
1618   Level: beginner
1619 
1620 .seealso: DMPlexCreate(), DMPlexGetSupport(), DMPlexSetChart(), DMPlexSetSupportSize(), DMSetUp()
1621 @*/
1622 PetscErrorCode DMPlexSetSupport(DM dm, PetscInt p, const PetscInt support[])
1623 {
1624   DM_Plex       *mesh = (DM_Plex*) dm->data;
1625   PetscInt       pStart, pEnd;
1626   PetscInt       dof, off, c;
1627   PetscErrorCode ierr;
1628 
1629   PetscFunctionBegin;
1630   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1631   ierr = PetscSectionGetChart(mesh->supportSection, &pStart, &pEnd);CHKERRQ(ierr);
1632   ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
1633   if (dof) PetscValidPointer(support, 3);
1634   ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
1635   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
1636   for (c = 0; c < dof; ++c) {
1637     if ((support[c] < pStart) || (support[c] >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support point %D is not in the valid range [%D, %D)", support[c], pStart, pEnd);
1638     mesh->supports[off+c] = support[c];
1639   }
1640   PetscFunctionReturn(0);
1641 }
1642 
1643 #undef __FUNCT__
1644 #define __FUNCT__ "DMPlexInsertSupport"
1645 PetscErrorCode DMPlexInsertSupport(DM dm, PetscInt p, PetscInt supportPos, PetscInt supportPoint)
1646 {
1647   DM_Plex       *mesh = (DM_Plex*) dm->data;
1648   PetscInt       pStart, pEnd;
1649   PetscInt       dof, off;
1650   PetscErrorCode ierr;
1651 
1652   PetscFunctionBegin;
1653   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1654   ierr = PetscSectionGetChart(mesh->supportSection, &pStart, &pEnd);CHKERRQ(ierr);
1655   ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
1656   ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
1657   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
1658   if ((supportPoint < pStart) || (supportPoint >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support point %D is not in the valid range [%D, %D)", supportPoint, pStart, pEnd);
1659   if (supportPos >= dof) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support position %D of point %D is not in the valid range [0, %D)", supportPos, p, dof);
1660   mesh->supports[off+supportPos] = supportPoint;
1661   PetscFunctionReturn(0);
1662 }
1663 
1664 #undef __FUNCT__
1665 #define __FUNCT__ "DMPlexGetTransitiveClosure"
1666 /*@C
1667   DMPlexGetTransitiveClosure - Return the points on the transitive closure of the in-edges or out-edges for this point in the Sieve DAG
1668 
1669   Not collective
1670 
1671   Input Parameters:
1672 + mesh - The DMPlex
1673 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1674 . useCone - PETSC_TRUE for in-edges,  otherwise use out-edges
1675 - points - If points is NULL on input, internal storage will be returned, otherwise the provided array is used
1676 
1677   Output Parameters:
1678 + numPoints - The number of points in the closure, so points[] is of size 2*numPoints
1679 - points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...]
1680 
1681   Note:
1682   If using internal storage (points is NULL on input), each call overwrites the last output.
1683 
1684   Fortran Notes:
1685   Since it returns an array, this routine is only available in Fortran 90, and you must
1686   include petsc.h90 in your code.
1687 
1688   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
1689 
1690   Level: beginner
1691 
1692 .seealso: DMPlexRestoreTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
1693 @*/
1694 PetscErrorCode DMPlexGetTransitiveClosure(DM dm, PetscInt p, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
1695 {
1696   DM_Plex        *mesh = (DM_Plex*) dm->data;
1697   PetscInt       *closure, *fifo;
1698   const PetscInt *tmp = NULL, *tmpO = NULL;
1699   PetscInt        tmpSize, t;
1700   PetscInt        depth       = 0, maxSize;
1701   PetscInt        closureSize = 2, fifoSize = 0, fifoStart = 0;
1702   PetscErrorCode  ierr;
1703 
1704   PetscFunctionBegin;
1705   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1706   ierr    = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
1707   /* This is only 1-level */
1708   if (useCone) {
1709     ierr = DMPlexGetConeSize(dm, p, &tmpSize);CHKERRQ(ierr);
1710     ierr = DMPlexGetCone(dm, p, &tmp);CHKERRQ(ierr);
1711     ierr = DMPlexGetConeOrientation(dm, p, &tmpO);CHKERRQ(ierr);
1712   } else {
1713     ierr = DMPlexGetSupportSize(dm, p, &tmpSize);CHKERRQ(ierr);
1714     ierr = DMPlexGetSupport(dm, p, &tmp);CHKERRQ(ierr);
1715   }
1716   if (depth == 1) {
1717     if (*points) {
1718       closure = *points;
1719     } else {
1720       maxSize = 2*(PetscMax(mesh->maxConeSize, mesh->maxSupportSize)+1);
1721       ierr = DMGetWorkArray(dm, maxSize, PETSC_INT, &closure);CHKERRQ(ierr);
1722     }
1723     closure[0] = p; closure[1] = 0;
1724     for (t = 0; t < tmpSize; ++t, closureSize += 2) {
1725       closure[closureSize]   = tmp[t];
1726       closure[closureSize+1] = tmpO ? tmpO[t] : 0;
1727     }
1728     if (numPoints) *numPoints = closureSize/2;
1729     if (points)    *points    = closure;
1730     PetscFunctionReturn(0);
1731   }
1732   maxSize = 2*PetscMax(PetscMax(PetscPowInt(mesh->maxConeSize,depth+1),PetscPowInt(mesh->maxSupportSize,depth+1)),depth+1);
1733   ierr    = DMGetWorkArray(dm, maxSize, PETSC_INT, &fifo);CHKERRQ(ierr);
1734   if (*points) {
1735     closure = *points;
1736   } else {
1737     ierr = DMGetWorkArray(dm, maxSize, PETSC_INT, &closure);CHKERRQ(ierr);
1738   }
1739   closure[0] = p; closure[1] = 0;
1740   for (t = 0; t < tmpSize; ++t, closureSize += 2, fifoSize += 2) {
1741     const PetscInt cp = tmp[t];
1742     const PetscInt co = tmpO ? tmpO[t] : 0;
1743 
1744     closure[closureSize]   = cp;
1745     closure[closureSize+1] = co;
1746     fifo[fifoSize]         = cp;
1747     fifo[fifoSize+1]       = co;
1748   }
1749   /* Should kick out early when depth is reached, rather than checking all vertices for empty cones */
1750   while (fifoSize - fifoStart) {
1751     const PetscInt q   = fifo[fifoStart];
1752     const PetscInt o   = fifo[fifoStart+1];
1753     const PetscInt rev = o >= 0 ? 0 : 1;
1754     const PetscInt off = rev ? -(o+1) : o;
1755 
1756     if (useCone) {
1757       ierr = DMPlexGetConeSize(dm, q, &tmpSize);CHKERRQ(ierr);
1758       ierr = DMPlexGetCone(dm, q, &tmp);CHKERRQ(ierr);
1759       ierr = DMPlexGetConeOrientation(dm, q, &tmpO);CHKERRQ(ierr);
1760     } else {
1761       ierr = DMPlexGetSupportSize(dm, q, &tmpSize);CHKERRQ(ierr);
1762       ierr = DMPlexGetSupport(dm, q, &tmp);CHKERRQ(ierr);
1763       tmpO = NULL;
1764     }
1765     for (t = 0; t < tmpSize; ++t) {
1766       const PetscInt i  = ((rev ? tmpSize-t : t) + off)%tmpSize;
1767       const PetscInt cp = tmp[i];
1768       /* Must propogate orientation: When we reverse orientation, we both reverse the direction of iteration and start at the other end of the chain. */
1769       /* HACK: It is worse to get the size here, than to change the interpretation of -(*+1)
1770        const PetscInt co = tmpO ? (rev ? -(tmpO[i]+1) : tmpO[i]) : 0; */
1771       PetscInt       co = tmpO ? tmpO[i] : 0;
1772       PetscInt       c;
1773 
1774       if (rev) {
1775         PetscInt childSize, coff;
1776         ierr = DMPlexGetConeSize(dm, cp, &childSize);CHKERRQ(ierr);
1777         coff = tmpO[i] < 0 ? -(tmpO[i]+1) : tmpO[i];
1778         co   = childSize ? -(((coff+childSize-1)%childSize)+1) : 0;
1779       }
1780       /* Check for duplicate */
1781       for (c = 0; c < closureSize; c += 2) {
1782         if (closure[c] == cp) break;
1783       }
1784       if (c == closureSize) {
1785         closure[closureSize]   = cp;
1786         closure[closureSize+1] = co;
1787         fifo[fifoSize]         = cp;
1788         fifo[fifoSize+1]       = co;
1789         closureSize           += 2;
1790         fifoSize              += 2;
1791       }
1792     }
1793     fifoStart += 2;
1794   }
1795   if (numPoints) *numPoints = closureSize/2;
1796   if (points)    *points    = closure;
1797   ierr = DMRestoreWorkArray(dm, maxSize, PETSC_INT, &fifo);CHKERRQ(ierr);
1798   PetscFunctionReturn(0);
1799 }
1800 
1801 #undef __FUNCT__
1802 #define __FUNCT__ "DMPlexGetTransitiveClosure_Internal"
1803 /*@C
1804   DMPlexGetTransitiveClosure_Internal - Return the points on the transitive closure of the in-edges or out-edges for this point in the Sieve DAG with a specified initial orientation
1805 
1806   Not collective
1807 
1808   Input Parameters:
1809 + mesh - The DMPlex
1810 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1811 . orientation - The orientation of the point
1812 . useCone - PETSC_TRUE for in-edges,  otherwise use out-edges
1813 - points - If points is NULL on input, internal storage will be returned, otherwise the provided array is used
1814 
1815   Output Parameters:
1816 + numPoints - The number of points in the closure, so points[] is of size 2*numPoints
1817 - points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...]
1818 
1819   Note:
1820   If using internal storage (points is NULL on input), each call overwrites the last output.
1821 
1822   Fortran Notes:
1823   Since it returns an array, this routine is only available in Fortran 90, and you must
1824   include petsc.h90 in your code.
1825 
1826   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
1827 
1828   Level: beginner
1829 
1830 .seealso: DMPlexRestoreTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
1831 @*/
1832 PetscErrorCode DMPlexGetTransitiveClosure_Internal(DM dm, PetscInt p, PetscInt ornt, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
1833 {
1834   DM_Plex        *mesh = (DM_Plex*) dm->data;
1835   PetscInt       *closure, *fifo;
1836   const PetscInt *tmp = NULL, *tmpO = NULL;
1837   PetscInt        tmpSize, t;
1838   PetscInt        depth       = 0, maxSize;
1839   PetscInt        closureSize = 2, fifoSize = 0, fifoStart = 0;
1840   PetscErrorCode  ierr;
1841 
1842   PetscFunctionBegin;
1843   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1844   ierr    = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
1845   /* This is only 1-level */
1846   if (useCone) {
1847     ierr = DMPlexGetConeSize(dm, p, &tmpSize);CHKERRQ(ierr);
1848     ierr = DMPlexGetCone(dm, p, &tmp);CHKERRQ(ierr);
1849     ierr = DMPlexGetConeOrientation(dm, p, &tmpO);CHKERRQ(ierr);
1850   } else {
1851     ierr = DMPlexGetSupportSize(dm, p, &tmpSize);CHKERRQ(ierr);
1852     ierr = DMPlexGetSupport(dm, p, &tmp);CHKERRQ(ierr);
1853   }
1854   if (depth == 1) {
1855     if (*points) {
1856       closure = *points;
1857     } else {
1858       maxSize = 2*(PetscMax(mesh->maxConeSize, mesh->maxSupportSize)+1);
1859       ierr = DMGetWorkArray(dm, maxSize, PETSC_INT, &closure);CHKERRQ(ierr);
1860     }
1861     closure[0] = p; closure[1] = ornt;
1862     for (t = 0; t < tmpSize; ++t, closureSize += 2) {
1863       const PetscInt i = ornt >= 0 ? (t+ornt)%tmpSize : (-(ornt+1) + tmpSize-t)%tmpSize;
1864       closure[closureSize]   = tmp[i];
1865       closure[closureSize+1] = tmpO ? tmpO[i] : 0;
1866     }
1867     if (numPoints) *numPoints = closureSize/2;
1868     if (points)    *points    = closure;
1869     PetscFunctionReturn(0);
1870   }
1871   maxSize = 2*PetscMax(PetscMax(PetscPowInt(mesh->maxConeSize,depth+1),PetscPowInt(mesh->maxSupportSize,depth+1)),depth+1);
1872   ierr    = DMGetWorkArray(dm, maxSize, PETSC_INT, &fifo);CHKERRQ(ierr);
1873   if (*points) {
1874     closure = *points;
1875   } else {
1876     ierr = DMGetWorkArray(dm, maxSize, PETSC_INT, &closure);CHKERRQ(ierr);
1877   }
1878   closure[0] = p; closure[1] = ornt;
1879   for (t = 0; t < tmpSize; ++t, closureSize += 2, fifoSize += 2) {
1880     const PetscInt i  = ornt >= 0 ? (t+ornt)%tmpSize : (-(ornt+1) + tmpSize-t)%tmpSize;
1881     const PetscInt cp = tmp[i];
1882     PetscInt       co = tmpO ? tmpO[i] : 0;
1883 
1884     if (ornt < 0) {
1885       PetscInt childSize, coff;
1886       ierr = DMPlexGetConeSize(dm, cp, &childSize);CHKERRQ(ierr);
1887       coff = tmpO[i] < 0 ? -(tmpO[i]+1) : tmpO[i];
1888       co   = childSize ? -(((coff+childSize-1)%childSize)+1) : 0;
1889     }
1890     closure[closureSize]   = cp;
1891     closure[closureSize+1] = co;
1892     fifo[fifoSize]         = cp;
1893     fifo[fifoSize+1]       = co;
1894   }
1895   /* Should kick out early when depth is reached, rather than checking all vertices for empty cones */
1896   while (fifoSize - fifoStart) {
1897     const PetscInt q   = fifo[fifoStart];
1898     const PetscInt o   = fifo[fifoStart+1];
1899     const PetscInt rev = o >= 0 ? 0 : 1;
1900     const PetscInt off = rev ? -(o+1) : o;
1901 
1902     if (useCone) {
1903       ierr = DMPlexGetConeSize(dm, q, &tmpSize);CHKERRQ(ierr);
1904       ierr = DMPlexGetCone(dm, q, &tmp);CHKERRQ(ierr);
1905       ierr = DMPlexGetConeOrientation(dm, q, &tmpO);CHKERRQ(ierr);
1906     } else {
1907       ierr = DMPlexGetSupportSize(dm, q, &tmpSize);CHKERRQ(ierr);
1908       ierr = DMPlexGetSupport(dm, q, &tmp);CHKERRQ(ierr);
1909       tmpO = NULL;
1910     }
1911     for (t = 0; t < tmpSize; ++t) {
1912       const PetscInt i  = ((rev ? tmpSize-t : t) + off)%tmpSize;
1913       const PetscInt cp = tmp[i];
1914       /* Must propogate orientation: When we reverse orientation, we both reverse the direction of iteration and start at the other end of the chain. */
1915       /* HACK: It is worse to get the size here, than to change the interpretation of -(*+1)
1916        const PetscInt co = tmpO ? (rev ? -(tmpO[i]+1) : tmpO[i]) : 0; */
1917       PetscInt       co = tmpO ? tmpO[i] : 0;
1918       PetscInt       c;
1919 
1920       if (rev) {
1921         PetscInt childSize, coff;
1922         ierr = DMPlexGetConeSize(dm, cp, &childSize);CHKERRQ(ierr);
1923         coff = tmpO[i] < 0 ? -(tmpO[i]+1) : tmpO[i];
1924         co   = childSize ? -(((coff+childSize-1)%childSize)+1) : 0;
1925       }
1926       /* Check for duplicate */
1927       for (c = 0; c < closureSize; c += 2) {
1928         if (closure[c] == cp) break;
1929       }
1930       if (c == closureSize) {
1931         closure[closureSize]   = cp;
1932         closure[closureSize+1] = co;
1933         fifo[fifoSize]         = cp;
1934         fifo[fifoSize+1]       = co;
1935         closureSize           += 2;
1936         fifoSize              += 2;
1937       }
1938     }
1939     fifoStart += 2;
1940   }
1941   if (numPoints) *numPoints = closureSize/2;
1942   if (points)    *points    = closure;
1943   ierr = DMRestoreWorkArray(dm, maxSize, PETSC_INT, &fifo);CHKERRQ(ierr);
1944   PetscFunctionReturn(0);
1945 }
1946 
1947 #undef __FUNCT__
1948 #define __FUNCT__ "DMPlexRestoreTransitiveClosure"
1949 /*@C
1950   DMPlexRestoreTransitiveClosure - Restore the array of points on the transitive closure of the in-edges or out-edges for this point in the Sieve DAG
1951 
1952   Not collective
1953 
1954   Input Parameters:
1955 + mesh - The DMPlex
1956 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1957 . useCone - PETSC_TRUE for in-edges,  otherwise use out-edges
1958 . numPoints - The number of points in the closure, so points[] is of size 2*numPoints, zeroed on exit
1959 - points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...], zeroed on exit
1960 
1961   Note:
1962   If not using internal storage (points is not NULL on input), this call is unnecessary
1963 
1964   Fortran Notes:
1965   Since it returns an array, this routine is only available in Fortran 90, and you must
1966   include petsc.h90 in your code.
1967 
1968   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
1969 
1970   Level: beginner
1971 
1972 .seealso: DMPlexGetTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
1973 @*/
1974 PetscErrorCode DMPlexRestoreTransitiveClosure(DM dm, PetscInt p, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
1975 {
1976   PetscErrorCode ierr;
1977 
1978   PetscFunctionBegin;
1979   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1980   if (numPoints) PetscValidIntPointer(numPoints,4);
1981   if (points) PetscValidPointer(points,5);
1982   ierr = DMRestoreWorkArray(dm, 0, PETSC_INT, points);CHKERRQ(ierr);
1983   if (numPoints) *numPoints = 0;
1984   PetscFunctionReturn(0);
1985 }
1986 
1987 #undef __FUNCT__
1988 #define __FUNCT__ "DMPlexGetMaxSizes"
1989 /*@
1990   DMPlexGetMaxSizes - Return the maximum number of in-edges (cone) and out-edges (support) for any point in the Sieve DAG
1991 
1992   Not collective
1993 
1994   Input Parameter:
1995 . mesh - The DMPlex
1996 
1997   Output Parameters:
1998 + maxConeSize - The maximum number of in-edges
1999 - maxSupportSize - The maximum number of out-edges
2000 
2001   Level: beginner
2002 
2003 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart()
2004 @*/
2005 PetscErrorCode DMPlexGetMaxSizes(DM dm, PetscInt *maxConeSize, PetscInt *maxSupportSize)
2006 {
2007   DM_Plex *mesh = (DM_Plex*) dm->data;
2008 
2009   PetscFunctionBegin;
2010   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2011   if (maxConeSize)    *maxConeSize    = mesh->maxConeSize;
2012   if (maxSupportSize) *maxSupportSize = mesh->maxSupportSize;
2013   PetscFunctionReturn(0);
2014 }
2015 
2016 #undef __FUNCT__
2017 #define __FUNCT__ "DMSetUp_Plex"
2018 PetscErrorCode DMSetUp_Plex(DM dm)
2019 {
2020   DM_Plex       *mesh = (DM_Plex*) dm->data;
2021   PetscInt       size;
2022   PetscErrorCode ierr;
2023 
2024   PetscFunctionBegin;
2025   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2026   ierr = PetscSectionSetUp(mesh->coneSection);CHKERRQ(ierr);
2027   ierr = PetscSectionGetStorageSize(mesh->coneSection, &size);CHKERRQ(ierr);
2028   ierr = PetscMalloc1(size, &mesh->cones);CHKERRQ(ierr);
2029   ierr = PetscCalloc1(size, &mesh->coneOrientations);CHKERRQ(ierr);
2030   if (mesh->maxSupportSize) {
2031     ierr = PetscSectionSetUp(mesh->supportSection);CHKERRQ(ierr);
2032     ierr = PetscSectionGetStorageSize(mesh->supportSection, &size);CHKERRQ(ierr);
2033     ierr = PetscMalloc1(size, &mesh->supports);CHKERRQ(ierr);
2034   }
2035   PetscFunctionReturn(0);
2036 }
2037 
2038 #undef __FUNCT__
2039 #define __FUNCT__ "DMCreateSubDM_Plex"
2040 PetscErrorCode DMCreateSubDM_Plex(DM dm, PetscInt numFields, PetscInt fields[], IS *is, DM *subdm)
2041 {
2042   PetscErrorCode ierr;
2043 
2044   PetscFunctionBegin;
2045   if (subdm) {ierr = DMClone(dm, subdm);CHKERRQ(ierr);}
2046   ierr = DMCreateSubDM_Section_Private(dm, numFields, fields, is, subdm);CHKERRQ(ierr);
2047   PetscFunctionReturn(0);
2048 }
2049 
2050 #undef __FUNCT__
2051 #define __FUNCT__ "DMPlexSymmetrize"
2052 /*@
2053   DMPlexSymmetrize - Creates support (out-edge) information from cone (in-edge) inoformation
2054 
2055   Not collective
2056 
2057   Input Parameter:
2058 . mesh - The DMPlex
2059 
2060   Output Parameter:
2061 
2062   Note:
2063   This should be called after all calls to DMPlexSetCone()
2064 
2065   Level: beginner
2066 
2067 .seealso: DMPlexCreate(), DMPlexSetChart(), DMPlexSetConeSize(), DMPlexSetCone()
2068 @*/
2069 PetscErrorCode DMPlexSymmetrize(DM dm)
2070 {
2071   DM_Plex       *mesh = (DM_Plex*) dm->data;
2072   PetscInt      *offsets;
2073   PetscInt       supportSize;
2074   PetscInt       pStart, pEnd, p;
2075   PetscErrorCode ierr;
2076 
2077   PetscFunctionBegin;
2078   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2079   if (mesh->supports) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONGSTATE, "Supports were already setup in this DMPlex");
2080   /* Calculate support sizes */
2081   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
2082   for (p = pStart; p < pEnd; ++p) {
2083     PetscInt dof, off, c;
2084 
2085     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2086     ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2087     for (c = off; c < off+dof; ++c) {
2088       ierr = PetscSectionAddDof(mesh->supportSection, mesh->cones[c], 1);CHKERRQ(ierr);
2089     }
2090   }
2091   for (p = pStart; p < pEnd; ++p) {
2092     PetscInt dof;
2093 
2094     ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
2095 
2096     mesh->maxSupportSize = PetscMax(mesh->maxSupportSize, dof);
2097   }
2098   ierr = PetscSectionSetUp(mesh->supportSection);CHKERRQ(ierr);
2099   /* Calculate supports */
2100   ierr = PetscSectionGetStorageSize(mesh->supportSection, &supportSize);CHKERRQ(ierr);
2101   ierr = PetscMalloc1(supportSize, &mesh->supports);CHKERRQ(ierr);
2102   ierr = PetscCalloc1(pEnd - pStart, &offsets);CHKERRQ(ierr);
2103   for (p = pStart; p < pEnd; ++p) {
2104     PetscInt dof, off, c;
2105 
2106     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2107     ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2108     for (c = off; c < off+dof; ++c) {
2109       const PetscInt q = mesh->cones[c];
2110       PetscInt       offS;
2111 
2112       ierr = PetscSectionGetOffset(mesh->supportSection, q, &offS);CHKERRQ(ierr);
2113 
2114       mesh->supports[offS+offsets[q]] = p;
2115       ++offsets[q];
2116     }
2117   }
2118   ierr = PetscFree(offsets);CHKERRQ(ierr);
2119   PetscFunctionReturn(0);
2120 }
2121 
2122 #undef __FUNCT__
2123 #define __FUNCT__ "DMPlexStratify"
2124 /*@
2125   DMPlexStratify - The Sieve DAG for most topologies is a graded poset (http://en.wikipedia.org/wiki/Graded_poset), and
2126   can be illustrated by Hasse Diagram (a http://en.wikipedia.org/wiki/Hasse_diagram). The strata group all points of the
2127   same grade, and this function calculates the strata. This grade can be seen as the height (or depth) of the point in
2128   the DAG.
2129 
2130   Not collective
2131 
2132   Input Parameter:
2133 . mesh - The DMPlex
2134 
2135   Output Parameter:
2136 
2137   Notes:
2138   The normal association for the point grade is element dimension (or co-dimension). For instance, all vertices would
2139   have depth 0, and all edges depth 1. Likewise, all cells heights would have height 0, and all faces height 1.
2140 
2141   This should be called after all calls to DMPlexSymmetrize()
2142 
2143   Level: beginner
2144 
2145 .seealso: DMPlexCreate(), DMPlexSymmetrize()
2146 @*/
2147 PetscErrorCode DMPlexStratify(DM dm)
2148 {
2149   DMLabel        label;
2150   PetscInt       pStart, pEnd, p;
2151   PetscInt       numRoots = 0, numLeaves = 0;
2152   PetscErrorCode ierr;
2153 
2154   PetscFunctionBegin;
2155   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2156   ierr = PetscLogEventBegin(DMPLEX_Stratify,dm,0,0,0);CHKERRQ(ierr);
2157   /* Calculate depth */
2158   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
2159   ierr = DMPlexCreateLabel(dm, "depth");CHKERRQ(ierr);
2160   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
2161   /* Initialize roots and count leaves */
2162   for (p = pStart; p < pEnd; ++p) {
2163     PetscInt coneSize, supportSize;
2164 
2165     ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
2166     ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
2167     if (!coneSize && supportSize) {
2168       ++numRoots;
2169       ierr = DMLabelSetValue(label, p, 0);CHKERRQ(ierr);
2170     } else if (!supportSize && coneSize) {
2171       ++numLeaves;
2172     } else if (!supportSize && !coneSize) {
2173       /* Isolated points */
2174       ierr = DMLabelSetValue(label, p, 0);CHKERRQ(ierr);
2175     }
2176   }
2177   if (numRoots + numLeaves == (pEnd - pStart)) {
2178     for (p = pStart; p < pEnd; ++p) {
2179       PetscInt coneSize, supportSize;
2180 
2181       ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
2182       ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
2183       if (!supportSize && coneSize) {
2184         ierr = DMLabelSetValue(label, p, 1);CHKERRQ(ierr);
2185       }
2186     }
2187   } else {
2188     IS       pointIS;
2189     PetscInt numPoints = 0, level = 0;
2190 
2191     ierr = DMLabelGetStratumIS(label, level, &pointIS);CHKERRQ(ierr);
2192     if (pointIS) {ierr = ISGetLocalSize(pointIS, &numPoints);CHKERRQ(ierr);}
2193     while (numPoints) {
2194       const PetscInt *points;
2195       const PetscInt  newLevel = level+1;
2196 
2197       ierr = ISGetIndices(pointIS, &points);CHKERRQ(ierr);
2198       for (p = 0; p < numPoints; ++p) {
2199         const PetscInt  point = points[p];
2200         const PetscInt *support;
2201         PetscInt        supportSize, s;
2202 
2203         ierr = DMPlexGetSupportSize(dm, point, &supportSize);CHKERRQ(ierr);
2204         ierr = DMPlexGetSupport(dm, point, &support);CHKERRQ(ierr);
2205         for (s = 0; s < supportSize; ++s) {
2206           ierr = DMLabelSetValue(label, support[s], newLevel);CHKERRQ(ierr);
2207         }
2208       }
2209       ierr = ISRestoreIndices(pointIS, &points);CHKERRQ(ierr);
2210       ++level;
2211       ierr = ISDestroy(&pointIS);CHKERRQ(ierr);
2212       ierr = DMLabelGetStratumIS(label, level, &pointIS);CHKERRQ(ierr);
2213       if (pointIS) {ierr = ISGetLocalSize(pointIS, &numPoints);CHKERRQ(ierr);}
2214       else         {numPoints = 0;}
2215     }
2216     ierr = ISDestroy(&pointIS);CHKERRQ(ierr);
2217   }
2218   ierr = PetscLogEventEnd(DMPLEX_Stratify,dm,0,0,0);CHKERRQ(ierr);
2219   PetscFunctionReturn(0);
2220 }
2221 
2222 #undef __FUNCT__
2223 #define __FUNCT__ "DMPlexGetJoin"
2224 /*@C
2225   DMPlexGetJoin - Get an array for the join of the set of points
2226 
2227   Not Collective
2228 
2229   Input Parameters:
2230 + dm - The DMPlex object
2231 . numPoints - The number of input points for the join
2232 - points - The input points
2233 
2234   Output Parameters:
2235 + numCoveredPoints - The number of points in the join
2236 - coveredPoints - The points in the join
2237 
2238   Level: intermediate
2239 
2240   Note: Currently, this is restricted to a single level join
2241 
2242   Fortran Notes:
2243   Since it returns an array, this routine is only available in Fortran 90, and you must
2244   include petsc.h90 in your code.
2245 
2246   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2247 
2248 .keywords: mesh
2249 .seealso: DMPlexRestoreJoin(), DMPlexGetMeet()
2250 @*/
2251 PetscErrorCode DMPlexGetJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
2252 {
2253   DM_Plex       *mesh = (DM_Plex*) dm->data;
2254   PetscInt      *join[2];
2255   PetscInt       joinSize, i = 0;
2256   PetscInt       dof, off, p, c, m;
2257   PetscErrorCode ierr;
2258 
2259   PetscFunctionBegin;
2260   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2261   PetscValidPointer(points, 2);
2262   PetscValidPointer(numCoveredPoints, 3);
2263   PetscValidPointer(coveredPoints, 4);
2264   ierr = DMGetWorkArray(dm, mesh->maxSupportSize, PETSC_INT, &join[0]);CHKERRQ(ierr);
2265   ierr = DMGetWorkArray(dm, mesh->maxSupportSize, PETSC_INT, &join[1]);CHKERRQ(ierr);
2266   /* Copy in support of first point */
2267   ierr = PetscSectionGetDof(mesh->supportSection, points[0], &dof);CHKERRQ(ierr);
2268   ierr = PetscSectionGetOffset(mesh->supportSection, points[0], &off);CHKERRQ(ierr);
2269   for (joinSize = 0; joinSize < dof; ++joinSize) {
2270     join[i][joinSize] = mesh->supports[off+joinSize];
2271   }
2272   /* Check each successive support */
2273   for (p = 1; p < numPoints; ++p) {
2274     PetscInt newJoinSize = 0;
2275 
2276     ierr = PetscSectionGetDof(mesh->supportSection, points[p], &dof);CHKERRQ(ierr);
2277     ierr = PetscSectionGetOffset(mesh->supportSection, points[p], &off);CHKERRQ(ierr);
2278     for (c = 0; c < dof; ++c) {
2279       const PetscInt point = mesh->supports[off+c];
2280 
2281       for (m = 0; m < joinSize; ++m) {
2282         if (point == join[i][m]) {
2283           join[1-i][newJoinSize++] = point;
2284           break;
2285         }
2286       }
2287     }
2288     joinSize = newJoinSize;
2289     i        = 1-i;
2290   }
2291   *numCoveredPoints = joinSize;
2292   *coveredPoints    = join[i];
2293   ierr              = DMRestoreWorkArray(dm, mesh->maxSupportSize, PETSC_INT, &join[1-i]);CHKERRQ(ierr);
2294   PetscFunctionReturn(0);
2295 }
2296 
2297 #undef __FUNCT__
2298 #define __FUNCT__ "DMPlexRestoreJoin"
2299 /*@C
2300   DMPlexRestoreJoin - Restore an array for the join of the set of points
2301 
2302   Not Collective
2303 
2304   Input Parameters:
2305 + dm - The DMPlex object
2306 . numPoints - The number of input points for the join
2307 - points - The input points
2308 
2309   Output Parameters:
2310 + numCoveredPoints - The number of points in the join
2311 - coveredPoints - The points in the join
2312 
2313   Fortran Notes:
2314   Since it returns an array, this routine is only available in Fortran 90, and you must
2315   include petsc.h90 in your code.
2316 
2317   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2318 
2319   Level: intermediate
2320 
2321 .keywords: mesh
2322 .seealso: DMPlexGetJoin(), DMPlexGetFullJoin(), DMPlexGetMeet()
2323 @*/
2324 PetscErrorCode DMPlexRestoreJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
2325 {
2326   PetscErrorCode ierr;
2327 
2328   PetscFunctionBegin;
2329   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2330   if (points) PetscValidIntPointer(points,3);
2331   if (numCoveredPoints) PetscValidIntPointer(numCoveredPoints,4);
2332   PetscValidPointer(coveredPoints, 5);
2333   ierr = DMRestoreWorkArray(dm, 0, PETSC_INT, (void*) coveredPoints);CHKERRQ(ierr);
2334   if (numCoveredPoints) *numCoveredPoints = 0;
2335   PetscFunctionReturn(0);
2336 }
2337 
2338 #undef __FUNCT__
2339 #define __FUNCT__ "DMPlexGetFullJoin"
2340 /*@C
2341   DMPlexGetFullJoin - Get an array for the join of the set of points
2342 
2343   Not Collective
2344 
2345   Input Parameters:
2346 + dm - The DMPlex object
2347 . numPoints - The number of input points for the join
2348 - points - The input points
2349 
2350   Output Parameters:
2351 + numCoveredPoints - The number of points in the join
2352 - coveredPoints - The points in the join
2353 
2354   Fortran Notes:
2355   Since it returns an array, this routine is only available in Fortran 90, and you must
2356   include petsc.h90 in your code.
2357 
2358   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2359 
2360   Level: intermediate
2361 
2362 .keywords: mesh
2363 .seealso: DMPlexGetJoin(), DMPlexRestoreJoin(), DMPlexGetMeet()
2364 @*/
2365 PetscErrorCode DMPlexGetFullJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
2366 {
2367   DM_Plex       *mesh = (DM_Plex*) dm->data;
2368   PetscInt      *offsets, **closures;
2369   PetscInt      *join[2];
2370   PetscInt       depth = 0, maxSize, joinSize = 0, i = 0;
2371   PetscInt       p, d, c, m;
2372   PetscErrorCode ierr;
2373 
2374   PetscFunctionBegin;
2375   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2376   PetscValidPointer(points, 2);
2377   PetscValidPointer(numCoveredPoints, 3);
2378   PetscValidPointer(coveredPoints, 4);
2379 
2380   ierr    = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
2381   ierr    = PetscCalloc1(numPoints, &closures);CHKERRQ(ierr);
2382   ierr    = DMGetWorkArray(dm, numPoints*(depth+2), PETSC_INT, &offsets);CHKERRQ(ierr);
2383   maxSize = PetscPowInt(mesh->maxSupportSize,depth+1);
2384   ierr    = DMGetWorkArray(dm, maxSize, PETSC_INT, &join[0]);CHKERRQ(ierr);
2385   ierr    = DMGetWorkArray(dm, maxSize, PETSC_INT, &join[1]);CHKERRQ(ierr);
2386 
2387   for (p = 0; p < numPoints; ++p) {
2388     PetscInt closureSize;
2389 
2390     ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_FALSE, &closureSize, &closures[p]);CHKERRQ(ierr);
2391 
2392     offsets[p*(depth+2)+0] = 0;
2393     for (d = 0; d < depth+1; ++d) {
2394       PetscInt pStart, pEnd, i;
2395 
2396       ierr = DMPlexGetDepthStratum(dm, d, &pStart, &pEnd);CHKERRQ(ierr);
2397       for (i = offsets[p*(depth+2)+d]; i < closureSize; ++i) {
2398         if ((pStart > closures[p][i*2]) || (pEnd <= closures[p][i*2])) {
2399           offsets[p*(depth+2)+d+1] = i;
2400           break;
2401         }
2402       }
2403       if (i == closureSize) offsets[p*(depth+2)+d+1] = i;
2404     }
2405     if (offsets[p*(depth+2)+depth+1] != closureSize) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Total size of closure %D should be %D", offsets[p*(depth+2)+depth+1], closureSize);
2406   }
2407   for (d = 0; d < depth+1; ++d) {
2408     PetscInt dof;
2409 
2410     /* Copy in support of first point */
2411     dof = offsets[d+1] - offsets[d];
2412     for (joinSize = 0; joinSize < dof; ++joinSize) {
2413       join[i][joinSize] = closures[0][(offsets[d]+joinSize)*2];
2414     }
2415     /* Check each successive cone */
2416     for (p = 1; p < numPoints && joinSize; ++p) {
2417       PetscInt newJoinSize = 0;
2418 
2419       dof = offsets[p*(depth+2)+d+1] - offsets[p*(depth+2)+d];
2420       for (c = 0; c < dof; ++c) {
2421         const PetscInt point = closures[p][(offsets[p*(depth+2)+d]+c)*2];
2422 
2423         for (m = 0; m < joinSize; ++m) {
2424           if (point == join[i][m]) {
2425             join[1-i][newJoinSize++] = point;
2426             break;
2427           }
2428         }
2429       }
2430       joinSize = newJoinSize;
2431       i        = 1-i;
2432     }
2433     if (joinSize) break;
2434   }
2435   *numCoveredPoints = joinSize;
2436   *coveredPoints    = join[i];
2437   for (p = 0; p < numPoints; ++p) {
2438     ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_FALSE, NULL, &closures[p]);CHKERRQ(ierr);
2439   }
2440   ierr = PetscFree(closures);CHKERRQ(ierr);
2441   ierr = DMRestoreWorkArray(dm, numPoints*(depth+2), PETSC_INT, &offsets);CHKERRQ(ierr);
2442   ierr = DMRestoreWorkArray(dm, mesh->maxSupportSize, PETSC_INT, &join[1-i]);CHKERRQ(ierr);
2443   PetscFunctionReturn(0);
2444 }
2445 
2446 #undef __FUNCT__
2447 #define __FUNCT__ "DMPlexGetMeet"
2448 /*@C
2449   DMPlexGetMeet - Get an array for the meet of the set of points
2450 
2451   Not Collective
2452 
2453   Input Parameters:
2454 + dm - The DMPlex object
2455 . numPoints - The number of input points for the meet
2456 - points - The input points
2457 
2458   Output Parameters:
2459 + numCoveredPoints - The number of points in the meet
2460 - coveredPoints - The points in the meet
2461 
2462   Level: intermediate
2463 
2464   Note: Currently, this is restricted to a single level meet
2465 
2466   Fortran Notes:
2467   Since it returns an array, this routine is only available in Fortran 90, and you must
2468   include petsc.h90 in your code.
2469 
2470   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2471 
2472 .keywords: mesh
2473 .seealso: DMPlexRestoreMeet(), DMPlexGetJoin()
2474 @*/
2475 PetscErrorCode DMPlexGetMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveringPoints, const PetscInt **coveringPoints)
2476 {
2477   DM_Plex       *mesh = (DM_Plex*) dm->data;
2478   PetscInt      *meet[2];
2479   PetscInt       meetSize, i = 0;
2480   PetscInt       dof, off, p, c, m;
2481   PetscErrorCode ierr;
2482 
2483   PetscFunctionBegin;
2484   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2485   PetscValidPointer(points, 2);
2486   PetscValidPointer(numCoveringPoints, 3);
2487   PetscValidPointer(coveringPoints, 4);
2488   ierr = DMGetWorkArray(dm, mesh->maxConeSize, PETSC_INT, &meet[0]);CHKERRQ(ierr);
2489   ierr = DMGetWorkArray(dm, mesh->maxConeSize, PETSC_INT, &meet[1]);CHKERRQ(ierr);
2490   /* Copy in cone of first point */
2491   ierr = PetscSectionGetDof(mesh->coneSection, points[0], &dof);CHKERRQ(ierr);
2492   ierr = PetscSectionGetOffset(mesh->coneSection, points[0], &off);CHKERRQ(ierr);
2493   for (meetSize = 0; meetSize < dof; ++meetSize) {
2494     meet[i][meetSize] = mesh->cones[off+meetSize];
2495   }
2496   /* Check each successive cone */
2497   for (p = 1; p < numPoints; ++p) {
2498     PetscInt newMeetSize = 0;
2499 
2500     ierr = PetscSectionGetDof(mesh->coneSection, points[p], &dof);CHKERRQ(ierr);
2501     ierr = PetscSectionGetOffset(mesh->coneSection, points[p], &off);CHKERRQ(ierr);
2502     for (c = 0; c < dof; ++c) {
2503       const PetscInt point = mesh->cones[off+c];
2504 
2505       for (m = 0; m < meetSize; ++m) {
2506         if (point == meet[i][m]) {
2507           meet[1-i][newMeetSize++] = point;
2508           break;
2509         }
2510       }
2511     }
2512     meetSize = newMeetSize;
2513     i        = 1-i;
2514   }
2515   *numCoveringPoints = meetSize;
2516   *coveringPoints    = meet[i];
2517   ierr               = DMRestoreWorkArray(dm, mesh->maxConeSize, PETSC_INT, &meet[1-i]);CHKERRQ(ierr);
2518   PetscFunctionReturn(0);
2519 }
2520 
2521 #undef __FUNCT__
2522 #define __FUNCT__ "DMPlexRestoreMeet"
2523 /*@C
2524   DMPlexRestoreMeet - Restore an array for the meet of the set of points
2525 
2526   Not Collective
2527 
2528   Input Parameters:
2529 + dm - The DMPlex object
2530 . numPoints - The number of input points for the meet
2531 - points - The input points
2532 
2533   Output Parameters:
2534 + numCoveredPoints - The number of points in the meet
2535 - coveredPoints - The points in the meet
2536 
2537   Level: intermediate
2538 
2539   Fortran Notes:
2540   Since it returns an array, this routine is only available in Fortran 90, and you must
2541   include petsc.h90 in your code.
2542 
2543   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2544 
2545 .keywords: mesh
2546 .seealso: DMPlexGetMeet(), DMPlexGetFullMeet(), DMPlexGetJoin()
2547 @*/
2548 PetscErrorCode DMPlexRestoreMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
2549 {
2550   PetscErrorCode ierr;
2551 
2552   PetscFunctionBegin;
2553   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2554   if (points) PetscValidIntPointer(points,3);
2555   if (numCoveredPoints) PetscValidIntPointer(numCoveredPoints,4);
2556   PetscValidPointer(coveredPoints,5);
2557   ierr = DMRestoreWorkArray(dm, 0, PETSC_INT, (void*) coveredPoints);CHKERRQ(ierr);
2558   if (numCoveredPoints) *numCoveredPoints = 0;
2559   PetscFunctionReturn(0);
2560 }
2561 
2562 #undef __FUNCT__
2563 #define __FUNCT__ "DMPlexGetFullMeet"
2564 /*@C
2565   DMPlexGetFullMeet - Get an array for the meet of the set of points
2566 
2567   Not Collective
2568 
2569   Input Parameters:
2570 + dm - The DMPlex object
2571 . numPoints - The number of input points for the meet
2572 - points - The input points
2573 
2574   Output Parameters:
2575 + numCoveredPoints - The number of points in the meet
2576 - coveredPoints - The points in the meet
2577 
2578   Level: intermediate
2579 
2580   Fortran Notes:
2581   Since it returns an array, this routine is only available in Fortran 90, and you must
2582   include petsc.h90 in your code.
2583 
2584   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2585 
2586 .keywords: mesh
2587 .seealso: DMPlexGetMeet(), DMPlexRestoreMeet(), DMPlexGetJoin()
2588 @*/
2589 PetscErrorCode DMPlexGetFullMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
2590 {
2591   DM_Plex       *mesh = (DM_Plex*) dm->data;
2592   PetscInt      *offsets, **closures;
2593   PetscInt      *meet[2];
2594   PetscInt       height = 0, maxSize, meetSize = 0, i = 0;
2595   PetscInt       p, h, c, m;
2596   PetscErrorCode ierr;
2597 
2598   PetscFunctionBegin;
2599   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2600   PetscValidPointer(points, 2);
2601   PetscValidPointer(numCoveredPoints, 3);
2602   PetscValidPointer(coveredPoints, 4);
2603 
2604   ierr    = DMPlexGetDepth(dm, &height);CHKERRQ(ierr);
2605   ierr    = PetscMalloc1(numPoints, &closures);CHKERRQ(ierr);
2606   ierr    = DMGetWorkArray(dm, numPoints*(height+2), PETSC_INT, &offsets);CHKERRQ(ierr);
2607   maxSize = PetscPowInt(mesh->maxConeSize,height+1);
2608   ierr    = DMGetWorkArray(dm, maxSize, PETSC_INT, &meet[0]);CHKERRQ(ierr);
2609   ierr    = DMGetWorkArray(dm, maxSize, PETSC_INT, &meet[1]);CHKERRQ(ierr);
2610 
2611   for (p = 0; p < numPoints; ++p) {
2612     PetscInt closureSize;
2613 
2614     ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_TRUE, &closureSize, &closures[p]);CHKERRQ(ierr);
2615 
2616     offsets[p*(height+2)+0] = 0;
2617     for (h = 0; h < height+1; ++h) {
2618       PetscInt pStart, pEnd, i;
2619 
2620       ierr = DMPlexGetHeightStratum(dm, h, &pStart, &pEnd);CHKERRQ(ierr);
2621       for (i = offsets[p*(height+2)+h]; i < closureSize; ++i) {
2622         if ((pStart > closures[p][i*2]) || (pEnd <= closures[p][i*2])) {
2623           offsets[p*(height+2)+h+1] = i;
2624           break;
2625         }
2626       }
2627       if (i == closureSize) offsets[p*(height+2)+h+1] = i;
2628     }
2629     if (offsets[p*(height+2)+height+1] != closureSize) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Total size of closure %D should be %D", offsets[p*(height+2)+height+1], closureSize);
2630   }
2631   for (h = 0; h < height+1; ++h) {
2632     PetscInt dof;
2633 
2634     /* Copy in cone of first point */
2635     dof = offsets[h+1] - offsets[h];
2636     for (meetSize = 0; meetSize < dof; ++meetSize) {
2637       meet[i][meetSize] = closures[0][(offsets[h]+meetSize)*2];
2638     }
2639     /* Check each successive cone */
2640     for (p = 1; p < numPoints && meetSize; ++p) {
2641       PetscInt newMeetSize = 0;
2642 
2643       dof = offsets[p*(height+2)+h+1] - offsets[p*(height+2)+h];
2644       for (c = 0; c < dof; ++c) {
2645         const PetscInt point = closures[p][(offsets[p*(height+2)+h]+c)*2];
2646 
2647         for (m = 0; m < meetSize; ++m) {
2648           if (point == meet[i][m]) {
2649             meet[1-i][newMeetSize++] = point;
2650             break;
2651           }
2652         }
2653       }
2654       meetSize = newMeetSize;
2655       i        = 1-i;
2656     }
2657     if (meetSize) break;
2658   }
2659   *numCoveredPoints = meetSize;
2660   *coveredPoints    = meet[i];
2661   for (p = 0; p < numPoints; ++p) {
2662     ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_TRUE, NULL, &closures[p]);CHKERRQ(ierr);
2663   }
2664   ierr = PetscFree(closures);CHKERRQ(ierr);
2665   ierr = DMRestoreWorkArray(dm, numPoints*(height+2), PETSC_INT, &offsets);CHKERRQ(ierr);
2666   ierr = DMRestoreWorkArray(dm, mesh->maxConeSize, PETSC_INT, &meet[1-i]);CHKERRQ(ierr);
2667   PetscFunctionReturn(0);
2668 }
2669 
2670 #undef __FUNCT__
2671 #define __FUNCT__ "DMPlexEqual"
2672 /*@C
2673   DMPlexEqual - Determine if two DMs have the same topology
2674 
2675   Not Collective
2676 
2677   Input Parameters:
2678 + dmA - A DMPlex object
2679 - dmB - A DMPlex object
2680 
2681   Output Parameters:
2682 . equal - PETSC_TRUE if the topologies are identical
2683 
2684   Level: intermediate
2685 
2686   Notes:
2687   We are not solving graph isomorphism, so we do not permutation.
2688 
2689 .keywords: mesh
2690 .seealso: DMPlexGetCone()
2691 @*/
2692 PetscErrorCode DMPlexEqual(DM dmA, DM dmB, PetscBool *equal)
2693 {
2694   PetscInt       depth, depthB, pStart, pEnd, pStartB, pEndB, p;
2695   PetscErrorCode ierr;
2696 
2697   PetscFunctionBegin;
2698   PetscValidHeaderSpecific(dmA, DM_CLASSID, 1);
2699   PetscValidHeaderSpecific(dmB, DM_CLASSID, 2);
2700   PetscValidPointer(equal, 3);
2701 
2702   *equal = PETSC_FALSE;
2703   ierr = DMPlexGetDepth(dmA, &depth);CHKERRQ(ierr);
2704   ierr = DMPlexGetDepth(dmB, &depthB);CHKERRQ(ierr);
2705   if (depth != depthB) PetscFunctionReturn(0);
2706   ierr = DMPlexGetChart(dmA, &pStart,  &pEnd);CHKERRQ(ierr);
2707   ierr = DMPlexGetChart(dmB, &pStartB, &pEndB);CHKERRQ(ierr);
2708   if ((pStart != pStartB) || (pEnd != pEndB)) PetscFunctionReturn(0);
2709   for (p = pStart; p < pEnd; ++p) {
2710     const PetscInt *cone, *coneB, *ornt, *orntB, *support, *supportB;
2711     PetscInt        coneSize, coneSizeB, c, supportSize, supportSizeB, s;
2712 
2713     ierr = DMPlexGetConeSize(dmA, p, &coneSize);CHKERRQ(ierr);
2714     ierr = DMPlexGetCone(dmA, p, &cone);CHKERRQ(ierr);
2715     ierr = DMPlexGetConeOrientation(dmA, p, &ornt);CHKERRQ(ierr);
2716     ierr = DMPlexGetConeSize(dmB, p, &coneSizeB);CHKERRQ(ierr);
2717     ierr = DMPlexGetCone(dmB, p, &coneB);CHKERRQ(ierr);
2718     ierr = DMPlexGetConeOrientation(dmB, p, &orntB);CHKERRQ(ierr);
2719     if (coneSize != coneSizeB) PetscFunctionReturn(0);
2720     for (c = 0; c < coneSize; ++c) {
2721       if (cone[c] != coneB[c]) PetscFunctionReturn(0);
2722       if (ornt[c] != orntB[c]) PetscFunctionReturn(0);
2723     }
2724     ierr = DMPlexGetSupportSize(dmA, p, &supportSize);CHKERRQ(ierr);
2725     ierr = DMPlexGetSupport(dmA, p, &support);CHKERRQ(ierr);
2726     ierr = DMPlexGetSupportSize(dmB, p, &supportSizeB);CHKERRQ(ierr);
2727     ierr = DMPlexGetSupport(dmB, p, &supportB);CHKERRQ(ierr);
2728     if (supportSize != supportSizeB) PetscFunctionReturn(0);
2729     for (s = 0; s < supportSize; ++s) {
2730       if (support[s] != supportB[s]) PetscFunctionReturn(0);
2731     }
2732   }
2733   *equal = PETSC_TRUE;
2734   PetscFunctionReturn(0);
2735 }
2736 
2737 #undef __FUNCT__
2738 #define __FUNCT__ "DMPlexGetNumFaceVertices"
2739 PetscErrorCode DMPlexGetNumFaceVertices(DM dm, PetscInt cellDim, PetscInt numCorners, PetscInt *numFaceVertices)
2740 {
2741   MPI_Comm       comm;
2742   PetscErrorCode ierr;
2743 
2744   PetscFunctionBegin;
2745   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
2746   PetscValidPointer(numFaceVertices,3);
2747   switch (cellDim) {
2748   case 0:
2749     *numFaceVertices = 0;
2750     break;
2751   case 1:
2752     *numFaceVertices = 1;
2753     break;
2754   case 2:
2755     switch (numCorners) {
2756     case 3: /* triangle */
2757       *numFaceVertices = 2; /* Edge has 2 vertices */
2758       break;
2759     case 4: /* quadrilateral */
2760       *numFaceVertices = 2; /* Edge has 2 vertices */
2761       break;
2762     case 6: /* quadratic triangle, tri and quad cohesive Lagrange cells */
2763       *numFaceVertices = 3; /* Edge has 3 vertices */
2764       break;
2765     case 9: /* quadratic quadrilateral, quadratic quad cohesive Lagrange cells */
2766       *numFaceVertices = 3; /* Edge has 3 vertices */
2767       break;
2768     default:
2769       SETERRQ2(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid number of face corners %d for dimension %d", numCorners, cellDim);
2770     }
2771     break;
2772   case 3:
2773     switch (numCorners) {
2774     case 4: /* tetradehdron */
2775       *numFaceVertices = 3; /* Face has 3 vertices */
2776       break;
2777     case 6: /* tet cohesive cells */
2778       *numFaceVertices = 4; /* Face has 4 vertices */
2779       break;
2780     case 8: /* hexahedron */
2781       *numFaceVertices = 4; /* Face has 4 vertices */
2782       break;
2783     case 9: /* tet cohesive Lagrange cells */
2784       *numFaceVertices = 6; /* Face has 6 vertices */
2785       break;
2786     case 10: /* quadratic tetrahedron */
2787       *numFaceVertices = 6; /* Face has 6 vertices */
2788       break;
2789     case 12: /* hex cohesive Lagrange cells */
2790       *numFaceVertices = 6; /* Face has 6 vertices */
2791       break;
2792     case 18: /* quadratic tet cohesive Lagrange cells */
2793       *numFaceVertices = 6; /* Face has 6 vertices */
2794       break;
2795     case 27: /* quadratic hexahedron, quadratic hex cohesive Lagrange cells */
2796       *numFaceVertices = 9; /* Face has 9 vertices */
2797       break;
2798     default:
2799       SETERRQ2(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid number of face corners %d for dimension %d", numCorners, cellDim);
2800     }
2801     break;
2802   default:
2803     SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid cell dimension %d", cellDim);
2804   }
2805   PetscFunctionReturn(0);
2806 }
2807 
2808 #undef __FUNCT__
2809 #define __FUNCT__ "DMPlexOrient"
2810 /* Trys to give the mesh a consistent orientation */
2811 PetscErrorCode DMPlexOrient(DM dm)
2812 {
2813   PetscBT        seenCells, flippedCells, seenFaces;
2814   PetscInt      *faceFIFO, fTop, fBottom;
2815   PetscInt       dim, h, cStart, cEnd, c, fStart, fEnd, face, maxConeSize, *revcone, *revconeO;
2816   PetscErrorCode ierr;
2817 
2818   PetscFunctionBegin;
2819   /* Truth Table
2820      mismatch    flips   do action   mismatch   flipA ^ flipB   action
2821          F       0 flips     no         F             F           F
2822          F       1 flip      yes        F             T           T
2823          F       2 flips     no         T             F           T
2824          T       0 flips     yes        T             T           F
2825          T       1 flip      no
2826          T       2 flips     yes
2827   */
2828   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
2829   ierr = DMPlexGetVTKCellHeight(dm, &h);CHKERRQ(ierr);
2830   ierr = DMPlexGetHeightStratum(dm, h,   &cStart, &cEnd);CHKERRQ(ierr);
2831   ierr = DMPlexGetHeightStratum(dm, h+1, &fStart, &fEnd);CHKERRQ(ierr);
2832   ierr = PetscBTCreate(cEnd - cStart, &seenCells);CHKERRQ(ierr);
2833   ierr = PetscBTMemzero(cEnd - cStart, seenCells);CHKERRQ(ierr);
2834   ierr = PetscBTCreate(cEnd - cStart, &flippedCells);CHKERRQ(ierr);
2835   ierr = PetscBTMemzero(cEnd - cStart, flippedCells);CHKERRQ(ierr);
2836   ierr = PetscBTCreate(fEnd - fStart, &seenFaces);CHKERRQ(ierr);
2837   ierr = PetscBTMemzero(fEnd - fStart, seenFaces);CHKERRQ(ierr);
2838   ierr = PetscMalloc1((fEnd - fStart), &faceFIFO);CHKERRQ(ierr);
2839   fTop = fBottom = 0;
2840   /* Initialize FIFO with first cell */
2841   if (cEnd > cStart) {
2842     const PetscInt *cone;
2843     PetscInt        coneSize;
2844 
2845     ierr = DMPlexGetConeSize(dm, cStart, &coneSize);CHKERRQ(ierr);
2846     ierr = DMPlexGetCone(dm, cStart, &cone);CHKERRQ(ierr);
2847     for (c = 0; c < coneSize; ++c) {
2848       faceFIFO[fBottom++] = cone[c];
2849       ierr = PetscBTSet(seenFaces, cone[c]-fStart);CHKERRQ(ierr);
2850     }
2851   }
2852   /* Consider each face in FIFO */
2853   while (fTop < fBottom) {
2854     const PetscInt *support, *coneA, *coneB, *coneOA, *coneOB;
2855     PetscInt        supportSize, coneSizeA, coneSizeB, posA = -1, posB = -1;
2856     PetscInt        seenA, flippedA, seenB, flippedB, mismatch;
2857 
2858     face = faceFIFO[fTop++];
2859     ierr = DMPlexGetSupportSize(dm, face, &supportSize);CHKERRQ(ierr);
2860     ierr = DMPlexGetSupport(dm, face, &support);CHKERRQ(ierr);
2861     if (supportSize < 2) continue;
2862     if (supportSize != 2) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Faces should separate only two cells, not %d", supportSize);
2863     seenA    = PetscBTLookup(seenCells,    support[0]-cStart);
2864     flippedA = PetscBTLookup(flippedCells, support[0]-cStart) ? 1 : 0;
2865     seenB    = PetscBTLookup(seenCells,    support[1]-cStart);
2866     flippedB = PetscBTLookup(flippedCells, support[1]-cStart) ? 1 : 0;
2867 
2868     ierr = DMPlexGetConeSize(dm, support[0], &coneSizeA);CHKERRQ(ierr);
2869     ierr = DMPlexGetConeSize(dm, support[1], &coneSizeB);CHKERRQ(ierr);
2870     ierr = DMPlexGetCone(dm, support[0], &coneA);CHKERRQ(ierr);
2871     ierr = DMPlexGetCone(dm, support[1], &coneB);CHKERRQ(ierr);
2872     ierr = DMPlexGetConeOrientation(dm, support[0], &coneOA);CHKERRQ(ierr);
2873     ierr = DMPlexGetConeOrientation(dm, support[1], &coneOB);CHKERRQ(ierr);
2874     for (c = 0; c < coneSizeA; ++c) {
2875       if (!PetscBTLookup(seenFaces, coneA[c]-fStart)) {
2876         faceFIFO[fBottom++] = coneA[c];
2877         ierr = PetscBTSet(seenFaces, coneA[c]-fStart);CHKERRQ(ierr);
2878       }
2879       if (coneA[c] == face) posA = c;
2880       if (fBottom > fEnd-fStart) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Face %d was pushed exceeding capacity %d > %d", coneA[c], fBottom, fEnd-fStart);
2881     }
2882     if (posA < 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %d could not be located in cell %d", face, support[0]);
2883     for (c = 0; c < coneSizeB; ++c) {
2884       if (!PetscBTLookup(seenFaces, coneB[c]-fStart)) {
2885         faceFIFO[fBottom++] = coneB[c];
2886         ierr = PetscBTSet(seenFaces, coneB[c]-fStart);CHKERRQ(ierr);
2887       }
2888       if (coneB[c] == face) posB = c;
2889       if (fBottom > fEnd-fStart) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Face %d was pushed exceeding capacity %d > %d", coneA[c], fBottom, fEnd-fStart);
2890     }
2891     if (posB < 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %d could not be located in cell %d", face, support[1]);
2892 
2893     if (dim == 1) {
2894       mismatch = posA == posB;
2895     } else {
2896       mismatch = coneOA[posA] == coneOB[posB];
2897     }
2898 
2899     if (mismatch ^ (flippedA ^ flippedB)) {
2900       if (seenA && seenB) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Previously seen cells %d and %d do not match: Fault mesh is non-orientable", support[0], support[1]);
2901       if (!seenA && !flippedA) {
2902         ierr = PetscBTSet(flippedCells, support[0]-cStart);CHKERRQ(ierr);
2903       } else if (!seenB && !flippedB) {
2904         ierr = PetscBTSet(flippedCells, support[1]-cStart);CHKERRQ(ierr);
2905       } else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Inconsistent mesh orientation: Fault mesh is non-orientable");
2906     } else if (mismatch && flippedA && flippedB) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Attempt to flip already flipped cell: Fault mesh is non-orientable");
2907     ierr = PetscBTSet(seenCells, support[0]-cStart);CHKERRQ(ierr);
2908     ierr = PetscBTSet(seenCells, support[1]-cStart);CHKERRQ(ierr);
2909   }
2910   /* Now all subdomains are oriented, but we need a consistent parallel orientation */
2911   {
2912     /* Find a representative face (edge) separating pairs of procs */
2913     PetscSF            sf;
2914     const PetscInt    *lpoints;
2915     const PetscSFNode *rpoints;
2916     PetscInt          *neighbors, *nranks;
2917     PetscInt           numLeaves, numRoots, numNeighbors = 0, l, n;
2918 
2919     ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr);
2920     ierr = PetscSFGetGraph(sf, &numRoots, &numLeaves, &lpoints, &rpoints);CHKERRQ(ierr);
2921     if (numLeaves >= 0) {
2922       const PetscInt *cone, *ornt, *support;
2923       PetscInt        coneSize, supportSize;
2924       int            *rornt, *lornt; /* PetscSF cannot handle smaller than int */
2925       PetscBool      *match, flipped = PETSC_FALSE;
2926 
2927       ierr = PetscMalloc1(numLeaves,&neighbors);CHKERRQ(ierr);
2928       /* I know this is p^2 time in general, but for bounded degree its alright */
2929       for (l = 0; l < numLeaves; ++l) {
2930         const PetscInt face = lpoints[l];
2931         if ((face >= fStart) && (face < fEnd)) {
2932           const PetscInt rank = rpoints[l].rank;
2933           for (n = 0; n < numNeighbors; ++n) if (rank == rpoints[neighbors[n]].rank) break;
2934           if (n >= numNeighbors) {
2935             PetscInt supportSize;
2936             ierr = DMPlexGetSupportSize(dm, face, &supportSize);CHKERRQ(ierr);
2937             if (supportSize != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Boundary faces should see one cell, not %d", supportSize);
2938             neighbors[numNeighbors++] = l;
2939           }
2940         }
2941       }
2942       ierr = PetscCalloc4(numNeighbors,&match,numNeighbors,&nranks,numRoots,&rornt,numRoots,&lornt);CHKERRQ(ierr);
2943       for (face = fStart; face < fEnd; ++face) {
2944         ierr = DMPlexGetSupportSize(dm, face, &supportSize);CHKERRQ(ierr);
2945         if (supportSize != 1) continue;
2946         ierr = DMPlexGetSupport(dm, face, &support);CHKERRQ(ierr);
2947 
2948         ierr = DMPlexGetCone(dm, support[0], &cone);CHKERRQ(ierr);
2949         ierr = DMPlexGetConeSize(dm, support[0], &coneSize);CHKERRQ(ierr);
2950         ierr = DMPlexGetConeOrientation(dm, support[0], &ornt);CHKERRQ(ierr);
2951         for (c = 0; c < coneSize; ++c) if (cone[c] == face) break;
2952         if (dim == 1) {
2953           /* Use cone position instead, shifted to -1 or 1 */
2954           rornt[face] = c*2-1;
2955         } else {
2956           if (PetscBTLookup(flippedCells, support[0]-cStart)) rornt[face] = ornt[c] < 0 ? -1 :  1;
2957           else                                                rornt[face] = ornt[c] < 0 ?  1 : -1;
2958         }
2959       }
2960       /* Mark each edge with match or nomatch */
2961       ierr = PetscSFBcastBegin(sf, MPI_INT, rornt, lornt);CHKERRQ(ierr);
2962       ierr = PetscSFBcastEnd(sf, MPI_INT, rornt, lornt);CHKERRQ(ierr);
2963       for (n = 0; n < numNeighbors; ++n) {
2964         const PetscInt face = lpoints[neighbors[n]];
2965 
2966         if (rornt[face]*lornt[face] < 0) match[n] = PETSC_TRUE;
2967         else                             match[n] = PETSC_FALSE;
2968         nranks[n] = rpoints[neighbors[n]].rank;
2969       }
2970       /* Collect the graph on 0 */
2971       {
2972         MPI_Comm     comm = PetscObjectComm((PetscObject) sf);
2973         PetscBT      seenProcs, flippedProcs;
2974         PetscInt    *procFIFO, pTop, pBottom;
2975         PetscInt    *adj = NULL;
2976         PetscBool   *val = NULL;
2977         PetscMPIInt *recvcounts = NULL, *displs = NULL, p;
2978         PetscMPIInt  N = numNeighbors, numProcs = 0, rank;
2979         PetscInt     debug = 0;
2980 
2981         ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
2982         if (!rank) {ierr = MPI_Comm_size(comm, &numProcs);CHKERRQ(ierr);}
2983         ierr = PetscCalloc2(numProcs,&recvcounts,numProcs+1,&displs);CHKERRQ(ierr);
2984         ierr = MPI_Gather(&N, 1, MPI_INT, recvcounts, 1, MPI_INT, 0, comm);CHKERRQ(ierr);
2985         for (p = 0; p < numProcs; ++p) {
2986           displs[p+1] = displs[p] + recvcounts[p];
2987         }
2988         if (!rank) {ierr = PetscMalloc2(displs[numProcs],&adj,displs[numProcs],&val);CHKERRQ(ierr);}
2989         ierr = MPI_Gatherv(nranks, numNeighbors, MPIU_INT, adj, recvcounts, displs, MPIU_INT, 0, comm);CHKERRQ(ierr);
2990         ierr = MPI_Gatherv(match, numNeighbors, MPIU_BOOL, val, recvcounts, displs, MPIU_BOOL, 0, comm);CHKERRQ(ierr);
2991         if (debug) {
2992           for (p = 0; p < numProcs; ++p) {
2993             ierr = PetscPrintf(comm, "Proc %d:\n", p);
2994             for (n = 0; n < recvcounts[p]; ++n) {
2995               ierr = PetscPrintf(comm, "  edge %d (%d):\n", adj[displs[p]+n], val[displs[p]+n]);
2996             }
2997           }
2998         }
2999         ierr = PetscBTCreate(numProcs, &seenProcs);CHKERRQ(ierr);
3000         ierr = PetscBTMemzero(numProcs, seenProcs);CHKERRQ(ierr);
3001         ierr = PetscBTCreate(numProcs, &flippedProcs);CHKERRQ(ierr);
3002         ierr = PetscBTMemzero(numProcs, flippedProcs);CHKERRQ(ierr);
3003         ierr = PetscMalloc1(numProcs,&procFIFO);CHKERRQ(ierr);
3004         pTop = pBottom = 0;
3005         for (p = 0; p < numProcs; ++p) {
3006           if (PetscBTLookup(seenProcs, p)) continue;
3007           /* Initialize FIFO with next proc */
3008           procFIFO[pBottom++] = p;
3009           ierr = PetscBTSet(seenProcs, p);CHKERRQ(ierr);
3010           /* Consider each proc in FIFO */
3011           while (pTop < pBottom) {
3012             PetscInt proc, nproc, seen, flippedA, flippedB, mismatch;
3013 
3014             proc     = procFIFO[pTop++];
3015             flippedA = PetscBTLookup(flippedProcs, proc) ? 1 : 0;
3016             /* Loop over neighboring procs */
3017             for (n = 0; n < recvcounts[proc]; ++n) {
3018               nproc    = adj[displs[proc]+n];
3019               mismatch = val[displs[proc]+n] ? 0 : 1;
3020               seen     = PetscBTLookup(seenProcs, nproc);
3021               flippedB = PetscBTLookup(flippedProcs, nproc) ? 1 : 0;
3022 
3023               if (mismatch ^ (flippedA ^ flippedB)) {
3024                 if (seen) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Previously seen procs %d and %d do not match: Fault mesh is non-orientable", proc, nproc);
3025                 if (!flippedB) {
3026                   ierr = PetscBTSet(flippedProcs, nproc);CHKERRQ(ierr);
3027               } else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Inconsistent mesh orientation: Fault mesh is non-orientable");
3028               } else if (mismatch && flippedA && flippedB) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Attempt to flip already flipped cell: Fault mesh is non-orientable");
3029               if (!seen) {
3030                 procFIFO[pBottom++] = nproc;
3031                 ierr = PetscBTSet(seenProcs, nproc);CHKERRQ(ierr);
3032               }
3033             }
3034           }
3035         }
3036         ierr = PetscFree(procFIFO);CHKERRQ(ierr);
3037 
3038         ierr = PetscFree2(recvcounts,displs);CHKERRQ(ierr);
3039         ierr = PetscFree2(adj,val);CHKERRQ(ierr);
3040         {
3041           PetscBool *flips;
3042 
3043           ierr = PetscMalloc1(numProcs,&flips);CHKERRQ(ierr);
3044           for (p = 0; p < numProcs; ++p) {
3045             flips[p] = PetscBTLookup(flippedProcs, p) ? PETSC_TRUE : PETSC_FALSE;
3046             if (debug && flips[p]) {ierr = PetscPrintf(comm, "Flipping Proc %d:\n", p);}
3047           }
3048           ierr = MPI_Scatter(flips, 1, MPIU_BOOL, &flipped, 1, MPIU_BOOL, 0, comm);CHKERRQ(ierr);
3049           ierr = PetscFree(flips);CHKERRQ(ierr);
3050         }
3051         ierr = PetscBTDestroy(&seenProcs);CHKERRQ(ierr);
3052         ierr = PetscBTDestroy(&flippedProcs);CHKERRQ(ierr);
3053       }
3054       ierr = PetscFree4(match,nranks,rornt,lornt);CHKERRQ(ierr);
3055       ierr = PetscFree(neighbors);CHKERRQ(ierr);
3056       if (flipped) {for (c = cStart; c < cEnd; ++c) {ierr = PetscBTNegate(flippedCells, c-cStart);CHKERRQ(ierr);}}
3057     }
3058   }
3059   /* Reverse flipped cells in the mesh */
3060   ierr = DMPlexGetMaxSizes(dm, &maxConeSize, NULL);CHKERRQ(ierr);
3061   ierr = DMGetWorkArray(dm, maxConeSize, PETSC_INT, &revcone);CHKERRQ(ierr);
3062   ierr = DMGetWorkArray(dm, maxConeSize, PETSC_INT, &revconeO);CHKERRQ(ierr);
3063   for (c = cStart; c < cEnd; ++c) {
3064     const PetscInt *cone, *coneO, *support;
3065     PetscInt        coneSize, supportSize, faceSize, cp, sp;
3066 
3067     if (!PetscBTLookup(flippedCells, c-cStart)) continue;
3068     ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
3069     ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
3070     ierr = DMPlexGetConeOrientation(dm, c, &coneO);CHKERRQ(ierr);
3071     for (cp = 0; cp < coneSize; ++cp) {
3072       const PetscInt rcp = coneSize-cp-1;
3073 
3074       ierr = DMPlexGetConeSize(dm, cone[rcp], &faceSize);CHKERRQ(ierr);
3075       revcone[cp]  = cone[rcp];
3076       revconeO[cp] = coneO[rcp] >= 0 ? -(faceSize-coneO[rcp]) : faceSize+coneO[rcp];
3077     }
3078     ierr = DMPlexSetCone(dm, c, revcone);CHKERRQ(ierr);
3079     ierr = DMPlexSetConeOrientation(dm, c, revconeO);CHKERRQ(ierr);
3080     /* Reverse orientations of support */
3081     faceSize = coneSize;
3082     ierr = DMPlexGetSupportSize(dm, c, &supportSize);CHKERRQ(ierr);
3083     ierr = DMPlexGetSupport(dm, c, &support);CHKERRQ(ierr);
3084     for (sp = 0; sp < supportSize; ++sp) {
3085       ierr = DMPlexGetConeSize(dm, support[sp], &coneSize);CHKERRQ(ierr);
3086       ierr = DMPlexGetCone(dm, support[sp], &cone);CHKERRQ(ierr);
3087       ierr = DMPlexGetConeOrientation(dm, support[sp], &coneO);CHKERRQ(ierr);
3088       for (cp = 0; cp < coneSize; ++cp) {
3089         if (cone[cp] != c) continue;
3090         ierr = DMPlexInsertConeOrientation(dm, support[sp], cp, coneO[cp] >= 0 ? -(faceSize-coneO[cp]) : faceSize+coneO[cp]);CHKERRQ(ierr);
3091       }
3092     }
3093   }
3094   ierr = DMRestoreWorkArray(dm, maxConeSize, PETSC_INT, &revcone);CHKERRQ(ierr);
3095   ierr = DMRestoreWorkArray(dm, maxConeSize, PETSC_INT, &revconeO);CHKERRQ(ierr);
3096   ierr = PetscBTDestroy(&seenCells);CHKERRQ(ierr);
3097   ierr = PetscBTDestroy(&flippedCells);CHKERRQ(ierr);
3098   ierr = PetscBTDestroy(&seenFaces);CHKERRQ(ierr);
3099   ierr = PetscFree(faceFIFO);CHKERRQ(ierr);
3100   PetscFunctionReturn(0);
3101 }
3102 
3103 #undef __FUNCT__
3104 #define __FUNCT__ "DMPlexInvertCell"
3105 /*@C
3106   DMPlexInvertCell - This flips tetrahedron and hexahedron orientation since Plex stores them internally with outward normals. Other cells are left untouched.
3107 
3108   Input Parameters:
3109 + numCorners - The number of vertices in a cell
3110 - cone - The incoming cone
3111 
3112   Output Parameter:
3113 . cone - The inverted cone (in-place)
3114 
3115   Level: developer
3116 
3117 .seealso: DMPlexGenerate()
3118 @*/
3119 PetscErrorCode DMPlexInvertCell(PetscInt dim, PetscInt numCorners, int cone[])
3120 {
3121   int tmpc;
3122 
3123   PetscFunctionBegin;
3124   if (dim != 3) PetscFunctionReturn(0);
3125   switch (numCorners) {
3126   case 4:
3127     tmpc    = cone[0];
3128     cone[0] = cone[1];
3129     cone[1] = tmpc;
3130     break;
3131   case 8:
3132     tmpc    = cone[1];
3133     cone[1] = cone[3];
3134     cone[3] = tmpc;
3135     break;
3136   default: break;
3137   }
3138   PetscFunctionReturn(0);
3139 }
3140 
3141 #undef __FUNCT__
3142 #define __FUNCT__ "DMPlexInvertCells_Internal"
3143 /* This is to fix the tetrahedron orientation from TetGen */
3144 PETSC_UNUSED static PetscErrorCode DMPlexInvertCells_Internal(PetscInt dim, PetscInt numCells, PetscInt numCorners, int cells[])
3145 {
3146   PetscInt       bound = numCells*numCorners, coff;
3147   PetscErrorCode ierr;
3148 
3149   PetscFunctionBegin;
3150   for (coff = 0; coff < bound; coff += numCorners) {
3151     ierr = DMPlexInvertCell(dim, numCorners, &cells[coff]);CHKERRQ(ierr);
3152   }
3153   PetscFunctionReturn(0);
3154 }
3155 
3156 #if defined(PETSC_HAVE_TRIANGLE)
3157 #include <triangle.h>
3158 
3159 #undef __FUNCT__
3160 #define __FUNCT__ "InitInput_Triangle"
3161 PetscErrorCode InitInput_Triangle(struct triangulateio *inputCtx)
3162 {
3163   PetscFunctionBegin;
3164   inputCtx->numberofpoints             = 0;
3165   inputCtx->numberofpointattributes    = 0;
3166   inputCtx->pointlist                  = NULL;
3167   inputCtx->pointattributelist         = NULL;
3168   inputCtx->pointmarkerlist            = NULL;
3169   inputCtx->numberofsegments           = 0;
3170   inputCtx->segmentlist                = NULL;
3171   inputCtx->segmentmarkerlist          = NULL;
3172   inputCtx->numberoftriangleattributes = 0;
3173   inputCtx->trianglelist               = NULL;
3174   inputCtx->numberofholes              = 0;
3175   inputCtx->holelist                   = NULL;
3176   inputCtx->numberofregions            = 0;
3177   inputCtx->regionlist                 = NULL;
3178   PetscFunctionReturn(0);
3179 }
3180 
3181 #undef __FUNCT__
3182 #define __FUNCT__ "InitOutput_Triangle"
3183 PetscErrorCode InitOutput_Triangle(struct triangulateio *outputCtx)
3184 {
3185   PetscFunctionBegin;
3186   outputCtx->numberofpoints        = 0;
3187   outputCtx->pointlist             = NULL;
3188   outputCtx->pointattributelist    = NULL;
3189   outputCtx->pointmarkerlist       = NULL;
3190   outputCtx->numberoftriangles     = 0;
3191   outputCtx->trianglelist          = NULL;
3192   outputCtx->triangleattributelist = NULL;
3193   outputCtx->neighborlist          = NULL;
3194   outputCtx->segmentlist           = NULL;
3195   outputCtx->segmentmarkerlist     = NULL;
3196   outputCtx->numberofedges         = 0;
3197   outputCtx->edgelist              = NULL;
3198   outputCtx->edgemarkerlist        = NULL;
3199   PetscFunctionReturn(0);
3200 }
3201 
3202 #undef __FUNCT__
3203 #define __FUNCT__ "FiniOutput_Triangle"
3204 PetscErrorCode FiniOutput_Triangle(struct triangulateio *outputCtx)
3205 {
3206   PetscFunctionBegin;
3207   free(outputCtx->pointlist);
3208   free(outputCtx->pointmarkerlist);
3209   free(outputCtx->segmentlist);
3210   free(outputCtx->segmentmarkerlist);
3211   free(outputCtx->edgelist);
3212   free(outputCtx->edgemarkerlist);
3213   free(outputCtx->trianglelist);
3214   free(outputCtx->neighborlist);
3215   PetscFunctionReturn(0);
3216 }
3217 
3218 #undef __FUNCT__
3219 #define __FUNCT__ "DMPlexGenerate_Triangle"
3220 PetscErrorCode DMPlexGenerate_Triangle(DM boundary, PetscBool interpolate, DM *dm)
3221 {
3222   MPI_Comm             comm;
3223   PetscInt             dim              = 2;
3224   const PetscBool      createConvexHull = PETSC_FALSE;
3225   const PetscBool      constrained      = PETSC_FALSE;
3226   struct triangulateio in;
3227   struct triangulateio out;
3228   PetscInt             vStart, vEnd, v, eStart, eEnd, e;
3229   PetscMPIInt          rank;
3230   PetscErrorCode       ierr;
3231 
3232   PetscFunctionBegin;
3233   ierr = PetscObjectGetComm((PetscObject)boundary,&comm);CHKERRQ(ierr);
3234   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
3235   ierr = InitInput_Triangle(&in);CHKERRQ(ierr);
3236   ierr = InitOutput_Triangle(&out);CHKERRQ(ierr);
3237   ierr = DMPlexGetDepthStratum(boundary, 0, &vStart, &vEnd);CHKERRQ(ierr);
3238 
3239   in.numberofpoints = vEnd - vStart;
3240   if (in.numberofpoints > 0) {
3241     PetscSection coordSection;
3242     Vec          coordinates;
3243     PetscScalar *array;
3244 
3245     ierr = PetscMalloc1(in.numberofpoints*dim, &in.pointlist);CHKERRQ(ierr);
3246     ierr = PetscMalloc1(in.numberofpoints, &in.pointmarkerlist);CHKERRQ(ierr);
3247     ierr = DMGetCoordinatesLocal(boundary, &coordinates);CHKERRQ(ierr);
3248     ierr = DMGetCoordinateSection(boundary, &coordSection);CHKERRQ(ierr);
3249     ierr = VecGetArray(coordinates, &array);CHKERRQ(ierr);
3250     for (v = vStart; v < vEnd; ++v) {
3251       const PetscInt idx = v - vStart;
3252       PetscInt       off, d;
3253 
3254       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
3255       for (d = 0; d < dim; ++d) {
3256         in.pointlist[idx*dim + d] = PetscRealPart(array[off+d]);
3257       }
3258       ierr = DMPlexGetLabelValue(boundary, "marker", v, &in.pointmarkerlist[idx]);CHKERRQ(ierr);
3259     }
3260     ierr = VecRestoreArray(coordinates, &array);CHKERRQ(ierr);
3261   }
3262   ierr  = DMPlexGetHeightStratum(boundary, 0, &eStart, &eEnd);CHKERRQ(ierr);
3263   in.numberofsegments = eEnd - eStart;
3264   if (in.numberofsegments > 0) {
3265     ierr = PetscMalloc1(in.numberofsegments*2, &in.segmentlist);CHKERRQ(ierr);
3266     ierr = PetscMalloc1(in.numberofsegments, &in.segmentmarkerlist);CHKERRQ(ierr);
3267     for (e = eStart; e < eEnd; ++e) {
3268       const PetscInt  idx = e - eStart;
3269       const PetscInt *cone;
3270 
3271       ierr = DMPlexGetCone(boundary, e, &cone);CHKERRQ(ierr);
3272 
3273       in.segmentlist[idx*2+0] = cone[0] - vStart;
3274       in.segmentlist[idx*2+1] = cone[1] - vStart;
3275 
3276       ierr = DMPlexGetLabelValue(boundary, "marker", e, &in.segmentmarkerlist[idx]);CHKERRQ(ierr);
3277     }
3278   }
3279 #if 0 /* Do not currently support holes */
3280   PetscReal *holeCoords;
3281   PetscInt   h, d;
3282 
3283   ierr = DMPlexGetHoles(boundary, &in.numberofholes, &holeCords);CHKERRQ(ierr);
3284   if (in.numberofholes > 0) {
3285     ierr = PetscMalloc1(in.numberofholes*dim, &in.holelist);CHKERRQ(ierr);
3286     for (h = 0; h < in.numberofholes; ++h) {
3287       for (d = 0; d < dim; ++d) {
3288         in.holelist[h*dim+d] = holeCoords[h*dim+d];
3289       }
3290     }
3291   }
3292 #endif
3293   if (!rank) {
3294     char args[32];
3295 
3296     /* Take away 'Q' for verbose output */
3297     ierr = PetscStrcpy(args, "pqezQ");CHKERRQ(ierr);
3298     if (createConvexHull) {
3299       ierr = PetscStrcat(args, "c");CHKERRQ(ierr);
3300     }
3301     if (constrained) {
3302       ierr = PetscStrcpy(args, "zepDQ");CHKERRQ(ierr);
3303     }
3304     triangulate(args, &in, &out, NULL);
3305   }
3306   ierr = PetscFree(in.pointlist);CHKERRQ(ierr);
3307   ierr = PetscFree(in.pointmarkerlist);CHKERRQ(ierr);
3308   ierr = PetscFree(in.segmentlist);CHKERRQ(ierr);
3309   ierr = PetscFree(in.segmentmarkerlist);CHKERRQ(ierr);
3310   ierr = PetscFree(in.holelist);CHKERRQ(ierr);
3311 
3312   {
3313     const PetscInt numCorners  = 3;
3314     const PetscInt numCells    = out.numberoftriangles;
3315     const PetscInt numVertices = out.numberofpoints;
3316     const int     *cells      = out.trianglelist;
3317     const double  *meshCoords = out.pointlist;
3318 
3319     ierr = DMPlexCreateFromCellList(comm, dim, numCells, numVertices, numCorners, interpolate, cells, dim, meshCoords, dm);CHKERRQ(ierr);
3320     /* Set labels */
3321     for (v = 0; v < numVertices; ++v) {
3322       if (out.pointmarkerlist[v]) {
3323         ierr = DMPlexSetLabelValue(*dm, "marker", v+numCells, out.pointmarkerlist[v]);CHKERRQ(ierr);
3324       }
3325     }
3326     if (interpolate) {
3327       for (e = 0; e < out.numberofedges; e++) {
3328         if (out.edgemarkerlist[e]) {
3329           const PetscInt  vertices[2] = {out.edgelist[e*2+0]+numCells, out.edgelist[e*2+1]+numCells};
3330           const PetscInt *edges;
3331           PetscInt        numEdges;
3332 
3333           ierr = DMPlexGetJoin(*dm, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3334           if (numEdges != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Two vertices must cover only one edge, not %D", numEdges);
3335           ierr = DMPlexSetLabelValue(*dm, "marker", edges[0], out.edgemarkerlist[e]);CHKERRQ(ierr);
3336           ierr = DMPlexRestoreJoin(*dm, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3337         }
3338       }
3339     }
3340     ierr = DMPlexSetRefinementUniform(*dm, PETSC_FALSE);CHKERRQ(ierr);
3341   }
3342 #if 0 /* Do not currently support holes */
3343   ierr = DMPlexCopyHoles(*dm, boundary);CHKERRQ(ierr);
3344 #endif
3345   ierr = FiniOutput_Triangle(&out);CHKERRQ(ierr);
3346   PetscFunctionReturn(0);
3347 }
3348 
3349 #undef __FUNCT__
3350 #define __FUNCT__ "DMPlexRefine_Triangle"
3351 PetscErrorCode DMPlexRefine_Triangle(DM dm, double *maxVolumes, DM *dmRefined)
3352 {
3353   MPI_Comm             comm;
3354   PetscInt             dim  = 2;
3355   struct triangulateio in;
3356   struct triangulateio out;
3357   PetscInt             vStart, vEnd, v, cStart, cEnd, c, depth, depthGlobal;
3358   PetscMPIInt          rank;
3359   PetscErrorCode       ierr;
3360 
3361   PetscFunctionBegin;
3362   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
3363   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
3364   ierr = InitInput_Triangle(&in);CHKERRQ(ierr);
3365   ierr = InitOutput_Triangle(&out);CHKERRQ(ierr);
3366   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3367   ierr = MPI_Allreduce(&depth, &depthGlobal, 1, MPIU_INT, MPI_MAX, comm);CHKERRQ(ierr);
3368   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
3369 
3370   in.numberofpoints = vEnd - vStart;
3371   if (in.numberofpoints > 0) {
3372     PetscSection coordSection;
3373     Vec          coordinates;
3374     PetscScalar *array;
3375 
3376     ierr = PetscMalloc1(in.numberofpoints*dim, &in.pointlist);CHKERRQ(ierr);
3377     ierr = PetscMalloc1(in.numberofpoints, &in.pointmarkerlist);CHKERRQ(ierr);
3378     ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
3379     ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
3380     ierr = VecGetArray(coordinates, &array);CHKERRQ(ierr);
3381     for (v = vStart; v < vEnd; ++v) {
3382       const PetscInt idx = v - vStart;
3383       PetscInt       off, d;
3384 
3385       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
3386       for (d = 0; d < dim; ++d) {
3387         in.pointlist[idx*dim + d] = PetscRealPart(array[off+d]);
3388       }
3389       ierr = DMPlexGetLabelValue(dm, "marker", v, &in.pointmarkerlist[idx]);CHKERRQ(ierr);
3390     }
3391     ierr = VecRestoreArray(coordinates, &array);CHKERRQ(ierr);
3392   }
3393   ierr  = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
3394 
3395   in.numberofcorners   = 3;
3396   in.numberoftriangles = cEnd - cStart;
3397 
3398   in.trianglearealist  = (double*) maxVolumes;
3399   if (in.numberoftriangles > 0) {
3400     ierr = PetscMalloc1(in.numberoftriangles*in.numberofcorners, &in.trianglelist);CHKERRQ(ierr);
3401     for (c = cStart; c < cEnd; ++c) {
3402       const PetscInt idx      = c - cStart;
3403       PetscInt      *closure = NULL;
3404       PetscInt       closureSize;
3405 
3406       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
3407       if ((closureSize != 4) && (closureSize != 7)) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Mesh has cell which is not a triangle, %D vertices in closure", closureSize);
3408       for (v = 0; v < 3; ++v) {
3409         in.trianglelist[idx*in.numberofcorners + v] = closure[(v+closureSize-3)*2] - vStart;
3410       }
3411       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
3412     }
3413   }
3414   /* TODO: Segment markers are missing on input */
3415 #if 0 /* Do not currently support holes */
3416   PetscReal *holeCoords;
3417   PetscInt   h, d;
3418 
3419   ierr = DMPlexGetHoles(boundary, &in.numberofholes, &holeCords);CHKERRQ(ierr);
3420   if (in.numberofholes > 0) {
3421     ierr = PetscMalloc1(in.numberofholes*dim, &in.holelist);CHKERRQ(ierr);
3422     for (h = 0; h < in.numberofholes; ++h) {
3423       for (d = 0; d < dim; ++d) {
3424         in.holelist[h*dim+d] = holeCoords[h*dim+d];
3425       }
3426     }
3427   }
3428 #endif
3429   if (!rank) {
3430     char args[32];
3431 
3432     /* Take away 'Q' for verbose output */
3433     ierr = PetscStrcpy(args, "pqezQra");CHKERRQ(ierr);
3434     triangulate(args, &in, &out, NULL);
3435   }
3436   ierr = PetscFree(in.pointlist);CHKERRQ(ierr);
3437   ierr = PetscFree(in.pointmarkerlist);CHKERRQ(ierr);
3438   ierr = PetscFree(in.segmentlist);CHKERRQ(ierr);
3439   ierr = PetscFree(in.segmentmarkerlist);CHKERRQ(ierr);
3440   ierr = PetscFree(in.trianglelist);CHKERRQ(ierr);
3441 
3442   {
3443     const PetscInt numCorners  = 3;
3444     const PetscInt numCells    = out.numberoftriangles;
3445     const PetscInt numVertices = out.numberofpoints;
3446     const int     *cells      = out.trianglelist;
3447     const double  *meshCoords = out.pointlist;
3448     PetscBool      interpolate = depthGlobal > 1 ? PETSC_TRUE : PETSC_FALSE;
3449 
3450     ierr = DMPlexCreateFromCellList(comm, dim, numCells, numVertices, numCorners, interpolate, cells, dim, meshCoords, dmRefined);CHKERRQ(ierr);
3451     /* Set labels */
3452     for (v = 0; v < numVertices; ++v) {
3453       if (out.pointmarkerlist[v]) {
3454         ierr = DMPlexSetLabelValue(*dmRefined, "marker", v+numCells, out.pointmarkerlist[v]);CHKERRQ(ierr);
3455       }
3456     }
3457     if (interpolate) {
3458       PetscInt e;
3459 
3460       for (e = 0; e < out.numberofedges; e++) {
3461         if (out.edgemarkerlist[e]) {
3462           const PetscInt  vertices[2] = {out.edgelist[e*2+0]+numCells, out.edgelist[e*2+1]+numCells};
3463           const PetscInt *edges;
3464           PetscInt        numEdges;
3465 
3466           ierr = DMPlexGetJoin(*dmRefined, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3467           if (numEdges != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Two vertices must cover only one edge, not %D", numEdges);
3468           ierr = DMPlexSetLabelValue(*dmRefined, "marker", edges[0], out.edgemarkerlist[e]);CHKERRQ(ierr);
3469           ierr = DMPlexRestoreJoin(*dmRefined, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3470         }
3471       }
3472     }
3473     ierr = DMPlexSetRefinementUniform(*dmRefined, PETSC_FALSE);CHKERRQ(ierr);
3474   }
3475 #if 0 /* Do not currently support holes */
3476   ierr = DMPlexCopyHoles(*dm, boundary);CHKERRQ(ierr);
3477 #endif
3478   ierr = FiniOutput_Triangle(&out);CHKERRQ(ierr);
3479   PetscFunctionReturn(0);
3480 }
3481 #endif
3482 
3483 #if defined(PETSC_HAVE_TETGEN)
3484 #include <tetgen.h>
3485 #undef __FUNCT__
3486 #define __FUNCT__ "DMPlexGenerate_Tetgen"
3487 PetscErrorCode DMPlexGenerate_Tetgen(DM boundary, PetscBool interpolate, DM *dm)
3488 {
3489   MPI_Comm       comm;
3490   const PetscInt dim  = 3;
3491   ::tetgenio     in;
3492   ::tetgenio     out;
3493   PetscInt       vStart, vEnd, v, fStart, fEnd, f;
3494   PetscMPIInt    rank;
3495   PetscErrorCode ierr;
3496 
3497   PetscFunctionBegin;
3498   ierr              = PetscObjectGetComm((PetscObject)boundary,&comm);CHKERRQ(ierr);
3499   ierr              = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
3500   ierr              = DMPlexGetDepthStratum(boundary, 0, &vStart, &vEnd);CHKERRQ(ierr);
3501   in.numberofpoints = vEnd - vStart;
3502   if (in.numberofpoints > 0) {
3503     PetscSection coordSection;
3504     Vec          coordinates;
3505     PetscScalar *array;
3506 
3507     in.pointlist       = new double[in.numberofpoints*dim];
3508     in.pointmarkerlist = new int[in.numberofpoints];
3509 
3510     ierr = DMGetCoordinatesLocal(boundary, &coordinates);CHKERRQ(ierr);
3511     ierr = DMGetCoordinateSection(boundary, &coordSection);CHKERRQ(ierr);
3512     ierr = VecGetArray(coordinates, &array);CHKERRQ(ierr);
3513     for (v = vStart; v < vEnd; ++v) {
3514       const PetscInt idx = v - vStart;
3515       PetscInt       off, d;
3516 
3517       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
3518       for (d = 0; d < dim; ++d) in.pointlist[idx*dim + d] = array[off+d];
3519       ierr = DMPlexGetLabelValue(boundary, "marker", v, &in.pointmarkerlist[idx]);CHKERRQ(ierr);
3520     }
3521     ierr = VecRestoreArray(coordinates, &array);CHKERRQ(ierr);
3522   }
3523   ierr  = DMPlexGetHeightStratum(boundary, 0, &fStart, &fEnd);CHKERRQ(ierr);
3524 
3525   in.numberoffacets = fEnd - fStart;
3526   if (in.numberoffacets > 0) {
3527     in.facetlist       = new tetgenio::facet[in.numberoffacets];
3528     in.facetmarkerlist = new int[in.numberoffacets];
3529     for (f = fStart; f < fEnd; ++f) {
3530       const PetscInt idx     = f - fStart;
3531       PetscInt      *points = NULL, numPoints, p, numVertices = 0, v;
3532 
3533       in.facetlist[idx].numberofpolygons = 1;
3534       in.facetlist[idx].polygonlist      = new tetgenio::polygon[in.facetlist[idx].numberofpolygons];
3535       in.facetlist[idx].numberofholes    = 0;
3536       in.facetlist[idx].holelist         = NULL;
3537 
3538       ierr = DMPlexGetTransitiveClosure(boundary, f, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
3539       for (p = 0; p < numPoints*2; p += 2) {
3540         const PetscInt point = points[p];
3541         if ((point >= vStart) && (point < vEnd)) points[numVertices++] = point;
3542       }
3543 
3544       tetgenio::polygon *poly = in.facetlist[idx].polygonlist;
3545       poly->numberofvertices = numVertices;
3546       poly->vertexlist       = new int[poly->numberofvertices];
3547       for (v = 0; v < numVertices; ++v) {
3548         const PetscInt vIdx = points[v] - vStart;
3549         poly->vertexlist[v] = vIdx;
3550       }
3551       ierr = DMPlexGetLabelValue(boundary, "marker", f, &in.facetmarkerlist[idx]);CHKERRQ(ierr);
3552       ierr = DMPlexRestoreTransitiveClosure(boundary, f, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
3553     }
3554   }
3555   if (!rank) {
3556     char args[32];
3557 
3558     /* Take away 'Q' for verbose output */
3559     ierr = PetscStrcpy(args, "pqezQ");CHKERRQ(ierr);
3560     ::tetrahedralize(args, &in, &out);
3561   }
3562   {
3563     const PetscInt numCorners  = 4;
3564     const PetscInt numCells    = out.numberoftetrahedra;
3565     const PetscInt numVertices = out.numberofpoints;
3566     const double   *meshCoords = out.pointlist;
3567     int            *cells      = out.tetrahedronlist;
3568 
3569     ierr = DMPlexInvertCells_Internal(dim, numCells, numCorners, cells);CHKERRQ(ierr);
3570     ierr = DMPlexCreateFromCellList(comm, dim, numCells, numVertices, numCorners, interpolate, cells, dim, meshCoords, dm);CHKERRQ(ierr);
3571     /* Set labels */
3572     for (v = 0; v < numVertices; ++v) {
3573       if (out.pointmarkerlist[v]) {
3574         ierr = DMPlexSetLabelValue(*dm, "marker", v+numCells, out.pointmarkerlist[v]);CHKERRQ(ierr);
3575       }
3576     }
3577     if (interpolate) {
3578       PetscInt e;
3579 
3580       for (e = 0; e < out.numberofedges; e++) {
3581         if (out.edgemarkerlist[e]) {
3582           const PetscInt  vertices[2] = {out.edgelist[e*2+0]+numCells, out.edgelist[e*2+1]+numCells};
3583           const PetscInt *edges;
3584           PetscInt        numEdges;
3585 
3586           ierr = DMPlexGetJoin(*dm, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3587           if (numEdges != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Two vertices must cover only one edge, not %D", numEdges);
3588           ierr = DMPlexSetLabelValue(*dm, "marker", edges[0], out.edgemarkerlist[e]);CHKERRQ(ierr);
3589           ierr = DMPlexRestoreJoin(*dm, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3590         }
3591       }
3592       for (f = 0; f < out.numberoftrifaces; f++) {
3593         if (out.trifacemarkerlist[f]) {
3594           const PetscInt  vertices[3] = {out.trifacelist[f*3+0]+numCells, out.trifacelist[f*3+1]+numCells, out.trifacelist[f*3+2]+numCells};
3595           const PetscInt *faces;
3596           PetscInt        numFaces;
3597 
3598           ierr = DMPlexGetJoin(*dm, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
3599           if (numFaces != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Three vertices must cover only one face, not %D", numFaces);
3600           ierr = DMPlexSetLabelValue(*dm, "marker", faces[0], out.trifacemarkerlist[f]);CHKERRQ(ierr);
3601           ierr = DMPlexRestoreJoin(*dm, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
3602         }
3603       }
3604     }
3605     ierr = DMPlexSetRefinementUniform(*dm, PETSC_FALSE);CHKERRQ(ierr);
3606   }
3607   PetscFunctionReturn(0);
3608 }
3609 
3610 #undef __FUNCT__
3611 #define __FUNCT__ "DMPlexRefine_Tetgen"
3612 PetscErrorCode DMPlexRefine_Tetgen(DM dm, double *maxVolumes, DM *dmRefined)
3613 {
3614   MPI_Comm       comm;
3615   const PetscInt dim  = 3;
3616   ::tetgenio     in;
3617   ::tetgenio     out;
3618   PetscInt       vStart, vEnd, v, cStart, cEnd, c, depth, depthGlobal;
3619   PetscMPIInt    rank;
3620   PetscErrorCode ierr;
3621 
3622   PetscFunctionBegin;
3623   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
3624   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
3625   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3626   ierr = MPI_Allreduce(&depth, &depthGlobal, 1, MPIU_INT, MPI_MAX, comm);CHKERRQ(ierr);
3627   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
3628 
3629   in.numberofpoints = vEnd - vStart;
3630   if (in.numberofpoints > 0) {
3631     PetscSection coordSection;
3632     Vec          coordinates;
3633     PetscScalar *array;
3634 
3635     in.pointlist       = new double[in.numberofpoints*dim];
3636     in.pointmarkerlist = new int[in.numberofpoints];
3637 
3638     ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
3639     ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
3640     ierr = VecGetArray(coordinates, &array);CHKERRQ(ierr);
3641     for (v = vStart; v < vEnd; ++v) {
3642       const PetscInt idx = v - vStart;
3643       PetscInt       off, d;
3644 
3645       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
3646       for (d = 0; d < dim; ++d) in.pointlist[idx*dim + d] = array[off+d];
3647       ierr = DMPlexGetLabelValue(dm, "marker", v, &in.pointmarkerlist[idx]);CHKERRQ(ierr);
3648     }
3649     ierr = VecRestoreArray(coordinates, &array);CHKERRQ(ierr);
3650   }
3651   ierr  = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
3652 
3653   in.numberofcorners       = 4;
3654   in.numberoftetrahedra    = cEnd - cStart;
3655   in.tetrahedronvolumelist = (double*) maxVolumes;
3656   if (in.numberoftetrahedra > 0) {
3657     in.tetrahedronlist = new int[in.numberoftetrahedra*in.numberofcorners];
3658     for (c = cStart; c < cEnd; ++c) {
3659       const PetscInt idx      = c - cStart;
3660       PetscInt      *closure = NULL;
3661       PetscInt       closureSize;
3662 
3663       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
3664       if ((closureSize != 5) && (closureSize != 15)) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Mesh has cell which is not a tetrahedron, %D vertices in closure", closureSize);
3665       for (v = 0; v < 4; ++v) {
3666         in.tetrahedronlist[idx*in.numberofcorners + v] = closure[(v+closureSize-4)*2] - vStart;
3667       }
3668       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
3669     }
3670   }
3671   /* TODO: Put in boundary faces with markers */
3672   if (!rank) {
3673     char args[32];
3674 
3675     /* Take away 'Q' for verbose output */
3676     /*ierr = PetscStrcpy(args, "qezQra");CHKERRQ(ierr); */
3677     ierr = PetscStrcpy(args, "qezraVVVV");CHKERRQ(ierr);
3678     ::tetrahedralize(args, &in, &out);
3679   }
3680   in.tetrahedronvolumelist = NULL;
3681 
3682   {
3683     const PetscInt numCorners  = 4;
3684     const PetscInt numCells    = out.numberoftetrahedra;
3685     const PetscInt numVertices = out.numberofpoints;
3686     const double   *meshCoords = out.pointlist;
3687     int            *cells      = out.tetrahedronlist;
3688 
3689     PetscBool      interpolate = depthGlobal > 1 ? PETSC_TRUE : PETSC_FALSE;
3690 
3691     ierr = DMPlexInvertCells_Internal(dim, numCells, numCorners, cells);CHKERRQ(ierr);
3692     ierr = DMPlexCreateFromCellList(comm, dim, numCells, numVertices, numCorners, interpolate, cells, dim, meshCoords, dmRefined);CHKERRQ(ierr);
3693     /* Set labels */
3694     for (v = 0; v < numVertices; ++v) {
3695       if (out.pointmarkerlist[v]) {
3696         ierr = DMPlexSetLabelValue(*dmRefined, "marker", v+numCells, out.pointmarkerlist[v]);CHKERRQ(ierr);
3697       }
3698     }
3699     if (interpolate) {
3700       PetscInt e, f;
3701 
3702       for (e = 0; e < out.numberofedges; e++) {
3703         if (out.edgemarkerlist[e]) {
3704           const PetscInt  vertices[2] = {out.edgelist[e*2+0]+numCells, out.edgelist[e*2+1]+numCells};
3705           const PetscInt *edges;
3706           PetscInt        numEdges;
3707 
3708           ierr = DMPlexGetJoin(*dmRefined, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3709           if (numEdges != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Two vertices must cover only one edge, not %D", numEdges);
3710           ierr = DMPlexSetLabelValue(*dmRefined, "marker", edges[0], out.edgemarkerlist[e]);CHKERRQ(ierr);
3711           ierr = DMPlexRestoreJoin(*dmRefined, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3712         }
3713       }
3714       for (f = 0; f < out.numberoftrifaces; f++) {
3715         if (out.trifacemarkerlist[f]) {
3716           const PetscInt  vertices[3] = {out.trifacelist[f*3+0]+numCells, out.trifacelist[f*3+1]+numCells, out.trifacelist[f*3+2]+numCells};
3717           const PetscInt *faces;
3718           PetscInt        numFaces;
3719 
3720           ierr = DMPlexGetJoin(*dmRefined, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
3721           if (numFaces != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Three vertices must cover only one face, not %D", numFaces);
3722           ierr = DMPlexSetLabelValue(*dmRefined, "marker", faces[0], out.trifacemarkerlist[f]);CHKERRQ(ierr);
3723           ierr = DMPlexRestoreJoin(*dmRefined, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
3724         }
3725       }
3726     }
3727     ierr = DMPlexSetRefinementUniform(*dmRefined, PETSC_FALSE);CHKERRQ(ierr);
3728   }
3729   PetscFunctionReturn(0);
3730 }
3731 #endif
3732 
3733 #if defined(PETSC_HAVE_CTETGEN)
3734 #include <ctetgen.h>
3735 
3736 #undef __FUNCT__
3737 #define __FUNCT__ "DMPlexGenerate_CTetgen"
3738 PetscErrorCode DMPlexGenerate_CTetgen(DM boundary, PetscBool interpolate, DM *dm)
3739 {
3740   MPI_Comm       comm;
3741   const PetscInt dim  = 3;
3742   PLC           *in, *out;
3743   PetscInt       verbose = 0, vStart, vEnd, v, fStart, fEnd, f;
3744   PetscMPIInt    rank;
3745   PetscErrorCode ierr;
3746 
3747   PetscFunctionBegin;
3748   ierr = PetscObjectGetComm((PetscObject)boundary,&comm);CHKERRQ(ierr);
3749   ierr = PetscOptionsGetInt(((PetscObject) boundary)->prefix, "-ctetgen_verbose", &verbose, NULL);CHKERRQ(ierr);
3750   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
3751   ierr = DMPlexGetDepthStratum(boundary, 0, &vStart, &vEnd);CHKERRQ(ierr);
3752   ierr = PLCCreate(&in);CHKERRQ(ierr);
3753   ierr = PLCCreate(&out);CHKERRQ(ierr);
3754 
3755   in->numberofpoints = vEnd - vStart;
3756   if (in->numberofpoints > 0) {
3757     PetscSection coordSection;
3758     Vec          coordinates;
3759     PetscScalar *array;
3760 
3761     ierr = PetscMalloc1(in->numberofpoints*dim, &in->pointlist);CHKERRQ(ierr);
3762     ierr = PetscMalloc1(in->numberofpoints,       &in->pointmarkerlist);CHKERRQ(ierr);
3763     ierr = DMGetCoordinatesLocal(boundary, &coordinates);CHKERRQ(ierr);
3764     ierr = DMGetCoordinateSection(boundary, &coordSection);CHKERRQ(ierr);
3765     ierr = VecGetArray(coordinates, &array);CHKERRQ(ierr);
3766     for (v = vStart; v < vEnd; ++v) {
3767       const PetscInt idx = v - vStart;
3768       PetscInt       off, d, m;
3769 
3770       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
3771       for (d = 0; d < dim; ++d) {
3772         in->pointlist[idx*dim + d] = PetscRealPart(array[off+d]);
3773       }
3774       ierr = DMPlexGetLabelValue(boundary, "marker", v, &m);CHKERRQ(ierr);
3775 
3776       in->pointmarkerlist[idx] = (int) m;
3777     }
3778     ierr = VecRestoreArray(coordinates, &array);CHKERRQ(ierr);
3779   }
3780   ierr  = DMPlexGetHeightStratum(boundary, 0, &fStart, &fEnd);CHKERRQ(ierr);
3781 
3782   in->numberoffacets = fEnd - fStart;
3783   if (in->numberoffacets > 0) {
3784     ierr = PetscMalloc1(in->numberoffacets, &in->facetlist);CHKERRQ(ierr);
3785     ierr = PetscMalloc1(in->numberoffacets,   &in->facetmarkerlist);CHKERRQ(ierr);
3786     for (f = fStart; f < fEnd; ++f) {
3787       const PetscInt idx     = f - fStart;
3788       PetscInt      *points = NULL, numPoints, p, numVertices = 0, v, m;
3789       polygon       *poly;
3790 
3791       in->facetlist[idx].numberofpolygons = 1;
3792 
3793       ierr = PetscMalloc1(in->facetlist[idx].numberofpolygons, &in->facetlist[idx].polygonlist);CHKERRQ(ierr);
3794 
3795       in->facetlist[idx].numberofholes    = 0;
3796       in->facetlist[idx].holelist         = NULL;
3797 
3798       ierr = DMPlexGetTransitiveClosure(boundary, f, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
3799       for (p = 0; p < numPoints*2; p += 2) {
3800         const PetscInt point = points[p];
3801         if ((point >= vStart) && (point < vEnd)) points[numVertices++] = point;
3802       }
3803 
3804       poly                   = in->facetlist[idx].polygonlist;
3805       poly->numberofvertices = numVertices;
3806       ierr                   = PetscMalloc1(poly->numberofvertices, &poly->vertexlist);CHKERRQ(ierr);
3807       for (v = 0; v < numVertices; ++v) {
3808         const PetscInt vIdx = points[v] - vStart;
3809         poly->vertexlist[v] = vIdx;
3810       }
3811       ierr                     = DMPlexGetLabelValue(boundary, "marker", f, &m);CHKERRQ(ierr);
3812       in->facetmarkerlist[idx] = (int) m;
3813       ierr                     = DMPlexRestoreTransitiveClosure(boundary, f, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
3814     }
3815   }
3816   if (!rank) {
3817     TetGenOpts t;
3818 
3819     ierr        = TetGenOptsInitialize(&t);CHKERRQ(ierr);
3820     t.in        = boundary; /* Should go away */
3821     t.plc       = 1;
3822     t.quality   = 1;
3823     t.edgesout  = 1;
3824     t.zeroindex = 1;
3825     t.quiet     = 1;
3826     t.verbose   = verbose;
3827     ierr        = TetGenCheckOpts(&t);CHKERRQ(ierr);
3828     ierr        = TetGenTetrahedralize(&t, in, out);CHKERRQ(ierr);
3829   }
3830   {
3831     const PetscInt numCorners  = 4;
3832     const PetscInt numCells    = out->numberoftetrahedra;
3833     const PetscInt numVertices = out->numberofpoints;
3834     const double   *meshCoords = out->pointlist;
3835     int            *cells      = out->tetrahedronlist;
3836 
3837     ierr = DMPlexInvertCells_Internal(dim, numCells, numCorners, cells);CHKERRQ(ierr);
3838     ierr = DMPlexCreateFromCellList(comm, dim, numCells, numVertices, numCorners, interpolate, cells, dim, meshCoords, dm);CHKERRQ(ierr);
3839     /* Set labels */
3840     for (v = 0; v < numVertices; ++v) {
3841       if (out->pointmarkerlist[v]) {
3842         ierr = DMPlexSetLabelValue(*dm, "marker", v+numCells, out->pointmarkerlist[v]);CHKERRQ(ierr);
3843       }
3844     }
3845     if (interpolate) {
3846       PetscInt e;
3847 
3848       for (e = 0; e < out->numberofedges; e++) {
3849         if (out->edgemarkerlist[e]) {
3850           const PetscInt  vertices[2] = {out->edgelist[e*2+0]+numCells, out->edgelist[e*2+1]+numCells};
3851           const PetscInt *edges;
3852           PetscInt        numEdges;
3853 
3854           ierr = DMPlexGetJoin(*dm, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3855           if (numEdges != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Two vertices must cover only one edge, not %D", numEdges);
3856           ierr = DMPlexSetLabelValue(*dm, "marker", edges[0], out->edgemarkerlist[e]);CHKERRQ(ierr);
3857           ierr = DMPlexRestoreJoin(*dm, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3858         }
3859       }
3860       for (f = 0; f < out->numberoftrifaces; f++) {
3861         if (out->trifacemarkerlist[f]) {
3862           const PetscInt  vertices[3] = {out->trifacelist[f*3+0]+numCells, out->trifacelist[f*3+1]+numCells, out->trifacelist[f*3+2]+numCells};
3863           const PetscInt *faces;
3864           PetscInt        numFaces;
3865 
3866           ierr = DMPlexGetFullJoin(*dm, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
3867           if (numFaces != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Three vertices must cover only one face, not %D", numFaces);
3868           ierr = DMPlexSetLabelValue(*dm, "marker", faces[0], out->trifacemarkerlist[f]);CHKERRQ(ierr);
3869           ierr = DMPlexRestoreJoin(*dm, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
3870         }
3871       }
3872     }
3873     ierr = DMPlexSetRefinementUniform(*dm, PETSC_FALSE);CHKERRQ(ierr);
3874   }
3875 
3876   ierr = PLCDestroy(&in);CHKERRQ(ierr);
3877   ierr = PLCDestroy(&out);CHKERRQ(ierr);
3878   PetscFunctionReturn(0);
3879 }
3880 
3881 #undef __FUNCT__
3882 #define __FUNCT__ "DMPlexRefine_CTetgen"
3883 PetscErrorCode DMPlexRefine_CTetgen(DM dm, PetscReal *maxVolumes, DM *dmRefined)
3884 {
3885   MPI_Comm       comm;
3886   const PetscInt dim  = 3;
3887   PLC           *in, *out;
3888   PetscInt       verbose = 0, vStart, vEnd, v, cStart, cEnd, c, depth, depthGlobal;
3889   PetscMPIInt    rank;
3890   PetscErrorCode ierr;
3891 
3892   PetscFunctionBegin;
3893   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
3894   ierr = PetscOptionsGetInt(((PetscObject) dm)->prefix, "-ctetgen_verbose", &verbose, NULL);CHKERRQ(ierr);
3895   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
3896   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3897   ierr = MPI_Allreduce(&depth, &depthGlobal, 1, MPIU_INT, MPI_MAX, comm);CHKERRQ(ierr);
3898   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
3899   ierr = PLCCreate(&in);CHKERRQ(ierr);
3900   ierr = PLCCreate(&out);CHKERRQ(ierr);
3901 
3902   in->numberofpoints = vEnd - vStart;
3903   if (in->numberofpoints > 0) {
3904     PetscSection coordSection;
3905     Vec          coordinates;
3906     PetscScalar *array;
3907 
3908     ierr = PetscMalloc1(in->numberofpoints*dim, &in->pointlist);CHKERRQ(ierr);
3909     ierr = PetscMalloc1(in->numberofpoints,       &in->pointmarkerlist);CHKERRQ(ierr);
3910     ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
3911     ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
3912     ierr = VecGetArray(coordinates, &array);CHKERRQ(ierr);
3913     for (v = vStart; v < vEnd; ++v) {
3914       const PetscInt idx = v - vStart;
3915       PetscInt       off, d, m;
3916 
3917       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
3918       for (d = 0; d < dim; ++d) {
3919         in->pointlist[idx*dim + d] = PetscRealPart(array[off+d]);
3920       }
3921       ierr = DMPlexGetLabelValue(dm, "marker", v, &m);CHKERRQ(ierr);
3922 
3923       in->pointmarkerlist[idx] = (int) m;
3924     }
3925     ierr = VecRestoreArray(coordinates, &array);CHKERRQ(ierr);
3926   }
3927   ierr  = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
3928 
3929   in->numberofcorners       = 4;
3930   in->numberoftetrahedra    = cEnd - cStart;
3931   in->tetrahedronvolumelist = maxVolumes;
3932   if (in->numberoftetrahedra > 0) {
3933     ierr = PetscMalloc1(in->numberoftetrahedra*in->numberofcorners, &in->tetrahedronlist);CHKERRQ(ierr);
3934     for (c = cStart; c < cEnd; ++c) {
3935       const PetscInt idx      = c - cStart;
3936       PetscInt      *closure = NULL;
3937       PetscInt       closureSize;
3938 
3939       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
3940       if ((closureSize != 5) && (closureSize != 15)) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Mesh has cell which is not a tetrahedron, %D vertices in closure", closureSize);
3941       for (v = 0; v < 4; ++v) {
3942         in->tetrahedronlist[idx*in->numberofcorners + v] = closure[(v+closureSize-4)*2] - vStart;
3943       }
3944       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
3945     }
3946   }
3947   if (!rank) {
3948     TetGenOpts t;
3949 
3950     ierr = TetGenOptsInitialize(&t);CHKERRQ(ierr);
3951 
3952     t.in        = dm; /* Should go away */
3953     t.refine    = 1;
3954     t.varvolume = 1;
3955     t.quality   = 1;
3956     t.edgesout  = 1;
3957     t.zeroindex = 1;
3958     t.quiet     = 1;
3959     t.verbose   = verbose; /* Change this */
3960 
3961     ierr = TetGenCheckOpts(&t);CHKERRQ(ierr);
3962     ierr = TetGenTetrahedralize(&t, in, out);CHKERRQ(ierr);
3963   }
3964   {
3965     const PetscInt numCorners  = 4;
3966     const PetscInt numCells    = out->numberoftetrahedra;
3967     const PetscInt numVertices = out->numberofpoints;
3968     const double   *meshCoords = out->pointlist;
3969     int            *cells      = out->tetrahedronlist;
3970     PetscBool      interpolate = depthGlobal > 1 ? PETSC_TRUE : PETSC_FALSE;
3971 
3972     ierr = DMPlexInvertCells_Internal(dim, numCells, numCorners, cells);CHKERRQ(ierr);
3973     ierr = DMPlexCreateFromCellList(comm, dim, numCells, numVertices, numCorners, interpolate, cells, dim, meshCoords, dmRefined);CHKERRQ(ierr);
3974     /* Set labels */
3975     for (v = 0; v < numVertices; ++v) {
3976       if (out->pointmarkerlist[v]) {
3977         ierr = DMPlexSetLabelValue(*dmRefined, "marker", v+numCells, out->pointmarkerlist[v]);CHKERRQ(ierr);
3978       }
3979     }
3980     if (interpolate) {
3981       PetscInt e, f;
3982 
3983       for (e = 0; e < out->numberofedges; e++) {
3984         if (out->edgemarkerlist[e]) {
3985           const PetscInt  vertices[2] = {out->edgelist[e*2+0]+numCells, out->edgelist[e*2+1]+numCells};
3986           const PetscInt *edges;
3987           PetscInt        numEdges;
3988 
3989           ierr = DMPlexGetJoin(*dmRefined, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3990           if (numEdges != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Two vertices must cover only one edge, not %D", numEdges);
3991           ierr = DMPlexSetLabelValue(*dmRefined, "marker", edges[0], out->edgemarkerlist[e]);CHKERRQ(ierr);
3992           ierr = DMPlexRestoreJoin(*dmRefined, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3993         }
3994       }
3995       for (f = 0; f < out->numberoftrifaces; f++) {
3996         if (out->trifacemarkerlist[f]) {
3997           const PetscInt  vertices[3] = {out->trifacelist[f*3+0]+numCells, out->trifacelist[f*3+1]+numCells, out->trifacelist[f*3+2]+numCells};
3998           const PetscInt *faces;
3999           PetscInt        numFaces;
4000 
4001           ierr = DMPlexGetFullJoin(*dmRefined, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
4002           if (numFaces != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Three vertices must cover only one face, not %D", numFaces);
4003           ierr = DMPlexSetLabelValue(*dmRefined, "marker", faces[0], out->trifacemarkerlist[f]);CHKERRQ(ierr);
4004           ierr = DMPlexRestoreJoin(*dmRefined, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
4005         }
4006       }
4007     }
4008     ierr = DMPlexSetRefinementUniform(*dmRefined, PETSC_FALSE);CHKERRQ(ierr);
4009   }
4010   ierr = PLCDestroy(&in);CHKERRQ(ierr);
4011   ierr = PLCDestroy(&out);CHKERRQ(ierr);
4012   PetscFunctionReturn(0);
4013 }
4014 #endif
4015 
4016 #undef __FUNCT__
4017 #define __FUNCT__ "DMPlexGenerate"
4018 /*@C
4019   DMPlexGenerate - Generates a mesh.
4020 
4021   Not Collective
4022 
4023   Input Parameters:
4024 + boundary - The DMPlex boundary object
4025 . name - The mesh generation package name
4026 - interpolate - Flag to create intermediate mesh elements
4027 
4028   Output Parameter:
4029 . mesh - The DMPlex object
4030 
4031   Level: intermediate
4032 
4033 .keywords: mesh, elements
4034 .seealso: DMPlexCreate(), DMRefine()
4035 @*/
4036 PetscErrorCode DMPlexGenerate(DM boundary, const char name[], PetscBool interpolate, DM *mesh)
4037 {
4038   PetscInt       dim;
4039   char           genname[1024];
4040   PetscBool      isTriangle = PETSC_FALSE, isTetgen = PETSC_FALSE, isCTetgen = PETSC_FALSE, flg;
4041   PetscErrorCode ierr;
4042 
4043   PetscFunctionBegin;
4044   PetscValidHeaderSpecific(boundary, DM_CLASSID, 1);
4045   PetscValidLogicalCollectiveBool(boundary, interpolate, 2);
4046   ierr = DMPlexGetDimension(boundary, &dim);CHKERRQ(ierr);
4047   ierr = PetscOptionsGetString(((PetscObject) boundary)->prefix, "-dm_plex_generator", genname, 1024, &flg);CHKERRQ(ierr);
4048   if (flg) name = genname;
4049   if (name) {
4050     ierr = PetscStrcmp(name, "triangle", &isTriangle);CHKERRQ(ierr);
4051     ierr = PetscStrcmp(name, "tetgen",   &isTetgen);CHKERRQ(ierr);
4052     ierr = PetscStrcmp(name, "ctetgen",  &isCTetgen);CHKERRQ(ierr);
4053   }
4054   switch (dim) {
4055   case 1:
4056     if (!name || isTriangle) {
4057 #if defined(PETSC_HAVE_TRIANGLE)
4058       ierr = DMPlexGenerate_Triangle(boundary, interpolate, mesh);CHKERRQ(ierr);
4059 #else
4060       SETERRQ(PetscObjectComm((PetscObject)boundary), PETSC_ERR_SUP, "Mesh generation needs external package support.\nPlease reconfigure with --download-triangle.");
4061 #endif
4062     } else SETERRQ1(PetscObjectComm((PetscObject)boundary), PETSC_ERR_SUP, "Unknown 2D mesh generation package %s", name);
4063     break;
4064   case 2:
4065     if (!name || isCTetgen) {
4066 #if defined(PETSC_HAVE_CTETGEN)
4067       ierr = DMPlexGenerate_CTetgen(boundary, interpolate, mesh);CHKERRQ(ierr);
4068 #else
4069       SETERRQ(PetscObjectComm((PetscObject)boundary), PETSC_ERR_SUP, "CTetgen needs external package support.\nPlease reconfigure with --download-ctetgen.");
4070 #endif
4071     } else if (isTetgen) {
4072 #if defined(PETSC_HAVE_TETGEN)
4073       ierr = DMPlexGenerate_Tetgen(boundary, interpolate, mesh);CHKERRQ(ierr);
4074 #else
4075       SETERRQ(PetscObjectComm((PetscObject)boundary), PETSC_ERR_SUP, "Tetgen needs external package support.\nPlease reconfigure with --with-c-language=cxx --download-tetgen.");
4076 #endif
4077     } else SETERRQ1(PetscObjectComm((PetscObject)boundary), PETSC_ERR_SUP, "Unknown 3D mesh generation package %s", name);
4078     break;
4079   default:
4080     SETERRQ1(PetscObjectComm((PetscObject)boundary), PETSC_ERR_SUP, "Mesh generation for a dimension %d boundary is not supported.", dim);
4081   }
4082   PetscFunctionReturn(0);
4083 }
4084 
4085 #undef __FUNCT__
4086 #define __FUNCT__ "DMRefine_Plex"
4087 PetscErrorCode DMRefine_Plex(DM dm, MPI_Comm comm, DM *dmRefined)
4088 {
4089   PetscReal      refinementLimit;
4090   PetscInt       dim, cStart, cEnd;
4091   char           genname[1024], *name = NULL;
4092   PetscBool      isUniform, isTriangle = PETSC_FALSE, isTetgen = PETSC_FALSE, isCTetgen = PETSC_FALSE, flg;
4093   PetscErrorCode ierr;
4094 
4095   PetscFunctionBegin;
4096   ierr = DMPlexGetRefinementUniform(dm, &isUniform);CHKERRQ(ierr);
4097   if (isUniform) {
4098     CellRefiner cellRefiner;
4099 
4100     ierr = DMPlexGetCellRefiner_Internal(dm, &cellRefiner);CHKERRQ(ierr);
4101     ierr = DMPlexRefineUniform_Internal(dm, cellRefiner, dmRefined);CHKERRQ(ierr);
4102     PetscFunctionReturn(0);
4103   }
4104   ierr = DMPlexGetRefinementLimit(dm, &refinementLimit);CHKERRQ(ierr);
4105   if (refinementLimit == 0.0) PetscFunctionReturn(0);
4106   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
4107   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
4108   ierr = PetscOptionsGetString(((PetscObject) dm)->prefix, "-dm_plex_generator", genname, 1024, &flg);CHKERRQ(ierr);
4109   if (flg) name = genname;
4110   if (name) {
4111     ierr = PetscStrcmp(name, "triangle", &isTriangle);CHKERRQ(ierr);
4112     ierr = PetscStrcmp(name, "tetgen",   &isTetgen);CHKERRQ(ierr);
4113     ierr = PetscStrcmp(name, "ctetgen",  &isCTetgen);CHKERRQ(ierr);
4114   }
4115   switch (dim) {
4116   case 2:
4117     if (!name || isTriangle) {
4118 #if defined(PETSC_HAVE_TRIANGLE)
4119       double  *maxVolumes;
4120       PetscInt c;
4121 
4122       ierr = PetscMalloc1((cEnd - cStart), &maxVolumes);CHKERRQ(ierr);
4123       for (c = 0; c < cEnd-cStart; ++c) maxVolumes[c] = refinementLimit;
4124       ierr = DMPlexRefine_Triangle(dm, maxVolumes, dmRefined);CHKERRQ(ierr);
4125       ierr = PetscFree(maxVolumes);CHKERRQ(ierr);
4126 #else
4127       SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Mesh refinement needs external package support.\nPlease reconfigure with --download-triangle.");
4128 #endif
4129     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Unknown 2D mesh generation package %s", name);
4130     break;
4131   case 3:
4132     if (!name || isCTetgen) {
4133 #if defined(PETSC_HAVE_CTETGEN)
4134       PetscReal *maxVolumes;
4135       PetscInt   c;
4136 
4137       ierr = PetscMalloc1((cEnd - cStart), &maxVolumes);CHKERRQ(ierr);
4138       for (c = 0; c < cEnd-cStart; ++c) maxVolumes[c] = refinementLimit;
4139       ierr = DMPlexRefine_CTetgen(dm, maxVolumes, dmRefined);CHKERRQ(ierr);
4140 #else
4141       SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "CTetgen needs external package support.\nPlease reconfigure with --download-ctetgen.");
4142 #endif
4143     } else if (isTetgen) {
4144 #if defined(PETSC_HAVE_TETGEN)
4145       double  *maxVolumes;
4146       PetscInt c;
4147 
4148       ierr = PetscMalloc1((cEnd - cStart), &maxVolumes);CHKERRQ(ierr);
4149       for (c = 0; c < cEnd-cStart; ++c) maxVolumes[c] = refinementLimit;
4150       ierr = DMPlexRefine_Tetgen(dm, maxVolumes, dmRefined);CHKERRQ(ierr);
4151 #else
4152       SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Tetgen needs external package support.\nPlease reconfigure with --with-c-language=cxx --download-tetgen.");
4153 #endif
4154     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Unknown 3D mesh generation package %s", name);
4155     break;
4156   default:
4157     SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Mesh refinement in dimension %d is not supported.", dim);
4158   }
4159   PetscFunctionReturn(0);
4160 }
4161 
4162 #undef __FUNCT__
4163 #define __FUNCT__ "DMRefineHierarchy_Plex"
4164 PetscErrorCode DMRefineHierarchy_Plex(DM dm, PetscInt nlevels, DM dmRefined[])
4165 {
4166   DM             cdm = dm;
4167   PetscInt       r;
4168   PetscBool      isUniform;
4169   PetscErrorCode ierr;
4170 
4171   PetscFunctionBegin;
4172   ierr = DMPlexGetRefinementUniform(dm, &isUniform);CHKERRQ(ierr);
4173   if (!isUniform) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Non-uniform refinement is incompatible with the hierarchy");
4174   for (r = 0; r < nlevels; ++r) {
4175     CellRefiner cellRefiner;
4176 
4177     ierr = DMPlexGetCellRefiner_Internal(cdm, &cellRefiner);CHKERRQ(ierr);
4178     ierr = DMPlexRefineUniform_Internal(cdm, cellRefiner, &dmRefined[r]);CHKERRQ(ierr);
4179     ierr = DMPlexSetCoarseDM(dmRefined[r], cdm);CHKERRQ(ierr);
4180     cdm  = dmRefined[r];
4181   }
4182   PetscFunctionReturn(0);
4183 }
4184 
4185 #undef __FUNCT__
4186 #define __FUNCT__ "DMCoarsen_Plex"
4187 PetscErrorCode DMCoarsen_Plex(DM dm, MPI_Comm comm, DM *dmCoarsened)
4188 {
4189   DM_Plex       *mesh = (DM_Plex*) dm->data;
4190   PetscErrorCode ierr;
4191 
4192   PetscFunctionBegin;
4193   ierr = PetscObjectReference((PetscObject) mesh->coarseMesh);CHKERRQ(ierr);
4194   *dmCoarsened = mesh->coarseMesh;
4195   PetscFunctionReturn(0);
4196 }
4197 
4198 #undef __FUNCT__
4199 #define __FUNCT__ "DMPlexLocalizeCoordinate_Internal"
4200 PetscErrorCode DMPlexLocalizeCoordinate_Internal(DM dm, PetscInt dim, const PetscScalar anchor[], const PetscScalar in[], PetscScalar out[])
4201 {
4202   PetscInt d;
4203 
4204   PetscFunctionBegin;
4205   if (!dm->maxCell) {
4206     for (d = 0; d < dim; ++d) out[d] = in[d];
4207   } else {
4208     for (d = 0; d < dim; ++d) {
4209       if (PetscAbsScalar(anchor[d] - in[d]) > dm->maxCell[d]) {
4210         out[d] = anchor[d] > in[d] ? dm->L[d] + in[d] : in[d] - dm->L[d];
4211       } else {
4212         out[d] = in[d];
4213       }
4214     }
4215   }
4216   PetscFunctionReturn(0);
4217 }
4218 
4219 #undef __FUNCT__
4220 #define __FUNCT__ "DMPlexLocalizeAddCoordinate_Internal"
4221 PetscErrorCode DMPlexLocalizeAddCoordinate_Internal(DM dm, PetscInt dim, const PetscScalar anchor[], const PetscScalar in[], PetscScalar out[])
4222 {
4223   PetscInt d;
4224 
4225   PetscFunctionBegin;
4226   if (!dm->maxCell) {
4227     for (d = 0; d < dim; ++d) out[d] += in[d];
4228   } else {
4229     for (d = 0; d < dim; ++d) {
4230       if (PetscAbsScalar(anchor[d] - in[d]) > dm->maxCell[d]) {
4231         out[d] += anchor[d] > in[d] ? dm->L[d] + in[d] : in[d] - dm->L[d];
4232       } else {
4233         out[d] += in[d];
4234       }
4235     }
4236   }
4237   PetscFunctionReturn(0);
4238 }
4239 
4240 #undef __FUNCT__
4241 #define __FUNCT__ "DMPlexLocalizeCoordinates"
4242 PetscErrorCode DMPlexLocalizeCoordinates(DM dm)
4243 {
4244   PetscSection   coordSection, cSection;
4245   Vec            coordinates,  cVec;
4246   PetscScalar   *coords, *coords2, *anchor;
4247   PetscInt       cStart, cEnd, c, vStart, vEnd, v, dof, d, off, off2, bs, coordSize;
4248   PetscErrorCode ierr;
4249 
4250   PetscFunctionBegin;
4251   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4252   if (!dm->maxCell) PetscFunctionReturn(0);
4253   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
4254   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
4255   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
4256   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
4257   ierr = PetscSectionCreate(PetscObjectComm((PetscObject) dm), &cSection);CHKERRQ(ierr);
4258   ierr = PetscSectionSetChart(cSection, cStart, vEnd);CHKERRQ(ierr);
4259   for (v = vStart; v < vEnd; ++v) {
4260     ierr = PetscSectionGetDof(coordSection, v, &dof);CHKERRQ(ierr);
4261     ierr = PetscSectionSetDof(cSection,     v,  dof);CHKERRQ(ierr);
4262   }
4263   for (c = cStart; c < cEnd; ++c) {
4264     ierr = DMPlexVecGetClosure(dm, coordSection, coordinates, c, &dof, NULL);CHKERRQ(ierr);
4265     ierr = PetscSectionSetDof(cSection, c, dof);CHKERRQ(ierr);
4266   }
4267   ierr = PetscSectionSetUp(cSection);CHKERRQ(ierr);
4268   ierr = PetscSectionGetStorageSize(cSection, &coordSize);CHKERRQ(ierr);
4269   ierr = VecCreate(PetscObjectComm((PetscObject) dm), &cVec);CHKERRQ(ierr);
4270   ierr = VecGetBlockSize(coordinates, &bs);CHKERRQ(ierr);
4271   ierr = VecSetBlockSize(cVec,         bs);CHKERRQ(ierr);
4272   ierr = VecSetSizes(cVec, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
4273   ierr = VecSetType(cVec,VECSTANDARD);CHKERRQ(ierr);
4274   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
4275   ierr = VecGetArray(cVec,        &coords2);CHKERRQ(ierr);
4276   for (v = vStart; v < vEnd; ++v) {
4277     ierr = PetscSectionGetDof(coordSection, v, &dof);CHKERRQ(ierr);
4278     ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
4279     ierr = PetscSectionGetOffset(cSection,     v, &off2);CHKERRQ(ierr);
4280     for (d = 0; d < dof; ++d) coords2[off2+d] = coords[off+d];
4281   }
4282   ierr = DMGetWorkArray(dm, 3, PETSC_SCALAR, &anchor);CHKERRQ(ierr);
4283   for (c = cStart; c < cEnd; ++c) {
4284     PetscScalar *cellCoords = NULL;
4285     PetscInt     b;
4286 
4287     ierr = DMPlexVecGetClosure(dm, coordSection, coordinates, c, &dof, &cellCoords);CHKERRQ(ierr);
4288     ierr = PetscSectionGetOffset(cSection, c, &off2);CHKERRQ(ierr);
4289     for (b = 0; b < bs; ++b) anchor[b] = cellCoords[b];
4290     for (d = 0; d < dof/bs; ++d) {ierr = DMPlexLocalizeCoordinate_Internal(dm, bs, anchor, &cellCoords[d*bs], &coords2[off2+d*bs]);CHKERRQ(ierr);}
4291     ierr = DMPlexVecRestoreClosure(dm, coordSection, coordinates, c, &dof, &cellCoords);CHKERRQ(ierr);
4292   }
4293   ierr = DMRestoreWorkArray(dm, 3, PETSC_SCALAR, &anchor);CHKERRQ(ierr);
4294   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
4295   ierr = VecRestoreArray(cVec,        &coords2);CHKERRQ(ierr);
4296   ierr = DMSetCoordinateSection(dm, cSection);CHKERRQ(ierr);
4297   ierr = DMSetCoordinatesLocal(dm, cVec);CHKERRQ(ierr);
4298   ierr = VecDestroy(&cVec);CHKERRQ(ierr);
4299   ierr = PetscSectionDestroy(&cSection);CHKERRQ(ierr);
4300   PetscFunctionReturn(0);
4301 }
4302 
4303 #undef __FUNCT__
4304 #define __FUNCT__ "DMPlexGetDepthLabel"
4305 /*@
4306   DMPlexGetDepthLabel - Get the DMLabel recording the depth of each point
4307 
4308   Not Collective
4309 
4310   Input Parameter:
4311 . dm    - The DMPlex object
4312 
4313   Output Parameter:
4314 . depthLabel - The DMLabel recording point depth
4315 
4316   Level: developer
4317 
4318 .keywords: mesh, points
4319 .seealso: DMPlexGetDepth(), DMPlexGetHeightStratum(), DMPlexGetDepthStratum()
4320 @*/
4321 PetscErrorCode DMPlexGetDepthLabel(DM dm, DMLabel *depthLabel)
4322 {
4323   DM_Plex       *mesh = (DM_Plex*) dm->data;
4324   PetscErrorCode ierr;
4325 
4326   PetscFunctionBegin;
4327   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4328   PetscValidPointer(depthLabel, 2);
4329   if (!mesh->depthLabel) {ierr = DMPlexGetLabel(dm, "depth", &mesh->depthLabel);CHKERRQ(ierr);}
4330   *depthLabel = mesh->depthLabel;
4331   PetscFunctionReturn(0);
4332 }
4333 
4334 #undef __FUNCT__
4335 #define __FUNCT__ "DMPlexGetDepth"
4336 /*@
4337   DMPlexGetDepth - Get the depth of the DAG representing this mesh
4338 
4339   Not Collective
4340 
4341   Input Parameter:
4342 . dm    - The DMPlex object
4343 
4344   Output Parameter:
4345 . depth - The number of strata (breadth first levels) in the DAG
4346 
4347   Level: developer
4348 
4349 .keywords: mesh, points
4350 .seealso: DMPlexGetDepthLabel(), DMPlexGetHeightStratum(), DMPlexGetDepthStratum()
4351 @*/
4352 PetscErrorCode DMPlexGetDepth(DM dm, PetscInt *depth)
4353 {
4354   DMLabel        label;
4355   PetscInt       d = 0;
4356   PetscErrorCode ierr;
4357 
4358   PetscFunctionBegin;
4359   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4360   PetscValidPointer(depth, 2);
4361   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
4362   if (label) {ierr = DMLabelGetNumValues(label, &d);CHKERRQ(ierr);}
4363   *depth = d-1;
4364   PetscFunctionReturn(0);
4365 }
4366 
4367 #undef __FUNCT__
4368 #define __FUNCT__ "DMPlexGetDepthStratum"
4369 /*@
4370   DMPlexGetDepthStratum - Get the bounds [start, end) for all points at a certain depth.
4371 
4372   Not Collective
4373 
4374   Input Parameters:
4375 + dm           - The DMPlex object
4376 - stratumValue - The requested depth
4377 
4378   Output Parameters:
4379 + start - The first point at this depth
4380 - end   - One beyond the last point at this depth
4381 
4382   Level: developer
4383 
4384 .keywords: mesh, points
4385 .seealso: DMPlexGetHeightStratum(), DMPlexGetDepth()
4386 @*/
4387 PetscErrorCode DMPlexGetDepthStratum(DM dm, PetscInt stratumValue, PetscInt *start, PetscInt *end)
4388 {
4389   DMLabel        label;
4390   PetscInt       pStart, pEnd;
4391   PetscErrorCode ierr;
4392 
4393   PetscFunctionBegin;
4394   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4395   if (start) {PetscValidPointer(start, 3); *start = 0;}
4396   if (end)   {PetscValidPointer(end,   4); *end   = 0;}
4397   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
4398   if (pStart == pEnd) PetscFunctionReturn(0);
4399   if (stratumValue < 0) {
4400     if (start) *start = pStart;
4401     if (end)   *end   = pEnd;
4402     PetscFunctionReturn(0);
4403   }
4404   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
4405   if (!label) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "No label named depth was found");
4406   ierr = DMLabelGetStratumBounds(label, stratumValue, start, end);CHKERRQ(ierr);
4407   PetscFunctionReturn(0);
4408 }
4409 
4410 #undef __FUNCT__
4411 #define __FUNCT__ "DMPlexGetHeightStratum"
4412 /*@
4413   DMPlexGetHeightStratum - Get the bounds [start, end) for all points at a certain height.
4414 
4415   Not Collective
4416 
4417   Input Parameters:
4418 + dm           - The DMPlex object
4419 - stratumValue - The requested height
4420 
4421   Output Parameters:
4422 + start - The first point at this height
4423 - end   - One beyond the last point at this height
4424 
4425   Level: developer
4426 
4427 .keywords: mesh, points
4428 .seealso: DMPlexGetDepthStratum(), DMPlexGetDepth()
4429 @*/
4430 PetscErrorCode DMPlexGetHeightStratum(DM dm, PetscInt stratumValue, PetscInt *start, PetscInt *end)
4431 {
4432   DMLabel        label;
4433   PetscInt       depth, pStart, pEnd;
4434   PetscErrorCode ierr;
4435 
4436   PetscFunctionBegin;
4437   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4438   if (start) {PetscValidPointer(start, 3); *start = 0;}
4439   if (end)   {PetscValidPointer(end,   4); *end   = 0;}
4440   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
4441   if (pStart == pEnd) PetscFunctionReturn(0);
4442   if (stratumValue < 0) {
4443     if (start) *start = pStart;
4444     if (end)   *end   = pEnd;
4445     PetscFunctionReturn(0);
4446   }
4447   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
4448   if (!label) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "No label named depth was found");CHKERRQ(ierr);
4449   ierr = DMLabelGetNumValues(label, &depth);CHKERRQ(ierr);
4450   ierr = DMLabelGetStratumBounds(label, depth-1-stratumValue, start, end);CHKERRQ(ierr);
4451   PetscFunctionReturn(0);
4452 }
4453 
4454 #undef __FUNCT__
4455 #define __FUNCT__ "DMPlexCreateSectionInitial"
4456 /* Set the number of dof on each point and separate by fields */
4457 PetscErrorCode DMPlexCreateSectionInitial(DM dm, PetscInt dim, PetscInt numFields,const PetscInt numComp[],const PetscInt numDof[], PetscSection *section)
4458 {
4459   PetscInt      *numDofTot;
4460   PetscInt       depth, pStart = 0, pEnd = 0;
4461   PetscInt       p, d, dep, f;
4462   PetscErrorCode ierr;
4463 
4464   PetscFunctionBegin;
4465   ierr = PetscMalloc1((dim+1), &numDofTot);CHKERRQ(ierr);
4466   for (d = 0; d <= dim; ++d) {
4467     numDofTot[d] = 0;
4468     for (f = 0; f < numFields; ++f) numDofTot[d] += numDof[f*(dim+1)+d];
4469   }
4470   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), section);CHKERRQ(ierr);
4471   if (numFields > 0) {
4472     ierr = PetscSectionSetNumFields(*section, numFields);CHKERRQ(ierr);
4473     if (numComp) {
4474       for (f = 0; f < numFields; ++f) {
4475         ierr = PetscSectionSetFieldComponents(*section, f, numComp[f]);CHKERRQ(ierr);
4476       }
4477     }
4478   }
4479   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
4480   ierr = PetscSectionSetChart(*section, pStart, pEnd);CHKERRQ(ierr);
4481   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
4482   for (dep = 0; dep <= depth; ++dep) {
4483     d    = dim == depth ? dep : (!dep ? 0 : dim);
4484     ierr = DMPlexGetDepthStratum(dm, dep, &pStart, &pEnd);CHKERRQ(ierr);
4485     for (p = pStart; p < pEnd; ++p) {
4486       for (f = 0; f < numFields; ++f) {
4487         ierr = PetscSectionSetFieldDof(*section, p, f, numDof[f*(dim+1)+d]);CHKERRQ(ierr);
4488       }
4489       ierr = PetscSectionSetDof(*section, p, numDofTot[d]);CHKERRQ(ierr);
4490     }
4491   }
4492   ierr = PetscFree(numDofTot);CHKERRQ(ierr);
4493   PetscFunctionReturn(0);
4494 }
4495 
4496 #undef __FUNCT__
4497 #define __FUNCT__ "DMPlexCreateSectionBCDof"
4498 /* Set the number of dof on each point and separate by fields
4499    If constDof is PETSC_DETERMINE, constrain every dof on the point
4500 */
4501 PetscErrorCode DMPlexCreateSectionBCDof(DM dm, PetscInt numBC,const PetscInt bcField[],const IS bcPoints[], PetscInt constDof, PetscSection section)
4502 {
4503   PetscInt       numFields;
4504   PetscInt       bc;
4505   PetscErrorCode ierr;
4506 
4507   PetscFunctionBegin;
4508   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
4509   for (bc = 0; bc < numBC; ++bc) {
4510     PetscInt        field = 0;
4511     const PetscInt *idx;
4512     PetscInt        n, i;
4513 
4514     if (numFields) field = bcField[bc];
4515     ierr = ISGetLocalSize(bcPoints[bc], &n);CHKERRQ(ierr);
4516     ierr = ISGetIndices(bcPoints[bc], &idx);CHKERRQ(ierr);
4517     for (i = 0; i < n; ++i) {
4518       const PetscInt p        = idx[i];
4519       PetscInt       numConst = constDof;
4520 
4521       /* Constrain every dof on the point */
4522       if (numConst < 0) {
4523         if (numFields) {
4524           ierr = PetscSectionGetFieldDof(section, p, field, &numConst);CHKERRQ(ierr);
4525         } else {
4526           ierr = PetscSectionGetDof(section, p, &numConst);CHKERRQ(ierr);
4527         }
4528       }
4529       if (numFields) {
4530         ierr = PetscSectionAddFieldConstraintDof(section, p, field, numConst);CHKERRQ(ierr);
4531       }
4532       ierr = PetscSectionAddConstraintDof(section, p, numConst);CHKERRQ(ierr);
4533     }
4534     ierr = ISRestoreIndices(bcPoints[bc], &idx);CHKERRQ(ierr);
4535   }
4536   PetscFunctionReturn(0);
4537 }
4538 
4539 #undef __FUNCT__
4540 #define __FUNCT__ "DMPlexCreateSectionBCIndicesAll"
4541 /* Set the constrained indices on each point and separate by fields */
4542 PetscErrorCode DMPlexCreateSectionBCIndicesAll(DM dm, PetscSection section)
4543 {
4544   PetscInt      *maxConstraints;
4545   PetscInt       numFields, f, pStart = 0, pEnd = 0, p;
4546   PetscErrorCode ierr;
4547 
4548   PetscFunctionBegin;
4549   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
4550   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
4551   ierr = PetscMalloc1((numFields+1), &maxConstraints);CHKERRQ(ierr);
4552   for (f = 0; f <= numFields; ++f) maxConstraints[f] = 0;
4553   for (p = pStart; p < pEnd; ++p) {
4554     PetscInt cdof;
4555 
4556     if (numFields) {
4557       for (f = 0; f < numFields; ++f) {
4558         ierr              = PetscSectionGetFieldConstraintDof(section, p, f, &cdof);CHKERRQ(ierr);
4559         maxConstraints[f] = PetscMax(maxConstraints[f], cdof);
4560       }
4561     } else {
4562       ierr              = PetscSectionGetConstraintDof(section, p, &cdof);CHKERRQ(ierr);
4563       maxConstraints[0] = PetscMax(maxConstraints[0], cdof);
4564     }
4565   }
4566   for (f = 0; f < numFields; ++f) {
4567     maxConstraints[numFields] += maxConstraints[f];
4568   }
4569   if (maxConstraints[numFields]) {
4570     PetscInt *indices;
4571 
4572     ierr = PetscMalloc1(maxConstraints[numFields], &indices);CHKERRQ(ierr);
4573     for (p = pStart; p < pEnd; ++p) {
4574       PetscInt cdof, d;
4575 
4576       ierr = PetscSectionGetConstraintDof(section, p, &cdof);CHKERRQ(ierr);
4577       if (cdof) {
4578         if (cdof > maxConstraints[numFields]) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_LIB, "Likely memory corruption, point %D cDof %D > maxConstraints %D", p, cdof, maxConstraints[numFields]);
4579         if (numFields) {
4580           PetscInt numConst = 0, foff = 0;
4581 
4582           for (f = 0; f < numFields; ++f) {
4583             PetscInt cfdof, fdof;
4584 
4585             ierr = PetscSectionGetFieldDof(section, p, f, &fdof);CHKERRQ(ierr);
4586             ierr = PetscSectionGetFieldConstraintDof(section, p, f, &cfdof);CHKERRQ(ierr);
4587             /* Change constraint numbering from absolute local dof number to field relative local dof number */
4588             for (d = 0; d < cfdof; ++d) indices[numConst+d] = d;
4589             ierr = PetscSectionSetFieldConstraintIndices(section, p, f, &indices[numConst]);CHKERRQ(ierr);
4590             for (d = 0; d < cfdof; ++d) indices[numConst+d] += foff;
4591             numConst += cfdof;
4592             foff     += fdof;
4593           }
4594           if (cdof != numConst) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_LIB, "Total number of field constraints %D should be %D", numConst, cdof);
4595         } else {
4596           for (d = 0; d < cdof; ++d) indices[d] = d;
4597         }
4598         ierr = PetscSectionSetConstraintIndices(section, p, indices);CHKERRQ(ierr);
4599       }
4600     }
4601     ierr = PetscFree(indices);CHKERRQ(ierr);
4602   }
4603   ierr = PetscFree(maxConstraints);CHKERRQ(ierr);
4604   PetscFunctionReturn(0);
4605 }
4606 
4607 #undef __FUNCT__
4608 #define __FUNCT__ "DMPlexCreateSectionBCIndicesField"
4609 /* Set the constrained field indices on each point */
4610 PetscErrorCode DMPlexCreateSectionBCIndicesField(DM dm, PetscInt field, IS bcPoints, IS constraintIndices, PetscSection section)
4611 {
4612   const PetscInt *points, *indices;
4613   PetscInt        numFields, maxDof, numPoints, p, numConstraints;
4614   PetscErrorCode  ierr;
4615 
4616   PetscFunctionBegin;
4617   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
4618   if ((field < 0) || (field >= numFields)) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Section field %d should be in [%d, %d)", field, 0, numFields);
4619 
4620   ierr = ISGetLocalSize(bcPoints, &numPoints);CHKERRQ(ierr);
4621   ierr = ISGetIndices(bcPoints, &points);CHKERRQ(ierr);
4622   if (!constraintIndices) {
4623     PetscInt *idx, i;
4624 
4625     ierr = PetscSectionGetMaxDof(section, &maxDof);CHKERRQ(ierr);
4626     ierr = PetscMalloc1(maxDof, &idx);CHKERRQ(ierr);
4627     for (i = 0; i < maxDof; ++i) idx[i] = i;
4628     for (p = 0; p < numPoints; ++p) {
4629       ierr = PetscSectionSetFieldConstraintIndices(section, points[p], field, idx);CHKERRQ(ierr);
4630     }
4631     ierr = PetscFree(idx);CHKERRQ(ierr);
4632   } else {
4633     ierr = ISGetLocalSize(constraintIndices, &numConstraints);CHKERRQ(ierr);
4634     ierr = ISGetIndices(constraintIndices, &indices);CHKERRQ(ierr);
4635     for (p = 0; p < numPoints; ++p) {
4636       PetscInt fcdof;
4637 
4638       ierr = PetscSectionGetFieldConstraintDof(section, points[p], field, &fcdof);CHKERRQ(ierr);
4639       if (fcdof != numConstraints) SETERRQ4(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Section point %d field %d has %d constraints, but yo ugave %d indices", p, field, fcdof, numConstraints);
4640       ierr = PetscSectionSetFieldConstraintIndices(section, points[p], field, indices);CHKERRQ(ierr);
4641     }
4642     ierr = ISRestoreIndices(constraintIndices, &indices);CHKERRQ(ierr);
4643   }
4644   ierr = ISRestoreIndices(bcPoints, &points);CHKERRQ(ierr);
4645   PetscFunctionReturn(0);
4646 }
4647 
4648 #undef __FUNCT__
4649 #define __FUNCT__ "DMPlexCreateSectionBCIndices"
4650 /* Set the constrained indices on each point and separate by fields */
4651 PetscErrorCode DMPlexCreateSectionBCIndices(DM dm, PetscSection section)
4652 {
4653   PetscInt      *indices;
4654   PetscInt       numFields, maxDof, f, pStart = 0, pEnd = 0, p;
4655   PetscErrorCode ierr;
4656 
4657   PetscFunctionBegin;
4658   ierr = PetscSectionGetMaxDof(section, &maxDof);CHKERRQ(ierr);
4659   ierr = PetscMalloc1(maxDof, &indices);CHKERRQ(ierr);
4660   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
4661   if (!numFields) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "This function only works after users have set field constraint indices.");
4662   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
4663   for (p = pStart; p < pEnd; ++p) {
4664     PetscInt cdof, d;
4665 
4666     ierr = PetscSectionGetConstraintDof(section, p, &cdof);CHKERRQ(ierr);
4667     if (cdof) {
4668       PetscInt numConst = 0, foff = 0;
4669 
4670       for (f = 0; f < numFields; ++f) {
4671         const PetscInt *fcind;
4672         PetscInt        fdof, fcdof;
4673 
4674         ierr = PetscSectionGetFieldDof(section, p, f, &fdof);CHKERRQ(ierr);
4675         ierr = PetscSectionGetFieldConstraintDof(section, p, f, &fcdof);CHKERRQ(ierr);
4676         if (fcdof) {ierr = PetscSectionGetFieldConstraintIndices(section, p, f, &fcind);CHKERRQ(ierr);}
4677         /* Change constraint numbering from field relative local dof number to absolute local dof number */
4678         for (d = 0; d < fcdof; ++d) indices[numConst+d] = fcind[d]+foff;
4679         foff     += fdof;
4680         numConst += fcdof;
4681       }
4682       if (cdof != numConst) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_LIB, "Total number of field constraints %D should be %D", numConst, cdof);
4683       ierr = PetscSectionSetConstraintIndices(section, p, indices);CHKERRQ(ierr);
4684     }
4685   }
4686   ierr = PetscFree(indices);CHKERRQ(ierr);
4687   PetscFunctionReturn(0);
4688 }
4689 
4690 #undef __FUNCT__
4691 #define __FUNCT__ "DMPlexCreateSection"
4692 /*@C
4693   DMPlexCreateSection - Create a PetscSection based upon the dof layout specification provided.
4694 
4695   Not Collective
4696 
4697   Input Parameters:
4698 + dm        - The DMPlex object
4699 . dim       - The spatial dimension of the problem
4700 . numFields - The number of fields in the problem
4701 . numComp   - An array of size numFields that holds the number of components for each field
4702 . numDof    - An array of size numFields*(dim+1) which holds the number of dof for each field on a mesh piece of dimension d
4703 . numBC     - The number of boundary conditions
4704 . bcField   - An array of size numBC giving the field number for each boundry condition
4705 . bcPoints  - An array of size numBC giving an IS holding the sieve points to which each boundary condition applies
4706 - perm      - Optional permutation of the chart, or NULL
4707 
4708   Output Parameter:
4709 . section - The PetscSection object
4710 
4711   Notes: numDof[f*(dim+1)+d] gives the number of dof for field f on sieve points of dimension d. For instance, numDof[1] is the
4712   number of dof for field 0 on each edge.
4713 
4714   The chart permutation is the same one set using PetscSectionSetPermutation()
4715 
4716   Level: developer
4717 
4718   Fortran Notes:
4719   A Fortran 90 version is available as DMPlexCreateSectionF90()
4720 
4721 .keywords: mesh, elements
4722 .seealso: DMPlexCreate(), PetscSectionCreate(), PetscSectionSetPermutation()
4723 @*/
4724 PetscErrorCode DMPlexCreateSection(DM dm, PetscInt dim, PetscInt numFields,const PetscInt numComp[],const PetscInt numDof[], PetscInt numBC,const PetscInt bcField[],const IS bcPoints[], IS perm, PetscSection *section)
4725 {
4726   PetscErrorCode ierr;
4727 
4728   PetscFunctionBegin;
4729   ierr = DMPlexCreateSectionInitial(dm, dim, numFields, numComp, numDof, section);CHKERRQ(ierr);
4730   ierr = DMPlexCreateSectionBCDof(dm, numBC, bcField, bcPoints, PETSC_DETERMINE, *section);CHKERRQ(ierr);
4731   if (perm) {ierr = PetscSectionSetPermutation(*section, perm);CHKERRQ(ierr);}
4732   ierr = PetscSectionSetUp(*section);CHKERRQ(ierr);
4733   if (numBC) {ierr = DMPlexCreateSectionBCIndicesAll(dm, *section);CHKERRQ(ierr);}
4734   ierr = PetscSectionViewFromOptions(*section,NULL,"-section_view");CHKERRQ(ierr);
4735   PetscFunctionReturn(0);
4736 }
4737 
4738 #undef __FUNCT__
4739 #define __FUNCT__ "DMCreateCoordinateDM_Plex"
4740 PetscErrorCode DMCreateCoordinateDM_Plex(DM dm, DM *cdm)
4741 {
4742   PetscSection   section;
4743   PetscErrorCode ierr;
4744 
4745   PetscFunctionBegin;
4746   ierr = DMClone(dm, cdm);CHKERRQ(ierr);
4747   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &section);CHKERRQ(ierr);
4748   ierr = DMSetDefaultSection(*cdm, section);CHKERRQ(ierr);
4749   ierr = PetscSectionDestroy(&section);CHKERRQ(ierr);
4750   PetscFunctionReturn(0);
4751 }
4752 
4753 #undef __FUNCT__
4754 #define __FUNCT__ "DMPlexGetConeSection"
4755 PetscErrorCode DMPlexGetConeSection(DM dm, PetscSection *section)
4756 {
4757   DM_Plex *mesh = (DM_Plex*) dm->data;
4758 
4759   PetscFunctionBegin;
4760   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4761   if (section) *section = mesh->coneSection;
4762   PetscFunctionReturn(0);
4763 }
4764 
4765 #undef __FUNCT__
4766 #define __FUNCT__ "DMPlexGetSupportSection"
4767 PetscErrorCode DMPlexGetSupportSection(DM dm, PetscSection *section)
4768 {
4769   DM_Plex *mesh = (DM_Plex*) dm->data;
4770 
4771   PetscFunctionBegin;
4772   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4773   if (section) *section = mesh->supportSection;
4774   PetscFunctionReturn(0);
4775 }
4776 
4777 #undef __FUNCT__
4778 #define __FUNCT__ "DMPlexGetCones"
4779 PetscErrorCode DMPlexGetCones(DM dm, PetscInt *cones[])
4780 {
4781   DM_Plex *mesh = (DM_Plex*) dm->data;
4782 
4783   PetscFunctionBegin;
4784   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4785   if (cones) *cones = mesh->cones;
4786   PetscFunctionReturn(0);
4787 }
4788 
4789 #undef __FUNCT__
4790 #define __FUNCT__ "DMPlexGetConeOrientations"
4791 PetscErrorCode DMPlexGetConeOrientations(DM dm, PetscInt *coneOrientations[])
4792 {
4793   DM_Plex *mesh = (DM_Plex*) dm->data;
4794 
4795   PetscFunctionBegin;
4796   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4797   if (coneOrientations) *coneOrientations = mesh->coneOrientations;
4798   PetscFunctionReturn(0);
4799 }
4800 
4801 /******************************** FEM Support **********************************/
4802 
4803 #undef __FUNCT__
4804 #define __FUNCT__ "DMPlexVecGetClosure_Depth1_Static"
4805 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecGetClosure_Depth1_Static(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
4806 {
4807   PetscScalar    *array, *vArray;
4808   const PetscInt *cone, *coneO;
4809   PetscInt        pStart, pEnd, p, numPoints, size = 0, offset = 0;
4810   PetscErrorCode  ierr;
4811 
4812   PetscFunctionBeginHot;
4813   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
4814   ierr = DMPlexGetConeSize(dm, point, &numPoints);CHKERRQ(ierr);
4815   ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
4816   ierr = DMPlexGetConeOrientation(dm, point, &coneO);CHKERRQ(ierr);
4817   if (!values || !*values) {
4818     if ((point >= pStart) && (point < pEnd)) {
4819       PetscInt dof;
4820 
4821       ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4822       size += dof;
4823     }
4824     for (p = 0; p < numPoints; ++p) {
4825       const PetscInt cp = cone[p];
4826       PetscInt       dof;
4827 
4828       if ((cp < pStart) || (cp >= pEnd)) continue;
4829       ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
4830       size += dof;
4831     }
4832     if (!values) {
4833       if (csize) *csize = size;
4834       PetscFunctionReturn(0);
4835     }
4836     ierr = DMGetWorkArray(dm, size, PETSC_SCALAR, &array);CHKERRQ(ierr);
4837   } else {
4838     array = *values;
4839   }
4840   size = 0;
4841   ierr = VecGetArray(v, &vArray);CHKERRQ(ierr);
4842   if ((point >= pStart) && (point < pEnd)) {
4843     PetscInt     dof, off, d;
4844     PetscScalar *varr;
4845 
4846     ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4847     ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
4848     varr = &vArray[off];
4849     for (d = 0; d < dof; ++d, ++offset) {
4850       array[offset] = varr[d];
4851     }
4852     size += dof;
4853   }
4854   for (p = 0; p < numPoints; ++p) {
4855     const PetscInt cp = cone[p];
4856     PetscInt       o  = coneO[p];
4857     PetscInt       dof, off, d;
4858     PetscScalar   *varr;
4859 
4860     if ((cp < pStart) || (cp >= pEnd)) continue;
4861     ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
4862     ierr = PetscSectionGetOffset(section, cp, &off);CHKERRQ(ierr);
4863     varr = &vArray[off];
4864     if (o >= 0) {
4865       for (d = 0; d < dof; ++d, ++offset) {
4866         array[offset] = varr[d];
4867       }
4868     } else {
4869       for (d = dof-1; d >= 0; --d, ++offset) {
4870         array[offset] = varr[d];
4871       }
4872     }
4873     size += dof;
4874   }
4875   ierr = VecRestoreArray(v, &vArray);CHKERRQ(ierr);
4876   if (!*values) {
4877     if (csize) *csize = size;
4878     *values = array;
4879   } else {
4880     if (size > *csize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Size of input array %d < actual size %d", *csize, size);
4881     *csize = size;
4882   }
4883   PetscFunctionReturn(0);
4884 }
4885 
4886 #undef __FUNCT__
4887 #define __FUNCT__ "DMPlexVecGetClosure_Static"
4888 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecGetClosure_Static(PetscSection section, PetscInt numPoints, const PetscInt points[], const PetscScalar vArray[], PetscInt *size, PetscScalar array[])
4889 {
4890   PetscInt       offset = 0, p;
4891   PetscErrorCode ierr;
4892 
4893   PetscFunctionBeginHot;
4894   *size = 0;
4895   for (p = 0; p < numPoints*2; p += 2) {
4896     const PetscInt point = points[p];
4897     const PetscInt o     = points[p+1];
4898     PetscInt       dof, off, d;
4899     const PetscScalar *varr;
4900 
4901     ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4902     ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
4903     varr = &vArray[off];
4904     if (o >= 0) {
4905       for (d = 0; d < dof; ++d, ++offset)    array[offset] = varr[d];
4906     } else {
4907       for (d = dof-1; d >= 0; --d, ++offset) array[offset] = varr[d];
4908     }
4909   }
4910   *size = offset;
4911   PetscFunctionReturn(0);
4912 }
4913 
4914 #undef __FUNCT__
4915 #define __FUNCT__ "DMPlexVecGetClosure_Fields_Static"
4916 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecGetClosure_Fields_Static(PetscSection section, PetscInt numPoints, const PetscInt points[], PetscInt numFields, const PetscScalar vArray[], PetscInt *size, PetscScalar array[])
4917 {
4918   PetscInt       offset = 0, f;
4919   PetscErrorCode ierr;
4920 
4921   PetscFunctionBeginHot;
4922   *size = 0;
4923   for (f = 0; f < numFields; ++f) {
4924     PetscInt fcomp, p;
4925 
4926     ierr = PetscSectionGetFieldComponents(section, f, &fcomp);CHKERRQ(ierr);
4927     for (p = 0; p < numPoints*2; p += 2) {
4928       const PetscInt point = points[p];
4929       const PetscInt o     = points[p+1];
4930       PetscInt       fdof, foff, d, c;
4931       const PetscScalar *varr;
4932 
4933       ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
4934       ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
4935       varr = &vArray[foff];
4936       if (o >= 0) {
4937         for (d = 0; d < fdof; ++d, ++offset) array[offset] = varr[d];
4938       } else {
4939         for (d = fdof/fcomp-1; d >= 0; --d) {
4940           for (c = 0; c < fcomp; ++c, ++offset) {
4941             array[offset] = varr[d*fcomp+c];
4942           }
4943         }
4944       }
4945     }
4946   }
4947   *size = offset;
4948   PetscFunctionReturn(0);
4949 }
4950 
4951 #undef __FUNCT__
4952 #define __FUNCT__ "DMPlexVecGetClosure"
4953 /*@C
4954   DMPlexVecGetClosure - Get an array of the values on the closure of 'point'
4955 
4956   Not collective
4957 
4958   Input Parameters:
4959 + dm - The DM
4960 . section - The section describing the layout in v, or NULL to use the default section
4961 . v - The local vector
4962 - point - The sieve point in the DM
4963 
4964   Output Parameters:
4965 + csize - The number of values in the closure, or NULL
4966 - values - The array of values, which is a borrowed array and should not be freed
4967 
4968   Fortran Notes:
4969   Since it returns an array, this routine is only available in Fortran 90, and you must
4970   include petsc.h90 in your code.
4971 
4972   The csize argument is not present in the Fortran 90 binding since it is internal to the array.
4973 
4974   Level: intermediate
4975 
4976 .seealso DMPlexVecRestoreClosure(), DMPlexVecSetClosure(), DMPlexMatSetClosure()
4977 @*/
4978 PetscErrorCode DMPlexVecGetClosure(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
4979 {
4980   PetscSection    clSection;
4981   IS              clPoints;
4982   PetscScalar    *array, *vArray;
4983   PetscInt       *points = NULL;
4984   const PetscInt *clp;
4985   PetscInt        depth, numFields, numPoints, size;
4986   PetscErrorCode  ierr;
4987 
4988   PetscFunctionBeginHot;
4989   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4990   if (!section) {ierr = DMGetDefaultSection(dm, &section);CHKERRQ(ierr);}
4991   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
4992   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
4993   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
4994   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
4995   if (depth == 1 && numFields < 2) {
4996     ierr = DMPlexVecGetClosure_Depth1_Static(dm, section, v, point, csize, values);CHKERRQ(ierr);
4997     PetscFunctionReturn(0);
4998   }
4999   /* Get points */
5000   ierr = PetscSectionGetClosureIndex(section, (PetscObject) dm, &clSection, &clPoints);CHKERRQ(ierr);
5001   if (!clPoints) {
5002     PetscInt pStart, pEnd, p, q;
5003 
5004     ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
5005     ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
5006     /* Compress out points not in the section */
5007     for (p = 0, q = 0; p < numPoints*2; p += 2) {
5008       if ((points[p] >= pStart) && (points[p] < pEnd)) {
5009         points[q*2]   = points[p];
5010         points[q*2+1] = points[p+1];
5011         ++q;
5012       }
5013     }
5014     numPoints = q;
5015   } else {
5016     PetscInt dof, off;
5017 
5018     ierr = PetscSectionGetDof(clSection, point, &dof);CHKERRQ(ierr);
5019     ierr = PetscSectionGetOffset(clSection, point, &off);CHKERRQ(ierr);
5020     ierr = ISGetIndices(clPoints, &clp);CHKERRQ(ierr);
5021     numPoints = dof/2;
5022     points    = (PetscInt *) &clp[off];
5023   }
5024   /* Get array */
5025   if (!values || !*values) {
5026     PetscInt asize = 0, dof, p;
5027 
5028     for (p = 0; p < numPoints*2; p += 2) {
5029       ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5030       asize += dof;
5031     }
5032     if (!values) {
5033       if (!clPoints) {ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);}
5034       else           {ierr = ISRestoreIndices(clPoints, &clp);CHKERRQ(ierr);}
5035       if (csize) *csize = asize;
5036       PetscFunctionReturn(0);
5037     }
5038     ierr = DMGetWorkArray(dm, asize, PETSC_SCALAR, &array);CHKERRQ(ierr);
5039   } else {
5040     array = *values;
5041   }
5042   ierr = VecGetArray(v, &vArray);CHKERRQ(ierr);
5043   /* Get values */
5044   if (numFields > 0) {ierr = DMPlexVecGetClosure_Fields_Static(section, numPoints, points, numFields, vArray, &size, array);CHKERRQ(ierr);}
5045   else               {ierr = DMPlexVecGetClosure_Static(section, numPoints, points, vArray, &size, array);CHKERRQ(ierr);}
5046   /* Cleanup points */
5047   if (!clPoints) {ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);}
5048   else           {ierr = ISRestoreIndices(clPoints, &clp);CHKERRQ(ierr);}
5049   /* Cleanup array */
5050   ierr = VecRestoreArray(v, &vArray);CHKERRQ(ierr);
5051   if (!*values) {
5052     if (csize) *csize = size;
5053     *values = array;
5054   } else {
5055     if (size > *csize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Size of input array %d < actual size %d", *csize, size);
5056     *csize = size;
5057   }
5058   PetscFunctionReturn(0);
5059 }
5060 
5061 #undef __FUNCT__
5062 #define __FUNCT__ "DMPlexVecRestoreClosure"
5063 /*@C
5064   DMPlexVecRestoreClosure - Restore the array of the values on the closure of 'point'
5065 
5066   Not collective
5067 
5068   Input Parameters:
5069 + dm - The DM
5070 . section - The section describing the layout in v, or NULL to use the default section
5071 . v - The local vector
5072 . point - The sieve point in the DM
5073 . csize - The number of values in the closure, or NULL
5074 - values - The array of values, which is a borrowed array and should not be freed
5075 
5076   Fortran Notes:
5077   Since it returns an array, this routine is only available in Fortran 90, and you must
5078   include petsc.h90 in your code.
5079 
5080   The csize argument is not present in the Fortran 90 binding since it is internal to the array.
5081 
5082   Level: intermediate
5083 
5084 .seealso DMPlexVecGetClosure(), DMPlexVecSetClosure(), DMPlexMatSetClosure()
5085 @*/
5086 PetscErrorCode DMPlexVecRestoreClosure(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
5087 {
5088   PetscInt       size = 0;
5089   PetscErrorCode ierr;
5090 
5091   PetscFunctionBegin;
5092   /* Should work without recalculating size */
5093   ierr = DMRestoreWorkArray(dm, size, PETSC_SCALAR, (void*) values);CHKERRQ(ierr);
5094   PetscFunctionReturn(0);
5095 }
5096 
5097 PETSC_STATIC_INLINE void add   (PetscScalar *x, PetscScalar y) {*x += y;}
5098 PETSC_STATIC_INLINE void insert(PetscScalar *x, PetscScalar y) {*x  = y;}
5099 
5100 #undef __FUNCT__
5101 #define __FUNCT__ "updatePoint_private"
5102 PETSC_STATIC_INLINE PetscErrorCode updatePoint_private(PetscSection section, PetscInt point, PetscInt dof, void (*fuse)(PetscScalar*, PetscScalar), PetscBool setBC, PetscInt orientation, const PetscScalar values[], PetscScalar array[])
5103 {
5104   PetscInt        cdof;   /* The number of constraints on this point */
5105   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
5106   PetscScalar    *a;
5107   PetscInt        off, cind = 0, k;
5108   PetscErrorCode  ierr;
5109 
5110   PetscFunctionBegin;
5111   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
5112   ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
5113   a    = &array[off];
5114   if (!cdof || setBC) {
5115     if (orientation >= 0) {
5116       for (k = 0; k < dof; ++k) {
5117         fuse(&a[k], values[k]);
5118       }
5119     } else {
5120       for (k = 0; k < dof; ++k) {
5121         fuse(&a[k], values[dof-k-1]);
5122       }
5123     }
5124   } else {
5125     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
5126     if (orientation >= 0) {
5127       for (k = 0; k < dof; ++k) {
5128         if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
5129         fuse(&a[k], values[k]);
5130       }
5131     } else {
5132       for (k = 0; k < dof; ++k) {
5133         if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
5134         fuse(&a[k], values[dof-k-1]);
5135       }
5136     }
5137   }
5138   PetscFunctionReturn(0);
5139 }
5140 
5141 #undef __FUNCT__
5142 #define __FUNCT__ "updatePointBC_private"
5143 PETSC_STATIC_INLINE PetscErrorCode updatePointBC_private(PetscSection section, PetscInt point, PetscInt dof, void (*fuse)(PetscScalar*, PetscScalar), PetscInt orientation, const PetscScalar values[], PetscScalar array[])
5144 {
5145   PetscInt        cdof;   /* The number of constraints on this point */
5146   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
5147   PetscScalar    *a;
5148   PetscInt        off, cind = 0, k;
5149   PetscErrorCode  ierr;
5150 
5151   PetscFunctionBegin;
5152   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
5153   ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
5154   a    = &array[off];
5155   if (cdof) {
5156     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
5157     if (orientation >= 0) {
5158       for (k = 0; k < dof; ++k) {
5159         if ((cind < cdof) && (k == cdofs[cind])) {
5160           fuse(&a[k], values[k]);
5161           ++cind;
5162         }
5163       }
5164     } else {
5165       for (k = 0; k < dof; ++k) {
5166         if ((cind < cdof) && (k == cdofs[cind])) {
5167           fuse(&a[k], values[dof-k-1]);
5168           ++cind;
5169         }
5170       }
5171     }
5172   }
5173   PetscFunctionReturn(0);
5174 }
5175 
5176 #undef __FUNCT__
5177 #define __FUNCT__ "updatePointFields_private"
5178 PETSC_STATIC_INLINE PetscErrorCode updatePointFields_private(PetscSection section, PetscInt point, PetscInt o, PetscInt f, PetscInt fcomp, void (*fuse)(PetscScalar*, PetscScalar), PetscBool setBC, const PetscScalar values[], PetscInt *offset, PetscScalar array[])
5179 {
5180   PetscScalar    *a;
5181   PetscInt        fdof, foff, fcdof, foffset = *offset;
5182   const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
5183   PetscInt        cind = 0, k, c;
5184   PetscErrorCode  ierr;
5185 
5186   PetscFunctionBegin;
5187   ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
5188   ierr = PetscSectionGetFieldConstraintDof(section, point, f, &fcdof);CHKERRQ(ierr);
5189   ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
5190   a    = &array[foff];
5191   if (!fcdof || setBC) {
5192     if (o >= 0) {
5193       for (k = 0; k < fdof; ++k) fuse(&a[k], values[foffset+k]);
5194     } else {
5195       for (k = fdof/fcomp-1; k >= 0; --k) {
5196         for (c = 0; c < fcomp; ++c) {
5197           fuse(&a[(fdof/fcomp-1-k)*fcomp+c], values[foffset+k*fcomp+c]);
5198         }
5199       }
5200     }
5201   } else {
5202     ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
5203     if (o >= 0) {
5204       for (k = 0; k < fdof; ++k) {
5205         if ((cind < fcdof) && (k == fcdofs[cind])) {++cind; continue;}
5206         fuse(&a[k], values[foffset+k]);
5207       }
5208     } else {
5209       for (k = fdof/fcomp-1; k >= 0; --k) {
5210         for (c = 0; c < fcomp; ++c) {
5211           if ((cind < fcdof) && (k*fcomp+c == fcdofs[cind])) {++cind; continue;}
5212           fuse(&a[(fdof/fcomp-1-k)*fcomp+c], values[foffset+k*fcomp+c]);
5213         }
5214       }
5215     }
5216   }
5217   *offset += fdof;
5218   PetscFunctionReturn(0);
5219 }
5220 
5221 #undef __FUNCT__
5222 #define __FUNCT__ "updatePointFieldsBC_private"
5223 PETSC_STATIC_INLINE PetscErrorCode updatePointFieldsBC_private(PetscSection section, PetscInt point, PetscInt o, PetscInt f, PetscInt fcomp, void (*fuse)(PetscScalar*, PetscScalar), const PetscScalar values[], PetscInt *offset, PetscScalar array[])
5224 {
5225   PetscScalar    *a;
5226   PetscInt        fdof, foff, fcdof, foffset = *offset;
5227   const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
5228   PetscInt        cind = 0, k, c;
5229   PetscErrorCode  ierr;
5230 
5231   PetscFunctionBegin;
5232   ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
5233   ierr = PetscSectionGetFieldConstraintDof(section, point, f, &fcdof);CHKERRQ(ierr);
5234   ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
5235   a    = &array[foff];
5236   if (fcdof) {
5237     ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
5238     if (o >= 0) {
5239       for (k = 0; k < fdof; ++k) {
5240         if ((cind < fcdof) && (k == fcdofs[cind])) {
5241           fuse(&a[k], values[foffset+k]);
5242           ++cind;
5243         }
5244       }
5245     } else {
5246       for (k = fdof/fcomp-1; k >= 0; --k) {
5247         for (c = 0; c < fcomp; ++c) {
5248           if ((cind < fcdof) && (k*fcomp+c == fcdofs[cind])) {
5249             fuse(&a[(fdof/fcomp-1-k)*fcomp+c], values[foffset+k*fcomp+c]);
5250             ++cind;
5251           }
5252         }
5253       }
5254     }
5255   }
5256   *offset += fdof;
5257   PetscFunctionReturn(0);
5258 }
5259 
5260 #undef __FUNCT__
5261 #define __FUNCT__ "DMPlexVecSetClosure_Static"
5262 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecSetClosure_Static(DM dm, PetscSection section, Vec v, PetscInt point, const PetscScalar values[], InsertMode mode)
5263 {
5264   PetscScalar    *array;
5265   const PetscInt *cone, *coneO;
5266   PetscInt        pStart, pEnd, p, numPoints, off, dof;
5267   PetscErrorCode  ierr;
5268 
5269   PetscFunctionBeginHot;
5270   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
5271   ierr = DMPlexGetConeSize(dm, point, &numPoints);CHKERRQ(ierr);
5272   ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
5273   ierr = DMPlexGetConeOrientation(dm, point, &coneO);CHKERRQ(ierr);
5274   ierr = VecGetArray(v, &array);CHKERRQ(ierr);
5275   for (p = 0, off = 0; p <= numPoints; ++p, off += dof) {
5276     const PetscInt cp = !p ? point : cone[p-1];
5277     const PetscInt o  = !p ? 0     : coneO[p-1];
5278 
5279     if ((cp < pStart) || (cp >= pEnd)) {dof = 0; continue;}
5280     ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
5281     /* ADD_VALUES */
5282     {
5283       const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
5284       PetscScalar    *a;
5285       PetscInt        cdof, coff, cind = 0, k;
5286 
5287       ierr = PetscSectionGetConstraintDof(section, cp, &cdof);CHKERRQ(ierr);
5288       ierr = PetscSectionGetOffset(section, cp, &coff);CHKERRQ(ierr);
5289       a    = &array[coff];
5290       if (!cdof) {
5291         if (o >= 0) {
5292           for (k = 0; k < dof; ++k) {
5293             a[k] += values[off+k];
5294           }
5295         } else {
5296           for (k = 0; k < dof; ++k) {
5297             a[k] += values[off+dof-k-1];
5298           }
5299         }
5300       } else {
5301         ierr = PetscSectionGetConstraintIndices(section, cp, &cdofs);CHKERRQ(ierr);
5302         if (o >= 0) {
5303           for (k = 0; k < dof; ++k) {
5304             if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
5305             a[k] += values[off+k];
5306           }
5307         } else {
5308           for (k = 0; k < dof; ++k) {
5309             if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
5310             a[k] += values[off+dof-k-1];
5311           }
5312         }
5313       }
5314     }
5315   }
5316   ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
5317   PetscFunctionReturn(0);
5318 }
5319 
5320 #undef __FUNCT__
5321 #define __FUNCT__ "DMPlexVecSetClosure"
5322 /*@C
5323   DMPlexVecSetClosure - Set an array of the values on the closure of 'point'
5324 
5325   Not collective
5326 
5327   Input Parameters:
5328 + dm - The DM
5329 . section - The section describing the layout in v, or NULL to use the default section
5330 . v - The local vector
5331 . point - The sieve point in the DM
5332 . values - The array of values
5333 - mode - The insert mode, where INSERT_ALL_VALUES and ADD_ALL_VALUES also overwrite boundary conditions
5334 
5335   Fortran Notes:
5336   This routine is only available in Fortran 90, and you must include petsc.h90 in your code.
5337 
5338   Level: intermediate
5339 
5340 .seealso DMPlexVecGetClosure(), DMPlexMatSetClosure()
5341 @*/
5342 PetscErrorCode DMPlexVecSetClosure(DM dm, PetscSection section, Vec v, PetscInt point, const PetscScalar values[], InsertMode mode)
5343 {
5344   PetscSection    clSection;
5345   IS              clPoints;
5346   PetscScalar    *array;
5347   PetscInt       *points = NULL;
5348   const PetscInt *clp;
5349   PetscInt        depth, numFields, numPoints, p;
5350   PetscErrorCode  ierr;
5351 
5352   PetscFunctionBeginHot;
5353   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5354   if (!section) {ierr = DMGetDefaultSection(dm, &section);CHKERRQ(ierr);}
5355   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
5356   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
5357   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
5358   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5359   if (depth == 1 && numFields < 2 && mode == ADD_VALUES) {
5360     ierr = DMPlexVecSetClosure_Static(dm, section, v, point, values, mode);CHKERRQ(ierr);
5361     PetscFunctionReturn(0);
5362   }
5363   /* Get points */
5364   ierr = PetscSectionGetClosureIndex(section, (PetscObject) dm, &clSection, &clPoints);CHKERRQ(ierr);
5365   if (!clPoints) {
5366     PetscInt pStart, pEnd, q;
5367 
5368     ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
5369     ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
5370     /* Compress out points not in the section */
5371     for (p = 0, q = 0; p < numPoints*2; p += 2) {
5372       if ((points[p] >= pStart) && (points[p] < pEnd)) {
5373         points[q*2]   = points[p];
5374         points[q*2+1] = points[p+1];
5375         ++q;
5376       }
5377     }
5378     numPoints = q;
5379   } else {
5380     PetscInt dof, off;
5381 
5382     ierr = PetscSectionGetDof(clSection, point, &dof);CHKERRQ(ierr);
5383     ierr = PetscSectionGetOffset(clSection, point, &off);CHKERRQ(ierr);
5384     ierr = ISGetIndices(clPoints, &clp);CHKERRQ(ierr);
5385     numPoints = dof/2;
5386     points    = (PetscInt *) &clp[off];
5387   }
5388   /* Get array */
5389   ierr = VecGetArray(v, &array);CHKERRQ(ierr);
5390   /* Get values */
5391   if (numFields > 0) {
5392     PetscInt offset = 0, fcomp, f;
5393     for (f = 0; f < numFields; ++f) {
5394       ierr = PetscSectionGetFieldComponents(section, f, &fcomp);CHKERRQ(ierr);
5395       switch (mode) {
5396       case INSERT_VALUES:
5397         for (p = 0; p < numPoints*2; p += 2) {
5398           const PetscInt point = points[p];
5399           const PetscInt o     = points[p+1];
5400           updatePointFields_private(section, point, o, f, fcomp, insert, PETSC_FALSE, values, &offset, array);
5401         } break;
5402       case INSERT_ALL_VALUES:
5403         for (p = 0; p < numPoints*2; p += 2) {
5404           const PetscInt point = points[p];
5405           const PetscInt o     = points[p+1];
5406           updatePointFields_private(section, point, o, f, fcomp, insert, PETSC_TRUE, values, &offset, array);
5407         } break;
5408       case INSERT_BC_VALUES:
5409         for (p = 0; p < numPoints*2; p += 2) {
5410           const PetscInt point = points[p];
5411           const PetscInt o     = points[p+1];
5412           updatePointFieldsBC_private(section, point, o, f, fcomp, insert, values, &offset, array);
5413         } break;
5414       case ADD_VALUES:
5415         for (p = 0; p < numPoints*2; p += 2) {
5416           const PetscInt point = points[p];
5417           const PetscInt o     = points[p+1];
5418           updatePointFields_private(section, point, o, f, fcomp, add, PETSC_FALSE, values, &offset, array);
5419         } break;
5420       case ADD_ALL_VALUES:
5421         for (p = 0; p < numPoints*2; p += 2) {
5422           const PetscInt point = points[p];
5423           const PetscInt o     = points[p+1];
5424           updatePointFields_private(section, point, o, f, fcomp, add, PETSC_TRUE, values, &offset, array);
5425         } break;
5426       default:
5427         SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %D", mode);
5428       }
5429     }
5430   } else {
5431     PetscInt dof, off;
5432 
5433     switch (mode) {
5434     case INSERT_VALUES:
5435       for (p = 0, off = 0; p < numPoints*2; p += 2, off += dof) {
5436         PetscInt o = points[p+1];
5437         ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5438         updatePoint_private(section, points[p], dof, insert, PETSC_FALSE, o, &values[off], array);
5439       } break;
5440     case INSERT_ALL_VALUES:
5441       for (p = 0, off = 0; p < numPoints*2; p += 2, off += dof) {
5442         PetscInt o = points[p+1];
5443         ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5444         updatePoint_private(section, points[p], dof, insert, PETSC_TRUE,  o, &values[off], array);
5445       } break;
5446     case INSERT_BC_VALUES:
5447       for (p = 0, off = 0; p < numPoints*2; p += 2, off += dof) {
5448         PetscInt o = points[p+1];
5449         ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5450         updatePointBC_private(section, points[p], dof, insert,  o, &values[off], array);
5451       } break;
5452     case ADD_VALUES:
5453       for (p = 0, off = 0; p < numPoints*2; p += 2, off += dof) {
5454         PetscInt o = points[p+1];
5455         ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5456         updatePoint_private(section, points[p], dof, add,    PETSC_FALSE, o, &values[off], array);
5457       } break;
5458     case ADD_ALL_VALUES:
5459       for (p = 0, off = 0; p < numPoints*2; p += 2, off += dof) {
5460         PetscInt o = points[p+1];
5461         ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5462         updatePoint_private(section, points[p], dof, add,    PETSC_TRUE,  o, &values[off], array);
5463       } break;
5464     default:
5465       SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %D", mode);
5466     }
5467   }
5468   /* Cleanup points */
5469   if (!clPoints) {ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);}
5470   else           {ierr = ISRestoreIndices(clPoints, &clp);CHKERRQ(ierr);}
5471   /* Cleanup array */
5472   ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
5473   PetscFunctionReturn(0);
5474 }
5475 
5476 #undef __FUNCT__
5477 #define __FUNCT__ "DMPlexPrintMatSetValues"
5478 PetscErrorCode DMPlexPrintMatSetValues(PetscViewer viewer, Mat A, PetscInt point, PetscInt numRIndices, const PetscInt rindices[], PetscInt numCIndices, const PetscInt cindices[], const PetscScalar values[])
5479 {
5480   PetscMPIInt    rank;
5481   PetscInt       i, j;
5482   PetscErrorCode ierr;
5483 
5484   PetscFunctionBegin;
5485   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRQ(ierr);
5486   ierr = PetscViewerASCIIPrintf(viewer, "[%D]mat for sieve point %D\n", rank, point);CHKERRQ(ierr);
5487   for (i = 0; i < numRIndices; i++) {ierr = PetscViewerASCIIPrintf(viewer, "[%D]mat row indices[%D] = %D\n", rank, i, rindices[i]);CHKERRQ(ierr);}
5488   for (i = 0; i < numCIndices; i++) {ierr = PetscViewerASCIIPrintf(viewer, "[%D]mat col indices[%D] = %D\n", rank, i, cindices[i]);CHKERRQ(ierr);}
5489   numCIndices = numCIndices ? numCIndices : numRIndices;
5490   for (i = 0; i < numRIndices; i++) {
5491     ierr = PetscViewerASCIIPrintf(viewer, "[%D]", rank);CHKERRQ(ierr);
5492     for (j = 0; j < numCIndices; j++) {
5493 #if defined(PETSC_USE_COMPLEX)
5494       ierr = PetscViewerASCIIPrintf(viewer, " (%g,%g)", (double)PetscRealPart(values[i*numCIndices+j]), (double)PetscImaginaryPart(values[i*numCIndices+j]));CHKERRQ(ierr);
5495 #else
5496       ierr = PetscViewerASCIIPrintf(viewer, " %g", (double)values[i*numCIndices+j]);CHKERRQ(ierr);
5497 #endif
5498     }
5499     ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
5500   }
5501   PetscFunctionReturn(0);
5502 }
5503 
5504 #undef __FUNCT__
5505 #define __FUNCT__ "indicesPoint_private"
5506 /* . off - The global offset of this point */
5507 PetscErrorCode indicesPoint_private(PetscSection section, PetscInt point, PetscInt off, PetscInt *loff, PetscBool setBC, PetscInt orientation, PetscInt indices[])
5508 {
5509   PetscInt        dof;    /* The number of unknowns on this point */
5510   PetscInt        cdof;   /* The number of constraints on this point */
5511   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
5512   PetscInt        cind = 0, k;
5513   PetscErrorCode  ierr;
5514 
5515   PetscFunctionBegin;
5516   ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
5517   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
5518   if (!cdof || setBC) {
5519     if (orientation >= 0) {
5520       for (k = 0; k < dof; ++k) indices[*loff+k] = off+k;
5521     } else {
5522       for (k = 0; k < dof; ++k) indices[*loff+dof-k-1] = off+k;
5523     }
5524   } else {
5525     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
5526     if (orientation >= 0) {
5527       for (k = 0; k < dof; ++k) {
5528         if ((cind < cdof) && (k == cdofs[cind])) {
5529           /* Insert check for returning constrained indices */
5530           indices[*loff+k] = -(off+k+1);
5531           ++cind;
5532         } else {
5533           indices[*loff+k] = off+k-cind;
5534         }
5535       }
5536     } else {
5537       for (k = 0; k < dof; ++k) {
5538         if ((cind < cdof) && (k == cdofs[cind])) {
5539           /* Insert check for returning constrained indices */
5540           indices[*loff+dof-k-1] = -(off+k+1);
5541           ++cind;
5542         } else {
5543           indices[*loff+dof-k-1] = off+k-cind;
5544         }
5545       }
5546     }
5547   }
5548   *loff += dof;
5549   PetscFunctionReturn(0);
5550 }
5551 
5552 #undef __FUNCT__
5553 #define __FUNCT__ "indicesPointFields_private"
5554 /* . off - The global offset of this point */
5555 PetscErrorCode indicesPointFields_private(PetscSection section, PetscInt point, PetscInt off, PetscInt foffs[], PetscBool setBC, PetscInt orientation, PetscInt indices[])
5556 {
5557   PetscInt       numFields, foff, f;
5558   PetscErrorCode ierr;
5559 
5560   PetscFunctionBegin;
5561   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5562   for (f = 0, foff = 0; f < numFields; ++f) {
5563     PetscInt        fdof, fcomp, cfdof;
5564     const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
5565     PetscInt        cind = 0, k, c;
5566 
5567     ierr = PetscSectionGetFieldComponents(section, f, &fcomp);CHKERRQ(ierr);
5568     ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
5569     ierr = PetscSectionGetFieldConstraintDof(section, point, f, &cfdof);CHKERRQ(ierr);
5570     if (!cfdof || setBC) {
5571       if (orientation >= 0) {
5572         for (k = 0; k < fdof; ++k) indices[foffs[f]+k] = off+foff+k;
5573       } else {
5574         for (k = fdof/fcomp-1; k >= 0; --k) {
5575           for (c = 0; c < fcomp; ++c) {
5576             indices[foffs[f]+k*fcomp+c] = off+foff+(fdof/fcomp-1-k)*fcomp+c;
5577           }
5578         }
5579       }
5580     } else {
5581       ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
5582       if (orientation >= 0) {
5583         for (k = 0; k < fdof; ++k) {
5584           if ((cind < cfdof) && (k == fcdofs[cind])) {
5585             indices[foffs[f]+k] = -(off+foff+k+1);
5586             ++cind;
5587           } else {
5588             indices[foffs[f]+k] = off+foff+k-cind;
5589           }
5590         }
5591       } else {
5592         for (k = fdof/fcomp-1; k >= 0; --k) {
5593           for (c = 0; c < fcomp; ++c) {
5594             if ((cind < cfdof) && ((fdof/fcomp-1-k)*fcomp+c == fcdofs[cind])) {
5595               indices[foffs[f]+k*fcomp+c] = -(off+foff+(fdof/fcomp-1-k)*fcomp+c+1);
5596               ++cind;
5597             } else {
5598               indices[foffs[f]+k*fcomp+c] = off+foff+(fdof/fcomp-1-k)*fcomp+c-cind;
5599             }
5600           }
5601         }
5602       }
5603     }
5604     foff     += fdof - cfdof;
5605     foffs[f] += fdof;
5606   }
5607   PetscFunctionReturn(0);
5608 }
5609 
5610 #undef __FUNCT__
5611 #define __FUNCT__ "DMPlexMatSetClosure"
5612 /*@C
5613   DMPlexMatSetClosure - Set an array of the values on the closure of 'point'
5614 
5615   Not collective
5616 
5617   Input Parameters:
5618 + dm - The DM
5619 . section - The section describing the layout in v, or NULL to use the default section
5620 . globalSection - The section describing the layout in v, or NULL to use the default global section
5621 . A - The matrix
5622 . point - The sieve point in the DM
5623 . values - The array of values
5624 - mode - The insert mode, where INSERT_ALL_VALUES and ADD_ALL_VALUES also overwrite boundary conditions
5625 
5626   Fortran Notes:
5627   This routine is only available in Fortran 90, and you must include petsc.h90 in your code.
5628 
5629   Level: intermediate
5630 
5631 .seealso DMPlexVecGetClosure(), DMPlexVecSetClosure()
5632 @*/
5633 PetscErrorCode DMPlexMatSetClosure(DM dm, PetscSection section, PetscSection globalSection, Mat A, PetscInt point, const PetscScalar values[], InsertMode mode)
5634 {
5635   DM_Plex        *mesh   = (DM_Plex*) dm->data;
5636   PetscSection    clSection;
5637   IS              clPoints;
5638   PetscInt       *points = NULL;
5639   const PetscInt *clp;
5640   PetscInt       *indices;
5641   PetscInt        offsets[32];
5642   PetscInt        numFields, numPoints, numIndices, dof, off, globalOff, pStart, pEnd, p, q, f;
5643   PetscErrorCode  ierr;
5644 
5645   PetscFunctionBegin;
5646   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5647   if (!section) {ierr = DMGetDefaultSection(dm, &section);CHKERRQ(ierr);}
5648   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
5649   if (!globalSection) {ierr = DMGetDefaultGlobalSection(dm, &globalSection);CHKERRQ(ierr);}
5650   PetscValidHeaderSpecific(globalSection, PETSC_SECTION_CLASSID, 3);
5651   PetscValidHeaderSpecific(A, MAT_CLASSID, 4);
5652   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5653   if (numFields > 31) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", numFields);
5654   ierr = PetscMemzero(offsets, 32 * sizeof(PetscInt));CHKERRQ(ierr);
5655   ierr = PetscSectionGetClosureIndex(section, (PetscObject) dm, &clSection, &clPoints);CHKERRQ(ierr);
5656   if (!clPoints) {
5657     ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
5658     /* Compress out points not in the section */
5659     ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
5660     for (p = 0, q = 0; p < numPoints*2; p += 2) {
5661       if ((points[p] >= pStart) && (points[p] < pEnd)) {
5662         points[q*2]   = points[p];
5663         points[q*2+1] = points[p+1];
5664         ++q;
5665       }
5666     }
5667     numPoints = q;
5668   } else {
5669     PetscInt dof, off;
5670 
5671     ierr = PetscSectionGetDof(clSection, point, &dof);CHKERRQ(ierr);
5672     numPoints = dof/2;
5673     ierr = PetscSectionGetOffset(clSection, point, &off);CHKERRQ(ierr);
5674     ierr = ISGetIndices(clPoints, &clp);CHKERRQ(ierr);
5675     points = (PetscInt *) &clp[off];
5676   }
5677   for (p = 0, numIndices = 0; p < numPoints*2; p += 2) {
5678     PetscInt fdof;
5679 
5680     ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5681     for (f = 0; f < numFields; ++f) {
5682       ierr          = PetscSectionGetFieldDof(section, points[p], f, &fdof);CHKERRQ(ierr);
5683       offsets[f+1] += fdof;
5684     }
5685     numIndices += dof;
5686   }
5687   for (f = 1; f < numFields; ++f) offsets[f+1] += offsets[f];
5688 
5689   if (numFields && offsets[numFields] != numIndices) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid size for closure %d should be %d", offsets[numFields], numIndices);
5690   ierr = DMGetWorkArray(dm, numIndices, PETSC_INT, &indices);CHKERRQ(ierr);
5691   if (numFields) {
5692     for (p = 0; p < numPoints*2; p += 2) {
5693       PetscInt o = points[p+1];
5694       ierr = PetscSectionGetOffset(globalSection, points[p], &globalOff);CHKERRQ(ierr);
5695       indicesPointFields_private(section, points[p], globalOff < 0 ? -(globalOff+1) : globalOff, offsets, PETSC_FALSE, o, indices);
5696     }
5697   } else {
5698     for (p = 0, off = 0; p < numPoints*2; p += 2) {
5699       PetscInt o = points[p+1];
5700       ierr = PetscSectionGetOffset(globalSection, points[p], &globalOff);CHKERRQ(ierr);
5701       indicesPoint_private(section, points[p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, o, indices);
5702     }
5703   }
5704   if (mesh->printSetValues) {ierr = DMPlexPrintMatSetValues(PETSC_VIEWER_STDOUT_SELF, A, point, numIndices, indices, 0, NULL, values);CHKERRQ(ierr);}
5705   ierr = MatSetValues(A, numIndices, indices, numIndices, indices, values, mode);
5706   if (ierr) {
5707     PetscMPIInt    rank;
5708     PetscErrorCode ierr2;
5709 
5710     ierr2 = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRQ(ierr2);
5711     ierr2 = (*PetscErrorPrintf)("[%D]ERROR in DMPlexMatSetClosure\n", rank);CHKERRQ(ierr2);
5712     ierr2 = DMPlexPrintMatSetValues(PETSC_VIEWER_STDERR_SELF, A, point, numIndices, indices, 0, NULL, values);CHKERRQ(ierr2);
5713     ierr2 = DMRestoreWorkArray(dm, numIndices, PETSC_INT, &indices);CHKERRQ(ierr2);
5714     CHKERRQ(ierr);
5715   }
5716   if (!clPoints) {
5717     ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
5718   } else {
5719     ierr = ISRestoreIndices(clPoints, &clp);CHKERRQ(ierr);
5720   }
5721   ierr = DMRestoreWorkArray(dm, numIndices, PETSC_INT, &indices);CHKERRQ(ierr);
5722   PetscFunctionReturn(0);
5723 }
5724 
5725 #undef __FUNCT__
5726 #define __FUNCT__ "DMPlexMatSetClosureRefined"
5727 PetscErrorCode DMPlexMatSetClosureRefined(DM dmf, PetscSection fsection, PetscSection globalFSection, DM dmc, PetscSection csection, PetscSection globalCSection, Mat A, PetscInt point, const PetscScalar values[], InsertMode mode)
5728 {
5729   DM_Plex        *mesh   = (DM_Plex*) dmf->data;
5730   PetscInt       *fpoints = NULL, *ftotpoints = NULL;
5731   PetscInt       *cpoints = NULL;
5732   PetscInt       *findices, *cindices;
5733   PetscInt        foffsets[32], coffsets[32];
5734   CellRefiner     cellRefiner;
5735   PetscInt        numFields, numSubcells, maxFPoints, numFPoints, numCPoints, numFIndices, numCIndices, dof, off, globalOff, pStart, pEnd, p, q, r, s, f;
5736   PetscErrorCode  ierr;
5737 
5738   PetscFunctionBegin;
5739   PetscValidHeaderSpecific(dmf, DM_CLASSID, 1);
5740   PetscValidHeaderSpecific(dmc, DM_CLASSID, 4);
5741   if (!fsection) {ierr = DMGetDefaultSection(dmf, &fsection);CHKERRQ(ierr);}
5742   PetscValidHeaderSpecific(fsection, PETSC_SECTION_CLASSID, 2);
5743   if (!csection) {ierr = DMGetDefaultSection(dmc, &csection);CHKERRQ(ierr);}
5744   PetscValidHeaderSpecific(csection, PETSC_SECTION_CLASSID, 5);
5745   if (!globalFSection) {ierr = DMGetDefaultGlobalSection(dmf, &globalFSection);CHKERRQ(ierr);}
5746   PetscValidHeaderSpecific(globalFSection, PETSC_SECTION_CLASSID, 3);
5747   if (!globalCSection) {ierr = DMGetDefaultGlobalSection(dmc, &globalCSection);CHKERRQ(ierr);}
5748   PetscValidHeaderSpecific(globalCSection, PETSC_SECTION_CLASSID, 6);
5749   PetscValidHeaderSpecific(A, MAT_CLASSID, 7);
5750   ierr = PetscSectionGetNumFields(fsection, &numFields);CHKERRQ(ierr);
5751   if (numFields > 31) SETERRQ1(PetscObjectComm((PetscObject)dmf), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", numFields);
5752   ierr = PetscMemzero(foffsets, 32 * sizeof(PetscInt));CHKERRQ(ierr);
5753   ierr = PetscMemzero(coffsets, 32 * sizeof(PetscInt));CHKERRQ(ierr);
5754   /* Column indices */
5755   ierr = DMPlexGetTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
5756   maxFPoints = numCPoints;
5757   /* Compress out points not in the section */
5758   /*   TODO: Squeeze out points with 0 dof as well */
5759   ierr = PetscSectionGetChart(csection, &pStart, &pEnd);CHKERRQ(ierr);
5760   for (p = 0, q = 0; p < numCPoints*2; p += 2) {
5761     if ((cpoints[p] >= pStart) && (cpoints[p] < pEnd)) {
5762       cpoints[q*2]   = cpoints[p];
5763       cpoints[q*2+1] = cpoints[p+1];
5764       ++q;
5765     }
5766   }
5767   numCPoints = q;
5768   for (p = 0, numCIndices = 0; p < numCPoints*2; p += 2) {
5769     PetscInt fdof;
5770 
5771     ierr = PetscSectionGetDof(csection, cpoints[p], &dof);CHKERRQ(ierr);
5772     if (!dof) continue;
5773     for (f = 0; f < numFields; ++f) {
5774       ierr           = PetscSectionGetFieldDof(csection, cpoints[p], f, &fdof);CHKERRQ(ierr);
5775       coffsets[f+1] += fdof;
5776     }
5777     numCIndices += dof;
5778   }
5779   for (f = 1; f < numFields; ++f) coffsets[f+1] += coffsets[f];
5780   /* Row indices */
5781   ierr = DMPlexGetCellRefiner_Internal(dmc, &cellRefiner);CHKERRQ(ierr);
5782   ierr = CellRefinerGetAffineTransforms_Internal(cellRefiner, &numSubcells, NULL, NULL, NULL);CHKERRQ(ierr);
5783   ierr = DMGetWorkArray(dmf, maxFPoints*2*numSubcells, PETSC_INT, &ftotpoints);CHKERRQ(ierr);
5784   for (r = 0, q = 0; r < numSubcells; ++r) {
5785     /* TODO Map from coarse to fine cells */
5786     ierr = DMPlexGetTransitiveClosure(dmf, point*numSubcells + r, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
5787     /* Compress out points not in the section */
5788     ierr = PetscSectionGetChart(fsection, &pStart, &pEnd);CHKERRQ(ierr);
5789     for (p = 0; p < numFPoints*2; p += 2) {
5790       if ((fpoints[p] >= pStart) && (fpoints[p] < pEnd)) {
5791         ierr = PetscSectionGetDof(fsection, fpoints[p], &dof);CHKERRQ(ierr);
5792         if (!dof) continue;
5793         for (s = 0; s < q; ++s) if (fpoints[p] == ftotpoints[s*2]) break;
5794         if (s < q) continue;
5795         ftotpoints[q*2]   = fpoints[p];
5796         ftotpoints[q*2+1] = fpoints[p+1];
5797         ++q;
5798       }
5799     }
5800     ierr = DMPlexRestoreTransitiveClosure(dmf, point, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
5801   }
5802   numFPoints = q;
5803   for (p = 0, numFIndices = 0; p < numFPoints*2; p += 2) {
5804     PetscInt fdof;
5805 
5806     ierr = PetscSectionGetDof(fsection, ftotpoints[p], &dof);CHKERRQ(ierr);
5807     if (!dof) continue;
5808     for (f = 0; f < numFields; ++f) {
5809       ierr           = PetscSectionGetFieldDof(fsection, ftotpoints[p], f, &fdof);CHKERRQ(ierr);
5810       foffsets[f+1] += fdof;
5811     }
5812     numFIndices += dof;
5813   }
5814   for (f = 1; f < numFields; ++f) foffsets[f+1] += foffsets[f];
5815 
5816   if (numFields && foffsets[numFields] != numFIndices) SETERRQ2(PetscObjectComm((PetscObject)dmf), PETSC_ERR_PLIB, "Invalid size for closure %d should be %d", foffsets[numFields], numFIndices);
5817   if (numFields && coffsets[numFields] != numCIndices) SETERRQ2(PetscObjectComm((PetscObject)dmc), PETSC_ERR_PLIB, "Invalid size for closure %d should be %d", coffsets[numFields], numCIndices);
5818   ierr = DMGetWorkArray(dmf, numFIndices, PETSC_INT, &findices);CHKERRQ(ierr);
5819   ierr = DMGetWorkArray(dmc, numCIndices, PETSC_INT, &cindices);CHKERRQ(ierr);
5820   if (numFields) {
5821     for (p = 0; p < numFPoints*2; p += 2) {
5822       PetscInt o = ftotpoints[p+1];
5823       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[p], &globalOff);CHKERRQ(ierr);
5824       indicesPointFields_private(fsection, ftotpoints[p], globalOff < 0 ? -(globalOff+1) : globalOff, foffsets, PETSC_FALSE, o, findices);
5825     }
5826     for (p = 0; p < numCPoints*2; p += 2) {
5827       PetscInt o = cpoints[p+1];
5828       ierr = PetscSectionGetOffset(globalCSection, cpoints[p], &globalOff);CHKERRQ(ierr);
5829       indicesPointFields_private(csection, cpoints[p], globalOff < 0 ? -(globalOff+1) : globalOff, coffsets, PETSC_FALSE, o, cindices);
5830     }
5831   } else {
5832     for (p = 0, off = 0; p < numFPoints*2; p += 2) {
5833       PetscInt o = ftotpoints[p+1];
5834       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[p], &globalOff);CHKERRQ(ierr);
5835       indicesPoint_private(fsection, ftotpoints[p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, o, findices);
5836     }
5837     for (p = 0, off = 0; p < numCPoints*2; p += 2) {
5838       PetscInt o = cpoints[p+1];
5839       ierr = PetscSectionGetOffset(globalCSection, cpoints[p], &globalOff);CHKERRQ(ierr);
5840       indicesPoint_private(csection, cpoints[p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, o, cindices);
5841     }
5842   }
5843   if (mesh->printSetValues) {ierr = DMPlexPrintMatSetValues(PETSC_VIEWER_STDOUT_SELF, A, point, numFIndices, findices, numCIndices, cindices, values);CHKERRQ(ierr);}
5844   ierr = MatSetValues(A, numFIndices, findices, numCIndices, cindices, values, mode);
5845   if (ierr) {
5846     PetscMPIInt    rank;
5847     PetscErrorCode ierr2;
5848 
5849     ierr2 = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRQ(ierr2);
5850     ierr2 = (*PetscErrorPrintf)("[%D]ERROR in DMPlexMatSetClosure\n", rank);CHKERRQ(ierr2);
5851     ierr2 = DMPlexPrintMatSetValues(PETSC_VIEWER_STDERR_SELF, A, point, numFIndices, findices, numCIndices, cindices, values);CHKERRQ(ierr2);
5852     ierr2 = DMRestoreWorkArray(dmf, numFIndices, PETSC_INT, &findices);CHKERRQ(ierr2);
5853     ierr2 = DMRestoreWorkArray(dmc, numCIndices, PETSC_INT, &cindices);CHKERRQ(ierr2);
5854     CHKERRQ(ierr);
5855   }
5856   ierr = DMRestoreWorkArray(dmf, numCPoints*2*4, PETSC_INT, &ftotpoints);CHKERRQ(ierr);
5857   ierr = DMPlexRestoreTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
5858   ierr = DMRestoreWorkArray(dmf, numFIndices, PETSC_INT, &findices);CHKERRQ(ierr);
5859   ierr = DMRestoreWorkArray(dmc, numCIndices, PETSC_INT, &cindices);CHKERRQ(ierr);
5860   PetscFunctionReturn(0);
5861 }
5862 
5863 #undef __FUNCT__
5864 #define __FUNCT__ "DMPlexGetHybridBounds"
5865 /*@
5866   DMPlexGetHybridBounds - Get the first mesh point of each dimension which is a hybrid
5867 
5868   Input Parameter:
5869 . dm - The DMPlex object
5870 
5871   Output Parameters:
5872 + cMax - The first hybrid cell
5873 . cMax - The first hybrid face
5874 . cMax - The first hybrid edge
5875 - cMax - The first hybrid vertex
5876 
5877   Level: developer
5878 
5879 .seealso DMPlexCreateHybridMesh()
5880 @*/
5881 PetscErrorCode DMPlexGetHybridBounds(DM dm, PetscInt *cMax, PetscInt *fMax, PetscInt *eMax, PetscInt *vMax)
5882 {
5883   DM_Plex       *mesh = (DM_Plex*) dm->data;
5884   PetscInt       dim;
5885   PetscErrorCode ierr;
5886 
5887   PetscFunctionBegin;
5888   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5889   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
5890   if (cMax) *cMax = mesh->hybridPointMax[dim];
5891   if (fMax) *fMax = mesh->hybridPointMax[dim-1];
5892   if (eMax) *eMax = mesh->hybridPointMax[1];
5893   if (vMax) *vMax = mesh->hybridPointMax[0];
5894   PetscFunctionReturn(0);
5895 }
5896 
5897 #undef __FUNCT__
5898 #define __FUNCT__ "DMPlexSetHybridBounds"
5899 PetscErrorCode DMPlexSetHybridBounds(DM dm, PetscInt cMax, PetscInt fMax, PetscInt eMax, PetscInt vMax)
5900 {
5901   DM_Plex       *mesh = (DM_Plex*) dm->data;
5902   PetscInt       dim;
5903   PetscErrorCode ierr;
5904 
5905   PetscFunctionBegin;
5906   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5907   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
5908   if (cMax >= 0) mesh->hybridPointMax[dim]   = cMax;
5909   if (fMax >= 0) mesh->hybridPointMax[dim-1] = fMax;
5910   if (eMax >= 0) mesh->hybridPointMax[1]     = eMax;
5911   if (vMax >= 0) mesh->hybridPointMax[0]     = vMax;
5912   PetscFunctionReturn(0);
5913 }
5914 
5915 #undef __FUNCT__
5916 #define __FUNCT__ "DMPlexGetVTKCellHeight"
5917 PetscErrorCode DMPlexGetVTKCellHeight(DM dm, PetscInt *cellHeight)
5918 {
5919   DM_Plex *mesh = (DM_Plex*) dm->data;
5920 
5921   PetscFunctionBegin;
5922   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5923   PetscValidPointer(cellHeight, 2);
5924   *cellHeight = mesh->vtkCellHeight;
5925   PetscFunctionReturn(0);
5926 }
5927 
5928 #undef __FUNCT__
5929 #define __FUNCT__ "DMPlexSetVTKCellHeight"
5930 PetscErrorCode DMPlexSetVTKCellHeight(DM dm, PetscInt cellHeight)
5931 {
5932   DM_Plex *mesh = (DM_Plex*) dm->data;
5933 
5934   PetscFunctionBegin;
5935   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5936   mesh->vtkCellHeight = cellHeight;
5937   PetscFunctionReturn(0);
5938 }
5939 
5940 #undef __FUNCT__
5941 #define __FUNCT__ "DMPlexCreateNumbering_Private"
5942 /* We can easily have a form that takes an IS instead */
5943 PetscErrorCode DMPlexCreateNumbering_Private(DM dm, PetscInt pStart, PetscInt pEnd, PetscSF sf, IS *numbering)
5944 {
5945   PetscSection   section, globalSection;
5946   PetscInt      *numbers, p;
5947   PetscErrorCode ierr;
5948 
5949   PetscFunctionBegin;
5950   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &section);CHKERRQ(ierr);
5951   ierr = PetscSectionSetChart(section, pStart, pEnd);CHKERRQ(ierr);
5952   for (p = pStart; p < pEnd; ++p) {
5953     ierr = PetscSectionSetDof(section, p, 1);CHKERRQ(ierr);
5954   }
5955   ierr = PetscSectionSetUp(section);CHKERRQ(ierr);
5956   ierr = PetscSectionCreateGlobalSection(section, sf, PETSC_FALSE, &globalSection);CHKERRQ(ierr);
5957   ierr = PetscMalloc1((pEnd - pStart), &numbers);CHKERRQ(ierr);
5958   for (p = pStart; p < pEnd; ++p) {
5959     ierr = PetscSectionGetOffset(globalSection, p, &numbers[p-pStart]);CHKERRQ(ierr);
5960   }
5961   ierr = ISCreateGeneral(PetscObjectComm((PetscObject)dm), pEnd - pStart, numbers, PETSC_OWN_POINTER, numbering);CHKERRQ(ierr);
5962   ierr = PetscSectionDestroy(&section);CHKERRQ(ierr);
5963   ierr = PetscSectionDestroy(&globalSection);CHKERRQ(ierr);
5964   PetscFunctionReturn(0);
5965 }
5966 
5967 #undef __FUNCT__
5968 #define __FUNCT__ "DMPlexGetCellNumbering"
5969 PetscErrorCode DMPlexGetCellNumbering(DM dm, IS *globalCellNumbers)
5970 {
5971   DM_Plex       *mesh = (DM_Plex*) dm->data;
5972   PetscInt       cellHeight, cStart, cEnd, cMax;
5973   PetscErrorCode ierr;
5974 
5975   PetscFunctionBegin;
5976   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5977   if (!mesh->globalCellNumbers) {
5978     ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
5979     ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
5980     ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
5981     if (cMax >= 0) cEnd = PetscMin(cEnd, cMax);
5982     ierr = DMPlexCreateNumbering_Private(dm, cStart, cEnd, dm->sf, &mesh->globalCellNumbers);CHKERRQ(ierr);
5983   }
5984   *globalCellNumbers = mesh->globalCellNumbers;
5985   PetscFunctionReturn(0);
5986 }
5987 
5988 #undef __FUNCT__
5989 #define __FUNCT__ "DMPlexGetVertexNumbering"
5990 PetscErrorCode DMPlexGetVertexNumbering(DM dm, IS *globalVertexNumbers)
5991 {
5992   DM_Plex       *mesh = (DM_Plex*) dm->data;
5993   PetscInt       vStart, vEnd, vMax;
5994   PetscErrorCode ierr;
5995 
5996   PetscFunctionBegin;
5997   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5998   if (!mesh->globalVertexNumbers) {
5999     ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
6000     ierr = DMPlexGetHybridBounds(dm, NULL, NULL, NULL, &vMax);CHKERRQ(ierr);
6001     if (vMax >= 0) vEnd = PetscMin(vEnd, vMax);
6002     ierr = DMPlexCreateNumbering_Private(dm, vStart, vEnd, dm->sf, &mesh->globalVertexNumbers);CHKERRQ(ierr);
6003   }
6004   *globalVertexNumbers = mesh->globalVertexNumbers;
6005   PetscFunctionReturn(0);
6006 }
6007 
6008 
6009 #undef __FUNCT__
6010 #define __FUNCT__ "PetscSectionCreateGlobalSectionLabel"
6011 /*@C
6012   PetscSectionCreateGlobalSectionLabel - Create a section describing the global field layout using
6013   the local section and an SF describing the section point overlap.
6014 
6015   Input Parameters:
6016   + s - The PetscSection for the local field layout
6017   . sf - The SF describing parallel layout of the section points
6018   . includeConstraints - By default this is PETSC_FALSE, meaning that the global field vector will not possess constrained dofs
6019   . label - The label specifying the points
6020   - labelValue - The label stratum specifying the points
6021 
6022   Output Parameter:
6023   . gsection - The PetscSection for the global field layout
6024 
6025   Note: This gives negative sizes and offsets to points not owned by this process
6026 
6027   Level: developer
6028 
6029 .seealso: PetscSectionCreate()
6030 @*/
6031 PetscErrorCode PetscSectionCreateGlobalSectionLabel(PetscSection s, PetscSF sf, PetscBool includeConstraints, DMLabel label, PetscInt labelValue, PetscSection *gsection)
6032 {
6033   PetscInt      *neg = NULL, *tmpOff = NULL;
6034   PetscInt       pStart, pEnd, p, dof, cdof, off, globalOff = 0, nroots;
6035   PetscErrorCode ierr;
6036 
6037   PetscFunctionBegin;
6038   ierr = PetscSectionCreate(PetscObjectComm((PetscObject) s), gsection);CHKERRQ(ierr);
6039   ierr = PetscSectionGetChart(s, &pStart, &pEnd);CHKERRQ(ierr);
6040   ierr = PetscSectionSetChart(*gsection, pStart, pEnd);CHKERRQ(ierr);
6041   ierr = PetscSFGetGraph(sf, &nroots, NULL, NULL, NULL);CHKERRQ(ierr);
6042   if (nroots >= 0) {
6043     if (nroots < pEnd-pStart) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "PetscSF nroots %d < %d section size", nroots, pEnd-pStart);
6044     ierr = PetscCalloc1(nroots, &neg);CHKERRQ(ierr);
6045     if (nroots > pEnd-pStart) {
6046       ierr = PetscCalloc1(nroots, &tmpOff);CHKERRQ(ierr);
6047     } else {
6048       tmpOff = &(*gsection)->atlasDof[-pStart];
6049     }
6050   }
6051   /* Mark ghost points with negative dof */
6052   for (p = pStart; p < pEnd; ++p) {
6053     PetscInt value;
6054 
6055     ierr = DMLabelGetValue(label, p, &value);CHKERRQ(ierr);
6056     if (value != labelValue) continue;
6057     ierr = PetscSectionGetDof(s, p, &dof);CHKERRQ(ierr);
6058     ierr = PetscSectionSetDof(*gsection, p, dof);CHKERRQ(ierr);
6059     ierr = PetscSectionGetConstraintDof(s, p, &cdof);CHKERRQ(ierr);
6060     if (!includeConstraints && cdof > 0) {ierr = PetscSectionSetConstraintDof(*gsection, p, cdof);CHKERRQ(ierr);}
6061     if (neg) neg[p] = -(dof+1);
6062   }
6063   ierr = PetscSectionSetUpBC(*gsection);CHKERRQ(ierr);
6064   if (nroots >= 0) {
6065     ierr = PetscSFBcastBegin(sf, MPIU_INT, neg, tmpOff);CHKERRQ(ierr);
6066     ierr = PetscSFBcastEnd(sf, MPIU_INT, neg, tmpOff);CHKERRQ(ierr);
6067     if (nroots > pEnd-pStart) {
6068       for (p = pStart; p < pEnd; ++p) {if (tmpOff[p] < 0) (*gsection)->atlasDof[p-pStart] = tmpOff[p];}
6069     }
6070   }
6071   /* Calculate new sizes, get proccess offset, and calculate point offsets */
6072   for (p = 0, off = 0; p < pEnd-pStart; ++p) {
6073     cdof = (!includeConstraints && s->bc) ? s->bc->atlasDof[p] : 0;
6074     (*gsection)->atlasOff[p] = off;
6075     off += (*gsection)->atlasDof[p] > 0 ? (*gsection)->atlasDof[p]-cdof : 0;
6076   }
6077   ierr       = MPI_Scan(&off, &globalOff, 1, MPIU_INT, MPI_SUM, PetscObjectComm((PetscObject) s));CHKERRQ(ierr);
6078   globalOff -= off;
6079   for (p = 0, off = 0; p < pEnd-pStart; ++p) {
6080     (*gsection)->atlasOff[p] += globalOff;
6081     if (neg) neg[p] = -((*gsection)->atlasOff[p]+1);
6082   }
6083   /* Put in negative offsets for ghost points */
6084   if (nroots >= 0) {
6085     ierr = PetscSFBcastBegin(sf, MPIU_INT, neg, tmpOff);CHKERRQ(ierr);
6086     ierr = PetscSFBcastEnd(sf, MPIU_INT, neg, tmpOff);CHKERRQ(ierr);
6087     if (nroots > pEnd-pStart) {
6088       for (p = pStart; p < pEnd; ++p) {if (tmpOff[p] < 0) (*gsection)->atlasOff[p-pStart] = tmpOff[p];}
6089     }
6090   }
6091   if (nroots >= 0 && nroots > pEnd-pStart) {ierr = PetscFree(tmpOff);CHKERRQ(ierr);}
6092   ierr = PetscFree(neg);CHKERRQ(ierr);
6093   PetscFunctionReturn(0);
6094 }
6095 
6096 #undef __FUNCT__
6097 #define __FUNCT__ "DMPlexCheckSymmetry"
6098 /*@
6099   DMPlexCheckSymmetry - Check that the adjacency information in the mesh is symmetric.
6100 
6101   Input Parameters:
6102   + dm - The DMPlex object
6103 
6104   Note: This is a useful diagnostic when creating meshes programmatically.
6105 
6106   Level: developer
6107 
6108 .seealso: DMCreate(), DMCheckSkeleton(), DMCheckFaces()
6109 @*/
6110 PetscErrorCode DMPlexCheckSymmetry(DM dm)
6111 {
6112   PetscSection    coneSection, supportSection;
6113   const PetscInt *cone, *support;
6114   PetscInt        coneSize, c, supportSize, s;
6115   PetscInt        pStart, pEnd, p, csize, ssize;
6116   PetscErrorCode  ierr;
6117 
6118   PetscFunctionBegin;
6119   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6120   ierr = DMPlexGetConeSection(dm, &coneSection);CHKERRQ(ierr);
6121   ierr = DMPlexGetSupportSection(dm, &supportSection);CHKERRQ(ierr);
6122   /* Check that point p is found in the support of its cone points, and vice versa */
6123   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
6124   for (p = pStart; p < pEnd; ++p) {
6125     ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
6126     ierr = DMPlexGetCone(dm, p, &cone);CHKERRQ(ierr);
6127     for (c = 0; c < coneSize; ++c) {
6128       PetscBool dup = PETSC_FALSE;
6129       PetscInt  d;
6130       for (d = c-1; d >= 0; --d) {
6131         if (cone[c] == cone[d]) {dup = PETSC_TRUE; break;}
6132       }
6133       ierr = DMPlexGetSupportSize(dm, cone[c], &supportSize);CHKERRQ(ierr);
6134       ierr = DMPlexGetSupport(dm, cone[c], &support);CHKERRQ(ierr);
6135       for (s = 0; s < supportSize; ++s) {
6136         if (support[s] == p) break;
6137       }
6138       if ((s >= supportSize) || (dup && (support[s+1] != p))) {
6139         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %d cone: ", p);
6140         for (s = 0; s < coneSize; ++s) {
6141           ierr = PetscPrintf(PETSC_COMM_SELF, "%d, ", cone[s]);
6142         }
6143         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");
6144         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %d support: ", cone[c]);
6145         for (s = 0; s < supportSize; ++s) {
6146           ierr = PetscPrintf(PETSC_COMM_SELF, "%d, ", support[s]);
6147         }
6148         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");
6149         if (dup) {
6150           SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %d not repeatedly found in support of repeated cone point %d", p, cone[c]);
6151         } else {
6152           SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %d not found in support of cone point %d", p, cone[c]);
6153         }
6154       }
6155     }
6156     ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
6157     ierr = DMPlexGetSupport(dm, p, &support);CHKERRQ(ierr);
6158     for (s = 0; s < supportSize; ++s) {
6159       ierr = DMPlexGetConeSize(dm, support[s], &coneSize);CHKERRQ(ierr);
6160       ierr = DMPlexGetCone(dm, support[s], &cone);CHKERRQ(ierr);
6161       for (c = 0; c < coneSize; ++c) {
6162         if (cone[c] == p) break;
6163       }
6164       if (c >= coneSize) {
6165         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %d support: ", p);
6166         for (c = 0; c < supportSize; ++c) {
6167           ierr = PetscPrintf(PETSC_COMM_SELF, "%d, ", support[c]);
6168         }
6169         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");
6170         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %d cone: ", support[s]);
6171         for (c = 0; c < coneSize; ++c) {
6172           ierr = PetscPrintf(PETSC_COMM_SELF, "%d, ", cone[c]);
6173         }
6174         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");
6175         SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %d not found in cone of support point %d", p, support[s]);
6176       }
6177     }
6178   }
6179   ierr = PetscSectionGetStorageSize(coneSection, &csize);CHKERRQ(ierr);
6180   ierr = PetscSectionGetStorageSize(supportSection, &ssize);CHKERRQ(ierr);
6181   if (csize != ssize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "Total cone size %d != Total support size %d", csize, ssize);
6182   PetscFunctionReturn(0);
6183 }
6184 
6185 #undef __FUNCT__
6186 #define __FUNCT__ "DMPlexCheckSkeleton"
6187 /*@
6188   DMPlexCheckSkeleton - Check that each cell has the correct number of vertices
6189 
6190   Input Parameters:
6191 + dm - The DMPlex object
6192 . isSimplex - Are the cells simplices or tensor products
6193 - cellHeight - Normally 0
6194 
6195   Note: This is a useful diagnostic when creating meshes programmatically.
6196 
6197   Level: developer
6198 
6199 .seealso: DMCreate(), DMCheckSymmetry(), DMCheckFaces()
6200 @*/
6201 PetscErrorCode DMPlexCheckSkeleton(DM dm, PetscBool isSimplex, PetscInt cellHeight)
6202 {
6203   PetscInt       dim, numCorners, numHybridCorners, vStart, vEnd, cStart, cEnd, cMax, c;
6204   PetscErrorCode ierr;
6205 
6206   PetscFunctionBegin;
6207   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6208   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
6209   switch (dim) {
6210   case 1: numCorners = isSimplex ? 2 : 2; numHybridCorners = isSimplex ? 2 : 2; break;
6211   case 2: numCorners = isSimplex ? 3 : 4; numHybridCorners = isSimplex ? 4 : 4; break;
6212   case 3: numCorners = isSimplex ? 4 : 8; numHybridCorners = isSimplex ? 6 : 8; break;
6213   default:
6214     SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_OUTOFRANGE, "Cannot handle meshes of dimension %d", dim);
6215   }
6216   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
6217   ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
6218   ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
6219   cMax = cMax >= 0 ? cMax : cEnd;
6220   for (c = cStart; c < cMax; ++c) {
6221     PetscInt *closure = NULL, closureSize, cl, coneSize = 0;
6222 
6223     ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6224     for (cl = 0; cl < closureSize*2; cl += 2) {
6225       const PetscInt p = closure[cl];
6226       if ((p >= vStart) && (p < vEnd)) ++coneSize;
6227     }
6228     ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6229     if (coneSize != numCorners) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %d has  %d vertices != %d", c, coneSize, numCorners);
6230   }
6231   for (c = cMax; c < cEnd; ++c) {
6232     PetscInt *closure = NULL, closureSize, cl, coneSize = 0;
6233 
6234     ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6235     for (cl = 0; cl < closureSize*2; cl += 2) {
6236       const PetscInt p = closure[cl];
6237       if ((p >= vStart) && (p < vEnd)) ++coneSize;
6238     }
6239     ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6240     if (coneSize > numHybridCorners) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Hybrid cell %d has  %d vertices > %d", c, coneSize, numHybridCorners);
6241   }
6242   PetscFunctionReturn(0);
6243 }
6244 
6245 #undef __FUNCT__
6246 #define __FUNCT__ "DMPlexCheckFaces"
6247 /*@
6248   DMPlexCheckFaces - Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type
6249 
6250   Input Parameters:
6251 + dm - The DMPlex object
6252 . isSimplex - Are the cells simplices or tensor products
6253 - cellHeight - Normally 0
6254 
6255   Note: This is a useful diagnostic when creating meshes programmatically.
6256 
6257   Level: developer
6258 
6259 .seealso: DMCreate(), DMCheckSymmetry(), DMCheckSkeleton()
6260 @*/
6261 PetscErrorCode DMPlexCheckFaces(DM dm, PetscBool isSimplex, PetscInt cellHeight)
6262 {
6263   PetscInt       pMax[4];
6264   PetscInt       dim, vStart, vEnd, cStart, cEnd, c, h;
6265   PetscErrorCode ierr;
6266 
6267   PetscFunctionBegin;
6268   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6269   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
6270   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
6271   ierr = DMPlexGetHybridBounds(dm, &pMax[dim], &pMax[dim-1], &pMax[1], &pMax[0]);CHKERRQ(ierr);
6272   for (h = cellHeight; h < dim; ++h) {
6273     ierr = DMPlexGetHeightStratum(dm, h, &cStart, &cEnd);CHKERRQ(ierr);
6274     for (c = cStart; c < cEnd; ++c) {
6275       const PetscInt *cone, *ornt, *faces;
6276       PetscInt        numFaces, faceSize, coneSize,f;
6277       PetscInt       *closure = NULL, closureSize, cl, numCorners = 0;
6278 
6279       if (pMax[dim-h] >= 0 && c >= pMax[dim-h]) continue;
6280       ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
6281       ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
6282       ierr = DMPlexGetConeOrientation(dm, c, &ornt);CHKERRQ(ierr);
6283       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6284       for (cl = 0; cl < closureSize*2; cl += 2) {
6285         const PetscInt p = closure[cl];
6286         if ((p >= vStart) && (p < vEnd)) closure[numCorners++] = p;
6287       }
6288       ierr = DMPlexGetRawFaces_Internal(dm, dim-h, numCorners, closure, &numFaces, &faceSize, &faces);CHKERRQ(ierr);
6289       if (coneSize != numFaces) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %d has %d faces but should have %d", c, coneSize, numFaces);
6290       for (f = 0; f < numFaces; ++f) {
6291         PetscInt *fclosure = NULL, fclosureSize, cl, fnumCorners = 0, v;
6292 
6293         ierr = DMPlexGetTransitiveClosure_Internal(dm, cone[f], ornt[f], PETSC_TRUE, &fclosureSize, &fclosure);CHKERRQ(ierr);
6294         for (cl = 0; cl < fclosureSize*2; cl += 2) {
6295           const PetscInt p = fclosure[cl];
6296           if ((p >= vStart) && (p < vEnd)) fclosure[fnumCorners++] = p;
6297         }
6298         if (fnumCorners != faceSize) SETERRQ5(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %d (%d) of cell %d has %d vertices but should have %d", cone[f], f, c, fnumCorners, faceSize);
6299         for (v = 0; v < fnumCorners; ++v) {
6300           if (fclosure[v] != faces[f*faceSize+v]) SETERRQ6(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %d (%d) of cell %d vertex %d, %d != %d", cone[f], f, c, v, fclosure[v], faces[f*faceSize+v]);
6301         }
6302         ierr = DMPlexRestoreTransitiveClosure(dm, cone[f], PETSC_TRUE, &fclosureSize, &fclosure);CHKERRQ(ierr);
6303       }
6304       ierr = DMPlexRestoreFaces_Internal(dm, dim, c, &numFaces, &faceSize, &faces);CHKERRQ(ierr);
6305       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6306     }
6307   }
6308   PetscFunctionReturn(0);
6309 }
6310 
6311 #undef __FUNCT__
6312 #define __FUNCT__ "DMCreateInterpolation_Plex"
6313 /* Pointwise interpolation
6314      Just code FEM for now
6315      u^f = I u^c
6316      sum_k u^f_k phi^f_k = I sum_j u^c_j phi^c_j
6317      u^f_i = sum_j psi^f_i I phi^c_j u^c_j
6318      I_{ij} = psi^f_i phi^c_j
6319 */
6320 PetscErrorCode DMCreateInterpolation_Plex(DM dmCoarse, DM dmFine, Mat *interpolation, Vec *scaling)
6321 {
6322   PetscSection   gsc, gsf;
6323   PetscInt       m, n;
6324   void          *ctx;
6325   PetscErrorCode ierr;
6326 
6327   PetscFunctionBegin;
6328   /*
6329   Loop over coarse cells
6330     Loop over coarse basis functions
6331       Loop over fine cells in coarse cell
6332         Loop over fine dual basis functions
6333           Evaluate coarse basis on fine dual basis quad points
6334           Sum
6335           Update local element matrix
6336     Accumulate to interpolation matrix
6337 
6338    Can extend PetscFEIntegrateJacobian_Basic() to do a specialized cell loop
6339   */
6340   ierr = DMGetDefaultGlobalSection(dmFine, &gsf);CHKERRQ(ierr);
6341   ierr = PetscSectionGetConstrainedStorageSize(gsf, &m);CHKERRQ(ierr);
6342   ierr = DMGetDefaultGlobalSection(dmCoarse, &gsc);CHKERRQ(ierr);
6343   ierr = PetscSectionGetConstrainedStorageSize(gsc, &n);CHKERRQ(ierr);
6344   /* We need to preallocate properly */
6345   ierr = MatCreate(PetscObjectComm((PetscObject) dmCoarse), interpolation);CHKERRQ(ierr);
6346   ierr = MatSetSizes(*interpolation, m, n, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
6347   ierr = MatSetType(*interpolation, dmCoarse->mattype);CHKERRQ(ierr);
6348   ierr = MatSetUp(*interpolation);CHKERRQ(ierr);
6349   ierr = MatSetFromOptions(*interpolation);CHKERRQ(ierr);
6350   ierr = MatSetOption(*interpolation, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_FALSE);CHKERRQ(ierr);
6351   ierr = DMGetApplicationContext(dmFine, &ctx);CHKERRQ(ierr);
6352   ierr = DMPlexComputeInterpolatorFEM(dmCoarse, dmFine, *interpolation, ctx);CHKERRQ(ierr);
6353   /* Use naive scaling */
6354   ierr = DMCreateInterpolationScale(dmCoarse, dmFine, *interpolation, scaling);CHKERRQ(ierr);
6355   PetscFunctionReturn(0);
6356 }
6357 
6358 #undef __FUNCT__
6359 #define __FUNCT__ "DMCreateInjection_Plex"
6360 PetscErrorCode DMCreateInjection_Plex(DM dmCoarse, DM dmFine, VecScatter *ctx)
6361 {
6362   Vec             cv,  fv;
6363   IS              cis, fis, fpointIS;
6364   PetscSection    sc, gsc, gsf;
6365   const PetscInt *fpoints;
6366   PetscInt       *cindices, *findices;
6367   PetscInt        cpStart, cpEnd, m, off, cp;
6368   PetscErrorCode  ierr;
6369 
6370   PetscFunctionBegin;
6371   ierr = DMGetDefaultGlobalSection(dmFine, &gsf);CHKERRQ(ierr);
6372   ierr = DMGetGlobalVector(dmFine, &fv);CHKERRQ(ierr);
6373   ierr = DMGetDefaultSection(dmCoarse, &sc);CHKERRQ(ierr);
6374   ierr = DMGetDefaultGlobalSection(dmCoarse, &gsc);CHKERRQ(ierr);
6375   ierr = DMGetGlobalVector(dmCoarse, &cv);CHKERRQ(ierr);
6376   ierr = DMPlexCreateCoarsePointIS(dmCoarse, &fpointIS);CHKERRQ(ierr);
6377   ierr = PetscSectionGetConstrainedStorageSize(gsc, &m);CHKERRQ(ierr);
6378   ierr = PetscMalloc2(m,&cindices,m,&findices);CHKERRQ(ierr);
6379   ierr = PetscSectionGetChart(gsc, &cpStart, &cpEnd);CHKERRQ(ierr);
6380   ierr = ISGetIndices(fpointIS, &fpoints);CHKERRQ(ierr);
6381   for (cp = cpStart, off = 0; cp < cpEnd; ++cp) {
6382     const PetscInt *cdofsC = NULL;
6383     PetscInt        fp     = fpoints[cp-cpStart], dofC, cdofC, dofF, offC, offF, d, e;
6384 
6385     ierr = PetscSectionGetDof(gsc, cp, &dofC);CHKERRQ(ierr);
6386     if (dofC <= 0) continue;
6387     ierr = PetscSectionGetConstraintDof(sc, cp, &cdofC);CHKERRQ(ierr);
6388     ierr = PetscSectionGetDof(gsf, fp, &dofF);CHKERRQ(ierr);
6389     ierr = PetscSectionGetOffset(gsc, cp, &offC);CHKERRQ(ierr);
6390     ierr = PetscSectionGetOffset(gsf, fp, &offF);CHKERRQ(ierr);
6391     if (cdofC) {ierr = PetscSectionGetConstraintIndices(sc, cp, &cdofsC);CHKERRQ(ierr);}
6392     if (dofC != dofF) SETERRQ4(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Point %d (%d) has %d coarse dofs != %d fine dofs", cp, fp, dofC, dofF);
6393     if (offC < 0 || offF < 0) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Coarse point %d has invalid offset %d (%d)", cp, offC, offF);
6394     for (d = 0, e = 0; d < dofC; ++d) {
6395       if (cdofsC && cdofsC[e] == d) {++e; continue;}
6396       cindices[off+d-e] = offC+d; findices[off+d-e] = offF+d;
6397     }
6398     if (e != cdofC) SETERRQ4(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Point %d (%d) has invalid number of constraints %d != %d", cp, fp, e, cdofC);
6399     off += dofC-cdofC;
6400   }
6401   ierr = ISRestoreIndices(fpointIS, &fpoints);CHKERRQ(ierr);
6402   if (off != m) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of coarse dofs %d != %d", off, m);
6403   ierr = ISCreateGeneral(PETSC_COMM_SELF, m, cindices, PETSC_OWN_POINTER, &cis);CHKERRQ(ierr);
6404   ierr = ISCreateGeneral(PETSC_COMM_SELF, m, findices, PETSC_OWN_POINTER, &fis);CHKERRQ(ierr);
6405   ierr = VecScatterCreate(cv, cis, fv, fis, ctx);CHKERRQ(ierr);
6406   ierr = ISDestroy(&cis);CHKERRQ(ierr);
6407   ierr = ISDestroy(&fis);CHKERRQ(ierr);
6408   ierr = DMRestoreGlobalVector(dmFine, &fv);CHKERRQ(ierr);
6409   ierr = DMRestoreGlobalVector(dmCoarse, &cv);CHKERRQ(ierr);
6410   ierr = ISDestroy(&fpointIS);CHKERRQ(ierr);
6411   PetscFunctionReturn(0);
6412 }
6413 
6414 #undef __FUNCT__
6415 #define __FUNCT__ "DMCreateDefaultSection_Plex"
6416 /* Pointwise interpolation
6417      Just code FEM for now
6418      u^f = I u^c
6419      sum_k u^f_k phi^f_k = I sum_l u^c_l phi^c_l
6420      u^f_i = sum_l int psi^f_i I phi^c_l u^c_l
6421      I_{ij} = int psi^f_i phi^c_j
6422 */
6423 PetscErrorCode DMCreateDefaultSection_Plex(DM dm)
6424 {
6425   PetscSection   section;
6426   IS            *bcPoints;
6427   PetscInt      *bcFields, *numComp, *numDof;
6428   PetscInt       depth, dim, numBd, numBC = 0, numFields, bd, bc, f;
6429   PetscErrorCode ierr;
6430 
6431   PetscFunctionBegin;
6432   /* Handle boundary conditions */
6433   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
6434   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
6435   ierr = DMPlexGetNumBoundary(dm, &numBd);CHKERRQ(ierr);
6436   for (bd = 0; bd < numBd; ++bd) {
6437     PetscBool isEssential;
6438     ierr = DMPlexGetBoundary(dm, bd, &isEssential, NULL, NULL, NULL, NULL, NULL, NULL);CHKERRQ(ierr);
6439     if (isEssential) ++numBC;
6440   }
6441   ierr = PetscMalloc2(numBC,&bcFields,numBC,&bcPoints);CHKERRQ(ierr);
6442   for (bd = 0, bc = 0; bd < numBd; ++bd) {
6443     const char     *bdLabel;
6444     DMLabel         label;
6445     const PetscInt *values;
6446     PetscInt        field, numValues;
6447     PetscBool       isEssential, has;
6448 
6449     ierr = DMPlexGetBoundary(dm, bd, &isEssential, &bdLabel, &field, NULL, &numValues, &values, NULL);CHKERRQ(ierr);
6450     if (numValues != 1) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Bug me and I will fix this");
6451     ierr = DMPlexHasLabel(dm, bdLabel, &has);CHKERRQ(ierr);
6452     if (!has) {
6453       ierr = DMPlexCreateLabel(dm, bdLabel);CHKERRQ(ierr);
6454       ierr = DMPlexGetLabel(dm, bdLabel, &label);CHKERRQ(ierr);
6455       ierr = DMPlexMarkBoundaryFaces(dm, label);CHKERRQ(ierr);
6456     }
6457     ierr = DMPlexGetLabel(dm, bdLabel, &label);CHKERRQ(ierr);
6458     /* Only want to do this for FEM */
6459     ierr = DMPlexLabelComplete(dm, label);CHKERRQ(ierr);
6460     ierr = DMPlexLabelAddCells(dm, label);CHKERRQ(ierr);
6461     /* Filter out cells, if you actually want to constraint cells you need to do things by hand right now */
6462     if (isEssential) {
6463       IS              tmp;
6464       PetscInt       *newidx;
6465       const PetscInt *idx;
6466       PetscInt        cStart, cEnd, n, p, newn = 0;
6467 
6468       bcFields[bc] = field;
6469       ierr = DMPlexGetStratumIS(dm, bdLabel, values[0], &tmp);CHKERRQ(ierr);
6470       ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
6471       ierr = ISGetLocalSize(tmp, &n);CHKERRQ(ierr);
6472       ierr = ISGetIndices(tmp, &idx);CHKERRQ(ierr);
6473       for (p = 0; p < n; ++p) if ((idx[p] < cStart) || (idx[p] >= cEnd)) ++newn;
6474       ierr = PetscMalloc1(newn,&newidx);CHKERRQ(ierr);
6475       newn = 0;
6476       for (p = 0; p < n; ++p) if ((idx[p] < cStart) || (idx[p] >= cEnd)) newidx[newn++] = idx[p];
6477       ierr = ISCreateGeneral(PetscObjectComm((PetscObject) dm), newn, newidx, PETSC_OWN_POINTER, &bcPoints[bc++]);CHKERRQ(ierr);
6478       ierr = ISRestoreIndices(tmp, &idx);CHKERRQ(ierr);
6479       ierr = ISDestroy(&tmp);CHKERRQ(ierr);
6480     }
6481   }
6482   /* Handle discretization */
6483   ierr = DMGetNumFields(dm, &numFields);CHKERRQ(ierr);
6484   ierr = PetscMalloc2(numFields,&numComp,numFields*(dim+1),&numDof);CHKERRQ(ierr);
6485   for (f = 0; f < numFields; ++f) {
6486     PetscFE         fe;
6487     const PetscInt *numFieldDof;
6488     PetscInt        d;
6489 
6490     ierr = DMGetField(dm, f, (PetscObject *) &fe);CHKERRQ(ierr);
6491     ierr = PetscFEGetNumComponents(fe, &numComp[f]);CHKERRQ(ierr);
6492     ierr = PetscFEGetNumDof(fe, &numFieldDof);CHKERRQ(ierr);
6493     for (d = 0; d < dim+1; ++d) numDof[f*(dim+1)+d] = numFieldDof[d];
6494   }
6495   for (f = 0; f < numFields; ++f) {
6496     PetscInt d;
6497     for (d = 1; d < dim; ++d) {
6498       if ((numDof[f*(dim+1)+d] > 0) && (depth < dim)) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Mesh must be interpolated when unknowns are specified on edges or faces.");
6499     }
6500   }
6501   ierr = DMPlexCreateSection(dm, dim, numFields, numComp, numDof, numBC, bcFields, bcPoints, NULL, &section);CHKERRQ(ierr);
6502   for (f = 0; f < numFields; ++f) {
6503     PetscFE     fe;
6504     const char *name;
6505 
6506     ierr = DMGetField(dm, f, (PetscObject *) &fe);CHKERRQ(ierr);
6507     ierr = PetscObjectGetName((PetscObject) fe, &name);CHKERRQ(ierr);
6508     ierr = PetscSectionSetFieldName(section, f, name);CHKERRQ(ierr);
6509   }
6510   ierr = DMSetDefaultSection(dm, section);CHKERRQ(ierr);
6511   ierr = PetscSectionDestroy(&section);CHKERRQ(ierr);
6512   for (bc = 0; bc < numBC; ++bc) {ierr = ISDestroy(&bcPoints[bc]);CHKERRQ(ierr);}
6513   ierr = PetscFree2(bcFields,bcPoints);CHKERRQ(ierr);
6514   ierr = PetscFree2(numComp,numDof);CHKERRQ(ierr);
6515   PetscFunctionReturn(0);
6516 }
6517 
6518 #undef __FUNCT__
6519 #define __FUNCT__ "DMPlexGetCoarseDM"
6520 /*@
6521   DMPlexGetCoarseDM - Get the coarse mesh from which this was obtained by refinement
6522 
6523   Input Parameter:
6524 . dm - The DMPlex object
6525 
6526   Output Parameter:
6527 . cdm - The coarse DM
6528 
6529   Level: intermediate
6530 
6531 .seealso: DMPlexSetCoarseDM()
6532 @*/
6533 PetscErrorCode DMPlexGetCoarseDM(DM dm, DM *cdm)
6534 {
6535   PetscFunctionBegin;
6536   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6537   PetscValidPointer(cdm, 2);
6538   *cdm = ((DM_Plex *) dm->data)->coarseMesh;
6539   PetscFunctionReturn(0);
6540 }
6541 
6542 #undef __FUNCT__
6543 #define __FUNCT__ "DMPlexSetCoarseDM"
6544 /*@
6545   DMPlexSetCoarseDM - Set the coarse mesh from which this was obtained by refinement
6546 
6547   Input Parameters:
6548 + dm - The DMPlex object
6549 - cdm - The coarse DM
6550 
6551   Level: intermediate
6552 
6553 .seealso: DMPlexGetCoarseDM()
6554 @*/
6555 PetscErrorCode DMPlexSetCoarseDM(DM dm, DM cdm)
6556 {
6557   DM_Plex       *mesh;
6558   PetscErrorCode ierr;
6559 
6560   PetscFunctionBegin;
6561   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6562   if (cdm) PetscValidHeaderSpecific(cdm, DM_CLASSID, 2);
6563   mesh = (DM_Plex *) dm->data;
6564   ierr = DMDestroy(&mesh->coarseMesh);CHKERRQ(ierr);
6565   mesh->coarseMesh = cdm;
6566   ierr = PetscObjectReference((PetscObject) mesh->coarseMesh);CHKERRQ(ierr);
6567   PetscFunctionReturn(0);
6568 }
6569