xref: /petsc/src/dm/impls/plex/plex.c (revision f4d763aae1a2b76856d0541875bb6619284b60ab)
1 #include <petsc-private/dmpleximpl.h>   /*I      "petscdmplex.h"   I*/
2 #include <../src/sys/utils/hash.h>
3 #include <petsc-private/isimpl.h>
4 #include <petscsf.h>
5 #include <petscviewerhdf5.h>
6 
7 /* Logging support */
8 PetscLogEvent DMPLEX_Interpolate, DMPLEX_Partition, DMPLEX_Distribute, DMPLEX_DistributeCones, DMPLEX_DistributeLabels, DMPLEX_DistributeSF, DMPLEX_DistributeField, DMPLEX_DistributeData, DMPLEX_Stratify, DMPLEX_Preallocate, DMPLEX_ResidualFEM, DMPLEX_JacobianFEM;
9 
10 PETSC_EXTERN PetscErrorCode VecView_Seq(Vec, PetscViewer);
11 PETSC_EXTERN PetscErrorCode VecView_MPI(Vec, PetscViewer);
12 PETSC_EXTERN PetscErrorCode VecLoad_Default(Vec, PetscViewer);
13 PETSC_EXTERN PetscErrorCode DMTSGetTimeStepNumber(DM,PetscInt*);
14 
15 #undef __FUNCT__
16 #define __FUNCT__ "GetField_Static"
17 static PetscErrorCode GetField_Static(DM dm, PetscSection section, PetscSection sectionGlobal, Vec v, PetscInt field, PetscInt pStart, PetscInt pEnd, IS *is, Vec *subv)
18 {
19   PetscInt      *subIndices;
20   PetscInt       Nc, subSize = 0, subOff = 0, p;
21   PetscErrorCode ierr;
22 
23   PetscFunctionBegin;
24   ierr = PetscSectionGetFieldComponents(section, field, &Nc);CHKERRQ(ierr);
25   for (p = pStart; p < pEnd; ++p) {
26     PetscInt gdof, fdof = 0;
27 
28     ierr = PetscSectionGetDof(sectionGlobal, p, &gdof);CHKERRQ(ierr);
29     if (gdof > 0) {ierr = PetscSectionGetFieldDof(section, p, field, &fdof);CHKERRQ(ierr);}
30     subSize += fdof;
31   }
32   ierr = PetscMalloc1(subSize, &subIndices);CHKERRQ(ierr);
33   for (p = pStart; p < pEnd; ++p) {
34     PetscInt gdof, goff;
35 
36     ierr = PetscSectionGetDof(sectionGlobal, p, &gdof);CHKERRQ(ierr);
37     if (gdof > 0) {
38       PetscInt fdof, fc, f2, poff = 0;
39 
40       ierr = PetscSectionGetOffset(sectionGlobal, p, &goff);CHKERRQ(ierr);
41       /* Can get rid of this loop by storing field information in the global section */
42       for (f2 = 0; f2 < field; ++f2) {
43         ierr  = PetscSectionGetFieldDof(section, p, f2, &fdof);CHKERRQ(ierr);
44         poff += fdof;
45       }
46       ierr = PetscSectionGetFieldDof(section, p, field, &fdof);CHKERRQ(ierr);
47       for (fc = 0; fc < fdof; ++fc, ++subOff) subIndices[subOff] = goff+poff+fc;
48     }
49   }
50   ierr = ISCreateGeneral(PetscObjectComm((PetscObject) dm), subSize, subIndices, PETSC_OWN_POINTER, is);CHKERRQ(ierr);
51   ierr = VecGetSubVector(v, *is, subv);CHKERRQ(ierr);
52   ierr = VecSetBlockSize(*subv, Nc);CHKERRQ(ierr);
53   PetscFunctionReturn(0);
54 }
55 
56 #undef __FUNCT__
57 #define __FUNCT__ "RestoreField_Static"
58 static PetscErrorCode RestoreField_Static(DM dm, PetscSection section, PetscSection sectionGlobal, Vec v, PetscInt field, PetscInt pStart, PetscInt pEnd, IS *is, Vec *subv)
59 {
60   PetscErrorCode ierr;
61 
62   PetscFunctionBegin;
63   ierr = VecRestoreSubVector(v, *is, subv);CHKERRQ(ierr);
64   ierr = ISDestroy(is);CHKERRQ(ierr);
65   PetscFunctionReturn(0);
66 }
67 
68 #undef __FUNCT__
69 #define __FUNCT__ "GetFieldType_Static"
70 static PetscErrorCode GetFieldType_Static(DM dm, PetscSection section, PetscInt field, PetscInt *sStart, PetscInt *sEnd, PetscViewerVTKFieldType *ft)
71 {
72   PetscInt       dim, pStart, pEnd, vStart, vEnd, cStart, cEnd, vdof = 0, cdof = 0;
73   PetscErrorCode ierr;
74 
75   PetscFunctionBegin;
76   *ft  = PETSC_VTK_POINT_FIELD;
77   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
78   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
79   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
80   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
81   if (field >= 0) {
82     if ((vStart >= pStart) && (vStart < pEnd)) {ierr = PetscSectionGetFieldDof(section, vStart, field, &vdof);CHKERRQ(ierr);}
83     if ((cStart >= pStart) && (cStart < pEnd)) {ierr = PetscSectionGetFieldDof(section, cStart, field, &cdof);CHKERRQ(ierr);}
84   } else {
85     if ((vStart >= pStart) && (vStart < pEnd)) {ierr = PetscSectionGetDof(section, vStart, &vdof);CHKERRQ(ierr);}
86     if ((cStart >= pStart) && (cStart < pEnd)) {ierr = PetscSectionGetDof(section, cStart, &cdof);CHKERRQ(ierr);}
87   }
88   if (vdof) {
89     *sStart = vStart;
90     *sEnd   = vEnd;
91     if (vdof == dim) *ft = PETSC_VTK_POINT_VECTOR_FIELD;
92     else             *ft = PETSC_VTK_POINT_FIELD;
93   } else if (cdof) {
94     *sStart = cStart;
95     *sEnd   = cEnd;
96     if (cdof == dim) *ft = PETSC_VTK_CELL_VECTOR_FIELD;
97     else             *ft = PETSC_VTK_CELL_FIELD;
98   } else SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Could not classify input Vec for VTK");
99   PetscFunctionReturn(0);
100 }
101 
102 #if defined(PETSC_HAVE_HDF5)
103 #undef __FUNCT__
104 #define __FUNCT__ "VecView_Plex_Local_HDF5"
105 static PetscErrorCode VecView_Plex_Local_HDF5(Vec v, PetscViewer viewer)
106 {
107   DM                      dm;
108   DM                      dmBC;
109   PetscSection            section, sectionGlobal;
110   Vec                     gv;
111   const char             *name;
112   PetscViewerVTKFieldType ft;
113   PetscViewerFormat       format;
114   PetscInt                timestep;
115   PetscBool               isseq;
116   PetscErrorCode          ierr;
117 
118   PetscFunctionBegin;
119   ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
120   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
121   ierr = DMGetDefaultSection(dm, &section);CHKERRQ(ierr);
122   ierr = DMTSGetTimeStepNumber(dm, &timestep);CHKERRQ(ierr);
123   ierr = PetscViewerHDF5SetTimestep(viewer, timestep);CHKERRQ(ierr);
124   ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
125   ierr = DMGetOutputDM(dm, &dmBC);CHKERRQ(ierr);
126   ierr = DMGetDefaultGlobalSection(dmBC, &sectionGlobal);CHKERRQ(ierr);
127   ierr = DMGetGlobalVector(dmBC, &gv);CHKERRQ(ierr);
128   ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
129   ierr = PetscObjectSetName((PetscObject) gv, name);CHKERRQ(ierr);
130   ierr = DMLocalToGlobalBegin(dmBC, v, INSERT_VALUES, gv);CHKERRQ(ierr);
131   ierr = DMLocalToGlobalEnd(dmBC, v, INSERT_VALUES, gv);CHKERRQ(ierr);
132   ierr = PetscObjectTypeCompare((PetscObject) gv, VECSEQ, &isseq);CHKERRQ(ierr);
133   if (isseq) {ierr = VecView_Seq(gv, viewer);CHKERRQ(ierr);}
134   else       {ierr = VecView_MPI(gv, viewer);CHKERRQ(ierr);}
135   if (format == PETSC_VIEWER_HDF5_VIZ) {
136     /* Output visualization representation */
137     PetscInt numFields, f;
138 
139     ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
140     for (f = 0; f < numFields; ++f) {
141       Vec         subv;
142       IS          is;
143       const char *fname, *fgroup;
144       char        group[PETSC_MAX_PATH_LEN];
145       PetscInt    pStart, pEnd;
146 
147       ierr = GetFieldType_Static(dm, section, f, &pStart, &pEnd, &ft);CHKERRQ(ierr);
148       fgroup = (ft == PETSC_VTK_POINT_VECTOR_FIELD) || (ft == PETSC_VTK_POINT_FIELD) ? "/vertex_fields" : "/cell_fields";
149       ierr = PetscSectionGetFieldName(section, f, &fname);CHKERRQ(ierr);
150       ierr = PetscViewerHDF5PushGroup(viewer, fgroup);CHKERRQ(ierr);
151       ierr = GetField_Static(dmBC, section, sectionGlobal, gv, f, pStart, pEnd, &is, &subv);CHKERRQ(ierr);
152       ierr = PetscObjectSetName((PetscObject) subv, fname);CHKERRQ(ierr);
153       if (isseq) {ierr = VecView_Seq(subv, viewer);CHKERRQ(ierr);}
154       else       {ierr = VecView_MPI(subv, viewer);CHKERRQ(ierr);}
155       ierr = RestoreField_Static(dmBC, section, sectionGlobal, gv, f, pStart, pEnd, &is, &subv);CHKERRQ(ierr);
156       ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
157       ierr = PetscSNPrintf(group, PETSC_MAX_PATH_LEN, "%s/%s", fgroup, fname);CHKERRQ(ierr);
158       if ((ft == PETSC_VTK_POINT_VECTOR_FIELD) || (ft == PETSC_VTK_CELL_VECTOR_FIELD)) {
159         ierr = PetscViewerHDF5WriteAttribute(viewer, group, "vector_field_type", PETSC_STRING, "vector");CHKERRQ(ierr);
160       } else {
161         ierr = PetscViewerHDF5WriteAttribute(viewer, group, "vector_field_type", PETSC_STRING, "scalar");CHKERRQ(ierr);
162       }
163     }
164   }
165   ierr = DMRestoreGlobalVector(dmBC, &gv);CHKERRQ(ierr);
166   PetscFunctionReturn(0);
167 }
168 #endif
169 
170 #undef __FUNCT__
171 #define __FUNCT__ "VecView_Plex_Local"
172 PetscErrorCode VecView_Plex_Local(Vec v, PetscViewer viewer)
173 {
174   DM             dm;
175   PetscBool      isvtk, ishdf5, isseq;
176   PetscErrorCode ierr;
177 
178   PetscFunctionBegin;
179   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
180   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
181   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,  &isvtk);CHKERRQ(ierr);
182   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
183   ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
184   if (isvtk || ishdf5) {ierr = DMPlexInsertBoundaryValues(dm, v);CHKERRQ(ierr);}
185   if (isvtk) {
186     PetscSection            section;
187     PetscViewerVTKFieldType ft;
188     PetscInt                pStart, pEnd;
189 
190     ierr = DMGetDefaultSection(dm, &section);CHKERRQ(ierr);
191     ierr = GetFieldType_Static(dm, section, PETSC_DETERMINE, &pStart, &pEnd, &ft);CHKERRQ(ierr);
192     ierr = PetscObjectReference((PetscObject) dm);CHKERRQ(ierr); /* viewer drops reference */
193     ierr = PetscObjectReference((PetscObject) v);CHKERRQ(ierr);  /* viewer drops reference */
194     ierr = PetscViewerVTKAddField(viewer, (PetscObject) dm, DMPlexVTKWriteAll, ft, (PetscObject) v);CHKERRQ(ierr);
195   } else if (ishdf5) {
196 #if defined(PETSC_HAVE_HDF5)
197     ierr = VecView_Plex_Local_HDF5(v, viewer);CHKERRQ(ierr);
198 #else
199     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
200 #endif
201   } else {
202     if (isseq) {ierr = VecView_Seq(v, viewer);CHKERRQ(ierr);}
203     else       {ierr = VecView_MPI(v, viewer);CHKERRQ(ierr);}
204   }
205   PetscFunctionReturn(0);
206 }
207 
208 #if defined(PETSC_HAVE_HDF5)
209 #undef __FUNCT__
210 #define __FUNCT__ "VecView_Plex_HDF5"
211 static PetscErrorCode VecView_Plex_HDF5(Vec v, PetscViewer viewer)
212 {
213   DM             dm;
214   Vec            locv;
215   const char    *name;
216   PetscErrorCode ierr;
217 
218   PetscFunctionBegin;
219   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
220   ierr = DMGetLocalVector(dm, &locv);CHKERRQ(ierr);
221   ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
222   ierr = PetscObjectSetName((PetscObject) locv, name);CHKERRQ(ierr);
223   ierr = DMGlobalToLocalBegin(dm, v, INSERT_VALUES, locv);CHKERRQ(ierr);
224   ierr = DMGlobalToLocalEnd(dm, v, INSERT_VALUES, locv);CHKERRQ(ierr);
225   ierr = PetscViewerHDF5PushGroup(viewer, "/fields");CHKERRQ(ierr);
226   ierr = PetscViewerPushFormat(viewer, PETSC_VIEWER_HDF5_VIZ);CHKERRQ(ierr);
227   ierr = VecView_Plex_Local(locv, viewer);CHKERRQ(ierr);
228   ierr = PetscViewerPopFormat(viewer);CHKERRQ(ierr);
229   ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
230   ierr = DMRestoreLocalVector(dm, &locv);CHKERRQ(ierr);
231   PetscFunctionReturn(0);
232 }
233 #endif
234 
235 #undef __FUNCT__
236 #define __FUNCT__ "VecView_Plex"
237 PetscErrorCode VecView_Plex(Vec v, PetscViewer viewer)
238 {
239   DM             dm;
240   PetscBool      isvtk, ishdf5, isseq;
241   PetscErrorCode ierr;
242 
243   PetscFunctionBegin;
244   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
245   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
246   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,  &isvtk);CHKERRQ(ierr);
247   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
248   ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
249   if (isvtk) {
250     Vec         locv;
251     const char *name;
252 
253     ierr = DMGetLocalVector(dm, &locv);CHKERRQ(ierr);
254     ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
255     ierr = PetscObjectSetName((PetscObject) locv, name);CHKERRQ(ierr);
256     ierr = DMGlobalToLocalBegin(dm, v, INSERT_VALUES, locv);CHKERRQ(ierr);
257     ierr = DMGlobalToLocalEnd(dm, v, INSERT_VALUES, locv);CHKERRQ(ierr);
258     ierr = VecView_Plex_Local(locv, viewer);CHKERRQ(ierr);
259     ierr = DMRestoreLocalVector(dm, &locv);CHKERRQ(ierr);
260   } else if (ishdf5) {
261 #if defined(PETSC_HAVE_HDF5)
262     ierr = VecView_Plex_HDF5(v, viewer);CHKERRQ(ierr);
263 #else
264     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
265 #endif
266   } else {
267     if (isseq) {ierr = VecView_Seq(v, viewer);CHKERRQ(ierr);}
268     else       {ierr = VecView_MPI(v, viewer);CHKERRQ(ierr);}
269   }
270   PetscFunctionReturn(0);
271 }
272 
273 #undef __FUNCT__
274 #define __FUNCT__ "VecLoad_Plex_Local"
275 PetscErrorCode VecLoad_Plex_Local(Vec v, PetscViewer viewer)
276 {
277   DM             dm;
278   PetscBool      ishdf5;
279   PetscErrorCode ierr;
280 
281   PetscFunctionBegin;
282   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
283   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
284   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
285   if (ishdf5) {
286     DM          dmBC;
287     Vec         gv;
288     const char *name;
289 
290     ierr = DMGetOutputDM(dm, &dmBC);CHKERRQ(ierr);
291     ierr = DMGetGlobalVector(dmBC, &gv);CHKERRQ(ierr);
292     ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
293     ierr = PetscObjectSetName((PetscObject) gv, name);CHKERRQ(ierr);
294     ierr = VecLoad_Default(gv, viewer);CHKERRQ(ierr);
295     ierr = DMGlobalToLocalBegin(dmBC, gv, INSERT_VALUES, v);CHKERRQ(ierr);
296     ierr = DMGlobalToLocalEnd(dmBC, gv, INSERT_VALUES, v);CHKERRQ(ierr);
297     ierr = DMRestoreGlobalVector(dmBC, &gv);CHKERRQ(ierr);
298   } else {
299     ierr = VecLoad_Default(v, viewer);CHKERRQ(ierr);
300   }
301   PetscFunctionReturn(0);
302 }
303 
304 #if defined(PETSC_HAVE_HDF5)
305 #undef __FUNCT__
306 #define __FUNCT__ "VecLoad_Plex_HDF5"
307 PetscErrorCode VecLoad_Plex_HDF5(Vec v, PetscViewer viewer)
308 {
309   DM             dm;
310   Vec            locv;
311   const char    *name;
312   PetscInt       timestep;
313   PetscErrorCode ierr;
314 
315   PetscFunctionBegin;
316   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
317   ierr = DMGetLocalVector(dm, &locv);CHKERRQ(ierr);
318   ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
319   ierr = PetscObjectSetName((PetscObject) locv, name);CHKERRQ(ierr);
320   ierr = DMTSGetTimeStepNumber(dm, &timestep);CHKERRQ(ierr);
321   ierr = PetscViewerHDF5SetTimestep(viewer, timestep);CHKERRQ(ierr);
322   ierr = PetscViewerHDF5PushGroup(viewer, "/fields");CHKERRQ(ierr);
323   ierr = VecLoad_Plex_Local(locv, viewer);CHKERRQ(ierr);
324   ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
325   ierr = DMLocalToGlobalBegin(dm, locv, INSERT_VALUES, v);CHKERRQ(ierr);
326   ierr = DMLocalToGlobalEnd(dm, locv, INSERT_VALUES, v);CHKERRQ(ierr);
327   ierr = DMRestoreLocalVector(dm, &locv);CHKERRQ(ierr);
328   PetscFunctionReturn(0);
329 }
330 #endif
331 
332 #undef __FUNCT__
333 #define __FUNCT__ "VecLoad_Plex"
334 PetscErrorCode VecLoad_Plex(Vec v, PetscViewer viewer)
335 {
336   DM             dm;
337   PetscBool      ishdf5;
338   PetscErrorCode ierr;
339 
340   PetscFunctionBegin;
341   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
342   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
343   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
344   if (ishdf5) {
345 #if defined(PETSC_HAVE_HDF5)
346     ierr = VecLoad_Plex_HDF5(v, viewer);CHKERRQ(ierr);
347 #else
348     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
349 #endif
350   } else {
351     ierr = VecLoad_Default(v, viewer);CHKERRQ(ierr);
352   }
353   PetscFunctionReturn(0);
354 }
355 
356 #undef __FUNCT__
357 #define __FUNCT__ "DMPlexView_Ascii"
358 PetscErrorCode DMPlexView_Ascii(DM dm, PetscViewer viewer)
359 {
360   DM_Plex          *mesh = (DM_Plex*) dm->data;
361   DM                cdm;
362   DMLabel           markers;
363   PetscSection      coordSection;
364   Vec               coordinates;
365   PetscViewerFormat format;
366   PetscErrorCode    ierr;
367 
368   PetscFunctionBegin;
369   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
370   ierr = DMGetDefaultSection(cdm, &coordSection);CHKERRQ(ierr);
371   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
372   ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
373   if (format == PETSC_VIEWER_ASCII_INFO_DETAIL) {
374     const char *name;
375     PetscInt    maxConeSize, maxSupportSize;
376     PetscInt    pStart, pEnd, p;
377     PetscMPIInt rank, size;
378 
379     ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRQ(ierr);
380     ierr = MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size);CHKERRQ(ierr);
381     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
382     ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
383     ierr = DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize);CHKERRQ(ierr);
384     ierr = PetscViewerASCIISynchronizedAllow(viewer, PETSC_TRUE);CHKERRQ(ierr);
385     ierr = PetscViewerASCIIPrintf(viewer, "Mesh '%s':\n", name);CHKERRQ(ierr);
386     ierr = PetscViewerASCIISynchronizedPrintf(viewer, "Max sizes cone: %D support: %D\n", maxConeSize, maxSupportSize);CHKERRQ(ierr);
387     ierr = PetscViewerASCIIPrintf(viewer, "orientation is missing\n", name);CHKERRQ(ierr);
388     ierr = PetscViewerASCIIPrintf(viewer, "cap --> base:\n", name);CHKERRQ(ierr);
389     for (p = pStart; p < pEnd; ++p) {
390       PetscInt dof, off, s;
391 
392       ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
393       ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
394       for (s = off; s < off+dof; ++s) {
395         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%D]: %D ----> %D\n", rank, p, mesh->supports[s]);CHKERRQ(ierr);
396       }
397     }
398     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
399     ierr = PetscViewerASCIIPrintf(viewer, "base <-- cap:\n", name);CHKERRQ(ierr);
400     for (p = pStart; p < pEnd; ++p) {
401       PetscInt dof, off, c;
402 
403       ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
404       ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
405       for (c = off; c < off+dof; ++c) {
406         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%D]: %D <---- %D (%D)\n", rank, p, mesh->cones[c], mesh->coneOrientations[c]);CHKERRQ(ierr);
407       }
408     }
409     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
410     ierr = PetscSectionGetChart(coordSection, &pStart, NULL);CHKERRQ(ierr);
411     if (pStart >= 0) {ierr = PetscSectionVecView(coordSection, coordinates, viewer);CHKERRQ(ierr);}
412     ierr = DMPlexGetLabel(dm, "marker", &markers);CHKERRQ(ierr);
413     ierr = DMLabelView(markers,viewer);CHKERRQ(ierr);
414     if (size > 1) {
415       PetscSF sf;
416 
417       ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr);
418       ierr = PetscSFView(sf, viewer);CHKERRQ(ierr);
419     }
420     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
421   } else if (format == PETSC_VIEWER_ASCII_LATEX) {
422     const char  *name;
423     const char  *colors[3] = {"red", "blue", "green"};
424     const int    numColors  = 3;
425     PetscReal    scale      = 2.0;
426     PetscScalar *coords;
427     PetscInt     depth, cStart, cEnd, c, vStart, vEnd, v, eStart = 0, eEnd = 0, e, p;
428     PetscMPIInt  rank, size;
429 
430     ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
431     ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRQ(ierr);
432     ierr = MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size);CHKERRQ(ierr);
433     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
434     ierr = PetscViewerASCIISynchronizedAllow(viewer, PETSC_TRUE);CHKERRQ(ierr);
435     ierr = PetscViewerASCIIPrintf(viewer, "\
436 \\documentclass[crop,multi=false]{standalone}\n\n\
437 \\usepackage{tikz}\n\
438 \\usepackage{pgflibraryshapes}\n\
439 \\usetikzlibrary{backgrounds}\n\
440 \\usetikzlibrary{arrows}\n\
441 \\begin{document}\n\
442 \\section{%s}\n\
443 \\begin{center}\n", name, 8.0/scale);CHKERRQ(ierr);
444     ierr = PetscViewerASCIIPrintf(viewer, "Mesh for process ");CHKERRQ(ierr);
445     for (p = 0; p < size; ++p) {
446       if (p > 0 && p == size-1) {
447         ierr = PetscViewerASCIIPrintf(viewer, ", and ", colors[p%numColors], p);CHKERRQ(ierr);
448       } else if (p > 0) {
449         ierr = PetscViewerASCIIPrintf(viewer, ", ", colors[p%numColors], p);CHKERRQ(ierr);
450       }
451       ierr = PetscViewerASCIIPrintf(viewer, "{\\textcolor{%s}%D}", colors[p%numColors], p);CHKERRQ(ierr);
452     }
453     ierr = PetscViewerASCIIPrintf(viewer, ".\n\n\n\
454 \\begin{tikzpicture}[scale = %g,font=\\fontsize{8}{8}\\selectfont]\n");CHKERRQ(ierr);
455     /* Plot vertices */
456     ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
457     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
458     ierr = PetscViewerASCIIPrintf(viewer, "\\path\n");CHKERRQ(ierr);
459     for (v = vStart; v < vEnd; ++v) {
460       PetscInt off, dof, d;
461 
462       ierr = PetscSectionGetDof(coordSection, v, &dof);CHKERRQ(ierr);
463       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
464       ierr = PetscViewerASCIISynchronizedPrintf(viewer, "(");CHKERRQ(ierr);
465       for (d = 0; d < dof; ++d) {
466         if (d > 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, ",");CHKERRQ(ierr);}
467         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "%g", (double)(scale*PetscRealPart(coords[off+d])));CHKERRQ(ierr);
468       }
469       ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%D) [draw,shape=circle,color=%s] {%D} --\n", v, rank, colors[rank%numColors], v);CHKERRQ(ierr);
470     }
471     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
472     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
473     ierr = PetscViewerASCIIPrintf(viewer, "(0,0);\n");CHKERRQ(ierr);
474     /* Plot edges */
475     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
476     ierr = PetscViewerASCIIPrintf(viewer, "\\path\n");CHKERRQ(ierr);
477     if (depth > 1) {ierr = DMPlexGetDepthStratum(dm, 1, &eStart, &eEnd);CHKERRQ(ierr);}
478     for (e = eStart; e < eEnd; ++e) {
479       const PetscInt *cone;
480       PetscInt        coneSize, offA, offB, dof, d;
481 
482       ierr = DMPlexGetConeSize(dm, e, &coneSize);CHKERRQ(ierr);
483       if (coneSize != 2) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Edge %d cone should have two vertices, not %d", e, coneSize);
484       ierr = DMPlexGetCone(dm, e, &cone);CHKERRQ(ierr);
485       ierr = PetscSectionGetDof(coordSection, cone[0], &dof);CHKERRQ(ierr);
486       ierr = PetscSectionGetOffset(coordSection, cone[0], &offA);CHKERRQ(ierr);
487       ierr = PetscSectionGetOffset(coordSection, cone[1], &offB);CHKERRQ(ierr);
488       ierr = PetscViewerASCIISynchronizedPrintf(viewer, "(");CHKERRQ(ierr);
489       for (d = 0; d < dof; ++d) {
490         if (d > 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, ",");CHKERRQ(ierr);}
491         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "%g", (double)(scale*0.5*PetscRealPart(coords[offA+d]+coords[offB+d])));CHKERRQ(ierr);
492       }
493       ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%D) [draw,shape=circle,color=%s] {%D} --\n", e, rank, colors[rank%numColors], e);CHKERRQ(ierr);
494     }
495     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
496     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
497     ierr = PetscViewerASCIIPrintf(viewer, "(0,0);\n");CHKERRQ(ierr);
498     /* Plot cells */
499     ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
500     for (c = cStart; c < cEnd; ++c) {
501       PetscInt *closure = NULL;
502       PetscInt  closureSize, firstPoint = -1;
503 
504       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
505       ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\draw[color=%s] ", colors[rank%numColors]);CHKERRQ(ierr);
506       for (p = 0; p < closureSize*2; p += 2) {
507         const PetscInt point = closure[p];
508 
509         if ((point < vStart) || (point >= vEnd)) continue;
510         if (firstPoint >= 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- ");CHKERRQ(ierr);}
511         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "(%D_%D)", point, rank);CHKERRQ(ierr);
512         if (firstPoint < 0) firstPoint = point;
513       }
514       /* Why doesn't this work? ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- cycle;\n");CHKERRQ(ierr); */
515       ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- (%D_%D);\n", firstPoint, rank);CHKERRQ(ierr);
516       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
517     }
518     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
519     ierr = PetscViewerASCIIPrintf(viewer, "\\end{tikzpicture}\n\\end{center}\n");CHKERRQ(ierr);
520     ierr = PetscViewerASCIIPrintf(viewer, "\\end{document}\n", name);CHKERRQ(ierr);
521   } else {
522     MPI_Comm    comm;
523     PetscInt   *sizes, *hybsizes;
524     PetscInt    locDepth, depth, dim, d, pMax[4];
525     PetscInt    pStart, pEnd, p;
526     PetscInt    numLabels, l;
527     const char *name;
528     PetscMPIInt size;
529 
530     ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
531     ierr = MPI_Comm_size(comm, &size);CHKERRQ(ierr);
532     ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
533     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
534     if (name) {ierr = PetscViewerASCIIPrintf(viewer, "%s in %D dimensions:\n", name, dim);CHKERRQ(ierr);}
535     else      {ierr = PetscViewerASCIIPrintf(viewer, "Mesh in %D dimensions:\n", dim);CHKERRQ(ierr);}
536     ierr = DMPlexGetDepth(dm, &locDepth);CHKERRQ(ierr);
537     ierr = MPI_Allreduce(&locDepth, &depth, 1, MPIU_INT, MPI_MAX, comm);CHKERRQ(ierr);
538     ierr = DMPlexGetHybridBounds(dm, &pMax[depth], &pMax[depth-1], &pMax[1], &pMax[0]);CHKERRQ(ierr);
539     ierr = PetscMalloc2(size,&sizes,size,&hybsizes);CHKERRQ(ierr);
540     if (depth == 1) {
541       ierr = DMPlexGetDepthStratum(dm, 0, &pStart, &pEnd);CHKERRQ(ierr);
542       pEnd = pEnd - pStart;
543       ierr = MPI_Gather(&pEnd, 1, MPIU_INT, sizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
544       ierr = PetscViewerASCIIPrintf(viewer, "  %D-cells:", 0);CHKERRQ(ierr);
545       for (p = 0; p < size; ++p) {ierr = PetscViewerASCIIPrintf(viewer, " %D", sizes[p]);CHKERRQ(ierr);}
546       ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
547       ierr = DMPlexGetHeightStratum(dm, 0, &pStart, &pEnd);CHKERRQ(ierr);
548       pEnd = pEnd - pStart;
549       ierr = MPI_Gather(&pEnd, 1, MPIU_INT, sizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
550       ierr = PetscViewerASCIIPrintf(viewer, "  %D-cells:", dim);CHKERRQ(ierr);
551       for (p = 0; p < size; ++p) {ierr = PetscViewerASCIIPrintf(viewer, " %D", sizes[p]);CHKERRQ(ierr);}
552       ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
553     } else {
554       for (d = 0; d <= dim; d++) {
555         ierr = DMPlexGetDepthStratum(dm, d, &pStart, &pEnd);CHKERRQ(ierr);
556         pEnd    -= pStart;
557         pMax[d] -= pStart;
558         ierr = MPI_Gather(&pEnd, 1, MPIU_INT, sizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
559         ierr = MPI_Gather(&pMax[d], 1, MPIU_INT, hybsizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
560         ierr = PetscViewerASCIIPrintf(viewer, "  %D-cells:", d);CHKERRQ(ierr);
561         for (p = 0; p < size; ++p) {
562           if (hybsizes[p] >= 0) {ierr = PetscViewerASCIIPrintf(viewer, " %D (%D)", sizes[p], sizes[p] - hybsizes[p]);CHKERRQ(ierr);}
563           else                  {ierr = PetscViewerASCIIPrintf(viewer, " %D", sizes[p]);CHKERRQ(ierr);}
564         }
565         ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
566       }
567     }
568     ierr = PetscFree2(sizes,hybsizes);CHKERRQ(ierr);
569     ierr = DMPlexGetNumLabels(dm, &numLabels);CHKERRQ(ierr);
570     if (numLabels) {ierr = PetscViewerASCIIPrintf(viewer, "Labels:\n");CHKERRQ(ierr);}
571     for (l = 0; l < numLabels; ++l) {
572       DMLabel         label;
573       const char     *name;
574       IS              valueIS;
575       const PetscInt *values;
576       PetscInt        numValues, v;
577 
578       ierr = DMPlexGetLabelName(dm, l, &name);CHKERRQ(ierr);
579       ierr = DMPlexGetLabel(dm, name, &label);CHKERRQ(ierr);
580       ierr = DMLabelGetNumValues(label, &numValues);CHKERRQ(ierr);
581       ierr = PetscViewerASCIIPrintf(viewer, "  %s: %d strata of sizes (", name, numValues);CHKERRQ(ierr);
582       ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
583       ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
584       for (v = 0; v < numValues; ++v) {
585         PetscInt size;
586 
587         ierr = DMLabelGetStratumSize(label, values[v], &size);CHKERRQ(ierr);
588         if (v > 0) {ierr = PetscViewerASCIIPrintf(viewer, ", ");CHKERRQ(ierr);}
589         ierr = PetscViewerASCIIPrintf(viewer, "%d", size);CHKERRQ(ierr);
590       }
591       ierr = PetscViewerASCIIPrintf(viewer, ")\n");CHKERRQ(ierr);
592       ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
593       ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
594     }
595   }
596   PetscFunctionReturn(0);
597 }
598 
599 #if defined(PETSC_HAVE_HDF5)
600 #undef __FUNCT__
601 #define __FUNCT__ "DMPlexView_HDF5"
602 /* We only write cells and vertices. Does this screw up parallel reading? */
603 PetscErrorCode DMPlexView_HDF5(DM dm, PetscViewer viewer)
604 {
605   DM              cdm;
606   Vec             coordinates, newcoords;
607   Vec             coneVec, cellVec;
608   IS              globalVertexNumbers;
609   const PetscInt *gvertex;
610   PetscScalar    *sizes, *vertices;
611   PetscReal       lengthScale;
612   const char     *label   = NULL;
613   PetscInt        labelId = 0, dim;
614   char            group[PETSC_MAX_PATH_LEN];
615   PetscInt        vStart, vEnd, v, cellHeight, cStart, cEnd, cMax, cell, conesSize = 0, numCornersLocal = 0, numCorners, numLabels, l;
616   hid_t           fileId, groupId;
617   herr_t          status;
618   PetscErrorCode  ierr;
619 
620   PetscFunctionBegin;
621   /* Write coordinates */
622   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
623   ierr = DMPlexGetScale(dm, PETSC_UNIT_LENGTH, &lengthScale);CHKERRQ(ierr);
624   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
625   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
626   ierr = VecDuplicate(coordinates, &newcoords);CHKERRQ(ierr);
627   ierr = PetscObjectSetName((PetscObject) newcoords, "vertices");CHKERRQ(ierr);
628   ierr = VecCopy(coordinates, newcoords);CHKERRQ(ierr);
629   ierr = VecScale(newcoords, lengthScale);CHKERRQ(ierr);
630   /* Use the local version to bypass the default group setting */
631   ierr = PetscViewerHDF5PushGroup(viewer, "/geometry");CHKERRQ(ierr);
632   ierr = VecView(newcoords, viewer);CHKERRQ(ierr);
633   ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
634   ierr = VecDestroy(&newcoords);CHKERRQ(ierr);
635   /* Write toplogy */
636   ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
637   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
638   ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
639   ierr = DMPlexGetHybridBounds(dm, &cMax, PETSC_NULL, PETSC_NULL, PETSC_NULL);CHKERRQ(ierr);
640   if (cMax >= 0) cEnd = PetscMin(cEnd, cMax);
641 
642   ierr = VecCreate(PetscObjectComm((PetscObject) dm), &coneVec);CHKERRQ(ierr);
643   ierr = VecSetSizes(coneVec, cEnd-cStart, PETSC_DETERMINE);CHKERRQ(ierr);
644   ierr = VecSetBlockSize(coneVec, 1);CHKERRQ(ierr);
645   ierr = VecSetFromOptions(coneVec);CHKERRQ(ierr);
646   ierr = PetscObjectSetName((PetscObject) coneVec, "coneSize");CHKERRQ(ierr);
647   ierr = VecGetArray(coneVec, &sizes);CHKERRQ(ierr);
648   for (cell = cStart; cell < cEnd; ++cell) {
649     PetscInt *closure = NULL;
650     PetscInt  closureSize, v, Nc = 0;
651 
652     if (label) {
653       PetscInt value;
654       ierr = DMPlexGetLabelValue(dm, label, cell, &value);CHKERRQ(ierr);
655       if (value == labelId) continue;
656     }
657     ierr = DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
658     for (v = 0; v < closureSize*2; v += 2) {
659       if ((closure[v] >= vStart) && (closure[v] < vEnd)) ++Nc;
660     }
661     ierr = DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
662     conesSize += Nc;
663     if (!numCornersLocal)           numCornersLocal = Nc;
664     else if (numCornersLocal != Nc) numCornersLocal = 1;
665   }
666   ierr = VecRestoreArray(coneVec, &sizes);CHKERRQ(ierr);
667   ierr = MPI_Allreduce(&numCornersLocal, &numCorners, 1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject) dm));CHKERRQ(ierr);
668   if (numCornersLocal && numCornersLocal != numCorners) numCorners = 1;
669 
670   ierr = DMPlexGetVertexNumbering(dm, &globalVertexNumbers);CHKERRQ(ierr);
671   ierr = ISGetIndices(globalVertexNumbers, &gvertex);CHKERRQ(ierr);
672   ierr = VecCreate(PetscObjectComm((PetscObject) dm), &cellVec);CHKERRQ(ierr);
673   ierr = VecSetSizes(cellVec, conesSize, PETSC_DETERMINE);CHKERRQ(ierr);
674   ierr = VecSetBlockSize(cellVec, numCorners);CHKERRQ(ierr);
675   ierr = VecSetFromOptions(cellVec);CHKERRQ(ierr);
676   ierr = PetscObjectSetName((PetscObject) cellVec, "cells");CHKERRQ(ierr);
677   ierr = VecGetArray(cellVec, &vertices);CHKERRQ(ierr);
678   for (cell = cStart, v = 0; cell < cEnd; ++cell) {
679     PetscInt *closure = NULL;
680     PetscInt  closureSize, Nc = 0, p;
681 
682     if (label) {
683       PetscInt value;
684       ierr = DMPlexGetLabelValue(dm, label, cell, &value);CHKERRQ(ierr);
685       if (value == labelId) continue;
686     }
687     ierr = DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
688     for (p = 0; p < closureSize*2; p += 2) {
689       if ((closure[p] >= vStart) && (closure[p] < vEnd)) {
690         closure[Nc++] = closure[p];
691         }
692     }
693     ierr = DMPlexInvertCell(dim, Nc, closure);CHKERRQ(ierr);
694     for (p = 0; p < Nc; ++p) {
695       const PetscInt gv = gvertex[closure[p] - vStart];
696       vertices[v++] = gv < 0 ? -(gv+1) : gv;
697     }
698     ierr = DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
699   }
700   if (v != conesSize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_LIB, "Total number of cell vertices %d != %d", v, conesSize);
701   ierr = VecRestoreArray(cellVec, &vertices);CHKERRQ(ierr);
702   ierr = PetscViewerHDF5PushGroup(viewer, "/topology");CHKERRQ(ierr);
703   ierr = VecView(cellVec, viewer);CHKERRQ(ierr);
704   if (numCorners == 1) {
705     ierr = VecView(coneVec, viewer);CHKERRQ(ierr);
706   } else {
707     ierr = PetscViewerHDF5WriteAttribute(viewer, "/topology/cells", "cell_corners", PETSC_INT, (void *) &numCorners);CHKERRQ(ierr);
708   }
709   ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
710   ierr = VecDestroy(&cellVec);CHKERRQ(ierr);
711   ierr = VecDestroy(&coneVec);CHKERRQ(ierr);
712   ierr = ISRestoreIndices(globalVertexNumbers, &gvertex);CHKERRQ(ierr);
713 
714   ierr = PetscViewerHDF5WriteAttribute(viewer, "/topology/cells", "cell_dim", PETSC_INT, (void *) &dim);CHKERRQ(ierr);
715   /* Write Labels*/
716   ierr = PetscViewerHDF5PushGroup(viewer, "/labels");CHKERRQ(ierr);
717   ierr = PetscViewerHDF5OpenGroup(viewer, &fileId, &groupId);CHKERRQ(ierr);
718   if (groupId != fileId) {status = H5Gclose(groupId);CHKERRQ(status);}
719   ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
720   ierr = DMPlexGetNumLabels(dm, &numLabels);CHKERRQ(ierr);
721   for (l = 0; l < numLabels; ++l) {
722     DMLabel         label;
723     const char     *name;
724     IS              valueIS;
725     const PetscInt *values;
726     PetscInt        numValues, v;
727     PetscBool       isDepth;
728 
729     ierr = DMPlexGetLabelName(dm, l, &name);CHKERRQ(ierr);
730     ierr = PetscStrncmp(name, "depth", 10, &isDepth);CHKERRQ(ierr);
731     if (isDepth) continue;
732     ierr = DMPlexGetLabel(dm, name, &label);CHKERRQ(ierr);
733     ierr = DMLabelGetNumValues(label, &numValues);CHKERRQ(ierr);
734     ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
735     ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
736     ierr = PetscSNPrintf(group, PETSC_MAX_PATH_LEN, "/labels/%s", name);CHKERRQ(ierr);
737     ierr = PetscViewerHDF5PushGroup(viewer, group);CHKERRQ(ierr);
738     ierr = PetscViewerHDF5OpenGroup(viewer, &fileId, &groupId);CHKERRQ(ierr);
739     if (groupId != fileId) {status = H5Gclose(groupId);CHKERRQ(status);}
740     ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
741     /* TODO: Need to actually loop over the union of label values, ISAllGather() */
742     for (v = 0; v < numValues; ++v) {
743       IS   stratumIS;
744 
745       ierr = PetscSNPrintf(group, PETSC_MAX_PATH_LEN, "/labels/%s/%d", name, values[v]);CHKERRQ(ierr);
746       ierr = DMLabelGetStratumIS(label, values[v], &stratumIS);CHKERRQ(ierr);
747       /* TODO: Need to globalize point names and remove unowned points */
748       ierr = PetscViewerHDF5PushGroup(viewer, group);CHKERRQ(ierr);
749       ierr = ISView(stratumIS, viewer);CHKERRQ(ierr);
750       ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
751       ierr = ISDestroy(&stratumIS);CHKERRQ(ierr);
752     }
753     ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
754     ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
755   }
756   PetscFunctionReturn(0);
757 }
758 #endif
759 
760 #undef __FUNCT__
761 #define __FUNCT__ "DMView_Plex"
762 PetscErrorCode DMView_Plex(DM dm, PetscViewer viewer)
763 {
764   PetscBool      iascii, ishdf5;
765   PetscErrorCode ierr;
766 
767   PetscFunctionBegin;
768   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
769   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
770   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERASCII, &iascii);CHKERRQ(ierr);
771   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,  &ishdf5);CHKERRQ(ierr);
772   if (iascii) {
773     ierr = DMPlexView_Ascii(dm, viewer);CHKERRQ(ierr);
774   } else if (ishdf5) {
775 #if defined(PETSC_HAVE_HDF5)
776     ierr = DMPlexView_HDF5(dm, viewer);CHKERRQ(ierr);
777 #else
778     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
779 #endif
780   }
781   PetscFunctionReturn(0);
782 }
783 
784 #if defined(PETSC_HAVE_HDF5)
785 typedef struct {
786   DM          dm;
787   PetscViewer viewer;
788   DMLabel     label;
789 } LabelCtx;
790 
791 static herr_t ReadLabelStratumHDF5_Static(hid_t g_id, const char *name, const H5L_info_t *info, void *op_data)
792 {
793   PetscViewer     viewer = ((LabelCtx *) op_data)->viewer;
794   DMLabel         label  = ((LabelCtx *) op_data)->label;
795   IS              stratumIS;
796   const PetscInt *ind;
797   PetscInt        value, N, i;
798   const char     *lname;
799   char            group[PETSC_MAX_PATH_LEN];
800   PetscErrorCode  ierr;
801 
802   ierr = PetscOptionsStringToInt(name, &value);
803   ierr = ISCreate(PetscObjectComm((PetscObject) viewer), &stratumIS);
804   ierr = PetscObjectSetName((PetscObject) stratumIS, "indices");
805   ierr = DMLabelGetName(label, &lname);
806   ierr = PetscSNPrintf(group, PETSC_MAX_PATH_LEN, "/labels/%s/%s", lname, name);CHKERRQ(ierr);
807   ierr = PetscViewerHDF5PushGroup(viewer, group);CHKERRQ(ierr);
808   ierr = ISLoad(stratumIS, viewer);
809   ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
810   ierr = ISGetSize(stratumIS, &N);
811   ierr = ISGetIndices(stratumIS, &ind);
812   for (i = 0; i < N; ++i) {ierr = DMLabelSetValue(label, ind[i], value);}
813   ierr = ISRestoreIndices(stratumIS, &ind);
814   ierr = ISDestroy(&stratumIS);
815   return 0;
816 }
817 
818 static herr_t ReadLabelHDF5_Static(hid_t g_id, const char *name, const H5L_info_t *info, void *op_data)
819 {
820   DM             dm = ((LabelCtx *) op_data)->dm;
821   hsize_t        idx;
822   herr_t         status;
823   PetscErrorCode ierr;
824 
825   ierr = DMPlexCreateLabel(dm, name); if (ierr) return (herr_t) ierr;
826   ierr = DMPlexGetLabel(dm, name, &((LabelCtx *) op_data)->label); if (ierr) return (herr_t) ierr;
827   status = H5Literate_by_name(g_id, name, H5_INDEX_NAME, H5_ITER_NATIVE, &idx, ReadLabelStratumHDF5_Static, op_data, 0);
828   return status;
829 }
830 
831 #undef __FUNCT__
832 #define __FUNCT__ "DMPlexLoad_HDF5"
833 /* The first version will read everything onto proc 0, letting the user distribute
834    The next will create a naive partition, and then rebalance after reading
835 */
836 PetscErrorCode DMPlexLoad_HDF5(DM dm, PetscViewer viewer)
837 {
838   LabelCtx        ctx;
839   PetscSection    coordSection;
840   Vec             coordinates;
841   Vec             cellVec;
842   PetscScalar    *cells;
843   PetscReal       lengthScale;
844   PetscInt       *cone;
845   PetscInt        dim, spatialDim, numVertices, v, numCorners, numCells, cell, c;
846   hid_t           fileId, groupId;
847   hsize_t         idx;
848   herr_t          status;
849   PetscErrorCode  ierr;
850 
851   PetscFunctionBegin;
852   /* Read geometry */
853   ierr = PetscViewerHDF5PushGroup(viewer, "/geometry");CHKERRQ(ierr);
854   ierr = VecCreate(PetscObjectComm((PetscObject) dm), &coordinates);CHKERRQ(ierr);
855   ierr = PetscObjectSetName((PetscObject) coordinates, "vertices");CHKERRQ(ierr);
856   ierr = VecLoad(coordinates, viewer);CHKERRQ(ierr);
857   ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
858   ierr = DMPlexGetScale(dm, PETSC_UNIT_LENGTH, &lengthScale);CHKERRQ(ierr);
859   ierr = VecScale(coordinates, 1.0/lengthScale);CHKERRQ(ierr);
860   ierr = VecGetSize(coordinates, &numVertices);CHKERRQ(ierr);
861   ierr = VecGetBlockSize(coordinates, &spatialDim);CHKERRQ(ierr);
862   numVertices /= spatialDim;
863   /* Read toplogy */
864   ierr = PetscViewerHDF5ReadAttribute(viewer, "/topology/cells", "cell_dim", PETSC_INT, (void *) &dim);CHKERRQ(ierr);
865   ierr = DMPlexSetDimension(dm, dim);CHKERRQ(ierr);
866   /*   TODO Check for coneSize vector rather than this attribute */
867   ierr = PetscViewerHDF5ReadAttribute(viewer, "/topology/cells", "cell_corners", PETSC_INT, (void *) &numCorners);CHKERRQ(ierr);
868   ierr = PetscViewerHDF5PushGroup(viewer, "/topology");CHKERRQ(ierr);
869   ierr = VecCreate(PetscObjectComm((PetscObject) dm), &cellVec);CHKERRQ(ierr);
870   ierr = VecSetBlockSize(cellVec, numCorners);CHKERRQ(ierr);
871   ierr = PetscObjectSetName((PetscObject) cellVec, "cells");CHKERRQ(ierr);
872   ierr = VecLoad(cellVec, viewer);CHKERRQ(ierr);
873   ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
874   ierr = VecGetSize(cellVec, &numCells);CHKERRQ(ierr);
875   numCells /= numCorners;
876   /* Create Plex */
877   ierr = DMPlexSetChart(dm, 0, numCells+numVertices);CHKERRQ(ierr);
878   for (cell = 0; cell < numCells; ++cell) {ierr = DMPlexSetConeSize(dm, cell, numCorners);CHKERRQ(ierr);}
879   ierr = DMSetUp(dm);CHKERRQ(ierr);
880   ierr = PetscMalloc1(numCorners,&cone);CHKERRQ(ierr);
881   ierr = VecGetArray(cellVec, &cells);CHKERRQ(ierr);
882   for (cell = 0; cell < numCells; ++cell) {
883     for (c = 0; c < numCorners; ++c) {cone[c] = numCells + cells[cell*numCorners+c];}
884     ierr = DMPlexSetCone(dm, cell, cone);CHKERRQ(ierr);
885   }
886   ierr = VecRestoreArray(cellVec, &cells);CHKERRQ(ierr);
887   ierr = PetscFree(cone);CHKERRQ(ierr);
888   ierr = VecDestroy(&cellVec);CHKERRQ(ierr);
889   ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
890   ierr = DMPlexStratify(dm);CHKERRQ(ierr);
891   /* Create coordinates */
892   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
893   ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
894   ierr = PetscSectionSetFieldComponents(coordSection, 0, spatialDim);CHKERRQ(ierr);
895   ierr = PetscSectionSetChart(coordSection, numCells, numCells+numVertices);CHKERRQ(ierr);
896   for (v = numCells; v < numCells+numVertices; ++v) {
897     ierr = PetscSectionSetDof(coordSection, v, spatialDim);CHKERRQ(ierr);
898     ierr = PetscSectionSetFieldDof(coordSection, v, 0, spatialDim);CHKERRQ(ierr);
899   }
900   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
901   ierr = DMSetCoordinates(dm, coordinates);CHKERRQ(ierr);
902   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
903   /* Read Labels*/
904   ctx.dm     = dm;
905   ctx.viewer = viewer;
906   ierr = PetscViewerHDF5PushGroup(viewer, "/labels");CHKERRQ(ierr);
907   ierr = PetscViewerHDF5OpenGroup(viewer, &fileId, &groupId);CHKERRQ(ierr);
908   status = H5Literate(groupId, H5_INDEX_NAME, H5_ITER_NATIVE, &idx, ReadLabelHDF5_Static, &ctx);CHKERRQ(status);
909   status = H5Gclose(groupId);CHKERRQ(status);
910   ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
911   PetscFunctionReturn(0);
912 }
913 #endif
914 
915 #undef __FUNCT__
916 #define __FUNCT__ "DMLoad_Plex"
917 PetscErrorCode DMLoad_Plex(DM dm, PetscViewer viewer)
918 {
919   PetscBool      isbinary, ishdf5;
920   PetscErrorCode ierr;
921 
922   PetscFunctionBegin;
923   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
924   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
925   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERBINARY, &isbinary);CHKERRQ(ierr);
926   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,   &ishdf5);CHKERRQ(ierr);
927   if (isbinary) {SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Do not yet support binary viewers");}
928   else if (ishdf5) {
929 #if defined(PETSC_HAVE_HDF5)
930     DM odm;
931     ierr = DMCreate(PetscObjectComm((PetscObject) dm), &odm);CHKERRQ(ierr);
932     ierr = DMSetType(odm, DMPLEX);CHKERRQ(ierr);
933     ierr = DMPlexLoad_HDF5(odm, viewer);CHKERRQ(ierr);
934     ierr = DMPlexInterpolate(odm, &dm);CHKERRQ(ierr);
935     ierr = DMPlexCopyCoordinates(odm, dm);CHKERRQ(ierr);
936     ierr = DMPlexCopyLabels(odm, dm);CHKERRQ(ierr);
937     ierr = DMDestroy(&odm);CHKERRQ(ierr);
938 #else
939     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
940 #endif
941   }
942   PetscFunctionReturn(0);
943 }
944 
945 #undef __FUNCT__
946 #define __FUNCT__ "BoundaryDestroy"
947 static PetscErrorCode BoundaryDestroy(DMBoundary *boundary)
948 {
949   DMBoundary     b, next;
950   PetscErrorCode ierr;
951 
952   PetscFunctionBeginUser;
953   if (!boundary) PetscFunctionReturn(0);
954   b = *boundary;
955   *boundary = NULL;
956   for (; b; b = next) {
957     next = b->next;
958     ierr = PetscFree(b->ids);CHKERRQ(ierr);
959     ierr = PetscFree(b->name);CHKERRQ(ierr);
960     ierr = PetscFree(b);CHKERRQ(ierr);
961   }
962   PetscFunctionReturn(0);
963 }
964 
965 #undef __FUNCT__
966 #define __FUNCT__ "DMDestroy_Plex"
967 PetscErrorCode DMDestroy_Plex(DM dm)
968 {
969   DM_Plex       *mesh = (DM_Plex*) dm->data;
970   DMLabel        next  = mesh->labels;
971   PetscErrorCode ierr;
972 
973   PetscFunctionBegin;
974   if (--mesh->refct > 0) PetscFunctionReturn(0);
975   ierr = PetscSectionDestroy(&mesh->coneSection);CHKERRQ(ierr);
976   ierr = PetscFree(mesh->cones);CHKERRQ(ierr);
977   ierr = PetscFree(mesh->coneOrientations);CHKERRQ(ierr);
978   ierr = PetscSectionDestroy(&mesh->supportSection);CHKERRQ(ierr);
979   ierr = PetscFree(mesh->supports);CHKERRQ(ierr);
980   ierr = PetscFree(mesh->facesTmp);CHKERRQ(ierr);
981   while (next) {
982     DMLabel tmp = next->next;
983 
984     ierr = DMLabelDestroy(&next);CHKERRQ(ierr);
985     next = tmp;
986   }
987   ierr = DMDestroy(&mesh->coarseMesh);CHKERRQ(ierr);
988   ierr = DMLabelDestroy(&mesh->subpointMap);CHKERRQ(ierr);
989   ierr = ISDestroy(&mesh->globalVertexNumbers);CHKERRQ(ierr);
990   ierr = ISDestroy(&mesh->globalCellNumbers);CHKERRQ(ierr);
991   ierr = BoundaryDestroy(&mesh->boundary);CHKERRQ(ierr);
992   /* This was originally freed in DMDestroy(), but that prevents reference counting of backend objects */
993   ierr = PetscFree(mesh);CHKERRQ(ierr);
994   PetscFunctionReturn(0);
995 }
996 
997 #undef __FUNCT__
998 #define __FUNCT__ "DMCreateMatrix_Plex"
999 PetscErrorCode DMCreateMatrix_Plex(DM dm, Mat *J)
1000 {
1001   PetscSection   section, sectionGlobal;
1002   PetscInt       bs = -1;
1003   PetscInt       localSize;
1004   PetscBool      isShell, isBlock, isSeqBlock, isMPIBlock, isSymBlock, isSymSeqBlock, isSymMPIBlock;
1005   PetscErrorCode ierr;
1006   MatType        mtype;
1007 
1008   PetscFunctionBegin;
1009   ierr = MatInitializePackage();CHKERRQ(ierr);
1010   mtype = dm->mattype;
1011   ierr = DMGetDefaultSection(dm, &section);CHKERRQ(ierr);
1012   ierr = DMGetDefaultGlobalSection(dm, &sectionGlobal);CHKERRQ(ierr);
1013   /* ierr = PetscSectionGetStorageSize(sectionGlobal, &localSize);CHKERRQ(ierr); */
1014   ierr = PetscSectionGetConstrainedStorageSize(sectionGlobal, &localSize);CHKERRQ(ierr);
1015   ierr = MatCreate(PetscObjectComm((PetscObject)dm), J);CHKERRQ(ierr);
1016   ierr = MatSetSizes(*J, localSize, localSize, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
1017   ierr = MatSetType(*J, mtype);CHKERRQ(ierr);
1018   ierr = MatSetFromOptions(*J);CHKERRQ(ierr);
1019   ierr = PetscStrcmp(mtype, MATSHELL, &isShell);CHKERRQ(ierr);
1020   ierr = PetscStrcmp(mtype, MATBAIJ, &isBlock);CHKERRQ(ierr);
1021   ierr = PetscStrcmp(mtype, MATSEQBAIJ, &isSeqBlock);CHKERRQ(ierr);
1022   ierr = PetscStrcmp(mtype, MATMPIBAIJ, &isMPIBlock);CHKERRQ(ierr);
1023   ierr = PetscStrcmp(mtype, MATSBAIJ, &isSymBlock);CHKERRQ(ierr);
1024   ierr = PetscStrcmp(mtype, MATSEQSBAIJ, &isSymSeqBlock);CHKERRQ(ierr);
1025   ierr = PetscStrcmp(mtype, MATMPISBAIJ, &isSymMPIBlock);CHKERRQ(ierr);
1026   if (!isShell) {
1027     PetscBool fillMatrix = (PetscBool) !dm->prealloc_only;
1028     PetscInt *dnz, *onz, *dnzu, *onzu, bsLocal, bsMax, bsMin;
1029 
1030     if (bs < 0) {
1031       if (isBlock || isSeqBlock || isMPIBlock || isSymBlock || isSymSeqBlock || isSymMPIBlock) {
1032         PetscInt pStart, pEnd, p, dof, cdof;
1033 
1034         ierr = PetscSectionGetChart(sectionGlobal, &pStart, &pEnd);CHKERRQ(ierr);
1035         for (p = pStart; p < pEnd; ++p) {
1036           ierr = PetscSectionGetDof(sectionGlobal, p, &dof);CHKERRQ(ierr);
1037           ierr = PetscSectionGetConstraintDof(sectionGlobal, p, &cdof);CHKERRQ(ierr);
1038           if (dof-cdof) {
1039             if (bs < 0) {
1040               bs = dof-cdof;
1041             } else if (bs != dof-cdof) {
1042               /* Layout does not admit a pointwise block size */
1043               bs = 1;
1044               break;
1045             }
1046           }
1047         }
1048         /* Must have same blocksize on all procs (some might have no points) */
1049         bsLocal = bs;
1050         ierr = MPI_Allreduce(&bsLocal, &bsMax, 1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
1051         bsLocal = bs < 0 ? bsMax : bs;
1052         ierr = MPI_Allreduce(&bsLocal, &bsMin, 1, MPIU_INT, MPI_MIN, PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
1053         if (bsMin != bsMax) {
1054           bs = 1;
1055         } else {
1056           bs = bsMax;
1057         }
1058       } else {
1059         bs = 1;
1060       }
1061     }
1062     ierr = PetscCalloc4(localSize/bs, &dnz, localSize/bs, &onz, localSize/bs, &dnzu, localSize/bs, &onzu);CHKERRQ(ierr);
1063     ierr = DMPlexPreallocateOperator(dm, bs, section, sectionGlobal, dnz, onz, dnzu, onzu, *J, fillMatrix);CHKERRQ(ierr);
1064     ierr = PetscFree4(dnz, onz, dnzu, onzu);CHKERRQ(ierr);
1065   }
1066   PetscFunctionReturn(0);
1067 }
1068 
1069 #undef __FUNCT__
1070 #define __FUNCT__ "DMPlexGetDimension"
1071 /*@
1072   DMPlexGetDimension - Return the topological mesh dimension
1073 
1074   Not collective
1075 
1076   Input Parameter:
1077 . mesh - The DMPlex
1078 
1079   Output Parameter:
1080 . dim - The topological mesh dimension
1081 
1082   Level: beginner
1083 
1084 .seealso: DMPlexCreate()
1085 @*/
1086 PetscErrorCode DMPlexGetDimension(DM dm, PetscInt *dim)
1087 {
1088   DM_Plex *mesh = (DM_Plex*) dm->data;
1089 
1090   PetscFunctionBegin;
1091   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1092   PetscValidPointer(dim, 2);
1093   *dim = mesh->dim;
1094   PetscFunctionReturn(0);
1095 }
1096 
1097 #undef __FUNCT__
1098 #define __FUNCT__ "DMPlexSetDimension"
1099 /*@
1100   DMPlexSetDimension - Set the topological mesh dimension
1101 
1102   Collective on mesh
1103 
1104   Input Parameters:
1105 + mesh - The DMPlex
1106 - dim - The topological mesh dimension
1107 
1108   Level: beginner
1109 
1110 .seealso: DMPlexCreate()
1111 @*/
1112 PetscErrorCode DMPlexSetDimension(DM dm, PetscInt dim)
1113 {
1114   DM_Plex *mesh = (DM_Plex*) dm->data;
1115 
1116   PetscFunctionBegin;
1117   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1118   PetscValidLogicalCollectiveInt(dm, dim, 2);
1119   mesh->dim = dim;
1120   PetscFunctionReturn(0);
1121 }
1122 
1123 #undef __FUNCT__
1124 #define __FUNCT__ "DMPlexGetChart"
1125 /*@
1126   DMPlexGetChart - Return the interval for all mesh points [pStart, pEnd)
1127 
1128   Not collective
1129 
1130   Input Parameter:
1131 . mesh - The DMPlex
1132 
1133   Output Parameters:
1134 + pStart - The first mesh point
1135 - pEnd   - The upper bound for mesh points
1136 
1137   Level: beginner
1138 
1139 .seealso: DMPlexCreate(), DMPlexSetChart()
1140 @*/
1141 PetscErrorCode DMPlexGetChart(DM dm, PetscInt *pStart, PetscInt *pEnd)
1142 {
1143   DM_Plex       *mesh = (DM_Plex*) dm->data;
1144   PetscErrorCode ierr;
1145 
1146   PetscFunctionBegin;
1147   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1148   ierr = PetscSectionGetChart(mesh->coneSection, pStart, pEnd);CHKERRQ(ierr);
1149   PetscFunctionReturn(0);
1150 }
1151 
1152 #undef __FUNCT__
1153 #define __FUNCT__ "DMPlexSetChart"
1154 /*@
1155   DMPlexSetChart - Set the interval for all mesh points [pStart, pEnd)
1156 
1157   Not collective
1158 
1159   Input Parameters:
1160 + mesh - The DMPlex
1161 . pStart - The first mesh point
1162 - pEnd   - The upper bound for mesh points
1163 
1164   Output Parameters:
1165 
1166   Level: beginner
1167 
1168 .seealso: DMPlexCreate(), DMPlexGetChart()
1169 @*/
1170 PetscErrorCode DMPlexSetChart(DM dm, PetscInt pStart, PetscInt pEnd)
1171 {
1172   DM_Plex       *mesh = (DM_Plex*) dm->data;
1173   PetscErrorCode ierr;
1174 
1175   PetscFunctionBegin;
1176   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1177   ierr = PetscSectionSetChart(mesh->coneSection, pStart, pEnd);CHKERRQ(ierr);
1178   ierr = PetscSectionSetChart(mesh->supportSection, pStart, pEnd);CHKERRQ(ierr);
1179   PetscFunctionReturn(0);
1180 }
1181 
1182 #undef __FUNCT__
1183 #define __FUNCT__ "DMPlexGetConeSize"
1184 /*@
1185   DMPlexGetConeSize - Return the number of in-edges for this point in the Sieve DAG
1186 
1187   Not collective
1188 
1189   Input Parameters:
1190 + mesh - The DMPlex
1191 - p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1192 
1193   Output Parameter:
1194 . size - The cone size for point p
1195 
1196   Level: beginner
1197 
1198 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart()
1199 @*/
1200 PetscErrorCode DMPlexGetConeSize(DM dm, PetscInt p, PetscInt *size)
1201 {
1202   DM_Plex       *mesh = (DM_Plex*) dm->data;
1203   PetscErrorCode ierr;
1204 
1205   PetscFunctionBegin;
1206   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1207   PetscValidPointer(size, 3);
1208   ierr = PetscSectionGetDof(mesh->coneSection, p, size);CHKERRQ(ierr);
1209   PetscFunctionReturn(0);
1210 }
1211 
1212 #undef __FUNCT__
1213 #define __FUNCT__ "DMPlexSetConeSize"
1214 /*@
1215   DMPlexSetConeSize - Set the number of in-edges for this point in the Sieve DAG
1216 
1217   Not collective
1218 
1219   Input Parameters:
1220 + mesh - The DMPlex
1221 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1222 - size - The cone size for point p
1223 
1224   Output Parameter:
1225 
1226   Note:
1227   This should be called after DMPlexSetChart().
1228 
1229   Level: beginner
1230 
1231 .seealso: DMPlexCreate(), DMPlexGetConeSize(), DMPlexSetChart()
1232 @*/
1233 PetscErrorCode DMPlexSetConeSize(DM dm, PetscInt p, PetscInt size)
1234 {
1235   DM_Plex       *mesh = (DM_Plex*) dm->data;
1236   PetscErrorCode ierr;
1237 
1238   PetscFunctionBegin;
1239   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1240   ierr = PetscSectionSetDof(mesh->coneSection, p, size);CHKERRQ(ierr);
1241 
1242   mesh->maxConeSize = PetscMax(mesh->maxConeSize, size);
1243   PetscFunctionReturn(0);
1244 }
1245 
1246 #undef __FUNCT__
1247 #define __FUNCT__ "DMPlexGetCone"
1248 /*@C
1249   DMPlexGetCone - Return the points on the in-edges for this point in the Sieve DAG
1250 
1251   Not collective
1252 
1253   Input Parameters:
1254 + mesh - The DMPlex
1255 - p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1256 
1257   Output Parameter:
1258 . cone - An array of points which are on the in-edges for point p
1259 
1260   Level: beginner
1261 
1262   Fortran Notes:
1263   Since it returns an array, this routine is only available in Fortran 90, and you must
1264   include petsc.h90 in your code.
1265 
1266   You must also call DMPlexRestoreCone() after you finish using the returned array.
1267 
1268 .seealso: DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart()
1269 @*/
1270 PetscErrorCode DMPlexGetCone(DM dm, PetscInt p, const PetscInt *cone[])
1271 {
1272   DM_Plex       *mesh = (DM_Plex*) dm->data;
1273   PetscInt       off;
1274   PetscErrorCode ierr;
1275 
1276   PetscFunctionBegin;
1277   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1278   PetscValidPointer(cone, 3);
1279   ierr  = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1280   *cone = &mesh->cones[off];
1281   PetscFunctionReturn(0);
1282 }
1283 
1284 #undef __FUNCT__
1285 #define __FUNCT__ "DMPlexSetCone"
1286 /*@
1287   DMPlexSetCone - Set the points on the in-edges for this point in the Sieve DAG
1288 
1289   Not collective
1290 
1291   Input Parameters:
1292 + mesh - The DMPlex
1293 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1294 - cone - An array of points which are on the in-edges for point p
1295 
1296   Output Parameter:
1297 
1298   Note:
1299   This should be called after all calls to DMPlexSetConeSize() and DMSetUp().
1300 
1301   Level: beginner
1302 
1303 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
1304 @*/
1305 PetscErrorCode DMPlexSetCone(DM dm, PetscInt p, const PetscInt cone[])
1306 {
1307   DM_Plex       *mesh = (DM_Plex*) dm->data;
1308   PetscInt       pStart, pEnd;
1309   PetscInt       dof, off, c;
1310   PetscErrorCode ierr;
1311 
1312   PetscFunctionBegin;
1313   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1314   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
1315   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
1316   if (dof) PetscValidPointer(cone, 3);
1317   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1318   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
1319   for (c = 0; c < dof; ++c) {
1320     if ((cone[c] < pStart) || (cone[c] >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone point %D is not in the valid range [%D, %D)", cone[c], pStart, pEnd);
1321     mesh->cones[off+c] = cone[c];
1322   }
1323   PetscFunctionReturn(0);
1324 }
1325 
1326 #undef __FUNCT__
1327 #define __FUNCT__ "DMPlexGetConeOrientation"
1328 /*@C
1329   DMPlexGetConeOrientation - Return the orientations on the in-edges for this point in the Sieve DAG
1330 
1331   Not collective
1332 
1333   Input Parameters:
1334 + mesh - The DMPlex
1335 - p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1336 
1337   Output Parameter:
1338 . coneOrientation - An array of orientations which are on the in-edges for point p. An orientation is an
1339                     integer giving the prescription for cone traversal. If it is negative, the cone is
1340                     traversed in the opposite direction. Its value 'o', or if negative '-(o+1)', gives
1341                     the index of the cone point on which to start.
1342 
1343   Level: beginner
1344 
1345   Fortran Notes:
1346   Since it returns an array, this routine is only available in Fortran 90, and you must
1347   include petsc.h90 in your code.
1348 
1349   You must also call DMPlexRestoreConeOrientation() after you finish using the returned array.
1350 
1351 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetCone(), DMPlexSetChart()
1352 @*/
1353 PetscErrorCode DMPlexGetConeOrientation(DM dm, PetscInt p, const PetscInt *coneOrientation[])
1354 {
1355   DM_Plex       *mesh = (DM_Plex*) dm->data;
1356   PetscInt       off;
1357   PetscErrorCode ierr;
1358 
1359   PetscFunctionBegin;
1360   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1361 #if defined(PETSC_USE_DEBUG)
1362   {
1363     PetscInt dof;
1364     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
1365     if (dof) PetscValidPointer(coneOrientation, 3);
1366   }
1367 #endif
1368   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1369 
1370   *coneOrientation = &mesh->coneOrientations[off];
1371   PetscFunctionReturn(0);
1372 }
1373 
1374 #undef __FUNCT__
1375 #define __FUNCT__ "DMPlexSetConeOrientation"
1376 /*@
1377   DMPlexSetConeOrientation - Set the orientations on the in-edges for this point in the Sieve DAG
1378 
1379   Not collective
1380 
1381   Input Parameters:
1382 + mesh - The DMPlex
1383 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1384 - coneOrientation - An array of orientations which are on the in-edges for point p. An orientation is an
1385                     integer giving the prescription for cone traversal. If it is negative, the cone is
1386                     traversed in the opposite direction. Its value 'o', or if negative '-(o+1)', gives
1387                     the index of the cone point on which to start.
1388 
1389   Output Parameter:
1390 
1391   Note:
1392   This should be called after all calls to DMPlexSetConeSize() and DMSetUp().
1393 
1394   Level: beginner
1395 
1396 .seealso: DMPlexCreate(), DMPlexGetConeOrientation(), DMPlexSetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
1397 @*/
1398 PetscErrorCode DMPlexSetConeOrientation(DM dm, PetscInt p, const PetscInt coneOrientation[])
1399 {
1400   DM_Plex       *mesh = (DM_Plex*) dm->data;
1401   PetscInt       pStart, pEnd;
1402   PetscInt       dof, off, c;
1403   PetscErrorCode ierr;
1404 
1405   PetscFunctionBegin;
1406   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1407   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
1408   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
1409   if (dof) PetscValidPointer(coneOrientation, 3);
1410   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1411   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
1412   for (c = 0; c < dof; ++c) {
1413     PetscInt cdof, o = coneOrientation[c];
1414 
1415     ierr = PetscSectionGetDof(mesh->coneSection, mesh->cones[off+c], &cdof);CHKERRQ(ierr);
1416     if (o && ((o < -(cdof+1)) || (o >= cdof))) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone orientation %D is not in the valid range [%D. %D)", o, -(cdof+1), cdof);
1417     mesh->coneOrientations[off+c] = o;
1418   }
1419   PetscFunctionReturn(0);
1420 }
1421 
1422 #undef __FUNCT__
1423 #define __FUNCT__ "DMPlexInsertCone"
1424 PetscErrorCode DMPlexInsertCone(DM dm, PetscInt p, PetscInt conePos, PetscInt conePoint)
1425 {
1426   DM_Plex       *mesh = (DM_Plex*) dm->data;
1427   PetscInt       pStart, pEnd;
1428   PetscInt       dof, off;
1429   PetscErrorCode ierr;
1430 
1431   PetscFunctionBegin;
1432   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1433   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
1434   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
1435   if ((conePoint < pStart) || (conePoint >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone point %D is not in the valid range [%D, %D)", conePoint, pStart, pEnd);
1436   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
1437   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1438   if ((conePos < 0) || (conePos >= dof)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone position %D of point %D is not in the valid range [0, %D)", conePos, p, dof);
1439   mesh->cones[off+conePos] = conePoint;
1440   PetscFunctionReturn(0);
1441 }
1442 
1443 #undef __FUNCT__
1444 #define __FUNCT__ "DMPlexInsertConeOrientation"
1445 PetscErrorCode DMPlexInsertConeOrientation(DM dm, PetscInt p, PetscInt conePos, PetscInt coneOrientation)
1446 {
1447   DM_Plex       *mesh = (DM_Plex*) dm->data;
1448   PetscInt       pStart, pEnd;
1449   PetscInt       dof, off;
1450   PetscErrorCode ierr;
1451 
1452   PetscFunctionBegin;
1453   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1454   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
1455   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
1456   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
1457   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1458   if ((conePos < 0) || (conePos >= dof)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone position %D of point %D is not in the valid range [0, %D)", conePos, p, dof);
1459   mesh->coneOrientations[off+conePos] = coneOrientation;
1460   PetscFunctionReturn(0);
1461 }
1462 
1463 #undef __FUNCT__
1464 #define __FUNCT__ "DMPlexGetSupportSize"
1465 /*@
1466   DMPlexGetSupportSize - Return the number of out-edges for this point in the Sieve DAG
1467 
1468   Not collective
1469 
1470   Input Parameters:
1471 + mesh - The DMPlex
1472 - p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1473 
1474   Output Parameter:
1475 . size - The support size for point p
1476 
1477   Level: beginner
1478 
1479 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart(), DMPlexGetConeSize()
1480 @*/
1481 PetscErrorCode DMPlexGetSupportSize(DM dm, PetscInt p, PetscInt *size)
1482 {
1483   DM_Plex       *mesh = (DM_Plex*) dm->data;
1484   PetscErrorCode ierr;
1485 
1486   PetscFunctionBegin;
1487   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1488   PetscValidPointer(size, 3);
1489   ierr = PetscSectionGetDof(mesh->supportSection, p, size);CHKERRQ(ierr);
1490   PetscFunctionReturn(0);
1491 }
1492 
1493 #undef __FUNCT__
1494 #define __FUNCT__ "DMPlexSetSupportSize"
1495 /*@
1496   DMPlexSetSupportSize - Set the number of out-edges for this point in the Sieve DAG
1497 
1498   Not collective
1499 
1500   Input Parameters:
1501 + mesh - The DMPlex
1502 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1503 - size - The support size for point p
1504 
1505   Output Parameter:
1506 
1507   Note:
1508   This should be called after DMPlexSetChart().
1509 
1510   Level: beginner
1511 
1512 .seealso: DMPlexCreate(), DMPlexGetSupportSize(), DMPlexSetChart()
1513 @*/
1514 PetscErrorCode DMPlexSetSupportSize(DM dm, PetscInt p, PetscInt size)
1515 {
1516   DM_Plex       *mesh = (DM_Plex*) dm->data;
1517   PetscErrorCode ierr;
1518 
1519   PetscFunctionBegin;
1520   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1521   ierr = PetscSectionSetDof(mesh->supportSection, p, size);CHKERRQ(ierr);
1522 
1523   mesh->maxSupportSize = PetscMax(mesh->maxSupportSize, size);
1524   PetscFunctionReturn(0);
1525 }
1526 
1527 #undef __FUNCT__
1528 #define __FUNCT__ "DMPlexGetSupport"
1529 /*@C
1530   DMPlexGetSupport - Return the points on the out-edges for this point in the Sieve DAG
1531 
1532   Not collective
1533 
1534   Input Parameters:
1535 + mesh - The DMPlex
1536 - p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1537 
1538   Output Parameter:
1539 . support - An array of points which are on the out-edges for point p
1540 
1541   Level: beginner
1542 
1543   Fortran Notes:
1544   Since it returns an array, this routine is only available in Fortran 90, and you must
1545   include petsc.h90 in your code.
1546 
1547   You must also call DMPlexRestoreSupport() after you finish using the returned array.
1548 
1549 .seealso: DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
1550 @*/
1551 PetscErrorCode DMPlexGetSupport(DM dm, PetscInt p, const PetscInt *support[])
1552 {
1553   DM_Plex       *mesh = (DM_Plex*) dm->data;
1554   PetscInt       off;
1555   PetscErrorCode ierr;
1556 
1557   PetscFunctionBegin;
1558   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1559   PetscValidPointer(support, 3);
1560   ierr     = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
1561   *support = &mesh->supports[off];
1562   PetscFunctionReturn(0);
1563 }
1564 
1565 #undef __FUNCT__
1566 #define __FUNCT__ "DMPlexSetSupport"
1567 /*@
1568   DMPlexSetSupport - Set the points on the out-edges for this point in the Sieve DAG
1569 
1570   Not collective
1571 
1572   Input Parameters:
1573 + mesh - The DMPlex
1574 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1575 - support - An array of points which are on the in-edges for point p
1576 
1577   Output Parameter:
1578 
1579   Note:
1580   This should be called after all calls to DMPlexSetSupportSize() and DMSetUp().
1581 
1582   Level: beginner
1583 
1584 .seealso: DMPlexCreate(), DMPlexGetSupport(), DMPlexSetChart(), DMPlexSetSupportSize(), DMSetUp()
1585 @*/
1586 PetscErrorCode DMPlexSetSupport(DM dm, PetscInt p, const PetscInt support[])
1587 {
1588   DM_Plex       *mesh = (DM_Plex*) dm->data;
1589   PetscInt       pStart, pEnd;
1590   PetscInt       dof, off, c;
1591   PetscErrorCode ierr;
1592 
1593   PetscFunctionBegin;
1594   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1595   ierr = PetscSectionGetChart(mesh->supportSection, &pStart, &pEnd);CHKERRQ(ierr);
1596   ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
1597   if (dof) PetscValidPointer(support, 3);
1598   ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
1599   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
1600   for (c = 0; c < dof; ++c) {
1601     if ((support[c] < pStart) || (support[c] >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support point %D is not in the valid range [%D, %D)", support[c], pStart, pEnd);
1602     mesh->supports[off+c] = support[c];
1603   }
1604   PetscFunctionReturn(0);
1605 }
1606 
1607 #undef __FUNCT__
1608 #define __FUNCT__ "DMPlexInsertSupport"
1609 PetscErrorCode DMPlexInsertSupport(DM dm, PetscInt p, PetscInt supportPos, PetscInt supportPoint)
1610 {
1611   DM_Plex       *mesh = (DM_Plex*) dm->data;
1612   PetscInt       pStart, pEnd;
1613   PetscInt       dof, off;
1614   PetscErrorCode ierr;
1615 
1616   PetscFunctionBegin;
1617   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1618   ierr = PetscSectionGetChart(mesh->supportSection, &pStart, &pEnd);CHKERRQ(ierr);
1619   ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
1620   ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
1621   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
1622   if ((supportPoint < pStart) || (supportPoint >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support point %D is not in the valid range [%D, %D)", supportPoint, pStart, pEnd);
1623   if (supportPos >= dof) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support position %D of point %D is not in the valid range [0, %D)", supportPos, p, dof);
1624   mesh->supports[off+supportPos] = supportPoint;
1625   PetscFunctionReturn(0);
1626 }
1627 
1628 #undef __FUNCT__
1629 #define __FUNCT__ "DMPlexGetTransitiveClosure"
1630 /*@C
1631   DMPlexGetTransitiveClosure - Return the points on the transitive closure of the in-edges or out-edges for this point in the Sieve DAG
1632 
1633   Not collective
1634 
1635   Input Parameters:
1636 + mesh - The DMPlex
1637 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1638 . useCone - PETSC_TRUE for in-edges,  otherwise use out-edges
1639 - points - If points is NULL on input, internal storage will be returned, otherwise the provided array is used
1640 
1641   Output Parameters:
1642 + numPoints - The number of points in the closure, so points[] is of size 2*numPoints
1643 - points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...]
1644 
1645   Note:
1646   If using internal storage (points is NULL on input), each call overwrites the last output.
1647 
1648   Fortran Notes:
1649   Since it returns an array, this routine is only available in Fortran 90, and you must
1650   include petsc.h90 in your code.
1651 
1652   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
1653 
1654   Level: beginner
1655 
1656 .seealso: DMPlexRestoreTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
1657 @*/
1658 PetscErrorCode DMPlexGetTransitiveClosure(DM dm, PetscInt p, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
1659 {
1660   DM_Plex        *mesh = (DM_Plex*) dm->data;
1661   PetscInt       *closure, *fifo;
1662   const PetscInt *tmp = NULL, *tmpO = NULL;
1663   PetscInt        tmpSize, t;
1664   PetscInt        depth       = 0, maxSize;
1665   PetscInt        closureSize = 2, fifoSize = 0, fifoStart = 0;
1666   PetscErrorCode  ierr;
1667 
1668   PetscFunctionBegin;
1669   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1670   ierr    = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
1671   /* This is only 1-level */
1672   if (useCone) {
1673     ierr = DMPlexGetConeSize(dm, p, &tmpSize);CHKERRQ(ierr);
1674     ierr = DMPlexGetCone(dm, p, &tmp);CHKERRQ(ierr);
1675     ierr = DMPlexGetConeOrientation(dm, p, &tmpO);CHKERRQ(ierr);
1676   } else {
1677     ierr = DMPlexGetSupportSize(dm, p, &tmpSize);CHKERRQ(ierr);
1678     ierr = DMPlexGetSupport(dm, p, &tmp);CHKERRQ(ierr);
1679   }
1680   if (depth == 1) {
1681     if (*points) {
1682       closure = *points;
1683     } else {
1684       maxSize = 2*(PetscMax(mesh->maxConeSize, mesh->maxSupportSize)+1);
1685       ierr = DMGetWorkArray(dm, maxSize, PETSC_INT, &closure);CHKERRQ(ierr);
1686     }
1687     closure[0] = p; closure[1] = 0;
1688     for (t = 0; t < tmpSize; ++t, closureSize += 2) {
1689       closure[closureSize]   = tmp[t];
1690       closure[closureSize+1] = tmpO ? tmpO[t] : 0;
1691     }
1692     if (numPoints) *numPoints = closureSize/2;
1693     if (points)    *points    = closure;
1694     PetscFunctionReturn(0);
1695   }
1696   maxSize = 2*PetscMax(PetscMax(PetscPowInt(mesh->maxConeSize,depth+1),PetscPowInt(mesh->maxSupportSize,depth+1)),depth+1);
1697   ierr    = DMGetWorkArray(dm, maxSize, PETSC_INT, &fifo);CHKERRQ(ierr);
1698   if (*points) {
1699     closure = *points;
1700   } else {
1701     ierr = DMGetWorkArray(dm, maxSize, PETSC_INT, &closure);CHKERRQ(ierr);
1702   }
1703   closure[0] = p; closure[1] = 0;
1704   for (t = 0; t < tmpSize; ++t, closureSize += 2, fifoSize += 2) {
1705     const PetscInt cp = tmp[t];
1706     const PetscInt co = tmpO ? tmpO[t] : 0;
1707 
1708     closure[closureSize]   = cp;
1709     closure[closureSize+1] = co;
1710     fifo[fifoSize]         = cp;
1711     fifo[fifoSize+1]       = co;
1712   }
1713   /* Should kick out early when depth is reached, rather than checking all vertices for empty cones */
1714   while (fifoSize - fifoStart) {
1715     const PetscInt q   = fifo[fifoStart];
1716     const PetscInt o   = fifo[fifoStart+1];
1717     const PetscInt rev = o >= 0 ? 0 : 1;
1718     const PetscInt off = rev ? -(o+1) : o;
1719 
1720     if (useCone) {
1721       ierr = DMPlexGetConeSize(dm, q, &tmpSize);CHKERRQ(ierr);
1722       ierr = DMPlexGetCone(dm, q, &tmp);CHKERRQ(ierr);
1723       ierr = DMPlexGetConeOrientation(dm, q, &tmpO);CHKERRQ(ierr);
1724     } else {
1725       ierr = DMPlexGetSupportSize(dm, q, &tmpSize);CHKERRQ(ierr);
1726       ierr = DMPlexGetSupport(dm, q, &tmp);CHKERRQ(ierr);
1727       tmpO = NULL;
1728     }
1729     for (t = 0; t < tmpSize; ++t) {
1730       const PetscInt i  = ((rev ? tmpSize-t : t) + off)%tmpSize;
1731       const PetscInt cp = tmp[i];
1732       /* Must propogate orientation: When we reverse orientation, we both reverse the direction of iteration and start at the other end of the chain. */
1733       /* HACK: It is worse to get the size here, than to change the interpretation of -(*+1)
1734        const PetscInt co = tmpO ? (rev ? -(tmpO[i]+1) : tmpO[i]) : 0; */
1735       PetscInt       co = tmpO ? tmpO[i] : 0;
1736       PetscInt       c;
1737 
1738       if (rev) {
1739         PetscInt childSize, coff;
1740         ierr = DMPlexGetConeSize(dm, cp, &childSize);CHKERRQ(ierr);
1741         coff = tmpO[i] < 0 ? -(tmpO[i]+1) : tmpO[i];
1742         co   = childSize ? -(((coff+childSize-1)%childSize)+1) : 0;
1743       }
1744       /* Check for duplicate */
1745       for (c = 0; c < closureSize; c += 2) {
1746         if (closure[c] == cp) break;
1747       }
1748       if (c == closureSize) {
1749         closure[closureSize]   = cp;
1750         closure[closureSize+1] = co;
1751         fifo[fifoSize]         = cp;
1752         fifo[fifoSize+1]       = co;
1753         closureSize           += 2;
1754         fifoSize              += 2;
1755       }
1756     }
1757     fifoStart += 2;
1758   }
1759   if (numPoints) *numPoints = closureSize/2;
1760   if (points)    *points    = closure;
1761   ierr = DMRestoreWorkArray(dm, maxSize, PETSC_INT, &fifo);CHKERRQ(ierr);
1762   PetscFunctionReturn(0);
1763 }
1764 
1765 #undef __FUNCT__
1766 #define __FUNCT__ "DMPlexGetTransitiveClosure_Internal"
1767 /*@C
1768   DMPlexGetTransitiveClosure_Internal - Return the points on the transitive closure of the in-edges or out-edges for this point in the Sieve DAG with a specified initial orientation
1769 
1770   Not collective
1771 
1772   Input Parameters:
1773 + mesh - The DMPlex
1774 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1775 . orientation - The orientation of the point
1776 . useCone - PETSC_TRUE for in-edges,  otherwise use out-edges
1777 - points - If points is NULL on input, internal storage will be returned, otherwise the provided array is used
1778 
1779   Output Parameters:
1780 + numPoints - The number of points in the closure, so points[] is of size 2*numPoints
1781 - points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...]
1782 
1783   Note:
1784   If using internal storage (points is NULL on input), each call overwrites the last output.
1785 
1786   Fortran Notes:
1787   Since it returns an array, this routine is only available in Fortran 90, and you must
1788   include petsc.h90 in your code.
1789 
1790   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
1791 
1792   Level: beginner
1793 
1794 .seealso: DMPlexRestoreTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
1795 @*/
1796 PetscErrorCode DMPlexGetTransitiveClosure_Internal(DM dm, PetscInt p, PetscInt ornt, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
1797 {
1798   DM_Plex        *mesh = (DM_Plex*) dm->data;
1799   PetscInt       *closure, *fifo;
1800   const PetscInt *tmp = NULL, *tmpO = NULL;
1801   PetscInt        tmpSize, t;
1802   PetscInt        depth       = 0, maxSize;
1803   PetscInt        closureSize = 2, fifoSize = 0, fifoStart = 0;
1804   PetscErrorCode  ierr;
1805 
1806   PetscFunctionBegin;
1807   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1808   ierr    = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
1809   /* This is only 1-level */
1810   if (useCone) {
1811     ierr = DMPlexGetConeSize(dm, p, &tmpSize);CHKERRQ(ierr);
1812     ierr = DMPlexGetCone(dm, p, &tmp);CHKERRQ(ierr);
1813     ierr = DMPlexGetConeOrientation(dm, p, &tmpO);CHKERRQ(ierr);
1814   } else {
1815     ierr = DMPlexGetSupportSize(dm, p, &tmpSize);CHKERRQ(ierr);
1816     ierr = DMPlexGetSupport(dm, p, &tmp);CHKERRQ(ierr);
1817   }
1818   if (depth == 1) {
1819     if (*points) {
1820       closure = *points;
1821     } else {
1822       maxSize = 2*(PetscMax(mesh->maxConeSize, mesh->maxSupportSize)+1);
1823       ierr = DMGetWorkArray(dm, maxSize, PETSC_INT, &closure);CHKERRQ(ierr);
1824     }
1825     closure[0] = p; closure[1] = ornt;
1826     for (t = 0; t < tmpSize; ++t, closureSize += 2) {
1827       const PetscInt i = ornt >= 0 ? (t+ornt)%tmpSize : (-(ornt+1) + tmpSize-t)%tmpSize;
1828       closure[closureSize]   = tmp[i];
1829       closure[closureSize+1] = tmpO ? tmpO[i] : 0;
1830     }
1831     if (numPoints) *numPoints = closureSize/2;
1832     if (points)    *points    = closure;
1833     PetscFunctionReturn(0);
1834   }
1835   maxSize = 2*PetscMax(PetscMax(PetscPowInt(mesh->maxConeSize,depth+1),PetscPowInt(mesh->maxSupportSize,depth+1)),depth+1);
1836   ierr    = DMGetWorkArray(dm, maxSize, PETSC_INT, &fifo);CHKERRQ(ierr);
1837   if (*points) {
1838     closure = *points;
1839   } else {
1840     ierr = DMGetWorkArray(dm, maxSize, PETSC_INT, &closure);CHKERRQ(ierr);
1841   }
1842   closure[0] = p; closure[1] = ornt;
1843   for (t = 0; t < tmpSize; ++t, closureSize += 2, fifoSize += 2) {
1844     const PetscInt i  = ornt >= 0 ? (t+ornt)%tmpSize : (-(ornt+1) + tmpSize-t)%tmpSize;
1845     const PetscInt cp = tmp[i];
1846     PetscInt       co = tmpO ? tmpO[i] : 0;
1847 
1848     if (ornt < 0) {
1849       PetscInt childSize, coff;
1850       ierr = DMPlexGetConeSize(dm, cp, &childSize);CHKERRQ(ierr);
1851       coff = tmpO[i] < 0 ? -(tmpO[i]+1) : tmpO[i];
1852       co   = childSize ? -(((coff+childSize-1)%childSize)+1) : 0;
1853     }
1854     closure[closureSize]   = cp;
1855     closure[closureSize+1] = co;
1856     fifo[fifoSize]         = cp;
1857     fifo[fifoSize+1]       = co;
1858   }
1859   /* Should kick out early when depth is reached, rather than checking all vertices for empty cones */
1860   while (fifoSize - fifoStart) {
1861     const PetscInt q   = fifo[fifoStart];
1862     const PetscInt o   = fifo[fifoStart+1];
1863     const PetscInt rev = o >= 0 ? 0 : 1;
1864     const PetscInt off = rev ? -(o+1) : o;
1865 
1866     if (useCone) {
1867       ierr = DMPlexGetConeSize(dm, q, &tmpSize);CHKERRQ(ierr);
1868       ierr = DMPlexGetCone(dm, q, &tmp);CHKERRQ(ierr);
1869       ierr = DMPlexGetConeOrientation(dm, q, &tmpO);CHKERRQ(ierr);
1870     } else {
1871       ierr = DMPlexGetSupportSize(dm, q, &tmpSize);CHKERRQ(ierr);
1872       ierr = DMPlexGetSupport(dm, q, &tmp);CHKERRQ(ierr);
1873       tmpO = NULL;
1874     }
1875     for (t = 0; t < tmpSize; ++t) {
1876       const PetscInt i  = ((rev ? tmpSize-t : t) + off)%tmpSize;
1877       const PetscInt cp = tmp[i];
1878       /* Must propogate orientation: When we reverse orientation, we both reverse the direction of iteration and start at the other end of the chain. */
1879       /* HACK: It is worse to get the size here, than to change the interpretation of -(*+1)
1880        const PetscInt co = tmpO ? (rev ? -(tmpO[i]+1) : tmpO[i]) : 0; */
1881       PetscInt       co = tmpO ? tmpO[i] : 0;
1882       PetscInt       c;
1883 
1884       if (rev) {
1885         PetscInt childSize, coff;
1886         ierr = DMPlexGetConeSize(dm, cp, &childSize);CHKERRQ(ierr);
1887         coff = tmpO[i] < 0 ? -(tmpO[i]+1) : tmpO[i];
1888         co   = childSize ? -(((coff+childSize-1)%childSize)+1) : 0;
1889       }
1890       /* Check for duplicate */
1891       for (c = 0; c < closureSize; c += 2) {
1892         if (closure[c] == cp) break;
1893       }
1894       if (c == closureSize) {
1895         closure[closureSize]   = cp;
1896         closure[closureSize+1] = co;
1897         fifo[fifoSize]         = cp;
1898         fifo[fifoSize+1]       = co;
1899         closureSize           += 2;
1900         fifoSize              += 2;
1901       }
1902     }
1903     fifoStart += 2;
1904   }
1905   if (numPoints) *numPoints = closureSize/2;
1906   if (points)    *points    = closure;
1907   ierr = DMRestoreWorkArray(dm, maxSize, PETSC_INT, &fifo);CHKERRQ(ierr);
1908   PetscFunctionReturn(0);
1909 }
1910 
1911 #undef __FUNCT__
1912 #define __FUNCT__ "DMPlexRestoreTransitiveClosure"
1913 /*@C
1914   DMPlexRestoreTransitiveClosure - Restore the array of points on the transitive closure of the in-edges or out-edges for this point in the Sieve DAG
1915 
1916   Not collective
1917 
1918   Input Parameters:
1919 + mesh - The DMPlex
1920 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1921 . useCone - PETSC_TRUE for in-edges,  otherwise use out-edges
1922 . numPoints - The number of points in the closure, so points[] is of size 2*numPoints, zeroed on exit
1923 - points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...], zeroed on exit
1924 
1925   Note:
1926   If not using internal storage (points is not NULL on input), this call is unnecessary
1927 
1928   Fortran Notes:
1929   Since it returns an array, this routine is only available in Fortran 90, and you must
1930   include petsc.h90 in your code.
1931 
1932   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
1933 
1934   Level: beginner
1935 
1936 .seealso: DMPlexGetTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
1937 @*/
1938 PetscErrorCode DMPlexRestoreTransitiveClosure(DM dm, PetscInt p, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
1939 {
1940   PetscErrorCode ierr;
1941 
1942   PetscFunctionBegin;
1943   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1944   if (numPoints) PetscValidIntPointer(numPoints,4);
1945   if (points) PetscValidPointer(points,5);
1946   ierr = DMRestoreWorkArray(dm, 0, PETSC_INT, points);CHKERRQ(ierr);
1947   if (numPoints) *numPoints = 0;
1948   PetscFunctionReturn(0);
1949 }
1950 
1951 #undef __FUNCT__
1952 #define __FUNCT__ "DMPlexGetMaxSizes"
1953 /*@
1954   DMPlexGetMaxSizes - Return the maximum number of in-edges (cone) and out-edges (support) for any point in the Sieve DAG
1955 
1956   Not collective
1957 
1958   Input Parameter:
1959 . mesh - The DMPlex
1960 
1961   Output Parameters:
1962 + maxConeSize - The maximum number of in-edges
1963 - maxSupportSize - The maximum number of out-edges
1964 
1965   Level: beginner
1966 
1967 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart()
1968 @*/
1969 PetscErrorCode DMPlexGetMaxSizes(DM dm, PetscInt *maxConeSize, PetscInt *maxSupportSize)
1970 {
1971   DM_Plex *mesh = (DM_Plex*) dm->data;
1972 
1973   PetscFunctionBegin;
1974   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1975   if (maxConeSize)    *maxConeSize    = mesh->maxConeSize;
1976   if (maxSupportSize) *maxSupportSize = mesh->maxSupportSize;
1977   PetscFunctionReturn(0);
1978 }
1979 
1980 #undef __FUNCT__
1981 #define __FUNCT__ "DMSetUp_Plex"
1982 PetscErrorCode DMSetUp_Plex(DM dm)
1983 {
1984   DM_Plex       *mesh = (DM_Plex*) dm->data;
1985   PetscInt       size;
1986   PetscErrorCode ierr;
1987 
1988   PetscFunctionBegin;
1989   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1990   ierr = PetscSectionSetUp(mesh->coneSection);CHKERRQ(ierr);
1991   ierr = PetscSectionGetStorageSize(mesh->coneSection, &size);CHKERRQ(ierr);
1992   ierr = PetscMalloc1(size, &mesh->cones);CHKERRQ(ierr);
1993   ierr = PetscCalloc1(size, &mesh->coneOrientations);CHKERRQ(ierr);
1994   if (mesh->maxSupportSize) {
1995     ierr = PetscSectionSetUp(mesh->supportSection);CHKERRQ(ierr);
1996     ierr = PetscSectionGetStorageSize(mesh->supportSection, &size);CHKERRQ(ierr);
1997     ierr = PetscMalloc1(size, &mesh->supports);CHKERRQ(ierr);
1998   }
1999   PetscFunctionReturn(0);
2000 }
2001 
2002 #undef __FUNCT__
2003 #define __FUNCT__ "DMCreateSubDM_Plex"
2004 PetscErrorCode DMCreateSubDM_Plex(DM dm, PetscInt numFields, PetscInt fields[], IS *is, DM *subdm)
2005 {
2006   PetscErrorCode ierr;
2007 
2008   PetscFunctionBegin;
2009   if (subdm) {ierr = DMClone(dm, subdm);CHKERRQ(ierr);}
2010   ierr = DMCreateSubDM_Section_Private(dm, numFields, fields, is, subdm);CHKERRQ(ierr);
2011   PetscFunctionReturn(0);
2012 }
2013 
2014 #undef __FUNCT__
2015 #define __FUNCT__ "DMPlexSymmetrize"
2016 /*@
2017   DMPlexSymmetrize - Creates support (out-edge) information from cone (in-edge) inoformation
2018 
2019   Not collective
2020 
2021   Input Parameter:
2022 . mesh - The DMPlex
2023 
2024   Output Parameter:
2025 
2026   Note:
2027   This should be called after all calls to DMPlexSetCone()
2028 
2029   Level: beginner
2030 
2031 .seealso: DMPlexCreate(), DMPlexSetChart(), DMPlexSetConeSize(), DMPlexSetCone()
2032 @*/
2033 PetscErrorCode DMPlexSymmetrize(DM dm)
2034 {
2035   DM_Plex       *mesh = (DM_Plex*) dm->data;
2036   PetscInt      *offsets;
2037   PetscInt       supportSize;
2038   PetscInt       pStart, pEnd, p;
2039   PetscErrorCode ierr;
2040 
2041   PetscFunctionBegin;
2042   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2043   if (mesh->supports) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONGSTATE, "Supports were already setup in this DMPlex");
2044   /* Calculate support sizes */
2045   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
2046   for (p = pStart; p < pEnd; ++p) {
2047     PetscInt dof, off, c;
2048 
2049     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2050     ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2051     for (c = off; c < off+dof; ++c) {
2052       ierr = PetscSectionAddDof(mesh->supportSection, mesh->cones[c], 1);CHKERRQ(ierr);
2053     }
2054   }
2055   for (p = pStart; p < pEnd; ++p) {
2056     PetscInt dof;
2057 
2058     ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
2059 
2060     mesh->maxSupportSize = PetscMax(mesh->maxSupportSize, dof);
2061   }
2062   ierr = PetscSectionSetUp(mesh->supportSection);CHKERRQ(ierr);
2063   /* Calculate supports */
2064   ierr = PetscSectionGetStorageSize(mesh->supportSection, &supportSize);CHKERRQ(ierr);
2065   ierr = PetscMalloc1(supportSize, &mesh->supports);CHKERRQ(ierr);
2066   ierr = PetscCalloc1(pEnd - pStart, &offsets);CHKERRQ(ierr);
2067   for (p = pStart; p < pEnd; ++p) {
2068     PetscInt dof, off, c;
2069 
2070     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2071     ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2072     for (c = off; c < off+dof; ++c) {
2073       const PetscInt q = mesh->cones[c];
2074       PetscInt       offS;
2075 
2076       ierr = PetscSectionGetOffset(mesh->supportSection, q, &offS);CHKERRQ(ierr);
2077 
2078       mesh->supports[offS+offsets[q]] = p;
2079       ++offsets[q];
2080     }
2081   }
2082   ierr = PetscFree(offsets);CHKERRQ(ierr);
2083   PetscFunctionReturn(0);
2084 }
2085 
2086 #undef __FUNCT__
2087 #define __FUNCT__ "DMPlexStratify"
2088 /*@
2089   DMPlexStratify - The Sieve DAG for most topologies is a graded poset (http://en.wikipedia.org/wiki/Graded_poset), and
2090   can be illustrated by Hasse Diagram (a http://en.wikipedia.org/wiki/Hasse_diagram). The strata group all points of the
2091   same grade, and this function calculates the strata. This grade can be seen as the height (or depth) of the point in
2092   the DAG.
2093 
2094   Not collective
2095 
2096   Input Parameter:
2097 . mesh - The DMPlex
2098 
2099   Output Parameter:
2100 
2101   Notes:
2102   The normal association for the point grade is element dimension (or co-dimension). For instance, all vertices would
2103   have depth 0, and all edges depth 1. Likewise, all cells heights would have height 0, and all faces height 1.
2104 
2105   This should be called after all calls to DMPlexSymmetrize()
2106 
2107   Level: beginner
2108 
2109 .seealso: DMPlexCreate(), DMPlexSymmetrize()
2110 @*/
2111 PetscErrorCode DMPlexStratify(DM dm)
2112 {
2113   DMLabel        label;
2114   PetscInt       pStart, pEnd, p;
2115   PetscInt       numRoots = 0, numLeaves = 0;
2116   PetscErrorCode ierr;
2117 
2118   PetscFunctionBegin;
2119   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2120   ierr = PetscLogEventBegin(DMPLEX_Stratify,dm,0,0,0);CHKERRQ(ierr);
2121   /* Calculate depth */
2122   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
2123   ierr = DMPlexCreateLabel(dm, "depth");CHKERRQ(ierr);
2124   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
2125   /* Initialize roots and count leaves */
2126   for (p = pStart; p < pEnd; ++p) {
2127     PetscInt coneSize, supportSize;
2128 
2129     ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
2130     ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
2131     if (!coneSize && supportSize) {
2132       ++numRoots;
2133       ierr = DMLabelSetValue(label, p, 0);CHKERRQ(ierr);
2134     } else if (!supportSize && coneSize) {
2135       ++numLeaves;
2136     } else if (!supportSize && !coneSize) {
2137       /* Isolated points */
2138       ierr = DMLabelSetValue(label, p, 0);CHKERRQ(ierr);
2139     }
2140   }
2141   if (numRoots + numLeaves == (pEnd - pStart)) {
2142     for (p = pStart; p < pEnd; ++p) {
2143       PetscInt coneSize, supportSize;
2144 
2145       ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
2146       ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
2147       if (!supportSize && coneSize) {
2148         ierr = DMLabelSetValue(label, p, 1);CHKERRQ(ierr);
2149       }
2150     }
2151   } else {
2152     IS       pointIS;
2153     PetscInt numPoints = 0, level = 0;
2154 
2155     ierr = DMLabelGetStratumIS(label, level, &pointIS);CHKERRQ(ierr);
2156     if (pointIS) {ierr = ISGetLocalSize(pointIS, &numPoints);CHKERRQ(ierr);}
2157     while (numPoints) {
2158       const PetscInt *points;
2159       const PetscInt  newLevel = level+1;
2160 
2161       ierr = ISGetIndices(pointIS, &points);CHKERRQ(ierr);
2162       for (p = 0; p < numPoints; ++p) {
2163         const PetscInt  point = points[p];
2164         const PetscInt *support;
2165         PetscInt        supportSize, s;
2166 
2167         ierr = DMPlexGetSupportSize(dm, point, &supportSize);CHKERRQ(ierr);
2168         ierr = DMPlexGetSupport(dm, point, &support);CHKERRQ(ierr);
2169         for (s = 0; s < supportSize; ++s) {
2170           ierr = DMLabelSetValue(label, support[s], newLevel);CHKERRQ(ierr);
2171         }
2172       }
2173       ierr = ISRestoreIndices(pointIS, &points);CHKERRQ(ierr);
2174       ++level;
2175       ierr = ISDestroy(&pointIS);CHKERRQ(ierr);
2176       ierr = DMLabelGetStratumIS(label, level, &pointIS);CHKERRQ(ierr);
2177       if (pointIS) {ierr = ISGetLocalSize(pointIS, &numPoints);CHKERRQ(ierr);}
2178       else         {numPoints = 0;}
2179     }
2180     ierr = ISDestroy(&pointIS);CHKERRQ(ierr);
2181   }
2182   ierr = PetscLogEventEnd(DMPLEX_Stratify,dm,0,0,0);CHKERRQ(ierr);
2183   PetscFunctionReturn(0);
2184 }
2185 
2186 #undef __FUNCT__
2187 #define __FUNCT__ "DMPlexGetJoin"
2188 /*@C
2189   DMPlexGetJoin - Get an array for the join of the set of points
2190 
2191   Not Collective
2192 
2193   Input Parameters:
2194 + dm - The DMPlex object
2195 . numPoints - The number of input points for the join
2196 - points - The input points
2197 
2198   Output Parameters:
2199 + numCoveredPoints - The number of points in the join
2200 - coveredPoints - The points in the join
2201 
2202   Level: intermediate
2203 
2204   Note: Currently, this is restricted to a single level join
2205 
2206   Fortran Notes:
2207   Since it returns an array, this routine is only available in Fortran 90, and you must
2208   include petsc.h90 in your code.
2209 
2210   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2211 
2212 .keywords: mesh
2213 .seealso: DMPlexRestoreJoin(), DMPlexGetMeet()
2214 @*/
2215 PetscErrorCode DMPlexGetJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
2216 {
2217   DM_Plex       *mesh = (DM_Plex*) dm->data;
2218   PetscInt      *join[2];
2219   PetscInt       joinSize, i = 0;
2220   PetscInt       dof, off, p, c, m;
2221   PetscErrorCode ierr;
2222 
2223   PetscFunctionBegin;
2224   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2225   PetscValidPointer(points, 2);
2226   PetscValidPointer(numCoveredPoints, 3);
2227   PetscValidPointer(coveredPoints, 4);
2228   ierr = DMGetWorkArray(dm, mesh->maxSupportSize, PETSC_INT, &join[0]);CHKERRQ(ierr);
2229   ierr = DMGetWorkArray(dm, mesh->maxSupportSize, PETSC_INT, &join[1]);CHKERRQ(ierr);
2230   /* Copy in support of first point */
2231   ierr = PetscSectionGetDof(mesh->supportSection, points[0], &dof);CHKERRQ(ierr);
2232   ierr = PetscSectionGetOffset(mesh->supportSection, points[0], &off);CHKERRQ(ierr);
2233   for (joinSize = 0; joinSize < dof; ++joinSize) {
2234     join[i][joinSize] = mesh->supports[off+joinSize];
2235   }
2236   /* Check each successive support */
2237   for (p = 1; p < numPoints; ++p) {
2238     PetscInt newJoinSize = 0;
2239 
2240     ierr = PetscSectionGetDof(mesh->supportSection, points[p], &dof);CHKERRQ(ierr);
2241     ierr = PetscSectionGetOffset(mesh->supportSection, points[p], &off);CHKERRQ(ierr);
2242     for (c = 0; c < dof; ++c) {
2243       const PetscInt point = mesh->supports[off+c];
2244 
2245       for (m = 0; m < joinSize; ++m) {
2246         if (point == join[i][m]) {
2247           join[1-i][newJoinSize++] = point;
2248           break;
2249         }
2250       }
2251     }
2252     joinSize = newJoinSize;
2253     i        = 1-i;
2254   }
2255   *numCoveredPoints = joinSize;
2256   *coveredPoints    = join[i];
2257   ierr              = DMRestoreWorkArray(dm, mesh->maxSupportSize, PETSC_INT, &join[1-i]);CHKERRQ(ierr);
2258   PetscFunctionReturn(0);
2259 }
2260 
2261 #undef __FUNCT__
2262 #define __FUNCT__ "DMPlexRestoreJoin"
2263 /*@C
2264   DMPlexRestoreJoin - Restore an array for the join of the set of points
2265 
2266   Not Collective
2267 
2268   Input Parameters:
2269 + dm - The DMPlex object
2270 . numPoints - The number of input points for the join
2271 - points - The input points
2272 
2273   Output Parameters:
2274 + numCoveredPoints - The number of points in the join
2275 - coveredPoints - The points in the join
2276 
2277   Fortran Notes:
2278   Since it returns an array, this routine is only available in Fortran 90, and you must
2279   include petsc.h90 in your code.
2280 
2281   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2282 
2283   Level: intermediate
2284 
2285 .keywords: mesh
2286 .seealso: DMPlexGetJoin(), DMPlexGetFullJoin(), DMPlexGetMeet()
2287 @*/
2288 PetscErrorCode DMPlexRestoreJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
2289 {
2290   PetscErrorCode ierr;
2291 
2292   PetscFunctionBegin;
2293   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2294   if (points) PetscValidIntPointer(points,3);
2295   if (numCoveredPoints) PetscValidIntPointer(numCoveredPoints,4);
2296   PetscValidPointer(coveredPoints, 5);
2297   ierr = DMRestoreWorkArray(dm, 0, PETSC_INT, (void*) coveredPoints);CHKERRQ(ierr);
2298   if (numCoveredPoints) *numCoveredPoints = 0;
2299   PetscFunctionReturn(0);
2300 }
2301 
2302 #undef __FUNCT__
2303 #define __FUNCT__ "DMPlexGetFullJoin"
2304 /*@C
2305   DMPlexGetFullJoin - Get an array for the join of the set of points
2306 
2307   Not Collective
2308 
2309   Input Parameters:
2310 + dm - The DMPlex object
2311 . numPoints - The number of input points for the join
2312 - points - The input points
2313 
2314   Output Parameters:
2315 + numCoveredPoints - The number of points in the join
2316 - coveredPoints - The points in the join
2317 
2318   Fortran Notes:
2319   Since it returns an array, this routine is only available in Fortran 90, and you must
2320   include petsc.h90 in your code.
2321 
2322   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2323 
2324   Level: intermediate
2325 
2326 .keywords: mesh
2327 .seealso: DMPlexGetJoin(), DMPlexRestoreJoin(), DMPlexGetMeet()
2328 @*/
2329 PetscErrorCode DMPlexGetFullJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
2330 {
2331   DM_Plex       *mesh = (DM_Plex*) dm->data;
2332   PetscInt      *offsets, **closures;
2333   PetscInt      *join[2];
2334   PetscInt       depth = 0, maxSize, joinSize = 0, i = 0;
2335   PetscInt       p, d, c, m;
2336   PetscErrorCode ierr;
2337 
2338   PetscFunctionBegin;
2339   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2340   PetscValidPointer(points, 2);
2341   PetscValidPointer(numCoveredPoints, 3);
2342   PetscValidPointer(coveredPoints, 4);
2343 
2344   ierr    = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
2345   ierr    = PetscCalloc1(numPoints, &closures);CHKERRQ(ierr);
2346   ierr    = DMGetWorkArray(dm, numPoints*(depth+2), PETSC_INT, &offsets);CHKERRQ(ierr);
2347   maxSize = PetscPowInt(mesh->maxSupportSize,depth+1);
2348   ierr    = DMGetWorkArray(dm, maxSize, PETSC_INT, &join[0]);CHKERRQ(ierr);
2349   ierr    = DMGetWorkArray(dm, maxSize, PETSC_INT, &join[1]);CHKERRQ(ierr);
2350 
2351   for (p = 0; p < numPoints; ++p) {
2352     PetscInt closureSize;
2353 
2354     ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_FALSE, &closureSize, &closures[p]);CHKERRQ(ierr);
2355 
2356     offsets[p*(depth+2)+0] = 0;
2357     for (d = 0; d < depth+1; ++d) {
2358       PetscInt pStart, pEnd, i;
2359 
2360       ierr = DMPlexGetDepthStratum(dm, d, &pStart, &pEnd);CHKERRQ(ierr);
2361       for (i = offsets[p*(depth+2)+d]; i < closureSize; ++i) {
2362         if ((pStart > closures[p][i*2]) || (pEnd <= closures[p][i*2])) {
2363           offsets[p*(depth+2)+d+1] = i;
2364           break;
2365         }
2366       }
2367       if (i == closureSize) offsets[p*(depth+2)+d+1] = i;
2368     }
2369     if (offsets[p*(depth+2)+depth+1] != closureSize) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Total size of closure %D should be %D", offsets[p*(depth+2)+depth+1], closureSize);
2370   }
2371   for (d = 0; d < depth+1; ++d) {
2372     PetscInt dof;
2373 
2374     /* Copy in support of first point */
2375     dof = offsets[d+1] - offsets[d];
2376     for (joinSize = 0; joinSize < dof; ++joinSize) {
2377       join[i][joinSize] = closures[0][(offsets[d]+joinSize)*2];
2378     }
2379     /* Check each successive cone */
2380     for (p = 1; p < numPoints && joinSize; ++p) {
2381       PetscInt newJoinSize = 0;
2382 
2383       dof = offsets[p*(depth+2)+d+1] - offsets[p*(depth+2)+d];
2384       for (c = 0; c < dof; ++c) {
2385         const PetscInt point = closures[p][(offsets[p*(depth+2)+d]+c)*2];
2386 
2387         for (m = 0; m < joinSize; ++m) {
2388           if (point == join[i][m]) {
2389             join[1-i][newJoinSize++] = point;
2390             break;
2391           }
2392         }
2393       }
2394       joinSize = newJoinSize;
2395       i        = 1-i;
2396     }
2397     if (joinSize) break;
2398   }
2399   *numCoveredPoints = joinSize;
2400   *coveredPoints    = join[i];
2401   for (p = 0; p < numPoints; ++p) {
2402     ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_FALSE, NULL, &closures[p]);CHKERRQ(ierr);
2403   }
2404   ierr = PetscFree(closures);CHKERRQ(ierr);
2405   ierr = DMRestoreWorkArray(dm, numPoints*(depth+2), PETSC_INT, &offsets);CHKERRQ(ierr);
2406   ierr = DMRestoreWorkArray(dm, mesh->maxSupportSize, PETSC_INT, &join[1-i]);CHKERRQ(ierr);
2407   PetscFunctionReturn(0);
2408 }
2409 
2410 #undef __FUNCT__
2411 #define __FUNCT__ "DMPlexGetMeet"
2412 /*@C
2413   DMPlexGetMeet - Get an array for the meet of the set of points
2414 
2415   Not Collective
2416 
2417   Input Parameters:
2418 + dm - The DMPlex object
2419 . numPoints - The number of input points for the meet
2420 - points - The input points
2421 
2422   Output Parameters:
2423 + numCoveredPoints - The number of points in the meet
2424 - coveredPoints - The points in the meet
2425 
2426   Level: intermediate
2427 
2428   Note: Currently, this is restricted to a single level meet
2429 
2430   Fortran Notes:
2431   Since it returns an array, this routine is only available in Fortran 90, and you must
2432   include petsc.h90 in your code.
2433 
2434   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2435 
2436 .keywords: mesh
2437 .seealso: DMPlexRestoreMeet(), DMPlexGetJoin()
2438 @*/
2439 PetscErrorCode DMPlexGetMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveringPoints, const PetscInt **coveringPoints)
2440 {
2441   DM_Plex       *mesh = (DM_Plex*) dm->data;
2442   PetscInt      *meet[2];
2443   PetscInt       meetSize, i = 0;
2444   PetscInt       dof, off, p, c, m;
2445   PetscErrorCode ierr;
2446 
2447   PetscFunctionBegin;
2448   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2449   PetscValidPointer(points, 2);
2450   PetscValidPointer(numCoveringPoints, 3);
2451   PetscValidPointer(coveringPoints, 4);
2452   ierr = DMGetWorkArray(dm, mesh->maxConeSize, PETSC_INT, &meet[0]);CHKERRQ(ierr);
2453   ierr = DMGetWorkArray(dm, mesh->maxConeSize, PETSC_INT, &meet[1]);CHKERRQ(ierr);
2454   /* Copy in cone of first point */
2455   ierr = PetscSectionGetDof(mesh->coneSection, points[0], &dof);CHKERRQ(ierr);
2456   ierr = PetscSectionGetOffset(mesh->coneSection, points[0], &off);CHKERRQ(ierr);
2457   for (meetSize = 0; meetSize < dof; ++meetSize) {
2458     meet[i][meetSize] = mesh->cones[off+meetSize];
2459   }
2460   /* Check each successive cone */
2461   for (p = 1; p < numPoints; ++p) {
2462     PetscInt newMeetSize = 0;
2463 
2464     ierr = PetscSectionGetDof(mesh->coneSection, points[p], &dof);CHKERRQ(ierr);
2465     ierr = PetscSectionGetOffset(mesh->coneSection, points[p], &off);CHKERRQ(ierr);
2466     for (c = 0; c < dof; ++c) {
2467       const PetscInt point = mesh->cones[off+c];
2468 
2469       for (m = 0; m < meetSize; ++m) {
2470         if (point == meet[i][m]) {
2471           meet[1-i][newMeetSize++] = point;
2472           break;
2473         }
2474       }
2475     }
2476     meetSize = newMeetSize;
2477     i        = 1-i;
2478   }
2479   *numCoveringPoints = meetSize;
2480   *coveringPoints    = meet[i];
2481   ierr               = DMRestoreWorkArray(dm, mesh->maxConeSize, PETSC_INT, &meet[1-i]);CHKERRQ(ierr);
2482   PetscFunctionReturn(0);
2483 }
2484 
2485 #undef __FUNCT__
2486 #define __FUNCT__ "DMPlexRestoreMeet"
2487 /*@C
2488   DMPlexRestoreMeet - Restore an array for the meet of the set of points
2489 
2490   Not Collective
2491 
2492   Input Parameters:
2493 + dm - The DMPlex object
2494 . numPoints - The number of input points for the meet
2495 - points - The input points
2496 
2497   Output Parameters:
2498 + numCoveredPoints - The number of points in the meet
2499 - coveredPoints - The points in the meet
2500 
2501   Level: intermediate
2502 
2503   Fortran Notes:
2504   Since it returns an array, this routine is only available in Fortran 90, and you must
2505   include petsc.h90 in your code.
2506 
2507   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2508 
2509 .keywords: mesh
2510 .seealso: DMPlexGetMeet(), DMPlexGetFullMeet(), DMPlexGetJoin()
2511 @*/
2512 PetscErrorCode DMPlexRestoreMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
2513 {
2514   PetscErrorCode ierr;
2515 
2516   PetscFunctionBegin;
2517   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2518   if (points) PetscValidIntPointer(points,3);
2519   if (numCoveredPoints) PetscValidIntPointer(numCoveredPoints,4);
2520   PetscValidPointer(coveredPoints,5);
2521   ierr = DMRestoreWorkArray(dm, 0, PETSC_INT, (void*) coveredPoints);CHKERRQ(ierr);
2522   if (numCoveredPoints) *numCoveredPoints = 0;
2523   PetscFunctionReturn(0);
2524 }
2525 
2526 #undef __FUNCT__
2527 #define __FUNCT__ "DMPlexGetFullMeet"
2528 /*@C
2529   DMPlexGetFullMeet - Get an array for the meet of the set of points
2530 
2531   Not Collective
2532 
2533   Input Parameters:
2534 + dm - The DMPlex object
2535 . numPoints - The number of input points for the meet
2536 - points - The input points
2537 
2538   Output Parameters:
2539 + numCoveredPoints - The number of points in the meet
2540 - coveredPoints - The points in the meet
2541 
2542   Level: intermediate
2543 
2544   Fortran Notes:
2545   Since it returns an array, this routine is only available in Fortran 90, and you must
2546   include petsc.h90 in your code.
2547 
2548   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2549 
2550 .keywords: mesh
2551 .seealso: DMPlexGetMeet(), DMPlexRestoreMeet(), DMPlexGetJoin()
2552 @*/
2553 PetscErrorCode DMPlexGetFullMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
2554 {
2555   DM_Plex       *mesh = (DM_Plex*) dm->data;
2556   PetscInt      *offsets, **closures;
2557   PetscInt      *meet[2];
2558   PetscInt       height = 0, maxSize, meetSize = 0, i = 0;
2559   PetscInt       p, h, c, m;
2560   PetscErrorCode ierr;
2561 
2562   PetscFunctionBegin;
2563   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2564   PetscValidPointer(points, 2);
2565   PetscValidPointer(numCoveredPoints, 3);
2566   PetscValidPointer(coveredPoints, 4);
2567 
2568   ierr    = DMPlexGetDepth(dm, &height);CHKERRQ(ierr);
2569   ierr    = PetscMalloc1(numPoints, &closures);CHKERRQ(ierr);
2570   ierr    = DMGetWorkArray(dm, numPoints*(height+2), PETSC_INT, &offsets);CHKERRQ(ierr);
2571   maxSize = PetscPowInt(mesh->maxConeSize,height+1);
2572   ierr    = DMGetWorkArray(dm, maxSize, PETSC_INT, &meet[0]);CHKERRQ(ierr);
2573   ierr    = DMGetWorkArray(dm, maxSize, PETSC_INT, &meet[1]);CHKERRQ(ierr);
2574 
2575   for (p = 0; p < numPoints; ++p) {
2576     PetscInt closureSize;
2577 
2578     ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_TRUE, &closureSize, &closures[p]);CHKERRQ(ierr);
2579 
2580     offsets[p*(height+2)+0] = 0;
2581     for (h = 0; h < height+1; ++h) {
2582       PetscInt pStart, pEnd, i;
2583 
2584       ierr = DMPlexGetHeightStratum(dm, h, &pStart, &pEnd);CHKERRQ(ierr);
2585       for (i = offsets[p*(height+2)+h]; i < closureSize; ++i) {
2586         if ((pStart > closures[p][i*2]) || (pEnd <= closures[p][i*2])) {
2587           offsets[p*(height+2)+h+1] = i;
2588           break;
2589         }
2590       }
2591       if (i == closureSize) offsets[p*(height+2)+h+1] = i;
2592     }
2593     if (offsets[p*(height+2)+height+1] != closureSize) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Total size of closure %D should be %D", offsets[p*(height+2)+height+1], closureSize);
2594   }
2595   for (h = 0; h < height+1; ++h) {
2596     PetscInt dof;
2597 
2598     /* Copy in cone of first point */
2599     dof = offsets[h+1] - offsets[h];
2600     for (meetSize = 0; meetSize < dof; ++meetSize) {
2601       meet[i][meetSize] = closures[0][(offsets[h]+meetSize)*2];
2602     }
2603     /* Check each successive cone */
2604     for (p = 1; p < numPoints && meetSize; ++p) {
2605       PetscInt newMeetSize = 0;
2606 
2607       dof = offsets[p*(height+2)+h+1] - offsets[p*(height+2)+h];
2608       for (c = 0; c < dof; ++c) {
2609         const PetscInt point = closures[p][(offsets[p*(height+2)+h]+c)*2];
2610 
2611         for (m = 0; m < meetSize; ++m) {
2612           if (point == meet[i][m]) {
2613             meet[1-i][newMeetSize++] = point;
2614             break;
2615           }
2616         }
2617       }
2618       meetSize = newMeetSize;
2619       i        = 1-i;
2620     }
2621     if (meetSize) break;
2622   }
2623   *numCoveredPoints = meetSize;
2624   *coveredPoints    = meet[i];
2625   for (p = 0; p < numPoints; ++p) {
2626     ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_TRUE, NULL, &closures[p]);CHKERRQ(ierr);
2627   }
2628   ierr = PetscFree(closures);CHKERRQ(ierr);
2629   ierr = DMRestoreWorkArray(dm, numPoints*(height+2), PETSC_INT, &offsets);CHKERRQ(ierr);
2630   ierr = DMRestoreWorkArray(dm, mesh->maxConeSize, PETSC_INT, &meet[1-i]);CHKERRQ(ierr);
2631   PetscFunctionReturn(0);
2632 }
2633 
2634 #undef __FUNCT__
2635 #define __FUNCT__ "DMPlexEqual"
2636 /*@C
2637   DMPlexEqual - Determine if two DMs have the same topology
2638 
2639   Not Collective
2640 
2641   Input Parameters:
2642 + dmA - A DMPlex object
2643 - dmB - A DMPlex object
2644 
2645   Output Parameters:
2646 . equal - PETSC_TRUE if the topologies are identical
2647 
2648   Level: intermediate
2649 
2650   Notes:
2651   We are not solving graph isomorphism, so we do not permutation.
2652 
2653 .keywords: mesh
2654 .seealso: DMPlexGetCone()
2655 @*/
2656 PetscErrorCode DMPlexEqual(DM dmA, DM dmB, PetscBool *equal)
2657 {
2658   PetscInt       depth, depthB, pStart, pEnd, pStartB, pEndB, p;
2659   PetscErrorCode ierr;
2660 
2661   PetscFunctionBegin;
2662   PetscValidHeaderSpecific(dmA, DM_CLASSID, 1);
2663   PetscValidHeaderSpecific(dmB, DM_CLASSID, 2);
2664   PetscValidPointer(equal, 3);
2665 
2666   *equal = PETSC_FALSE;
2667   ierr = DMPlexGetDepth(dmA, &depth);CHKERRQ(ierr);
2668   ierr = DMPlexGetDepth(dmB, &depthB);CHKERRQ(ierr);
2669   if (depth != depthB) PetscFunctionReturn(0);
2670   ierr = DMPlexGetChart(dmA, &pStart,  &pEnd);CHKERRQ(ierr);
2671   ierr = DMPlexGetChart(dmB, &pStartB, &pEndB);CHKERRQ(ierr);
2672   if ((pStart != pStartB) || (pEnd != pEndB)) PetscFunctionReturn(0);
2673   for (p = pStart; p < pEnd; ++p) {
2674     const PetscInt *cone, *coneB, *ornt, *orntB, *support, *supportB;
2675     PetscInt        coneSize, coneSizeB, c, supportSize, supportSizeB, s;
2676 
2677     ierr = DMPlexGetConeSize(dmA, p, &coneSize);CHKERRQ(ierr);
2678     ierr = DMPlexGetCone(dmA, p, &cone);CHKERRQ(ierr);
2679     ierr = DMPlexGetConeOrientation(dmA, p, &ornt);CHKERRQ(ierr);
2680     ierr = DMPlexGetConeSize(dmB, p, &coneSizeB);CHKERRQ(ierr);
2681     ierr = DMPlexGetCone(dmB, p, &coneB);CHKERRQ(ierr);
2682     ierr = DMPlexGetConeOrientation(dmB, p, &orntB);CHKERRQ(ierr);
2683     if (coneSize != coneSizeB) PetscFunctionReturn(0);
2684     for (c = 0; c < coneSize; ++c) {
2685       if (cone[c] != coneB[c]) PetscFunctionReturn(0);
2686       if (ornt[c] != orntB[c]) PetscFunctionReturn(0);
2687     }
2688     ierr = DMPlexGetSupportSize(dmA, p, &supportSize);CHKERRQ(ierr);
2689     ierr = DMPlexGetSupport(dmA, p, &support);CHKERRQ(ierr);
2690     ierr = DMPlexGetSupportSize(dmB, p, &supportSizeB);CHKERRQ(ierr);
2691     ierr = DMPlexGetSupport(dmB, p, &supportB);CHKERRQ(ierr);
2692     if (supportSize != supportSizeB) PetscFunctionReturn(0);
2693     for (s = 0; s < supportSize; ++s) {
2694       if (support[s] != supportB[s]) PetscFunctionReturn(0);
2695     }
2696   }
2697   *equal = PETSC_TRUE;
2698   PetscFunctionReturn(0);
2699 }
2700 
2701 #undef __FUNCT__
2702 #define __FUNCT__ "DMPlexGetNumFaceVertices"
2703 PetscErrorCode DMPlexGetNumFaceVertices(DM dm, PetscInt cellDim, PetscInt numCorners, PetscInt *numFaceVertices)
2704 {
2705   MPI_Comm       comm;
2706   PetscErrorCode ierr;
2707 
2708   PetscFunctionBegin;
2709   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
2710   PetscValidPointer(numFaceVertices,3);
2711   switch (cellDim) {
2712   case 0:
2713     *numFaceVertices = 0;
2714     break;
2715   case 1:
2716     *numFaceVertices = 1;
2717     break;
2718   case 2:
2719     switch (numCorners) {
2720     case 3: /* triangle */
2721       *numFaceVertices = 2; /* Edge has 2 vertices */
2722       break;
2723     case 4: /* quadrilateral */
2724       *numFaceVertices = 2; /* Edge has 2 vertices */
2725       break;
2726     case 6: /* quadratic triangle, tri and quad cohesive Lagrange cells */
2727       *numFaceVertices = 3; /* Edge has 3 vertices */
2728       break;
2729     case 9: /* quadratic quadrilateral, quadratic quad cohesive Lagrange cells */
2730       *numFaceVertices = 3; /* Edge has 3 vertices */
2731       break;
2732     default:
2733       SETERRQ2(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid number of face corners %d for dimension %d", numCorners, cellDim);
2734     }
2735     break;
2736   case 3:
2737     switch (numCorners) {
2738     case 4: /* tetradehdron */
2739       *numFaceVertices = 3; /* Face has 3 vertices */
2740       break;
2741     case 6: /* tet cohesive cells */
2742       *numFaceVertices = 4; /* Face has 4 vertices */
2743       break;
2744     case 8: /* hexahedron */
2745       *numFaceVertices = 4; /* Face has 4 vertices */
2746       break;
2747     case 9: /* tet cohesive Lagrange cells */
2748       *numFaceVertices = 6; /* Face has 6 vertices */
2749       break;
2750     case 10: /* quadratic tetrahedron */
2751       *numFaceVertices = 6; /* Face has 6 vertices */
2752       break;
2753     case 12: /* hex cohesive Lagrange cells */
2754       *numFaceVertices = 6; /* Face has 6 vertices */
2755       break;
2756     case 18: /* quadratic tet cohesive Lagrange cells */
2757       *numFaceVertices = 6; /* Face has 6 vertices */
2758       break;
2759     case 27: /* quadratic hexahedron, quadratic hex cohesive Lagrange cells */
2760       *numFaceVertices = 9; /* Face has 9 vertices */
2761       break;
2762     default:
2763       SETERRQ2(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid number of face corners %d for dimension %d", numCorners, cellDim);
2764     }
2765     break;
2766   default:
2767     SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid cell dimension %d", cellDim);
2768   }
2769   PetscFunctionReturn(0);
2770 }
2771 
2772 #undef __FUNCT__
2773 #define __FUNCT__ "DMPlexOrient"
2774 /* Trys to give the mesh a consistent orientation */
2775 PetscErrorCode DMPlexOrient(DM dm)
2776 {
2777   PetscBT        seenCells, flippedCells, seenFaces;
2778   PetscInt      *faceFIFO, fTop, fBottom;
2779   PetscInt       dim, h, cStart, cEnd, c, fStart, fEnd, face, maxConeSize, *revcone, *revconeO;
2780   PetscErrorCode ierr;
2781 
2782   PetscFunctionBegin;
2783   /* Truth Table
2784      mismatch    flips   do action   mismatch   flipA ^ flipB   action
2785          F       0 flips     no         F             F           F
2786          F       1 flip      yes        F             T           T
2787          F       2 flips     no         T             F           T
2788          T       0 flips     yes        T             T           F
2789          T       1 flip      no
2790          T       2 flips     yes
2791   */
2792   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
2793   ierr = DMPlexGetVTKCellHeight(dm, &h);CHKERRQ(ierr);
2794   ierr = DMPlexGetHeightStratum(dm, h,   &cStart, &cEnd);CHKERRQ(ierr);
2795   ierr = DMPlexGetHeightStratum(dm, h+1, &fStart, &fEnd);CHKERRQ(ierr);
2796   ierr = PetscBTCreate(cEnd - cStart, &seenCells);CHKERRQ(ierr);
2797   ierr = PetscBTMemzero(cEnd - cStart, seenCells);CHKERRQ(ierr);
2798   ierr = PetscBTCreate(cEnd - cStart, &flippedCells);CHKERRQ(ierr);
2799   ierr = PetscBTMemzero(cEnd - cStart, flippedCells);CHKERRQ(ierr);
2800   ierr = PetscBTCreate(fEnd - fStart, &seenFaces);CHKERRQ(ierr);
2801   ierr = PetscBTMemzero(fEnd - fStart, seenFaces);CHKERRQ(ierr);
2802   ierr = PetscMalloc1((fEnd - fStart), &faceFIFO);CHKERRQ(ierr);
2803   fTop = fBottom = 0;
2804   /* Initialize FIFO with first cell */
2805   if (cEnd > cStart) {
2806     const PetscInt *cone;
2807     PetscInt        coneSize;
2808 
2809     ierr = DMPlexGetConeSize(dm, cStart, &coneSize);CHKERRQ(ierr);
2810     ierr = DMPlexGetCone(dm, cStart, &cone);CHKERRQ(ierr);
2811     for (c = 0; c < coneSize; ++c) {
2812       faceFIFO[fBottom++] = cone[c];
2813       ierr = PetscBTSet(seenFaces, cone[c]-fStart);CHKERRQ(ierr);
2814     }
2815   }
2816   /* Consider each face in FIFO */
2817   while (fTop < fBottom) {
2818     const PetscInt *support, *coneA, *coneB, *coneOA, *coneOB;
2819     PetscInt        supportSize, coneSizeA, coneSizeB, posA = -1, posB = -1;
2820     PetscInt        seenA, flippedA, seenB, flippedB, mismatch;
2821 
2822     face = faceFIFO[fTop++];
2823     ierr = DMPlexGetSupportSize(dm, face, &supportSize);CHKERRQ(ierr);
2824     ierr = DMPlexGetSupport(dm, face, &support);CHKERRQ(ierr);
2825     if (supportSize < 2) continue;
2826     if (supportSize != 2) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Faces should separate only two cells, not %d", supportSize);
2827     seenA    = PetscBTLookup(seenCells,    support[0]-cStart);
2828     flippedA = PetscBTLookup(flippedCells, support[0]-cStart) ? 1 : 0;
2829     seenB    = PetscBTLookup(seenCells,    support[1]-cStart);
2830     flippedB = PetscBTLookup(flippedCells, support[1]-cStart) ? 1 : 0;
2831 
2832     ierr = DMPlexGetConeSize(dm, support[0], &coneSizeA);CHKERRQ(ierr);
2833     ierr = DMPlexGetConeSize(dm, support[1], &coneSizeB);CHKERRQ(ierr);
2834     ierr = DMPlexGetCone(dm, support[0], &coneA);CHKERRQ(ierr);
2835     ierr = DMPlexGetCone(dm, support[1], &coneB);CHKERRQ(ierr);
2836     ierr = DMPlexGetConeOrientation(dm, support[0], &coneOA);CHKERRQ(ierr);
2837     ierr = DMPlexGetConeOrientation(dm, support[1], &coneOB);CHKERRQ(ierr);
2838     for (c = 0; c < coneSizeA; ++c) {
2839       if (!PetscBTLookup(seenFaces, coneA[c]-fStart)) {
2840         faceFIFO[fBottom++] = coneA[c];
2841         ierr = PetscBTSet(seenFaces, coneA[c]-fStart);CHKERRQ(ierr);
2842       }
2843       if (coneA[c] == face) posA = c;
2844       if (fBottom > fEnd-fStart) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Face %d was pushed exceeding capacity %d > %d", coneA[c], fBottom, fEnd-fStart);
2845     }
2846     if (posA < 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %d could not be located in cell %d", face, support[0]);
2847     for (c = 0; c < coneSizeB; ++c) {
2848       if (!PetscBTLookup(seenFaces, coneB[c]-fStart)) {
2849         faceFIFO[fBottom++] = coneB[c];
2850         ierr = PetscBTSet(seenFaces, coneB[c]-fStart);CHKERRQ(ierr);
2851       }
2852       if (coneB[c] == face) posB = c;
2853       if (fBottom > fEnd-fStart) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Face %d was pushed exceeding capacity %d > %d", coneA[c], fBottom, fEnd-fStart);
2854     }
2855     if (posB < 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %d could not be located in cell %d", face, support[1]);
2856 
2857     if (dim == 1) {
2858       mismatch = posA == posB;
2859     } else {
2860       mismatch = coneOA[posA] == coneOB[posB];
2861     }
2862 
2863     if (mismatch ^ (flippedA ^ flippedB)) {
2864       if (seenA && seenB) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Previously seen cells %d and %d do not match: Fault mesh is non-orientable", support[0], support[1]);
2865       if (!seenA && !flippedA) {
2866         ierr = PetscBTSet(flippedCells, support[0]-cStart);CHKERRQ(ierr);
2867       } else if (!seenB && !flippedB) {
2868         ierr = PetscBTSet(flippedCells, support[1]-cStart);CHKERRQ(ierr);
2869       } else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Inconsistent mesh orientation: Fault mesh is non-orientable");
2870     } else if (mismatch && flippedA && flippedB) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Attempt to flip already flipped cell: Fault mesh is non-orientable");
2871     ierr = PetscBTSet(seenCells, support[0]-cStart);CHKERRQ(ierr);
2872     ierr = PetscBTSet(seenCells, support[1]-cStart);CHKERRQ(ierr);
2873   }
2874   /* Now all subdomains are oriented, but we need a consistent parallel orientation */
2875   {
2876     /* Find a representative face (edge) separating pairs of procs */
2877     PetscSF            sf;
2878     const PetscInt    *lpoints;
2879     const PetscSFNode *rpoints;
2880     PetscInt          *neighbors, *nranks;
2881     PetscInt           numLeaves, numRoots, numNeighbors = 0, l, n;
2882 
2883     ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr);
2884     ierr = PetscSFGetGraph(sf, &numRoots, &numLeaves, &lpoints, &rpoints);CHKERRQ(ierr);
2885     if (numLeaves >= 0) {
2886       const PetscInt *cone, *ornt, *support;
2887       PetscInt        coneSize, supportSize;
2888       int            *rornt, *lornt; /* PetscSF cannot handle smaller than int */
2889       PetscBool      *match, flipped = PETSC_FALSE;
2890 
2891       ierr = PetscMalloc1(numLeaves,&neighbors);CHKERRQ(ierr);
2892       /* I know this is p^2 time in general, but for bounded degree its alright */
2893       for (l = 0; l < numLeaves; ++l) {
2894         const PetscInt face = lpoints[l];
2895         if ((face >= fStart) && (face < fEnd)) {
2896           const PetscInt rank = rpoints[l].rank;
2897           for (n = 0; n < numNeighbors; ++n) if (rank == rpoints[neighbors[n]].rank) break;
2898           if (n >= numNeighbors) {
2899             PetscInt supportSize;
2900             ierr = DMPlexGetSupportSize(dm, face, &supportSize);CHKERRQ(ierr);
2901             if (supportSize != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Boundary faces should see one cell, not %d", supportSize);
2902             neighbors[numNeighbors++] = l;
2903           }
2904         }
2905       }
2906       ierr = PetscCalloc4(numNeighbors,&match,numNeighbors,&nranks,numRoots,&rornt,numRoots,&lornt);CHKERRQ(ierr);
2907       for (face = fStart; face < fEnd; ++face) {
2908         ierr = DMPlexGetSupportSize(dm, face, &supportSize);CHKERRQ(ierr);
2909         if (supportSize != 1) continue;
2910         ierr = DMPlexGetSupport(dm, face, &support);CHKERRQ(ierr);
2911 
2912         ierr = DMPlexGetCone(dm, support[0], &cone);CHKERRQ(ierr);
2913         ierr = DMPlexGetConeSize(dm, support[0], &coneSize);CHKERRQ(ierr);
2914         ierr = DMPlexGetConeOrientation(dm, support[0], &ornt);CHKERRQ(ierr);
2915         for (c = 0; c < coneSize; ++c) if (cone[c] == face) break;
2916         if (dim == 1) {
2917           /* Use cone position instead, shifted to -1 or 1 */
2918           rornt[face] = c*2-1;
2919         } else {
2920           if (PetscBTLookup(flippedCells, support[0]-cStart)) rornt[face] = ornt[c] < 0 ? -1 :  1;
2921           else                                                rornt[face] = ornt[c] < 0 ?  1 : -1;
2922         }
2923       }
2924       /* Mark each edge with match or nomatch */
2925       ierr = PetscSFBcastBegin(sf, MPI_INT, rornt, lornt);CHKERRQ(ierr);
2926       ierr = PetscSFBcastEnd(sf, MPI_INT, rornt, lornt);CHKERRQ(ierr);
2927       for (n = 0; n < numNeighbors; ++n) {
2928         const PetscInt face = lpoints[neighbors[n]];
2929 
2930         if (rornt[face]*lornt[face] < 0) match[n] = PETSC_TRUE;
2931         else                             match[n] = PETSC_FALSE;
2932         nranks[n] = rpoints[neighbors[n]].rank;
2933       }
2934       /* Collect the graph on 0 */
2935       {
2936         MPI_Comm     comm = PetscObjectComm((PetscObject) sf);
2937         PetscBT      seenProcs, flippedProcs;
2938         PetscInt    *procFIFO, pTop, pBottom;
2939         PetscInt    *adj = NULL;
2940         PetscBool   *val = NULL;
2941         PetscMPIInt *recvcounts = NULL, *displs = NULL, p;
2942         PetscMPIInt  N = numNeighbors, numProcs = 0, rank;
2943         PetscInt     debug = 0;
2944 
2945         ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
2946         if (!rank) {ierr = MPI_Comm_size(comm, &numProcs);CHKERRQ(ierr);}
2947         ierr = PetscCalloc2(numProcs,&recvcounts,numProcs+1,&displs);CHKERRQ(ierr);
2948         ierr = MPI_Gather(&N, 1, MPI_INT, recvcounts, 1, MPI_INT, 0, comm);CHKERRQ(ierr);
2949         for (p = 0; p < numProcs; ++p) {
2950           displs[p+1] = displs[p] + recvcounts[p];
2951         }
2952         if (!rank) {ierr = PetscMalloc2(displs[numProcs],&adj,displs[numProcs],&val);CHKERRQ(ierr);}
2953         ierr = MPI_Gatherv(nranks, numNeighbors, MPIU_INT, adj, recvcounts, displs, MPIU_INT, 0, comm);CHKERRQ(ierr);
2954         ierr = MPI_Gatherv(match, numNeighbors, MPIU_BOOL, val, recvcounts, displs, MPIU_BOOL, 0, comm);CHKERRQ(ierr);
2955         if (debug) {
2956           for (p = 0; p < numProcs; ++p) {
2957             ierr = PetscPrintf(comm, "Proc %d:\n", p);
2958             for (n = 0; n < recvcounts[p]; ++n) {
2959               ierr = PetscPrintf(comm, "  edge %d (%d):\n", adj[displs[p]+n], val[displs[p]+n]);
2960             }
2961           }
2962         }
2963         ierr = PetscBTCreate(numProcs, &seenProcs);CHKERRQ(ierr);
2964         ierr = PetscBTMemzero(numProcs, seenProcs);CHKERRQ(ierr);
2965         ierr = PetscBTCreate(numProcs, &flippedProcs);CHKERRQ(ierr);
2966         ierr = PetscBTMemzero(numProcs, flippedProcs);CHKERRQ(ierr);
2967         ierr = PetscMalloc1(numProcs,&procFIFO);CHKERRQ(ierr);
2968         pTop = pBottom = 0;
2969         for (p = 0; p < numProcs; ++p) {
2970           if (PetscBTLookup(seenProcs, p)) continue;
2971           /* Initialize FIFO with next proc */
2972           procFIFO[pBottom++] = p;
2973           ierr = PetscBTSet(seenProcs, p);CHKERRQ(ierr);
2974           /* Consider each proc in FIFO */
2975           while (pTop < pBottom) {
2976             PetscInt proc, nproc, seen, flippedA, flippedB, mismatch;
2977 
2978             proc     = procFIFO[pTop++];
2979             flippedA = PetscBTLookup(flippedProcs, proc) ? 1 : 0;
2980             /* Loop over neighboring procs */
2981             for (n = 0; n < recvcounts[proc]; ++n) {
2982               nproc    = adj[displs[proc]+n];
2983               mismatch = val[displs[proc]+n] ? 0 : 1;
2984               seen     = PetscBTLookup(seenProcs, nproc);
2985               flippedB = PetscBTLookup(flippedProcs, nproc) ? 1 : 0;
2986 
2987               if (mismatch ^ (flippedA ^ flippedB)) {
2988                 if (seen) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Previously seen procs %d and %d do not match: Fault mesh is non-orientable", proc, nproc);
2989                 if (!flippedB) {
2990                   ierr = PetscBTSet(flippedProcs, nproc);CHKERRQ(ierr);
2991               } else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Inconsistent mesh orientation: Fault mesh is non-orientable");
2992               } else if (mismatch && flippedA && flippedB) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Attempt to flip already flipped cell: Fault mesh is non-orientable");
2993               if (!seen) {
2994                 procFIFO[pBottom++] = nproc;
2995                 ierr = PetscBTSet(seenProcs, nproc);CHKERRQ(ierr);
2996               }
2997             }
2998           }
2999         }
3000         ierr = PetscFree(procFIFO);CHKERRQ(ierr);
3001 
3002         ierr = PetscFree2(recvcounts,displs);CHKERRQ(ierr);
3003         ierr = PetscFree2(adj,val);CHKERRQ(ierr);
3004         {
3005           PetscBool *flips;
3006 
3007           ierr = PetscMalloc1(numProcs,&flips);CHKERRQ(ierr);
3008           for (p = 0; p < numProcs; ++p) {
3009             flips[p] = PetscBTLookup(flippedProcs, p) ? PETSC_TRUE : PETSC_FALSE;
3010             if (debug && flips[p]) {ierr = PetscPrintf(comm, "Flipping Proc %d:\n", p);}
3011           }
3012           ierr = MPI_Scatter(flips, 1, MPIU_BOOL, &flipped, 1, MPIU_BOOL, 0, comm);CHKERRQ(ierr);
3013           ierr = PetscFree(flips);CHKERRQ(ierr);
3014         }
3015         ierr = PetscBTDestroy(&seenProcs);CHKERRQ(ierr);
3016         ierr = PetscBTDestroy(&flippedProcs);CHKERRQ(ierr);
3017       }
3018       ierr = PetscFree4(match,nranks,rornt,lornt);CHKERRQ(ierr);
3019       ierr = PetscFree(neighbors);CHKERRQ(ierr);
3020       if (flipped) {for (c = cStart; c < cEnd; ++c) {ierr = PetscBTNegate(flippedCells, c-cStart);CHKERRQ(ierr);}}
3021     }
3022   }
3023   /* Reverse flipped cells in the mesh */
3024   ierr = DMPlexGetMaxSizes(dm, &maxConeSize, NULL);CHKERRQ(ierr);
3025   ierr = DMGetWorkArray(dm, maxConeSize, PETSC_INT, &revcone);CHKERRQ(ierr);
3026   ierr = DMGetWorkArray(dm, maxConeSize, PETSC_INT, &revconeO);CHKERRQ(ierr);
3027   for (c = cStart; c < cEnd; ++c) {
3028     const PetscInt *cone, *coneO, *support;
3029     PetscInt        coneSize, supportSize, faceSize, cp, sp;
3030 
3031     if (!PetscBTLookup(flippedCells, c-cStart)) continue;
3032     ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
3033     ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
3034     ierr = DMPlexGetConeOrientation(dm, c, &coneO);CHKERRQ(ierr);
3035     for (cp = 0; cp < coneSize; ++cp) {
3036       const PetscInt rcp = coneSize-cp-1;
3037 
3038       ierr = DMPlexGetConeSize(dm, cone[rcp], &faceSize);CHKERRQ(ierr);
3039       revcone[cp]  = cone[rcp];
3040       revconeO[cp] = coneO[rcp] >= 0 ? -(faceSize-coneO[rcp]) : faceSize+coneO[rcp];
3041     }
3042     ierr = DMPlexSetCone(dm, c, revcone);CHKERRQ(ierr);
3043     ierr = DMPlexSetConeOrientation(dm, c, revconeO);CHKERRQ(ierr);
3044     /* Reverse orientations of support */
3045     faceSize = coneSize;
3046     ierr = DMPlexGetSupportSize(dm, c, &supportSize);CHKERRQ(ierr);
3047     ierr = DMPlexGetSupport(dm, c, &support);CHKERRQ(ierr);
3048     for (sp = 0; sp < supportSize; ++sp) {
3049       ierr = DMPlexGetConeSize(dm, support[sp], &coneSize);CHKERRQ(ierr);
3050       ierr = DMPlexGetCone(dm, support[sp], &cone);CHKERRQ(ierr);
3051       ierr = DMPlexGetConeOrientation(dm, support[sp], &coneO);CHKERRQ(ierr);
3052       for (cp = 0; cp < coneSize; ++cp) {
3053         if (cone[cp] != c) continue;
3054         ierr = DMPlexInsertConeOrientation(dm, support[sp], cp, coneO[cp] >= 0 ? -(faceSize-coneO[cp]) : faceSize+coneO[cp]);CHKERRQ(ierr);
3055       }
3056     }
3057   }
3058   ierr = DMRestoreWorkArray(dm, maxConeSize, PETSC_INT, &revcone);CHKERRQ(ierr);
3059   ierr = DMRestoreWorkArray(dm, maxConeSize, PETSC_INT, &revconeO);CHKERRQ(ierr);
3060   ierr = PetscBTDestroy(&seenCells);CHKERRQ(ierr);
3061   ierr = PetscBTDestroy(&flippedCells);CHKERRQ(ierr);
3062   ierr = PetscBTDestroy(&seenFaces);CHKERRQ(ierr);
3063   ierr = PetscFree(faceFIFO);CHKERRQ(ierr);
3064   PetscFunctionReturn(0);
3065 }
3066 
3067 #undef __FUNCT__
3068 #define __FUNCT__ "DMPlexInvertCell"
3069 /*@C
3070   DMPlexInvertCell - This flips tetrahedron and hexahedron orientation since Plex stores them internally with outward normals. Other cells are left untouched.
3071 
3072   Input Parameters:
3073 + numCorners - The number of vertices in a cell
3074 - cone - The incoming cone
3075 
3076   Output Parameter:
3077 . cone - The inverted cone (in-place)
3078 
3079   Level: developer
3080 
3081 .seealso: DMPlexGenerate()
3082 @*/
3083 PetscErrorCode DMPlexInvertCell(PetscInt dim, PetscInt numCorners, int cone[])
3084 {
3085   int tmpc;
3086 
3087   PetscFunctionBegin;
3088   if (dim != 3) PetscFunctionReturn(0);
3089   switch (numCorners) {
3090   case 4:
3091     tmpc    = cone[0];
3092     cone[0] = cone[1];
3093     cone[1] = tmpc;
3094     break;
3095   case 8:
3096     tmpc    = cone[1];
3097     cone[1] = cone[3];
3098     cone[3] = tmpc;
3099     break;
3100   default: break;
3101   }
3102   PetscFunctionReturn(0);
3103 }
3104 
3105 #undef __FUNCT__
3106 #define __FUNCT__ "DMPlexInvertCells_Internal"
3107 /* This is to fix the tetrahedron orientation from TetGen */
3108 PETSC_UNUSED static PetscErrorCode DMPlexInvertCells_Internal(PetscInt dim, PetscInt numCells, PetscInt numCorners, int cells[])
3109 {
3110   PetscInt       bound = numCells*numCorners, coff;
3111   PetscErrorCode ierr;
3112 
3113   PetscFunctionBegin;
3114   for (coff = 0; coff < bound; coff += numCorners) {
3115     ierr = DMPlexInvertCell(dim, numCorners, &cells[coff]);CHKERRQ(ierr);
3116   }
3117   PetscFunctionReturn(0);
3118 }
3119 
3120 #if defined(PETSC_HAVE_TRIANGLE)
3121 #include <triangle.h>
3122 
3123 #undef __FUNCT__
3124 #define __FUNCT__ "InitInput_Triangle"
3125 PetscErrorCode InitInput_Triangle(struct triangulateio *inputCtx)
3126 {
3127   PetscFunctionBegin;
3128   inputCtx->numberofpoints             = 0;
3129   inputCtx->numberofpointattributes    = 0;
3130   inputCtx->pointlist                  = NULL;
3131   inputCtx->pointattributelist         = NULL;
3132   inputCtx->pointmarkerlist            = NULL;
3133   inputCtx->numberofsegments           = 0;
3134   inputCtx->segmentlist                = NULL;
3135   inputCtx->segmentmarkerlist          = NULL;
3136   inputCtx->numberoftriangleattributes = 0;
3137   inputCtx->trianglelist               = NULL;
3138   inputCtx->numberofholes              = 0;
3139   inputCtx->holelist                   = NULL;
3140   inputCtx->numberofregions            = 0;
3141   inputCtx->regionlist                 = NULL;
3142   PetscFunctionReturn(0);
3143 }
3144 
3145 #undef __FUNCT__
3146 #define __FUNCT__ "InitOutput_Triangle"
3147 PetscErrorCode InitOutput_Triangle(struct triangulateio *outputCtx)
3148 {
3149   PetscFunctionBegin;
3150   outputCtx->numberofpoints        = 0;
3151   outputCtx->pointlist             = NULL;
3152   outputCtx->pointattributelist    = NULL;
3153   outputCtx->pointmarkerlist       = NULL;
3154   outputCtx->numberoftriangles     = 0;
3155   outputCtx->trianglelist          = NULL;
3156   outputCtx->triangleattributelist = NULL;
3157   outputCtx->neighborlist          = NULL;
3158   outputCtx->segmentlist           = NULL;
3159   outputCtx->segmentmarkerlist     = NULL;
3160   outputCtx->numberofedges         = 0;
3161   outputCtx->edgelist              = NULL;
3162   outputCtx->edgemarkerlist        = NULL;
3163   PetscFunctionReturn(0);
3164 }
3165 
3166 #undef __FUNCT__
3167 #define __FUNCT__ "FiniOutput_Triangle"
3168 PetscErrorCode FiniOutput_Triangle(struct triangulateio *outputCtx)
3169 {
3170   PetscFunctionBegin;
3171   free(outputCtx->pointlist);
3172   free(outputCtx->pointmarkerlist);
3173   free(outputCtx->segmentlist);
3174   free(outputCtx->segmentmarkerlist);
3175   free(outputCtx->edgelist);
3176   free(outputCtx->edgemarkerlist);
3177   free(outputCtx->trianglelist);
3178   free(outputCtx->neighborlist);
3179   PetscFunctionReturn(0);
3180 }
3181 
3182 #undef __FUNCT__
3183 #define __FUNCT__ "DMPlexGenerate_Triangle"
3184 PetscErrorCode DMPlexGenerate_Triangle(DM boundary, PetscBool interpolate, DM *dm)
3185 {
3186   MPI_Comm             comm;
3187   PetscInt             dim              = 2;
3188   const PetscBool      createConvexHull = PETSC_FALSE;
3189   const PetscBool      constrained      = PETSC_FALSE;
3190   struct triangulateio in;
3191   struct triangulateio out;
3192   PetscInt             vStart, vEnd, v, eStart, eEnd, e;
3193   PetscMPIInt          rank;
3194   PetscErrorCode       ierr;
3195 
3196   PetscFunctionBegin;
3197   ierr = PetscObjectGetComm((PetscObject)boundary,&comm);CHKERRQ(ierr);
3198   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
3199   ierr = InitInput_Triangle(&in);CHKERRQ(ierr);
3200   ierr = InitOutput_Triangle(&out);CHKERRQ(ierr);
3201   ierr = DMPlexGetDepthStratum(boundary, 0, &vStart, &vEnd);CHKERRQ(ierr);
3202 
3203   in.numberofpoints = vEnd - vStart;
3204   if (in.numberofpoints > 0) {
3205     PetscSection coordSection;
3206     Vec          coordinates;
3207     PetscScalar *array;
3208 
3209     ierr = PetscMalloc1(in.numberofpoints*dim, &in.pointlist);CHKERRQ(ierr);
3210     ierr = PetscMalloc1(in.numberofpoints, &in.pointmarkerlist);CHKERRQ(ierr);
3211     ierr = DMGetCoordinatesLocal(boundary, &coordinates);CHKERRQ(ierr);
3212     ierr = DMGetCoordinateSection(boundary, &coordSection);CHKERRQ(ierr);
3213     ierr = VecGetArray(coordinates, &array);CHKERRQ(ierr);
3214     for (v = vStart; v < vEnd; ++v) {
3215       const PetscInt idx = v - vStart;
3216       PetscInt       off, d;
3217 
3218       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
3219       for (d = 0; d < dim; ++d) {
3220         in.pointlist[idx*dim + d] = PetscRealPart(array[off+d]);
3221       }
3222       ierr = DMPlexGetLabelValue(boundary, "marker", v, &in.pointmarkerlist[idx]);CHKERRQ(ierr);
3223     }
3224     ierr = VecRestoreArray(coordinates, &array);CHKERRQ(ierr);
3225   }
3226   ierr  = DMPlexGetHeightStratum(boundary, 0, &eStart, &eEnd);CHKERRQ(ierr);
3227   in.numberofsegments = eEnd - eStart;
3228   if (in.numberofsegments > 0) {
3229     ierr = PetscMalloc1(in.numberofsegments*2, &in.segmentlist);CHKERRQ(ierr);
3230     ierr = PetscMalloc1(in.numberofsegments, &in.segmentmarkerlist);CHKERRQ(ierr);
3231     for (e = eStart; e < eEnd; ++e) {
3232       const PetscInt  idx = e - eStart;
3233       const PetscInt *cone;
3234 
3235       ierr = DMPlexGetCone(boundary, e, &cone);CHKERRQ(ierr);
3236 
3237       in.segmentlist[idx*2+0] = cone[0] - vStart;
3238       in.segmentlist[idx*2+1] = cone[1] - vStart;
3239 
3240       ierr = DMPlexGetLabelValue(boundary, "marker", e, &in.segmentmarkerlist[idx]);CHKERRQ(ierr);
3241     }
3242   }
3243 #if 0 /* Do not currently support holes */
3244   PetscReal *holeCoords;
3245   PetscInt   h, d;
3246 
3247   ierr = DMPlexGetHoles(boundary, &in.numberofholes, &holeCords);CHKERRQ(ierr);
3248   if (in.numberofholes > 0) {
3249     ierr = PetscMalloc1(in.numberofholes*dim, &in.holelist);CHKERRQ(ierr);
3250     for (h = 0; h < in.numberofholes; ++h) {
3251       for (d = 0; d < dim; ++d) {
3252         in.holelist[h*dim+d] = holeCoords[h*dim+d];
3253       }
3254     }
3255   }
3256 #endif
3257   if (!rank) {
3258     char args[32];
3259 
3260     /* Take away 'Q' for verbose output */
3261     ierr = PetscStrcpy(args, "pqezQ");CHKERRQ(ierr);
3262     if (createConvexHull) {
3263       ierr = PetscStrcat(args, "c");CHKERRQ(ierr);
3264     }
3265     if (constrained) {
3266       ierr = PetscStrcpy(args, "zepDQ");CHKERRQ(ierr);
3267     }
3268     triangulate(args, &in, &out, NULL);
3269   }
3270   ierr = PetscFree(in.pointlist);CHKERRQ(ierr);
3271   ierr = PetscFree(in.pointmarkerlist);CHKERRQ(ierr);
3272   ierr = PetscFree(in.segmentlist);CHKERRQ(ierr);
3273   ierr = PetscFree(in.segmentmarkerlist);CHKERRQ(ierr);
3274   ierr = PetscFree(in.holelist);CHKERRQ(ierr);
3275 
3276   {
3277     const PetscInt numCorners  = 3;
3278     const PetscInt numCells    = out.numberoftriangles;
3279     const PetscInt numVertices = out.numberofpoints;
3280     const int     *cells      = out.trianglelist;
3281     const double  *meshCoords = out.pointlist;
3282 
3283     ierr = DMPlexCreateFromCellList(comm, dim, numCells, numVertices, numCorners, interpolate, cells, dim, meshCoords, dm);CHKERRQ(ierr);
3284     /* Set labels */
3285     for (v = 0; v < numVertices; ++v) {
3286       if (out.pointmarkerlist[v]) {
3287         ierr = DMPlexSetLabelValue(*dm, "marker", v+numCells, out.pointmarkerlist[v]);CHKERRQ(ierr);
3288       }
3289     }
3290     if (interpolate) {
3291       for (e = 0; e < out.numberofedges; e++) {
3292         if (out.edgemarkerlist[e]) {
3293           const PetscInt  vertices[2] = {out.edgelist[e*2+0]+numCells, out.edgelist[e*2+1]+numCells};
3294           const PetscInt *edges;
3295           PetscInt        numEdges;
3296 
3297           ierr = DMPlexGetJoin(*dm, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3298           if (numEdges != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Two vertices must cover only one edge, not %D", numEdges);
3299           ierr = DMPlexSetLabelValue(*dm, "marker", edges[0], out.edgemarkerlist[e]);CHKERRQ(ierr);
3300           ierr = DMPlexRestoreJoin(*dm, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3301         }
3302       }
3303     }
3304     ierr = DMPlexSetRefinementUniform(*dm, PETSC_FALSE);CHKERRQ(ierr);
3305   }
3306 #if 0 /* Do not currently support holes */
3307   ierr = DMPlexCopyHoles(*dm, boundary);CHKERRQ(ierr);
3308 #endif
3309   ierr = FiniOutput_Triangle(&out);CHKERRQ(ierr);
3310   PetscFunctionReturn(0);
3311 }
3312 
3313 #undef __FUNCT__
3314 #define __FUNCT__ "DMPlexRefine_Triangle"
3315 PetscErrorCode DMPlexRefine_Triangle(DM dm, double *maxVolumes, DM *dmRefined)
3316 {
3317   MPI_Comm             comm;
3318   PetscInt             dim  = 2;
3319   struct triangulateio in;
3320   struct triangulateio out;
3321   PetscInt             vStart, vEnd, v, cStart, cEnd, c, depth, depthGlobal;
3322   PetscMPIInt          rank;
3323   PetscErrorCode       ierr;
3324 
3325   PetscFunctionBegin;
3326   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
3327   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
3328   ierr = InitInput_Triangle(&in);CHKERRQ(ierr);
3329   ierr = InitOutput_Triangle(&out);CHKERRQ(ierr);
3330   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3331   ierr = MPI_Allreduce(&depth, &depthGlobal, 1, MPIU_INT, MPI_MAX, comm);CHKERRQ(ierr);
3332   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
3333 
3334   in.numberofpoints = vEnd - vStart;
3335   if (in.numberofpoints > 0) {
3336     PetscSection coordSection;
3337     Vec          coordinates;
3338     PetscScalar *array;
3339 
3340     ierr = PetscMalloc1(in.numberofpoints*dim, &in.pointlist);CHKERRQ(ierr);
3341     ierr = PetscMalloc1(in.numberofpoints, &in.pointmarkerlist);CHKERRQ(ierr);
3342     ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
3343     ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
3344     ierr = VecGetArray(coordinates, &array);CHKERRQ(ierr);
3345     for (v = vStart; v < vEnd; ++v) {
3346       const PetscInt idx = v - vStart;
3347       PetscInt       off, d;
3348 
3349       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
3350       for (d = 0; d < dim; ++d) {
3351         in.pointlist[idx*dim + d] = PetscRealPart(array[off+d]);
3352       }
3353       ierr = DMPlexGetLabelValue(dm, "marker", v, &in.pointmarkerlist[idx]);CHKERRQ(ierr);
3354     }
3355     ierr = VecRestoreArray(coordinates, &array);CHKERRQ(ierr);
3356   }
3357   ierr  = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
3358 
3359   in.numberofcorners   = 3;
3360   in.numberoftriangles = cEnd - cStart;
3361 
3362   in.trianglearealist  = (double*) maxVolumes;
3363   if (in.numberoftriangles > 0) {
3364     ierr = PetscMalloc1(in.numberoftriangles*in.numberofcorners, &in.trianglelist);CHKERRQ(ierr);
3365     for (c = cStart; c < cEnd; ++c) {
3366       const PetscInt idx      = c - cStart;
3367       PetscInt      *closure = NULL;
3368       PetscInt       closureSize;
3369 
3370       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
3371       if ((closureSize != 4) && (closureSize != 7)) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Mesh has cell which is not a triangle, %D vertices in closure", closureSize);
3372       for (v = 0; v < 3; ++v) {
3373         in.trianglelist[idx*in.numberofcorners + v] = closure[(v+closureSize-3)*2] - vStart;
3374       }
3375       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
3376     }
3377   }
3378   /* TODO: Segment markers are missing on input */
3379 #if 0 /* Do not currently support holes */
3380   PetscReal *holeCoords;
3381   PetscInt   h, d;
3382 
3383   ierr = DMPlexGetHoles(boundary, &in.numberofholes, &holeCords);CHKERRQ(ierr);
3384   if (in.numberofholes > 0) {
3385     ierr = PetscMalloc1(in.numberofholes*dim, &in.holelist);CHKERRQ(ierr);
3386     for (h = 0; h < in.numberofholes; ++h) {
3387       for (d = 0; d < dim; ++d) {
3388         in.holelist[h*dim+d] = holeCoords[h*dim+d];
3389       }
3390     }
3391   }
3392 #endif
3393   if (!rank) {
3394     char args[32];
3395 
3396     /* Take away 'Q' for verbose output */
3397     ierr = PetscStrcpy(args, "pqezQra");CHKERRQ(ierr);
3398     triangulate(args, &in, &out, NULL);
3399   }
3400   ierr = PetscFree(in.pointlist);CHKERRQ(ierr);
3401   ierr = PetscFree(in.pointmarkerlist);CHKERRQ(ierr);
3402   ierr = PetscFree(in.segmentlist);CHKERRQ(ierr);
3403   ierr = PetscFree(in.segmentmarkerlist);CHKERRQ(ierr);
3404   ierr = PetscFree(in.trianglelist);CHKERRQ(ierr);
3405 
3406   {
3407     const PetscInt numCorners  = 3;
3408     const PetscInt numCells    = out.numberoftriangles;
3409     const PetscInt numVertices = out.numberofpoints;
3410     const int     *cells      = out.trianglelist;
3411     const double  *meshCoords = out.pointlist;
3412     PetscBool      interpolate = depthGlobal > 1 ? PETSC_TRUE : PETSC_FALSE;
3413 
3414     ierr = DMPlexCreateFromCellList(comm, dim, numCells, numVertices, numCorners, interpolate, cells, dim, meshCoords, dmRefined);CHKERRQ(ierr);
3415     /* Set labels */
3416     for (v = 0; v < numVertices; ++v) {
3417       if (out.pointmarkerlist[v]) {
3418         ierr = DMPlexSetLabelValue(*dmRefined, "marker", v+numCells, out.pointmarkerlist[v]);CHKERRQ(ierr);
3419       }
3420     }
3421     if (interpolate) {
3422       PetscInt e;
3423 
3424       for (e = 0; e < out.numberofedges; e++) {
3425         if (out.edgemarkerlist[e]) {
3426           const PetscInt  vertices[2] = {out.edgelist[e*2+0]+numCells, out.edgelist[e*2+1]+numCells};
3427           const PetscInt *edges;
3428           PetscInt        numEdges;
3429 
3430           ierr = DMPlexGetJoin(*dmRefined, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3431           if (numEdges != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Two vertices must cover only one edge, not %D", numEdges);
3432           ierr = DMPlexSetLabelValue(*dmRefined, "marker", edges[0], out.edgemarkerlist[e]);CHKERRQ(ierr);
3433           ierr = DMPlexRestoreJoin(*dmRefined, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3434         }
3435       }
3436     }
3437     ierr = DMPlexSetRefinementUniform(*dmRefined, PETSC_FALSE);CHKERRQ(ierr);
3438   }
3439 #if 0 /* Do not currently support holes */
3440   ierr = DMPlexCopyHoles(*dm, boundary);CHKERRQ(ierr);
3441 #endif
3442   ierr = FiniOutput_Triangle(&out);CHKERRQ(ierr);
3443   PetscFunctionReturn(0);
3444 }
3445 #endif
3446 
3447 #if defined(PETSC_HAVE_TETGEN)
3448 #include <tetgen.h>
3449 #undef __FUNCT__
3450 #define __FUNCT__ "DMPlexGenerate_Tetgen"
3451 PetscErrorCode DMPlexGenerate_Tetgen(DM boundary, PetscBool interpolate, DM *dm)
3452 {
3453   MPI_Comm       comm;
3454   const PetscInt dim  = 3;
3455   ::tetgenio     in;
3456   ::tetgenio     out;
3457   PetscInt       vStart, vEnd, v, fStart, fEnd, f;
3458   PetscMPIInt    rank;
3459   PetscErrorCode ierr;
3460 
3461   PetscFunctionBegin;
3462   ierr              = PetscObjectGetComm((PetscObject)boundary,&comm);CHKERRQ(ierr);
3463   ierr              = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
3464   ierr              = DMPlexGetDepthStratum(boundary, 0, &vStart, &vEnd);CHKERRQ(ierr);
3465   in.numberofpoints = vEnd - vStart;
3466   if (in.numberofpoints > 0) {
3467     PetscSection coordSection;
3468     Vec          coordinates;
3469     PetscScalar *array;
3470 
3471     in.pointlist       = new double[in.numberofpoints*dim];
3472     in.pointmarkerlist = new int[in.numberofpoints];
3473 
3474     ierr = DMGetCoordinatesLocal(boundary, &coordinates);CHKERRQ(ierr);
3475     ierr = DMGetCoordinateSection(boundary, &coordSection);CHKERRQ(ierr);
3476     ierr = VecGetArray(coordinates, &array);CHKERRQ(ierr);
3477     for (v = vStart; v < vEnd; ++v) {
3478       const PetscInt idx = v - vStart;
3479       PetscInt       off, d;
3480 
3481       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
3482       for (d = 0; d < dim; ++d) in.pointlist[idx*dim + d] = array[off+d];
3483       ierr = DMPlexGetLabelValue(boundary, "marker", v, &in.pointmarkerlist[idx]);CHKERRQ(ierr);
3484     }
3485     ierr = VecRestoreArray(coordinates, &array);CHKERRQ(ierr);
3486   }
3487   ierr  = DMPlexGetHeightStratum(boundary, 0, &fStart, &fEnd);CHKERRQ(ierr);
3488 
3489   in.numberoffacets = fEnd - fStart;
3490   if (in.numberoffacets > 0) {
3491     in.facetlist       = new tetgenio::facet[in.numberoffacets];
3492     in.facetmarkerlist = new int[in.numberoffacets];
3493     for (f = fStart; f < fEnd; ++f) {
3494       const PetscInt idx     = f - fStart;
3495       PetscInt      *points = NULL, numPoints, p, numVertices = 0, v;
3496 
3497       in.facetlist[idx].numberofpolygons = 1;
3498       in.facetlist[idx].polygonlist      = new tetgenio::polygon[in.facetlist[idx].numberofpolygons];
3499       in.facetlist[idx].numberofholes    = 0;
3500       in.facetlist[idx].holelist         = NULL;
3501 
3502       ierr = DMPlexGetTransitiveClosure(boundary, f, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
3503       for (p = 0; p < numPoints*2; p += 2) {
3504         const PetscInt point = points[p];
3505         if ((point >= vStart) && (point < vEnd)) points[numVertices++] = point;
3506       }
3507 
3508       tetgenio::polygon *poly = in.facetlist[idx].polygonlist;
3509       poly->numberofvertices = numVertices;
3510       poly->vertexlist       = new int[poly->numberofvertices];
3511       for (v = 0; v < numVertices; ++v) {
3512         const PetscInt vIdx = points[v] - vStart;
3513         poly->vertexlist[v] = vIdx;
3514       }
3515       ierr = DMPlexGetLabelValue(boundary, "marker", f, &in.facetmarkerlist[idx]);CHKERRQ(ierr);
3516       ierr = DMPlexRestoreTransitiveClosure(boundary, f, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
3517     }
3518   }
3519   if (!rank) {
3520     char args[32];
3521 
3522     /* Take away 'Q' for verbose output */
3523     ierr = PetscStrcpy(args, "pqezQ");CHKERRQ(ierr);
3524     ::tetrahedralize(args, &in, &out);
3525   }
3526   {
3527     const PetscInt numCorners  = 4;
3528     const PetscInt numCells    = out.numberoftetrahedra;
3529     const PetscInt numVertices = out.numberofpoints;
3530     const double   *meshCoords = out.pointlist;
3531     int            *cells      = out.tetrahedronlist;
3532 
3533     ierr = DMPlexInvertCells_Internal(dim, numCells, numCorners, cells);CHKERRQ(ierr);
3534     ierr = DMPlexCreateFromCellList(comm, dim, numCells, numVertices, numCorners, interpolate, cells, dim, meshCoords, dm);CHKERRQ(ierr);
3535     /* Set labels */
3536     for (v = 0; v < numVertices; ++v) {
3537       if (out.pointmarkerlist[v]) {
3538         ierr = DMPlexSetLabelValue(*dm, "marker", v+numCells, out.pointmarkerlist[v]);CHKERRQ(ierr);
3539       }
3540     }
3541     if (interpolate) {
3542       PetscInt e;
3543 
3544       for (e = 0; e < out.numberofedges; e++) {
3545         if (out.edgemarkerlist[e]) {
3546           const PetscInt  vertices[2] = {out.edgelist[e*2+0]+numCells, out.edgelist[e*2+1]+numCells};
3547           const PetscInt *edges;
3548           PetscInt        numEdges;
3549 
3550           ierr = DMPlexGetJoin(*dm, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3551           if (numEdges != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Two vertices must cover only one edge, not %D", numEdges);
3552           ierr = DMPlexSetLabelValue(*dm, "marker", edges[0], out.edgemarkerlist[e]);CHKERRQ(ierr);
3553           ierr = DMPlexRestoreJoin(*dm, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3554         }
3555       }
3556       for (f = 0; f < out.numberoftrifaces; f++) {
3557         if (out.trifacemarkerlist[f]) {
3558           const PetscInt  vertices[3] = {out.trifacelist[f*3+0]+numCells, out.trifacelist[f*3+1]+numCells, out.trifacelist[f*3+2]+numCells};
3559           const PetscInt *faces;
3560           PetscInt        numFaces;
3561 
3562           ierr = DMPlexGetJoin(*dm, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
3563           if (numFaces != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Three vertices must cover only one face, not %D", numFaces);
3564           ierr = DMPlexSetLabelValue(*dm, "marker", faces[0], out.trifacemarkerlist[f]);CHKERRQ(ierr);
3565           ierr = DMPlexRestoreJoin(*dm, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
3566         }
3567       }
3568     }
3569     ierr = DMPlexSetRefinementUniform(*dm, PETSC_FALSE);CHKERRQ(ierr);
3570   }
3571   PetscFunctionReturn(0);
3572 }
3573 
3574 #undef __FUNCT__
3575 #define __FUNCT__ "DMPlexRefine_Tetgen"
3576 PetscErrorCode DMPlexRefine_Tetgen(DM dm, double *maxVolumes, DM *dmRefined)
3577 {
3578   MPI_Comm       comm;
3579   const PetscInt dim  = 3;
3580   ::tetgenio     in;
3581   ::tetgenio     out;
3582   PetscInt       vStart, vEnd, v, cStart, cEnd, c, depth, depthGlobal;
3583   PetscMPIInt    rank;
3584   PetscErrorCode ierr;
3585 
3586   PetscFunctionBegin;
3587   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
3588   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
3589   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3590   ierr = MPI_Allreduce(&depth, &depthGlobal, 1, MPIU_INT, MPI_MAX, comm);CHKERRQ(ierr);
3591   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
3592 
3593   in.numberofpoints = vEnd - vStart;
3594   if (in.numberofpoints > 0) {
3595     PetscSection coordSection;
3596     Vec          coordinates;
3597     PetscScalar *array;
3598 
3599     in.pointlist       = new double[in.numberofpoints*dim];
3600     in.pointmarkerlist = new int[in.numberofpoints];
3601 
3602     ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
3603     ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
3604     ierr = VecGetArray(coordinates, &array);CHKERRQ(ierr);
3605     for (v = vStart; v < vEnd; ++v) {
3606       const PetscInt idx = v - vStart;
3607       PetscInt       off, d;
3608 
3609       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
3610       for (d = 0; d < dim; ++d) in.pointlist[idx*dim + d] = array[off+d];
3611       ierr = DMPlexGetLabelValue(dm, "marker", v, &in.pointmarkerlist[idx]);CHKERRQ(ierr);
3612     }
3613     ierr = VecRestoreArray(coordinates, &array);CHKERRQ(ierr);
3614   }
3615   ierr  = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
3616 
3617   in.numberofcorners       = 4;
3618   in.numberoftetrahedra    = cEnd - cStart;
3619   in.tetrahedronvolumelist = (double*) maxVolumes;
3620   if (in.numberoftetrahedra > 0) {
3621     in.tetrahedronlist = new int[in.numberoftetrahedra*in.numberofcorners];
3622     for (c = cStart; c < cEnd; ++c) {
3623       const PetscInt idx      = c - cStart;
3624       PetscInt      *closure = NULL;
3625       PetscInt       closureSize;
3626 
3627       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
3628       if ((closureSize != 5) && (closureSize != 15)) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Mesh has cell which is not a tetrahedron, %D vertices in closure", closureSize);
3629       for (v = 0; v < 4; ++v) {
3630         in.tetrahedronlist[idx*in.numberofcorners + v] = closure[(v+closureSize-4)*2] - vStart;
3631       }
3632       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
3633     }
3634   }
3635   /* TODO: Put in boundary faces with markers */
3636   if (!rank) {
3637     char args[32];
3638 
3639     /* Take away 'Q' for verbose output */
3640     /*ierr = PetscStrcpy(args, "qezQra");CHKERRQ(ierr); */
3641     ierr = PetscStrcpy(args, "qezraVVVV");CHKERRQ(ierr);
3642     ::tetrahedralize(args, &in, &out);
3643   }
3644   in.tetrahedronvolumelist = NULL;
3645 
3646   {
3647     const PetscInt numCorners  = 4;
3648     const PetscInt numCells    = out.numberoftetrahedra;
3649     const PetscInt numVertices = out.numberofpoints;
3650     const double   *meshCoords = out.pointlist;
3651     int            *cells      = out.tetrahedronlist;
3652 
3653     PetscBool      interpolate = depthGlobal > 1 ? PETSC_TRUE : PETSC_FALSE;
3654 
3655     ierr = DMPlexInvertCells_Internal(dim, numCells, numCorners, cells);CHKERRQ(ierr);
3656     ierr = DMPlexCreateFromCellList(comm, dim, numCells, numVertices, numCorners, interpolate, cells, dim, meshCoords, dmRefined);CHKERRQ(ierr);
3657     /* Set labels */
3658     for (v = 0; v < numVertices; ++v) {
3659       if (out.pointmarkerlist[v]) {
3660         ierr = DMPlexSetLabelValue(*dmRefined, "marker", v+numCells, out.pointmarkerlist[v]);CHKERRQ(ierr);
3661       }
3662     }
3663     if (interpolate) {
3664       PetscInt e, f;
3665 
3666       for (e = 0; e < out.numberofedges; e++) {
3667         if (out.edgemarkerlist[e]) {
3668           const PetscInt  vertices[2] = {out.edgelist[e*2+0]+numCells, out.edgelist[e*2+1]+numCells};
3669           const PetscInt *edges;
3670           PetscInt        numEdges;
3671 
3672           ierr = DMPlexGetJoin(*dmRefined, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3673           if (numEdges != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Two vertices must cover only one edge, not %D", numEdges);
3674           ierr = DMPlexSetLabelValue(*dmRefined, "marker", edges[0], out.edgemarkerlist[e]);CHKERRQ(ierr);
3675           ierr = DMPlexRestoreJoin(*dmRefined, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3676         }
3677       }
3678       for (f = 0; f < out.numberoftrifaces; f++) {
3679         if (out.trifacemarkerlist[f]) {
3680           const PetscInt  vertices[3] = {out.trifacelist[f*3+0]+numCells, out.trifacelist[f*3+1]+numCells, out.trifacelist[f*3+2]+numCells};
3681           const PetscInt *faces;
3682           PetscInt        numFaces;
3683 
3684           ierr = DMPlexGetJoin(*dmRefined, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
3685           if (numFaces != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Three vertices must cover only one face, not %D", numFaces);
3686           ierr = DMPlexSetLabelValue(*dmRefined, "marker", faces[0], out.trifacemarkerlist[f]);CHKERRQ(ierr);
3687           ierr = DMPlexRestoreJoin(*dmRefined, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
3688         }
3689       }
3690     }
3691     ierr = DMPlexSetRefinementUniform(*dmRefined, PETSC_FALSE);CHKERRQ(ierr);
3692   }
3693   PetscFunctionReturn(0);
3694 }
3695 #endif
3696 
3697 #if defined(PETSC_HAVE_CTETGEN)
3698 #include <ctetgen.h>
3699 
3700 #undef __FUNCT__
3701 #define __FUNCT__ "DMPlexGenerate_CTetgen"
3702 PetscErrorCode DMPlexGenerate_CTetgen(DM boundary, PetscBool interpolate, DM *dm)
3703 {
3704   MPI_Comm       comm;
3705   const PetscInt dim  = 3;
3706   PLC           *in, *out;
3707   PetscInt       verbose = 0, vStart, vEnd, v, fStart, fEnd, f;
3708   PetscMPIInt    rank;
3709   PetscErrorCode ierr;
3710 
3711   PetscFunctionBegin;
3712   ierr = PetscObjectGetComm((PetscObject)boundary,&comm);CHKERRQ(ierr);
3713   ierr = PetscOptionsGetInt(((PetscObject) boundary)->prefix, "-ctetgen_verbose", &verbose, NULL);CHKERRQ(ierr);
3714   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
3715   ierr = DMPlexGetDepthStratum(boundary, 0, &vStart, &vEnd);CHKERRQ(ierr);
3716   ierr = PLCCreate(&in);CHKERRQ(ierr);
3717   ierr = PLCCreate(&out);CHKERRQ(ierr);
3718 
3719   in->numberofpoints = vEnd - vStart;
3720   if (in->numberofpoints > 0) {
3721     PetscSection coordSection;
3722     Vec          coordinates;
3723     PetscScalar *array;
3724 
3725     ierr = PetscMalloc1(in->numberofpoints*dim, &in->pointlist);CHKERRQ(ierr);
3726     ierr = PetscMalloc1(in->numberofpoints,       &in->pointmarkerlist);CHKERRQ(ierr);
3727     ierr = DMGetCoordinatesLocal(boundary, &coordinates);CHKERRQ(ierr);
3728     ierr = DMGetCoordinateSection(boundary, &coordSection);CHKERRQ(ierr);
3729     ierr = VecGetArray(coordinates, &array);CHKERRQ(ierr);
3730     for (v = vStart; v < vEnd; ++v) {
3731       const PetscInt idx = v - vStart;
3732       PetscInt       off, d, m;
3733 
3734       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
3735       for (d = 0; d < dim; ++d) {
3736         in->pointlist[idx*dim + d] = PetscRealPart(array[off+d]);
3737       }
3738       ierr = DMPlexGetLabelValue(boundary, "marker", v, &m);CHKERRQ(ierr);
3739 
3740       in->pointmarkerlist[idx] = (int) m;
3741     }
3742     ierr = VecRestoreArray(coordinates, &array);CHKERRQ(ierr);
3743   }
3744   ierr  = DMPlexGetHeightStratum(boundary, 0, &fStart, &fEnd);CHKERRQ(ierr);
3745 
3746   in->numberoffacets = fEnd - fStart;
3747   if (in->numberoffacets > 0) {
3748     ierr = PetscMalloc1(in->numberoffacets, &in->facetlist);CHKERRQ(ierr);
3749     ierr = PetscMalloc1(in->numberoffacets,   &in->facetmarkerlist);CHKERRQ(ierr);
3750     for (f = fStart; f < fEnd; ++f) {
3751       const PetscInt idx     = f - fStart;
3752       PetscInt      *points = NULL, numPoints, p, numVertices = 0, v, m;
3753       polygon       *poly;
3754 
3755       in->facetlist[idx].numberofpolygons = 1;
3756 
3757       ierr = PetscMalloc1(in->facetlist[idx].numberofpolygons, &in->facetlist[idx].polygonlist);CHKERRQ(ierr);
3758 
3759       in->facetlist[idx].numberofholes    = 0;
3760       in->facetlist[idx].holelist         = NULL;
3761 
3762       ierr = DMPlexGetTransitiveClosure(boundary, f, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
3763       for (p = 0; p < numPoints*2; p += 2) {
3764         const PetscInt point = points[p];
3765         if ((point >= vStart) && (point < vEnd)) points[numVertices++] = point;
3766       }
3767 
3768       poly                   = in->facetlist[idx].polygonlist;
3769       poly->numberofvertices = numVertices;
3770       ierr                   = PetscMalloc1(poly->numberofvertices, &poly->vertexlist);CHKERRQ(ierr);
3771       for (v = 0; v < numVertices; ++v) {
3772         const PetscInt vIdx = points[v] - vStart;
3773         poly->vertexlist[v] = vIdx;
3774       }
3775       ierr                     = DMPlexGetLabelValue(boundary, "marker", f, &m);CHKERRQ(ierr);
3776       in->facetmarkerlist[idx] = (int) m;
3777       ierr                     = DMPlexRestoreTransitiveClosure(boundary, f, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
3778     }
3779   }
3780   if (!rank) {
3781     TetGenOpts t;
3782 
3783     ierr        = TetGenOptsInitialize(&t);CHKERRQ(ierr);
3784     t.in        = boundary; /* Should go away */
3785     t.plc       = 1;
3786     t.quality   = 1;
3787     t.edgesout  = 1;
3788     t.zeroindex = 1;
3789     t.quiet     = 1;
3790     t.verbose   = verbose;
3791     ierr        = TetGenCheckOpts(&t);CHKERRQ(ierr);
3792     ierr        = TetGenTetrahedralize(&t, in, out);CHKERRQ(ierr);
3793   }
3794   {
3795     const PetscInt numCorners  = 4;
3796     const PetscInt numCells    = out->numberoftetrahedra;
3797     const PetscInt numVertices = out->numberofpoints;
3798     const double   *meshCoords = out->pointlist;
3799     int            *cells      = out->tetrahedronlist;
3800 
3801     ierr = DMPlexInvertCells_Internal(dim, numCells, numCorners, cells);CHKERRQ(ierr);
3802     ierr = DMPlexCreateFromCellList(comm, dim, numCells, numVertices, numCorners, interpolate, cells, dim, meshCoords, dm);CHKERRQ(ierr);
3803     /* Set labels */
3804     for (v = 0; v < numVertices; ++v) {
3805       if (out->pointmarkerlist[v]) {
3806         ierr = DMPlexSetLabelValue(*dm, "marker", v+numCells, out->pointmarkerlist[v]);CHKERRQ(ierr);
3807       }
3808     }
3809     if (interpolate) {
3810       PetscInt e;
3811 
3812       for (e = 0; e < out->numberofedges; e++) {
3813         if (out->edgemarkerlist[e]) {
3814           const PetscInt  vertices[2] = {out->edgelist[e*2+0]+numCells, out->edgelist[e*2+1]+numCells};
3815           const PetscInt *edges;
3816           PetscInt        numEdges;
3817 
3818           ierr = DMPlexGetJoin(*dm, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3819           if (numEdges != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Two vertices must cover only one edge, not %D", numEdges);
3820           ierr = DMPlexSetLabelValue(*dm, "marker", edges[0], out->edgemarkerlist[e]);CHKERRQ(ierr);
3821           ierr = DMPlexRestoreJoin(*dm, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3822         }
3823       }
3824       for (f = 0; f < out->numberoftrifaces; f++) {
3825         if (out->trifacemarkerlist[f]) {
3826           const PetscInt  vertices[3] = {out->trifacelist[f*3+0]+numCells, out->trifacelist[f*3+1]+numCells, out->trifacelist[f*3+2]+numCells};
3827           const PetscInt *faces;
3828           PetscInt        numFaces;
3829 
3830           ierr = DMPlexGetFullJoin(*dm, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
3831           if (numFaces != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Three vertices must cover only one face, not %D", numFaces);
3832           ierr = DMPlexSetLabelValue(*dm, "marker", faces[0], out->trifacemarkerlist[f]);CHKERRQ(ierr);
3833           ierr = DMPlexRestoreJoin(*dm, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
3834         }
3835       }
3836     }
3837     ierr = DMPlexSetRefinementUniform(*dm, PETSC_FALSE);CHKERRQ(ierr);
3838   }
3839 
3840   ierr = PLCDestroy(&in);CHKERRQ(ierr);
3841   ierr = PLCDestroy(&out);CHKERRQ(ierr);
3842   PetscFunctionReturn(0);
3843 }
3844 
3845 #undef __FUNCT__
3846 #define __FUNCT__ "DMPlexRefine_CTetgen"
3847 PetscErrorCode DMPlexRefine_CTetgen(DM dm, PetscReal *maxVolumes, DM *dmRefined)
3848 {
3849   MPI_Comm       comm;
3850   const PetscInt dim  = 3;
3851   PLC           *in, *out;
3852   PetscInt       verbose = 0, vStart, vEnd, v, cStart, cEnd, c, depth, depthGlobal;
3853   PetscMPIInt    rank;
3854   PetscErrorCode ierr;
3855 
3856   PetscFunctionBegin;
3857   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
3858   ierr = PetscOptionsGetInt(((PetscObject) dm)->prefix, "-ctetgen_verbose", &verbose, NULL);CHKERRQ(ierr);
3859   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
3860   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3861   ierr = MPI_Allreduce(&depth, &depthGlobal, 1, MPIU_INT, MPI_MAX, comm);CHKERRQ(ierr);
3862   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
3863   ierr = PLCCreate(&in);CHKERRQ(ierr);
3864   ierr = PLCCreate(&out);CHKERRQ(ierr);
3865 
3866   in->numberofpoints = vEnd - vStart;
3867   if (in->numberofpoints > 0) {
3868     PetscSection coordSection;
3869     Vec          coordinates;
3870     PetscScalar *array;
3871 
3872     ierr = PetscMalloc1(in->numberofpoints*dim, &in->pointlist);CHKERRQ(ierr);
3873     ierr = PetscMalloc1(in->numberofpoints,       &in->pointmarkerlist);CHKERRQ(ierr);
3874     ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
3875     ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
3876     ierr = VecGetArray(coordinates, &array);CHKERRQ(ierr);
3877     for (v = vStart; v < vEnd; ++v) {
3878       const PetscInt idx = v - vStart;
3879       PetscInt       off, d, m;
3880 
3881       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
3882       for (d = 0; d < dim; ++d) {
3883         in->pointlist[idx*dim + d] = PetscRealPart(array[off+d]);
3884       }
3885       ierr = DMPlexGetLabelValue(dm, "marker", v, &m);CHKERRQ(ierr);
3886 
3887       in->pointmarkerlist[idx] = (int) m;
3888     }
3889     ierr = VecRestoreArray(coordinates, &array);CHKERRQ(ierr);
3890   }
3891   ierr  = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
3892 
3893   in->numberofcorners       = 4;
3894   in->numberoftetrahedra    = cEnd - cStart;
3895   in->tetrahedronvolumelist = maxVolumes;
3896   if (in->numberoftetrahedra > 0) {
3897     ierr = PetscMalloc1(in->numberoftetrahedra*in->numberofcorners, &in->tetrahedronlist);CHKERRQ(ierr);
3898     for (c = cStart; c < cEnd; ++c) {
3899       const PetscInt idx      = c - cStart;
3900       PetscInt      *closure = NULL;
3901       PetscInt       closureSize;
3902 
3903       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
3904       if ((closureSize != 5) && (closureSize != 15)) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Mesh has cell which is not a tetrahedron, %D vertices in closure", closureSize);
3905       for (v = 0; v < 4; ++v) {
3906         in->tetrahedronlist[idx*in->numberofcorners + v] = closure[(v+closureSize-4)*2] - vStart;
3907       }
3908       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
3909     }
3910   }
3911   if (!rank) {
3912     TetGenOpts t;
3913 
3914     ierr = TetGenOptsInitialize(&t);CHKERRQ(ierr);
3915 
3916     t.in        = dm; /* Should go away */
3917     t.refine    = 1;
3918     t.varvolume = 1;
3919     t.quality   = 1;
3920     t.edgesout  = 1;
3921     t.zeroindex = 1;
3922     t.quiet     = 1;
3923     t.verbose   = verbose; /* Change this */
3924 
3925     ierr = TetGenCheckOpts(&t);CHKERRQ(ierr);
3926     ierr = TetGenTetrahedralize(&t, in, out);CHKERRQ(ierr);
3927   }
3928   {
3929     const PetscInt numCorners  = 4;
3930     const PetscInt numCells    = out->numberoftetrahedra;
3931     const PetscInt numVertices = out->numberofpoints;
3932     const double   *meshCoords = out->pointlist;
3933     int            *cells      = out->tetrahedronlist;
3934     PetscBool      interpolate = depthGlobal > 1 ? PETSC_TRUE : PETSC_FALSE;
3935 
3936     ierr = DMPlexInvertCells_Internal(dim, numCells, numCorners, cells);CHKERRQ(ierr);
3937     ierr = DMPlexCreateFromCellList(comm, dim, numCells, numVertices, numCorners, interpolate, cells, dim, meshCoords, dmRefined);CHKERRQ(ierr);
3938     /* Set labels */
3939     for (v = 0; v < numVertices; ++v) {
3940       if (out->pointmarkerlist[v]) {
3941         ierr = DMPlexSetLabelValue(*dmRefined, "marker", v+numCells, out->pointmarkerlist[v]);CHKERRQ(ierr);
3942       }
3943     }
3944     if (interpolate) {
3945       PetscInt e, f;
3946 
3947       for (e = 0; e < out->numberofedges; e++) {
3948         if (out->edgemarkerlist[e]) {
3949           const PetscInt  vertices[2] = {out->edgelist[e*2+0]+numCells, out->edgelist[e*2+1]+numCells};
3950           const PetscInt *edges;
3951           PetscInt        numEdges;
3952 
3953           ierr = DMPlexGetJoin(*dmRefined, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3954           if (numEdges != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Two vertices must cover only one edge, not %D", numEdges);
3955           ierr = DMPlexSetLabelValue(*dmRefined, "marker", edges[0], out->edgemarkerlist[e]);CHKERRQ(ierr);
3956           ierr = DMPlexRestoreJoin(*dmRefined, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3957         }
3958       }
3959       for (f = 0; f < out->numberoftrifaces; f++) {
3960         if (out->trifacemarkerlist[f]) {
3961           const PetscInt  vertices[3] = {out->trifacelist[f*3+0]+numCells, out->trifacelist[f*3+1]+numCells, out->trifacelist[f*3+2]+numCells};
3962           const PetscInt *faces;
3963           PetscInt        numFaces;
3964 
3965           ierr = DMPlexGetFullJoin(*dmRefined, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
3966           if (numFaces != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Three vertices must cover only one face, not %D", numFaces);
3967           ierr = DMPlexSetLabelValue(*dmRefined, "marker", faces[0], out->trifacemarkerlist[f]);CHKERRQ(ierr);
3968           ierr = DMPlexRestoreJoin(*dmRefined, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
3969         }
3970       }
3971     }
3972     ierr = DMPlexSetRefinementUniform(*dmRefined, PETSC_FALSE);CHKERRQ(ierr);
3973   }
3974   ierr = PLCDestroy(&in);CHKERRQ(ierr);
3975   ierr = PLCDestroy(&out);CHKERRQ(ierr);
3976   PetscFunctionReturn(0);
3977 }
3978 #endif
3979 
3980 #undef __FUNCT__
3981 #define __FUNCT__ "DMPlexGenerate"
3982 /*@C
3983   DMPlexGenerate - Generates a mesh.
3984 
3985   Not Collective
3986 
3987   Input Parameters:
3988 + boundary - The DMPlex boundary object
3989 . name - The mesh generation package name
3990 - interpolate - Flag to create intermediate mesh elements
3991 
3992   Output Parameter:
3993 . mesh - The DMPlex object
3994 
3995   Level: intermediate
3996 
3997 .keywords: mesh, elements
3998 .seealso: DMPlexCreate(), DMRefine()
3999 @*/
4000 PetscErrorCode DMPlexGenerate(DM boundary, const char name[], PetscBool interpolate, DM *mesh)
4001 {
4002   PetscInt       dim;
4003   char           genname[1024];
4004   PetscBool      isTriangle = PETSC_FALSE, isTetgen = PETSC_FALSE, isCTetgen = PETSC_FALSE, flg;
4005   PetscErrorCode ierr;
4006 
4007   PetscFunctionBegin;
4008   PetscValidHeaderSpecific(boundary, DM_CLASSID, 1);
4009   PetscValidLogicalCollectiveBool(boundary, interpolate, 2);
4010   ierr = DMPlexGetDimension(boundary, &dim);CHKERRQ(ierr);
4011   ierr = PetscOptionsGetString(((PetscObject) boundary)->prefix, "-dm_plex_generator", genname, 1024, &flg);CHKERRQ(ierr);
4012   if (flg) name = genname;
4013   if (name) {
4014     ierr = PetscStrcmp(name, "triangle", &isTriangle);CHKERRQ(ierr);
4015     ierr = PetscStrcmp(name, "tetgen",   &isTetgen);CHKERRQ(ierr);
4016     ierr = PetscStrcmp(name, "ctetgen",  &isCTetgen);CHKERRQ(ierr);
4017   }
4018   switch (dim) {
4019   case 1:
4020     if (!name || isTriangle) {
4021 #if defined(PETSC_HAVE_TRIANGLE)
4022       ierr = DMPlexGenerate_Triangle(boundary, interpolate, mesh);CHKERRQ(ierr);
4023 #else
4024       SETERRQ(PetscObjectComm((PetscObject)boundary), PETSC_ERR_SUP, "Mesh generation needs external package support.\nPlease reconfigure with --download-triangle.");
4025 #endif
4026     } else SETERRQ1(PetscObjectComm((PetscObject)boundary), PETSC_ERR_SUP, "Unknown 2D mesh generation package %s", name);
4027     break;
4028   case 2:
4029     if (!name || isCTetgen) {
4030 #if defined(PETSC_HAVE_CTETGEN)
4031       ierr = DMPlexGenerate_CTetgen(boundary, interpolate, mesh);CHKERRQ(ierr);
4032 #else
4033       SETERRQ(PetscObjectComm((PetscObject)boundary), PETSC_ERR_SUP, "CTetgen needs external package support.\nPlease reconfigure with --download-ctetgen.");
4034 #endif
4035     } else if (isTetgen) {
4036 #if defined(PETSC_HAVE_TETGEN)
4037       ierr = DMPlexGenerate_Tetgen(boundary, interpolate, mesh);CHKERRQ(ierr);
4038 #else
4039       SETERRQ(PetscObjectComm((PetscObject)boundary), PETSC_ERR_SUP, "Tetgen needs external package support.\nPlease reconfigure with --with-c-language=cxx --download-tetgen.");
4040 #endif
4041     } else SETERRQ1(PetscObjectComm((PetscObject)boundary), PETSC_ERR_SUP, "Unknown 3D mesh generation package %s", name);
4042     break;
4043   default:
4044     SETERRQ1(PetscObjectComm((PetscObject)boundary), PETSC_ERR_SUP, "Mesh generation for a dimension %d boundary is not supported.", dim);
4045   }
4046   PetscFunctionReturn(0);
4047 }
4048 
4049 #undef __FUNCT__
4050 #define __FUNCT__ "DMRefine_Plex"
4051 PetscErrorCode DMRefine_Plex(DM dm, MPI_Comm comm, DM *dmRefined)
4052 {
4053   PetscReal      refinementLimit;
4054   PetscInt       dim, cStart, cEnd;
4055   char           genname[1024], *name = NULL;
4056   PetscBool      isUniform, isTriangle = PETSC_FALSE, isTetgen = PETSC_FALSE, isCTetgen = PETSC_FALSE, flg;
4057   PetscErrorCode ierr;
4058 
4059   PetscFunctionBegin;
4060   ierr = DMPlexGetRefinementUniform(dm, &isUniform);CHKERRQ(ierr);
4061   if (isUniform) {
4062     CellRefiner cellRefiner;
4063 
4064     ierr = DMPlexGetCellRefiner_Internal(dm, &cellRefiner);CHKERRQ(ierr);
4065     ierr = DMPlexRefineUniform_Internal(dm, cellRefiner, dmRefined);CHKERRQ(ierr);
4066     PetscFunctionReturn(0);
4067   }
4068   ierr = DMPlexGetRefinementLimit(dm, &refinementLimit);CHKERRQ(ierr);
4069   if (refinementLimit == 0.0) PetscFunctionReturn(0);
4070   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
4071   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
4072   ierr = PetscOptionsGetString(((PetscObject) dm)->prefix, "-dm_plex_generator", genname, 1024, &flg);CHKERRQ(ierr);
4073   if (flg) name = genname;
4074   if (name) {
4075     ierr = PetscStrcmp(name, "triangle", &isTriangle);CHKERRQ(ierr);
4076     ierr = PetscStrcmp(name, "tetgen",   &isTetgen);CHKERRQ(ierr);
4077     ierr = PetscStrcmp(name, "ctetgen",  &isCTetgen);CHKERRQ(ierr);
4078   }
4079   switch (dim) {
4080   case 2:
4081     if (!name || isTriangle) {
4082 #if defined(PETSC_HAVE_TRIANGLE)
4083       double  *maxVolumes;
4084       PetscInt c;
4085 
4086       ierr = PetscMalloc1((cEnd - cStart), &maxVolumes);CHKERRQ(ierr);
4087       for (c = 0; c < cEnd-cStart; ++c) maxVolumes[c] = refinementLimit;
4088       ierr = DMPlexRefine_Triangle(dm, maxVolumes, dmRefined);CHKERRQ(ierr);
4089       ierr = PetscFree(maxVolumes);CHKERRQ(ierr);
4090 #else
4091       SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Mesh refinement needs external package support.\nPlease reconfigure with --download-triangle.");
4092 #endif
4093     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Unknown 2D mesh generation package %s", name);
4094     break;
4095   case 3:
4096     if (!name || isCTetgen) {
4097 #if defined(PETSC_HAVE_CTETGEN)
4098       PetscReal *maxVolumes;
4099       PetscInt   c;
4100 
4101       ierr = PetscMalloc1((cEnd - cStart), &maxVolumes);CHKERRQ(ierr);
4102       for (c = 0; c < cEnd-cStart; ++c) maxVolumes[c] = refinementLimit;
4103       ierr = DMPlexRefine_CTetgen(dm, maxVolumes, dmRefined);CHKERRQ(ierr);
4104 #else
4105       SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "CTetgen needs external package support.\nPlease reconfigure with --download-ctetgen.");
4106 #endif
4107     } else if (isTetgen) {
4108 #if defined(PETSC_HAVE_TETGEN)
4109       double  *maxVolumes;
4110       PetscInt c;
4111 
4112       ierr = PetscMalloc1((cEnd - cStart), &maxVolumes);CHKERRQ(ierr);
4113       for (c = 0; c < cEnd-cStart; ++c) maxVolumes[c] = refinementLimit;
4114       ierr = DMPlexRefine_Tetgen(dm, maxVolumes, dmRefined);CHKERRQ(ierr);
4115 #else
4116       SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Tetgen needs external package support.\nPlease reconfigure with --with-c-language=cxx --download-tetgen.");
4117 #endif
4118     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Unknown 3D mesh generation package %s", name);
4119     break;
4120   default:
4121     SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Mesh refinement in dimension %d is not supported.", dim);
4122   }
4123   PetscFunctionReturn(0);
4124 }
4125 
4126 #undef __FUNCT__
4127 #define __FUNCT__ "DMRefineHierarchy_Plex"
4128 PetscErrorCode DMRefineHierarchy_Plex(DM dm, PetscInt nlevels, DM dmRefined[])
4129 {
4130   DM             cdm = dm;
4131   PetscInt       r;
4132   PetscBool      isUniform;
4133   PetscErrorCode ierr;
4134 
4135   PetscFunctionBegin;
4136   ierr = DMPlexGetRefinementUniform(dm, &isUniform);CHKERRQ(ierr);
4137   if (!isUniform) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Non-uniform refinement is incompatible with the hierarchy");
4138   for (r = 0; r < nlevels; ++r) {
4139     CellRefiner cellRefiner;
4140 
4141     ierr = DMPlexGetCellRefiner_Internal(cdm, &cellRefiner);CHKERRQ(ierr);
4142     ierr = DMPlexRefineUniform_Internal(cdm, cellRefiner, &dmRefined[r]);CHKERRQ(ierr);
4143     ierr = DMPlexSetCoarseDM(dmRefined[r], cdm);CHKERRQ(ierr);
4144     cdm  = dmRefined[r];
4145   }
4146   PetscFunctionReturn(0);
4147 }
4148 
4149 #undef __FUNCT__
4150 #define __FUNCT__ "DMCoarsen_Plex"
4151 PetscErrorCode DMCoarsen_Plex(DM dm, MPI_Comm comm, DM *dmCoarsened)
4152 {
4153   DM_Plex       *mesh = (DM_Plex*) dm->data;
4154   PetscErrorCode ierr;
4155 
4156   PetscFunctionBegin;
4157   ierr = PetscObjectReference((PetscObject) mesh->coarseMesh);CHKERRQ(ierr);
4158   *dmCoarsened = mesh->coarseMesh;
4159   PetscFunctionReturn(0);
4160 }
4161 
4162 #undef __FUNCT__
4163 #define __FUNCT__ "DMPlexGetDepthLabel"
4164 /*@
4165   DMPlexGetDepthLabel - Get the DMLabel recording the depth of each point
4166 
4167   Not Collective
4168 
4169   Input Parameter:
4170 . dm    - The DMPlex object
4171 
4172   Output Parameter:
4173 . depthLabel - The DMLabel recording point depth
4174 
4175   Level: developer
4176 
4177 .keywords: mesh, points
4178 .seealso: DMPlexGetDepth(), DMPlexGetHeightStratum(), DMPlexGetDepthStratum()
4179 @*/
4180 PetscErrorCode DMPlexGetDepthLabel(DM dm, DMLabel *depthLabel)
4181 {
4182   DM_Plex       *mesh = (DM_Plex*) dm->data;
4183   PetscErrorCode ierr;
4184 
4185   PetscFunctionBegin;
4186   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4187   PetscValidPointer(depthLabel, 2);
4188   if (!mesh->depthLabel) {ierr = DMPlexGetLabel(dm, "depth", &mesh->depthLabel);CHKERRQ(ierr);}
4189   *depthLabel = mesh->depthLabel;
4190   PetscFunctionReturn(0);
4191 }
4192 
4193 #undef __FUNCT__
4194 #define __FUNCT__ "DMPlexGetDepth"
4195 /*@
4196   DMPlexGetDepth - Get the depth of the DAG representing this mesh
4197 
4198   Not Collective
4199 
4200   Input Parameter:
4201 . dm    - The DMPlex object
4202 
4203   Output Parameter:
4204 . depth - The number of strata (breadth first levels) in the DAG
4205 
4206   Level: developer
4207 
4208 .keywords: mesh, points
4209 .seealso: DMPlexGetDepthLabel(), DMPlexGetHeightStratum(), DMPlexGetDepthStratum()
4210 @*/
4211 PetscErrorCode DMPlexGetDepth(DM dm, PetscInt *depth)
4212 {
4213   DMLabel        label;
4214   PetscInt       d = 0;
4215   PetscErrorCode ierr;
4216 
4217   PetscFunctionBegin;
4218   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4219   PetscValidPointer(depth, 2);
4220   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
4221   if (label) {ierr = DMLabelGetNumValues(label, &d);CHKERRQ(ierr);}
4222   *depth = d-1;
4223   PetscFunctionReturn(0);
4224 }
4225 
4226 #undef __FUNCT__
4227 #define __FUNCT__ "DMPlexGetDepthStratum"
4228 /*@
4229   DMPlexGetDepthStratum - Get the bounds [start, end) for all points at a certain depth.
4230 
4231   Not Collective
4232 
4233   Input Parameters:
4234 + dm           - The DMPlex object
4235 - stratumValue - The requested depth
4236 
4237   Output Parameters:
4238 + start - The first point at this depth
4239 - end   - One beyond the last point at this depth
4240 
4241   Level: developer
4242 
4243 .keywords: mesh, points
4244 .seealso: DMPlexGetHeightStratum(), DMPlexGetDepth()
4245 @*/
4246 PetscErrorCode DMPlexGetDepthStratum(DM dm, PetscInt stratumValue, PetscInt *start, PetscInt *end)
4247 {
4248   DMLabel        label;
4249   PetscInt       pStart, pEnd;
4250   PetscErrorCode ierr;
4251 
4252   PetscFunctionBegin;
4253   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4254   if (start) {PetscValidPointer(start, 3); *start = 0;}
4255   if (end)   {PetscValidPointer(end,   4); *end   = 0;}
4256   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
4257   if (pStart == pEnd) PetscFunctionReturn(0);
4258   if (stratumValue < 0) {
4259     if (start) *start = pStart;
4260     if (end)   *end   = pEnd;
4261     PetscFunctionReturn(0);
4262   }
4263   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
4264   if (!label) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "No label named depth was found");
4265   ierr = DMLabelGetStratumBounds(label, stratumValue, start, end);CHKERRQ(ierr);
4266   PetscFunctionReturn(0);
4267 }
4268 
4269 #undef __FUNCT__
4270 #define __FUNCT__ "DMPlexGetHeightStratum"
4271 /*@
4272   DMPlexGetHeightStratum - Get the bounds [start, end) for all points at a certain height.
4273 
4274   Not Collective
4275 
4276   Input Parameters:
4277 + dm           - The DMPlex object
4278 - stratumValue - The requested height
4279 
4280   Output Parameters:
4281 + start - The first point at this height
4282 - end   - One beyond the last point at this height
4283 
4284   Level: developer
4285 
4286 .keywords: mesh, points
4287 .seealso: DMPlexGetDepthStratum(), DMPlexGetDepth()
4288 @*/
4289 PetscErrorCode DMPlexGetHeightStratum(DM dm, PetscInt stratumValue, PetscInt *start, PetscInt *end)
4290 {
4291   DMLabel        label;
4292   PetscInt       depth, pStart, pEnd;
4293   PetscErrorCode ierr;
4294 
4295   PetscFunctionBegin;
4296   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4297   if (start) {PetscValidPointer(start, 3); *start = 0;}
4298   if (end)   {PetscValidPointer(end,   4); *end   = 0;}
4299   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
4300   if (pStart == pEnd) PetscFunctionReturn(0);
4301   if (stratumValue < 0) {
4302     if (start) *start = pStart;
4303     if (end)   *end   = pEnd;
4304     PetscFunctionReturn(0);
4305   }
4306   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
4307   if (!label) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "No label named depth was found");CHKERRQ(ierr);
4308   ierr = DMLabelGetNumValues(label, &depth);CHKERRQ(ierr);
4309   ierr = DMLabelGetStratumBounds(label, depth-1-stratumValue, start, end);CHKERRQ(ierr);
4310   PetscFunctionReturn(0);
4311 }
4312 
4313 #undef __FUNCT__
4314 #define __FUNCT__ "DMPlexCreateSectionInitial"
4315 /* Set the number of dof on each point and separate by fields */
4316 PetscErrorCode DMPlexCreateSectionInitial(DM dm, PetscInt dim, PetscInt numFields,const PetscInt numComp[],const PetscInt numDof[], PetscSection *section)
4317 {
4318   PetscInt      *numDofTot;
4319   PetscInt       depth, pStart = 0, pEnd = 0;
4320   PetscInt       p, d, dep, f;
4321   PetscErrorCode ierr;
4322 
4323   PetscFunctionBegin;
4324   ierr = PetscMalloc1((dim+1), &numDofTot);CHKERRQ(ierr);
4325   for (d = 0; d <= dim; ++d) {
4326     numDofTot[d] = 0;
4327     for (f = 0; f < numFields; ++f) numDofTot[d] += numDof[f*(dim+1)+d];
4328   }
4329   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), section);CHKERRQ(ierr);
4330   if (numFields > 0) {
4331     ierr = PetscSectionSetNumFields(*section, numFields);CHKERRQ(ierr);
4332     if (numComp) {
4333       for (f = 0; f < numFields; ++f) {
4334         ierr = PetscSectionSetFieldComponents(*section, f, numComp[f]);CHKERRQ(ierr);
4335       }
4336     }
4337   }
4338   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
4339   ierr = PetscSectionSetChart(*section, pStart, pEnd);CHKERRQ(ierr);
4340   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
4341   for (dep = 0; dep <= depth; ++dep) {
4342     d    = dim == depth ? dep : (!dep ? 0 : dim);
4343     ierr = DMPlexGetDepthStratum(dm, dep, &pStart, &pEnd);CHKERRQ(ierr);
4344     for (p = pStart; p < pEnd; ++p) {
4345       for (f = 0; f < numFields; ++f) {
4346         ierr = PetscSectionSetFieldDof(*section, p, f, numDof[f*(dim+1)+d]);CHKERRQ(ierr);
4347       }
4348       ierr = PetscSectionSetDof(*section, p, numDofTot[d]);CHKERRQ(ierr);
4349     }
4350   }
4351   ierr = PetscFree(numDofTot);CHKERRQ(ierr);
4352   PetscFunctionReturn(0);
4353 }
4354 
4355 #undef __FUNCT__
4356 #define __FUNCT__ "DMPlexCreateSectionBCDof"
4357 /* Set the number of dof on each point and separate by fields
4358    If constDof is PETSC_DETERMINE, constrain every dof on the point
4359 */
4360 PetscErrorCode DMPlexCreateSectionBCDof(DM dm, PetscInt numBC,const PetscInt bcField[],const IS bcPoints[], PetscInt constDof, PetscSection section)
4361 {
4362   PetscInt       numFields;
4363   PetscInt       bc;
4364   PetscErrorCode ierr;
4365 
4366   PetscFunctionBegin;
4367   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
4368   for (bc = 0; bc < numBC; ++bc) {
4369     PetscInt        field = 0;
4370     const PetscInt *idx;
4371     PetscInt        n, i;
4372 
4373     if (numFields) field = bcField[bc];
4374     ierr = ISGetLocalSize(bcPoints[bc], &n);CHKERRQ(ierr);
4375     ierr = ISGetIndices(bcPoints[bc], &idx);CHKERRQ(ierr);
4376     for (i = 0; i < n; ++i) {
4377       const PetscInt p        = idx[i];
4378       PetscInt       numConst = constDof;
4379 
4380       /* Constrain every dof on the point */
4381       if (numConst < 0) {
4382         if (numFields) {
4383           ierr = PetscSectionGetFieldDof(section, p, field, &numConst);CHKERRQ(ierr);
4384         } else {
4385           ierr = PetscSectionGetDof(section, p, &numConst);CHKERRQ(ierr);
4386         }
4387       }
4388       if (numFields) {
4389         ierr = PetscSectionAddFieldConstraintDof(section, p, field, numConst);CHKERRQ(ierr);
4390       }
4391       ierr = PetscSectionAddConstraintDof(section, p, numConst);CHKERRQ(ierr);
4392     }
4393     ierr = ISRestoreIndices(bcPoints[bc], &idx);CHKERRQ(ierr);
4394   }
4395   PetscFunctionReturn(0);
4396 }
4397 
4398 #undef __FUNCT__
4399 #define __FUNCT__ "DMPlexCreateSectionBCIndicesAll"
4400 /* Set the constrained indices on each point and separate by fields */
4401 PetscErrorCode DMPlexCreateSectionBCIndicesAll(DM dm, PetscSection section)
4402 {
4403   PetscInt      *maxConstraints;
4404   PetscInt       numFields, f, pStart = 0, pEnd = 0, p;
4405   PetscErrorCode ierr;
4406 
4407   PetscFunctionBegin;
4408   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
4409   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
4410   ierr = PetscMalloc1((numFields+1), &maxConstraints);CHKERRQ(ierr);
4411   for (f = 0; f <= numFields; ++f) maxConstraints[f] = 0;
4412   for (p = pStart; p < pEnd; ++p) {
4413     PetscInt cdof;
4414 
4415     if (numFields) {
4416       for (f = 0; f < numFields; ++f) {
4417         ierr              = PetscSectionGetFieldConstraintDof(section, p, f, &cdof);CHKERRQ(ierr);
4418         maxConstraints[f] = PetscMax(maxConstraints[f], cdof);
4419       }
4420     } else {
4421       ierr              = PetscSectionGetConstraintDof(section, p, &cdof);CHKERRQ(ierr);
4422       maxConstraints[0] = PetscMax(maxConstraints[0], cdof);
4423     }
4424   }
4425   for (f = 0; f < numFields; ++f) {
4426     maxConstraints[numFields] += maxConstraints[f];
4427   }
4428   if (maxConstraints[numFields]) {
4429     PetscInt *indices;
4430 
4431     ierr = PetscMalloc1(maxConstraints[numFields], &indices);CHKERRQ(ierr);
4432     for (p = pStart; p < pEnd; ++p) {
4433       PetscInt cdof, d;
4434 
4435       ierr = PetscSectionGetConstraintDof(section, p, &cdof);CHKERRQ(ierr);
4436       if (cdof) {
4437         if (cdof > maxConstraints[numFields]) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_LIB, "Likely memory corruption, point %D cDof %D > maxConstraints %D", p, cdof, maxConstraints[numFields]);
4438         if (numFields) {
4439           PetscInt numConst = 0, foff = 0;
4440 
4441           for (f = 0; f < numFields; ++f) {
4442             PetscInt cfdof, fdof;
4443 
4444             ierr = PetscSectionGetFieldDof(section, p, f, &fdof);CHKERRQ(ierr);
4445             ierr = PetscSectionGetFieldConstraintDof(section, p, f, &cfdof);CHKERRQ(ierr);
4446             /* Change constraint numbering from absolute local dof number to field relative local dof number */
4447             for (d = 0; d < cfdof; ++d) indices[numConst+d] = d;
4448             ierr = PetscSectionSetFieldConstraintIndices(section, p, f, &indices[numConst]);CHKERRQ(ierr);
4449             for (d = 0; d < cfdof; ++d) indices[numConst+d] += foff;
4450             numConst += cfdof;
4451             foff     += fdof;
4452           }
4453           if (cdof != numConst) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_LIB, "Total number of field constraints %D should be %D", numConst, cdof);
4454         } else {
4455           for (d = 0; d < cdof; ++d) indices[d] = d;
4456         }
4457         ierr = PetscSectionSetConstraintIndices(section, p, indices);CHKERRQ(ierr);
4458       }
4459     }
4460     ierr = PetscFree(indices);CHKERRQ(ierr);
4461   }
4462   ierr = PetscFree(maxConstraints);CHKERRQ(ierr);
4463   PetscFunctionReturn(0);
4464 }
4465 
4466 #undef __FUNCT__
4467 #define __FUNCT__ "DMPlexCreateSectionBCIndicesField"
4468 /* Set the constrained field indices on each point */
4469 PetscErrorCode DMPlexCreateSectionBCIndicesField(DM dm, PetscInt field, IS bcPoints, IS constraintIndices, PetscSection section)
4470 {
4471   const PetscInt *points, *indices;
4472   PetscInt        numFields, maxDof, numPoints, p, numConstraints;
4473   PetscErrorCode  ierr;
4474 
4475   PetscFunctionBegin;
4476   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
4477   if ((field < 0) || (field >= numFields)) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Section field %d should be in [%d, %d)", field, 0, numFields);
4478 
4479   ierr = ISGetLocalSize(bcPoints, &numPoints);CHKERRQ(ierr);
4480   ierr = ISGetIndices(bcPoints, &points);CHKERRQ(ierr);
4481   if (!constraintIndices) {
4482     PetscInt *idx, i;
4483 
4484     ierr = PetscSectionGetMaxDof(section, &maxDof);CHKERRQ(ierr);
4485     ierr = PetscMalloc1(maxDof, &idx);CHKERRQ(ierr);
4486     for (i = 0; i < maxDof; ++i) idx[i] = i;
4487     for (p = 0; p < numPoints; ++p) {
4488       ierr = PetscSectionSetFieldConstraintIndices(section, points[p], field, idx);CHKERRQ(ierr);
4489     }
4490     ierr = PetscFree(idx);CHKERRQ(ierr);
4491   } else {
4492     ierr = ISGetLocalSize(constraintIndices, &numConstraints);CHKERRQ(ierr);
4493     ierr = ISGetIndices(constraintIndices, &indices);CHKERRQ(ierr);
4494     for (p = 0; p < numPoints; ++p) {
4495       PetscInt fcdof;
4496 
4497       ierr = PetscSectionGetFieldConstraintDof(section, points[p], field, &fcdof);CHKERRQ(ierr);
4498       if (fcdof != numConstraints) SETERRQ4(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Section point %d field %d has %d constraints, but yo ugave %d indices", p, field, fcdof, numConstraints);
4499       ierr = PetscSectionSetFieldConstraintIndices(section, points[p], field, indices);CHKERRQ(ierr);
4500     }
4501     ierr = ISRestoreIndices(constraintIndices, &indices);CHKERRQ(ierr);
4502   }
4503   ierr = ISRestoreIndices(bcPoints, &points);CHKERRQ(ierr);
4504   PetscFunctionReturn(0);
4505 }
4506 
4507 #undef __FUNCT__
4508 #define __FUNCT__ "DMPlexCreateSectionBCIndices"
4509 /* Set the constrained indices on each point and separate by fields */
4510 PetscErrorCode DMPlexCreateSectionBCIndices(DM dm, PetscSection section)
4511 {
4512   PetscInt      *indices;
4513   PetscInt       numFields, maxDof, f, pStart = 0, pEnd = 0, p;
4514   PetscErrorCode ierr;
4515 
4516   PetscFunctionBegin;
4517   ierr = PetscSectionGetMaxDof(section, &maxDof);CHKERRQ(ierr);
4518   ierr = PetscMalloc1(maxDof, &indices);CHKERRQ(ierr);
4519   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
4520   if (!numFields) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "This function only works after users have set field constraint indices.");
4521   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
4522   for (p = pStart; p < pEnd; ++p) {
4523     PetscInt cdof, d;
4524 
4525     ierr = PetscSectionGetConstraintDof(section, p, &cdof);CHKERRQ(ierr);
4526     if (cdof) {
4527       PetscInt numConst = 0, foff = 0;
4528 
4529       for (f = 0; f < numFields; ++f) {
4530         const PetscInt *fcind;
4531         PetscInt        fdof, fcdof;
4532 
4533         ierr = PetscSectionGetFieldDof(section, p, f, &fdof);CHKERRQ(ierr);
4534         ierr = PetscSectionGetFieldConstraintDof(section, p, f, &fcdof);CHKERRQ(ierr);
4535         if (fcdof) {ierr = PetscSectionGetFieldConstraintIndices(section, p, f, &fcind);CHKERRQ(ierr);}
4536         /* Change constraint numbering from field relative local dof number to absolute local dof number */
4537         for (d = 0; d < fcdof; ++d) indices[numConst+d] = fcind[d]+foff;
4538         foff     += fdof;
4539         numConst += fcdof;
4540       }
4541       if (cdof != numConst) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_LIB, "Total number of field constraints %D should be %D", numConst, cdof);
4542       ierr = PetscSectionSetConstraintIndices(section, p, indices);CHKERRQ(ierr);
4543     }
4544   }
4545   ierr = PetscFree(indices);CHKERRQ(ierr);
4546   PetscFunctionReturn(0);
4547 }
4548 
4549 #undef __FUNCT__
4550 #define __FUNCT__ "DMPlexCreateSection"
4551 /*@C
4552   DMPlexCreateSection - Create a PetscSection based upon the dof layout specification provided.
4553 
4554   Not Collective
4555 
4556   Input Parameters:
4557 + dm        - The DMPlex object
4558 . dim       - The spatial dimension of the problem
4559 . numFields - The number of fields in the problem
4560 . numComp   - An array of size numFields that holds the number of components for each field
4561 . numDof    - An array of size numFields*(dim+1) which holds the number of dof for each field on a mesh piece of dimension d
4562 . numBC     - The number of boundary conditions
4563 . bcField   - An array of size numBC giving the field number for each boundry condition
4564 - bcPoints  - An array of size numBC giving an IS holding the sieve points to which each boundary condition applies
4565 
4566   Output Parameter:
4567 . section - The PetscSection object
4568 
4569   Notes: numDof[f*(dim+1)+d] gives the number of dof for field f on sieve points of dimension d. For instance, numDof[1] is the
4570   nubmer of dof for field 0 on each edge.
4571 
4572   Level: developer
4573 
4574   Fortran Notes:
4575   A Fortran 90 version is available as DMPlexCreateSectionF90()
4576 
4577 .keywords: mesh, elements
4578 .seealso: DMPlexCreate(), PetscSectionCreate()
4579 @*/
4580 PetscErrorCode DMPlexCreateSection(DM dm, PetscInt dim, PetscInt numFields,const PetscInt numComp[],const PetscInt numDof[], PetscInt numBC,const PetscInt bcField[],const IS bcPoints[], PetscSection *section)
4581 {
4582   PetscErrorCode ierr;
4583 
4584   PetscFunctionBegin;
4585   ierr = DMPlexCreateSectionInitial(dm, dim, numFields, numComp, numDof, section);CHKERRQ(ierr);
4586   ierr = DMPlexCreateSectionBCDof(dm, numBC, bcField, bcPoints, PETSC_DETERMINE, *section);CHKERRQ(ierr);
4587   ierr = PetscSectionSetUp(*section);CHKERRQ(ierr);
4588   if (numBC) {ierr = DMPlexCreateSectionBCIndicesAll(dm, *section);CHKERRQ(ierr);}
4589   ierr = PetscSectionViewFromOptions(*section,NULL,"-section_view");CHKERRQ(ierr);
4590   PetscFunctionReturn(0);
4591 }
4592 
4593 #undef __FUNCT__
4594 #define __FUNCT__ "DMCreateCoordinateDM_Plex"
4595 PetscErrorCode DMCreateCoordinateDM_Plex(DM dm, DM *cdm)
4596 {
4597   PetscSection   section;
4598   PetscErrorCode ierr;
4599 
4600   PetscFunctionBegin;
4601   ierr = DMClone(dm, cdm);CHKERRQ(ierr);
4602   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &section);CHKERRQ(ierr);
4603   ierr = DMSetDefaultSection(*cdm, section);CHKERRQ(ierr);
4604   ierr = PetscSectionDestroy(&section);CHKERRQ(ierr);
4605   PetscFunctionReturn(0);
4606 }
4607 
4608 #undef __FUNCT__
4609 #define __FUNCT__ "DMPlexGetConeSection"
4610 PetscErrorCode DMPlexGetConeSection(DM dm, PetscSection *section)
4611 {
4612   DM_Plex *mesh = (DM_Plex*) dm->data;
4613 
4614   PetscFunctionBegin;
4615   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4616   if (section) *section = mesh->coneSection;
4617   PetscFunctionReturn(0);
4618 }
4619 
4620 #undef __FUNCT__
4621 #define __FUNCT__ "DMPlexGetSupportSection"
4622 PetscErrorCode DMPlexGetSupportSection(DM dm, PetscSection *section)
4623 {
4624   DM_Plex *mesh = (DM_Plex*) dm->data;
4625 
4626   PetscFunctionBegin;
4627   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4628   if (section) *section = mesh->supportSection;
4629   PetscFunctionReturn(0);
4630 }
4631 
4632 #undef __FUNCT__
4633 #define __FUNCT__ "DMPlexGetCones"
4634 PetscErrorCode DMPlexGetCones(DM dm, PetscInt *cones[])
4635 {
4636   DM_Plex *mesh = (DM_Plex*) dm->data;
4637 
4638   PetscFunctionBegin;
4639   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4640   if (cones) *cones = mesh->cones;
4641   PetscFunctionReturn(0);
4642 }
4643 
4644 #undef __FUNCT__
4645 #define __FUNCT__ "DMPlexGetConeOrientations"
4646 PetscErrorCode DMPlexGetConeOrientations(DM dm, PetscInt *coneOrientations[])
4647 {
4648   DM_Plex *mesh = (DM_Plex*) dm->data;
4649 
4650   PetscFunctionBegin;
4651   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4652   if (coneOrientations) *coneOrientations = mesh->coneOrientations;
4653   PetscFunctionReturn(0);
4654 }
4655 
4656 /******************************** FEM Support **********************************/
4657 
4658 #undef __FUNCT__
4659 #define __FUNCT__ "DMPlexVecGetClosure_Depth1_Static"
4660 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecGetClosure_Depth1_Static(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
4661 {
4662   PetscScalar    *array, *vArray;
4663   const PetscInt *cone, *coneO;
4664   PetscInt        pStart, pEnd, p, numPoints, size = 0, offset = 0;
4665   PetscErrorCode  ierr;
4666 
4667   PetscFunctionBeginHot;
4668   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
4669   ierr = DMPlexGetConeSize(dm, point, &numPoints);CHKERRQ(ierr);
4670   ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
4671   ierr = DMPlexGetConeOrientation(dm, point, &coneO);CHKERRQ(ierr);
4672   if (!values || !*values) {
4673     if ((point >= pStart) && (point < pEnd)) {
4674       PetscInt dof;
4675 
4676       ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4677       size += dof;
4678     }
4679     for (p = 0; p < numPoints; ++p) {
4680       const PetscInt cp = cone[p];
4681       PetscInt       dof;
4682 
4683       if ((cp < pStart) || (cp >= pEnd)) continue;
4684       ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
4685       size += dof;
4686     }
4687     if (!values) {
4688       if (csize) *csize = size;
4689       PetscFunctionReturn(0);
4690     }
4691     ierr = DMGetWorkArray(dm, size, PETSC_SCALAR, &array);CHKERRQ(ierr);
4692   } else {
4693     array = *values;
4694   }
4695   size = 0;
4696   ierr = VecGetArray(v, &vArray);CHKERRQ(ierr);
4697   if ((point >= pStart) && (point < pEnd)) {
4698     PetscInt     dof, off, d;
4699     PetscScalar *varr;
4700 
4701     ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4702     ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
4703     varr = &vArray[off];
4704     for (d = 0; d < dof; ++d, ++offset) {
4705       array[offset] = varr[d];
4706     }
4707     size += dof;
4708   }
4709   for (p = 0; p < numPoints; ++p) {
4710     const PetscInt cp = cone[p];
4711     PetscInt       o  = coneO[p];
4712     PetscInt       dof, off, d;
4713     PetscScalar   *varr;
4714 
4715     if ((cp < pStart) || (cp >= pEnd)) continue;
4716     ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
4717     ierr = PetscSectionGetOffset(section, cp, &off);CHKERRQ(ierr);
4718     varr = &vArray[off];
4719     if (o >= 0) {
4720       for (d = 0; d < dof; ++d, ++offset) {
4721         array[offset] = varr[d];
4722       }
4723     } else {
4724       for (d = dof-1; d >= 0; --d, ++offset) {
4725         array[offset] = varr[d];
4726       }
4727     }
4728     size += dof;
4729   }
4730   ierr = VecRestoreArray(v, &vArray);CHKERRQ(ierr);
4731   if (!*values) {
4732     if (csize) *csize = size;
4733     *values = array;
4734   } else {
4735     if (size > *csize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Size of input array %d < actual size %d", *csize, size);
4736     *csize = size;
4737   }
4738   PetscFunctionReturn(0);
4739 }
4740 
4741 #undef __FUNCT__
4742 #define __FUNCT__ "DMPlexVecGetClosure_Static"
4743 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecGetClosure_Static(PetscSection section, PetscInt numPoints, const PetscInt points[], const PetscScalar vArray[], PetscInt *size, PetscScalar array[])
4744 {
4745   PetscInt       offset = 0, p;
4746   PetscErrorCode ierr;
4747 
4748   PetscFunctionBeginHot;
4749   *size = 0;
4750   for (p = 0; p < numPoints*2; p += 2) {
4751     const PetscInt point = points[p];
4752     const PetscInt o     = points[p+1];
4753     PetscInt       dof, off, d;
4754     const PetscScalar *varr;
4755 
4756     ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4757     ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
4758     varr = &vArray[off];
4759     if (o >= 0) {
4760       for (d = 0; d < dof; ++d, ++offset)    array[offset] = varr[d];
4761     } else {
4762       for (d = dof-1; d >= 0; --d, ++offset) array[offset] = varr[d];
4763     }
4764   }
4765   *size = offset;
4766   PetscFunctionReturn(0);
4767 }
4768 
4769 #undef __FUNCT__
4770 #define __FUNCT__ "DMPlexVecGetClosure_Fields_Static"
4771 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecGetClosure_Fields_Static(PetscSection section, PetscInt numPoints, const PetscInt points[], PetscInt numFields, const PetscScalar vArray[], PetscInt *size, PetscScalar array[])
4772 {
4773   PetscInt       offset = 0, f;
4774   PetscErrorCode ierr;
4775 
4776   PetscFunctionBeginHot;
4777   *size = 0;
4778   for (f = 0; f < numFields; ++f) {
4779     PetscInt fcomp, p;
4780 
4781     ierr = PetscSectionGetFieldComponents(section, f, &fcomp);CHKERRQ(ierr);
4782     for (p = 0; p < numPoints*2; p += 2) {
4783       const PetscInt point = points[p];
4784       const PetscInt o     = points[p+1];
4785       PetscInt       fdof, foff, d, c;
4786       const PetscScalar *varr;
4787 
4788       ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
4789       ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
4790       varr = &vArray[foff];
4791       if (o >= 0) {
4792         for (d = 0; d < fdof; ++d, ++offset) array[offset] = varr[d];
4793       } else {
4794         for (d = fdof/fcomp-1; d >= 0; --d) {
4795           for (c = 0; c < fcomp; ++c, ++offset) {
4796             array[offset] = varr[d*fcomp+c];
4797           }
4798         }
4799       }
4800     }
4801   }
4802   *size = offset;
4803   PetscFunctionReturn(0);
4804 }
4805 
4806 #undef __FUNCT__
4807 #define __FUNCT__ "DMPlexVecGetClosure"
4808 /*@C
4809   DMPlexVecGetClosure - Get an array of the values on the closure of 'point'
4810 
4811   Not collective
4812 
4813   Input Parameters:
4814 + dm - The DM
4815 . section - The section describing the layout in v, or NULL to use the default section
4816 . v - The local vector
4817 - point - The sieve point in the DM
4818 
4819   Output Parameters:
4820 + csize - The number of values in the closure, or NULL
4821 - values - The array of values, which is a borrowed array and should not be freed
4822 
4823   Fortran Notes:
4824   Since it returns an array, this routine is only available in Fortran 90, and you must
4825   include petsc.h90 in your code.
4826 
4827   The csize argument is not present in the Fortran 90 binding since it is internal to the array.
4828 
4829   Level: intermediate
4830 
4831 .seealso DMPlexVecRestoreClosure(), DMPlexVecSetClosure(), DMPlexMatSetClosure()
4832 @*/
4833 PetscErrorCode DMPlexVecGetClosure(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
4834 {
4835   PetscSection    clSection;
4836   IS              clPoints;
4837   PetscScalar    *array, *vArray;
4838   PetscInt       *points = NULL;
4839   const PetscInt *clp;
4840   PetscInt        depth, numFields, numPoints, size;
4841   PetscErrorCode  ierr;
4842 
4843   PetscFunctionBeginHot;
4844   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4845   if (!section) {ierr = DMGetDefaultSection(dm, &section);CHKERRQ(ierr);}
4846   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
4847   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
4848   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
4849   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
4850   if (depth == 1 && numFields < 2) {
4851     ierr = DMPlexVecGetClosure_Depth1_Static(dm, section, v, point, csize, values);CHKERRQ(ierr);
4852     PetscFunctionReturn(0);
4853   }
4854   /* Get points */
4855   ierr = PetscSectionGetClosureIndex(section, (PetscObject) dm, &clSection, &clPoints);CHKERRQ(ierr);
4856   if (!clPoints) {
4857     PetscInt pStart, pEnd, p, q;
4858 
4859     ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
4860     ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
4861     /* Compress out points not in the section */
4862     for (p = 0, q = 0; p < numPoints*2; p += 2) {
4863       if ((points[p] >= pStart) && (points[p] < pEnd)) {
4864         points[q*2]   = points[p];
4865         points[q*2+1] = points[p+1];
4866         ++q;
4867       }
4868     }
4869     numPoints = q;
4870   } else {
4871     PetscInt dof, off;
4872 
4873     ierr = PetscSectionGetDof(clSection, point, &dof);CHKERRQ(ierr);
4874     ierr = PetscSectionGetOffset(clSection, point, &off);CHKERRQ(ierr);
4875     ierr = ISGetIndices(clPoints, &clp);CHKERRQ(ierr);
4876     numPoints = dof/2;
4877     points    = (PetscInt *) &clp[off];
4878   }
4879   /* Get array */
4880   if (!values || !*values) {
4881     PetscInt asize = 0, dof, p;
4882 
4883     for (p = 0; p < numPoints*2; p += 2) {
4884       ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
4885       asize += dof;
4886     }
4887     if (!values) {
4888       if (!clPoints) {ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);}
4889       else           {ierr = ISRestoreIndices(clPoints, &clp);CHKERRQ(ierr);}
4890       if (csize) *csize = asize;
4891       PetscFunctionReturn(0);
4892     }
4893     ierr = DMGetWorkArray(dm, asize, PETSC_SCALAR, &array);CHKERRQ(ierr);
4894   } else {
4895     array = *values;
4896   }
4897   ierr = VecGetArray(v, &vArray);CHKERRQ(ierr);
4898   /* Get values */
4899   if (numFields > 0) {ierr = DMPlexVecGetClosure_Fields_Static(section, numPoints, points, numFields, vArray, &size, array);CHKERRQ(ierr);}
4900   else               {ierr = DMPlexVecGetClosure_Static(section, numPoints, points, vArray, &size, array);CHKERRQ(ierr);}
4901   /* Cleanup points */
4902   if (!clPoints) {ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);}
4903   else           {ierr = ISRestoreIndices(clPoints, &clp);CHKERRQ(ierr);}
4904   /* Cleanup array */
4905   ierr = VecRestoreArray(v, &vArray);CHKERRQ(ierr);
4906   if (!*values) {
4907     if (csize) *csize = size;
4908     *values = array;
4909   } else {
4910     if (size > *csize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Size of input array %d < actual size %d", *csize, size);
4911     *csize = size;
4912   }
4913   PetscFunctionReturn(0);
4914 }
4915 
4916 #undef __FUNCT__
4917 #define __FUNCT__ "DMPlexVecRestoreClosure"
4918 /*@C
4919   DMPlexVecRestoreClosure - Restore the array of the values on the closure of 'point'
4920 
4921   Not collective
4922 
4923   Input Parameters:
4924 + dm - The DM
4925 . section - The section describing the layout in v, or NULL to use the default section
4926 . v - The local vector
4927 . point - The sieve point in the DM
4928 . csize - The number of values in the closure, or NULL
4929 - values - The array of values, which is a borrowed array and should not be freed
4930 
4931   Fortran Notes:
4932   Since it returns an array, this routine is only available in Fortran 90, and you must
4933   include petsc.h90 in your code.
4934 
4935   The csize argument is not present in the Fortran 90 binding since it is internal to the array.
4936 
4937   Level: intermediate
4938 
4939 .seealso DMPlexVecGetClosure(), DMPlexVecSetClosure(), DMPlexMatSetClosure()
4940 @*/
4941 PetscErrorCode DMPlexVecRestoreClosure(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
4942 {
4943   PetscInt       size = 0;
4944   PetscErrorCode ierr;
4945 
4946   PetscFunctionBegin;
4947   /* Should work without recalculating size */
4948   ierr = DMRestoreWorkArray(dm, size, PETSC_SCALAR, (void*) values);CHKERRQ(ierr);
4949   PetscFunctionReturn(0);
4950 }
4951 
4952 PETSC_STATIC_INLINE void add   (PetscScalar *x, PetscScalar y) {*x += y;}
4953 PETSC_STATIC_INLINE void insert(PetscScalar *x, PetscScalar y) {*x  = y;}
4954 
4955 #undef __FUNCT__
4956 #define __FUNCT__ "updatePoint_private"
4957 PETSC_STATIC_INLINE PetscErrorCode updatePoint_private(PetscSection section, PetscInt point, PetscInt dof, void (*fuse)(PetscScalar*, PetscScalar), PetscBool setBC, PetscInt orientation, const PetscScalar values[], PetscScalar array[])
4958 {
4959   PetscInt        cdof;   /* The number of constraints on this point */
4960   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
4961   PetscScalar    *a;
4962   PetscInt        off, cind = 0, k;
4963   PetscErrorCode  ierr;
4964 
4965   PetscFunctionBegin;
4966   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
4967   ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
4968   a    = &array[off];
4969   if (!cdof || setBC) {
4970     if (orientation >= 0) {
4971       for (k = 0; k < dof; ++k) {
4972         fuse(&a[k], values[k]);
4973       }
4974     } else {
4975       for (k = 0; k < dof; ++k) {
4976         fuse(&a[k], values[dof-k-1]);
4977       }
4978     }
4979   } else {
4980     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
4981     if (orientation >= 0) {
4982       for (k = 0; k < dof; ++k) {
4983         if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
4984         fuse(&a[k], values[k]);
4985       }
4986     } else {
4987       for (k = 0; k < dof; ++k) {
4988         if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
4989         fuse(&a[k], values[dof-k-1]);
4990       }
4991     }
4992   }
4993   PetscFunctionReturn(0);
4994 }
4995 
4996 #undef __FUNCT__
4997 #define __FUNCT__ "updatePointBC_private"
4998 PETSC_STATIC_INLINE PetscErrorCode updatePointBC_private(PetscSection section, PetscInt point, PetscInt dof, void (*fuse)(PetscScalar*, PetscScalar), PetscInt orientation, const PetscScalar values[], PetscScalar array[])
4999 {
5000   PetscInt        cdof;   /* The number of constraints on this point */
5001   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
5002   PetscScalar    *a;
5003   PetscInt        off, cind = 0, k;
5004   PetscErrorCode  ierr;
5005 
5006   PetscFunctionBegin;
5007   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
5008   ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
5009   a    = &array[off];
5010   if (cdof) {
5011     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
5012     if (orientation >= 0) {
5013       for (k = 0; k < dof; ++k) {
5014         if ((cind < cdof) && (k == cdofs[cind])) {
5015           fuse(&a[k], values[k]);
5016           ++cind;
5017         }
5018       }
5019     } else {
5020       for (k = 0; k < dof; ++k) {
5021         if ((cind < cdof) && (k == cdofs[cind])) {
5022           fuse(&a[k], values[dof-k-1]);
5023           ++cind;
5024         }
5025       }
5026     }
5027   }
5028   PetscFunctionReturn(0);
5029 }
5030 
5031 #undef __FUNCT__
5032 #define __FUNCT__ "updatePointFields_private"
5033 PETSC_STATIC_INLINE PetscErrorCode updatePointFields_private(PetscSection section, PetscInt point, PetscInt o, PetscInt f, PetscInt fcomp, void (*fuse)(PetscScalar*, PetscScalar), PetscBool setBC, const PetscScalar values[], PetscInt *offset, PetscScalar array[])
5034 {
5035   PetscScalar    *a;
5036   PetscInt        fdof, foff, fcdof, foffset = *offset;
5037   const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
5038   PetscInt        cind = 0, k, c;
5039   PetscErrorCode  ierr;
5040 
5041   PetscFunctionBegin;
5042   ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
5043   ierr = PetscSectionGetFieldConstraintDof(section, point, f, &fcdof);CHKERRQ(ierr);
5044   ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
5045   a    = &array[foff];
5046   if (!fcdof || setBC) {
5047     if (o >= 0) {
5048       for (k = 0; k < fdof; ++k) fuse(&a[k], values[foffset+k]);
5049     } else {
5050       for (k = fdof/fcomp-1; k >= 0; --k) {
5051         for (c = 0; c < fcomp; ++c) {
5052           fuse(&a[(fdof/fcomp-1-k)*fcomp+c], values[foffset+k*fcomp+c]);
5053         }
5054       }
5055     }
5056   } else {
5057     ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
5058     if (o >= 0) {
5059       for (k = 0; k < fdof; ++k) {
5060         if ((cind < fcdof) && (k == fcdofs[cind])) {++cind; continue;}
5061         fuse(&a[k], values[foffset+k]);
5062       }
5063     } else {
5064       for (k = fdof/fcomp-1; k >= 0; --k) {
5065         for (c = 0; c < fcomp; ++c) {
5066           if ((cind < fcdof) && (k*fcomp+c == fcdofs[cind])) {++cind; continue;}
5067           fuse(&a[(fdof/fcomp-1-k)*fcomp+c], values[foffset+k*fcomp+c]);
5068         }
5069       }
5070     }
5071   }
5072   *offset += fdof;
5073   PetscFunctionReturn(0);
5074 }
5075 
5076 #undef __FUNCT__
5077 #define __FUNCT__ "updatePointFieldsBC_private"
5078 PETSC_STATIC_INLINE PetscErrorCode updatePointFieldsBC_private(PetscSection section, PetscInt point, PetscInt o, PetscInt f, PetscInt fcomp, void (*fuse)(PetscScalar*, PetscScalar), const PetscScalar values[], PetscInt *offset, PetscScalar array[])
5079 {
5080   PetscScalar    *a;
5081   PetscInt        fdof, foff, fcdof, foffset = *offset;
5082   const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
5083   PetscInt        cind = 0, k, c;
5084   PetscErrorCode  ierr;
5085 
5086   PetscFunctionBegin;
5087   ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
5088   ierr = PetscSectionGetFieldConstraintDof(section, point, f, &fcdof);CHKERRQ(ierr);
5089   ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
5090   a    = &array[foff];
5091   if (fcdof) {
5092     ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
5093     if (o >= 0) {
5094       for (k = 0; k < fdof; ++k) {
5095         if ((cind < fcdof) && (k == fcdofs[cind])) {
5096           fuse(&a[k], values[foffset+k]);
5097           ++cind;
5098         }
5099       }
5100     } else {
5101       for (k = fdof/fcomp-1; k >= 0; --k) {
5102         for (c = 0; c < fcomp; ++c) {
5103           if ((cind < fcdof) && (k*fcomp+c == fcdofs[cind])) {
5104             fuse(&a[(fdof/fcomp-1-k)*fcomp+c], values[foffset+k*fcomp+c]);
5105             ++cind;
5106           }
5107         }
5108       }
5109     }
5110   }
5111   *offset += fdof;
5112   PetscFunctionReturn(0);
5113 }
5114 
5115 #undef __FUNCT__
5116 #define __FUNCT__ "DMPlexVecSetClosure_Static"
5117 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecSetClosure_Static(DM dm, PetscSection section, Vec v, PetscInt point, const PetscScalar values[], InsertMode mode)
5118 {
5119   PetscScalar    *array;
5120   const PetscInt *cone, *coneO;
5121   PetscInt        pStart, pEnd, p, numPoints, off, dof;
5122   PetscErrorCode  ierr;
5123 
5124   PetscFunctionBeginHot;
5125   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
5126   ierr = DMPlexGetConeSize(dm, point, &numPoints);CHKERRQ(ierr);
5127   ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
5128   ierr = DMPlexGetConeOrientation(dm, point, &coneO);CHKERRQ(ierr);
5129   ierr = VecGetArray(v, &array);CHKERRQ(ierr);
5130   for (p = 0, off = 0; p <= numPoints; ++p, off += dof) {
5131     const PetscInt cp = !p ? point : cone[p-1];
5132     const PetscInt o  = !p ? 0     : coneO[p-1];
5133 
5134     if ((cp < pStart) || (cp >= pEnd)) {dof = 0; continue;}
5135     ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
5136     /* ADD_VALUES */
5137     {
5138       const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
5139       PetscScalar    *a;
5140       PetscInt        cdof, coff, cind = 0, k;
5141 
5142       ierr = PetscSectionGetConstraintDof(section, cp, &cdof);CHKERRQ(ierr);
5143       ierr = PetscSectionGetOffset(section, cp, &coff);CHKERRQ(ierr);
5144       a    = &array[coff];
5145       if (!cdof) {
5146         if (o >= 0) {
5147           for (k = 0; k < dof; ++k) {
5148             a[k] += values[off+k];
5149           }
5150         } else {
5151           for (k = 0; k < dof; ++k) {
5152             a[k] += values[off+dof-k-1];
5153           }
5154         }
5155       } else {
5156         ierr = PetscSectionGetConstraintIndices(section, cp, &cdofs);CHKERRQ(ierr);
5157         if (o >= 0) {
5158           for (k = 0; k < dof; ++k) {
5159             if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
5160             a[k] += values[off+k];
5161           }
5162         } else {
5163           for (k = 0; k < dof; ++k) {
5164             if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
5165             a[k] += values[off+dof-k-1];
5166           }
5167         }
5168       }
5169     }
5170   }
5171   ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
5172   PetscFunctionReturn(0);
5173 }
5174 
5175 #undef __FUNCT__
5176 #define __FUNCT__ "DMPlexVecSetClosure"
5177 /*@C
5178   DMPlexVecSetClosure - Set an array of the values on the closure of 'point'
5179 
5180   Not collective
5181 
5182   Input Parameters:
5183 + dm - The DM
5184 . section - The section describing the layout in v, or NULL to use the default section
5185 . v - The local vector
5186 . point - The sieve point in the DM
5187 . values - The array of values
5188 - mode - The insert mode, where INSERT_ALL_VALUES and ADD_ALL_VALUES also overwrite boundary conditions
5189 
5190   Fortran Notes:
5191   This routine is only available in Fortran 90, and you must include petsc.h90 in your code.
5192 
5193   Level: intermediate
5194 
5195 .seealso DMPlexVecGetClosure(), DMPlexMatSetClosure()
5196 @*/
5197 PetscErrorCode DMPlexVecSetClosure(DM dm, PetscSection section, Vec v, PetscInt point, const PetscScalar values[], InsertMode mode)
5198 {
5199   PetscSection    clSection;
5200   IS              clPoints;
5201   PetscScalar    *array;
5202   PetscInt       *points = NULL;
5203   const PetscInt *clp;
5204   PetscInt        depth, numFields, numPoints, p;
5205   PetscErrorCode  ierr;
5206 
5207   PetscFunctionBeginHot;
5208   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5209   if (!section) {ierr = DMGetDefaultSection(dm, &section);CHKERRQ(ierr);}
5210   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
5211   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
5212   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
5213   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5214   if (depth == 1 && numFields < 2 && mode == ADD_VALUES) {
5215     ierr = DMPlexVecSetClosure_Static(dm, section, v, point, values, mode);CHKERRQ(ierr);
5216     PetscFunctionReturn(0);
5217   }
5218   /* Get points */
5219   ierr = PetscSectionGetClosureIndex(section, (PetscObject) dm, &clSection, &clPoints);CHKERRQ(ierr);
5220   if (!clPoints) {
5221     PetscInt pStart, pEnd, q;
5222 
5223     ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
5224     ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
5225     /* Compress out points not in the section */
5226     for (p = 0, q = 0; p < numPoints*2; p += 2) {
5227       if ((points[p] >= pStart) && (points[p] < pEnd)) {
5228         points[q*2]   = points[p];
5229         points[q*2+1] = points[p+1];
5230         ++q;
5231       }
5232     }
5233     numPoints = q;
5234   } else {
5235     PetscInt dof, off;
5236 
5237     ierr = PetscSectionGetDof(clSection, point, &dof);CHKERRQ(ierr);
5238     ierr = PetscSectionGetOffset(clSection, point, &off);CHKERRQ(ierr);
5239     ierr = ISGetIndices(clPoints, &clp);CHKERRQ(ierr);
5240     numPoints = dof/2;
5241     points    = (PetscInt *) &clp[off];
5242   }
5243   /* Get array */
5244   ierr = VecGetArray(v, &array);CHKERRQ(ierr);
5245   /* Get values */
5246   if (numFields > 0) {
5247     PetscInt offset = 0, fcomp, f;
5248     for (f = 0; f < numFields; ++f) {
5249       ierr = PetscSectionGetFieldComponents(section, f, &fcomp);CHKERRQ(ierr);
5250       switch (mode) {
5251       case INSERT_VALUES:
5252         for (p = 0; p < numPoints*2; p += 2) {
5253           const PetscInt point = points[p];
5254           const PetscInt o     = points[p+1];
5255           updatePointFields_private(section, point, o, f, fcomp, insert, PETSC_FALSE, values, &offset, array);
5256         } break;
5257       case INSERT_ALL_VALUES:
5258         for (p = 0; p < numPoints*2; p += 2) {
5259           const PetscInt point = points[p];
5260           const PetscInt o     = points[p+1];
5261           updatePointFields_private(section, point, o, f, fcomp, insert, PETSC_TRUE, values, &offset, array);
5262         } break;
5263       case INSERT_BC_VALUES:
5264         for (p = 0; p < numPoints*2; p += 2) {
5265           const PetscInt point = points[p];
5266           const PetscInt o     = points[p+1];
5267           updatePointFieldsBC_private(section, point, o, f, fcomp, insert, values, &offset, array);
5268         } break;
5269       case ADD_VALUES:
5270         for (p = 0; p < numPoints*2; p += 2) {
5271           const PetscInt point = points[p];
5272           const PetscInt o     = points[p+1];
5273           updatePointFields_private(section, point, o, f, fcomp, add, PETSC_FALSE, values, &offset, array);
5274         } break;
5275       case ADD_ALL_VALUES:
5276         for (p = 0; p < numPoints*2; p += 2) {
5277           const PetscInt point = points[p];
5278           const PetscInt o     = points[p+1];
5279           updatePointFields_private(section, point, o, f, fcomp, add, PETSC_TRUE, values, &offset, array);
5280         } break;
5281       default:
5282         SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %D", mode);
5283       }
5284     }
5285   } else {
5286     PetscInt dof, off;
5287 
5288     switch (mode) {
5289     case INSERT_VALUES:
5290       for (p = 0, off = 0; p < numPoints*2; p += 2, off += dof) {
5291         PetscInt o = points[p+1];
5292         ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5293         updatePoint_private(section, points[p], dof, insert, PETSC_FALSE, o, &values[off], array);
5294       } break;
5295     case INSERT_ALL_VALUES:
5296       for (p = 0, off = 0; p < numPoints*2; p += 2, off += dof) {
5297         PetscInt o = points[p+1];
5298         ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5299         updatePoint_private(section, points[p], dof, insert, PETSC_TRUE,  o, &values[off], array);
5300       } break;
5301     case INSERT_BC_VALUES:
5302       for (p = 0, off = 0; p < numPoints*2; p += 2, off += dof) {
5303         PetscInt o = points[p+1];
5304         ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5305         updatePointBC_private(section, points[p], dof, insert,  o, &values[off], array);
5306       } break;
5307     case ADD_VALUES:
5308       for (p = 0, off = 0; p < numPoints*2; p += 2, off += dof) {
5309         PetscInt o = points[p+1];
5310         ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5311         updatePoint_private(section, points[p], dof, add,    PETSC_FALSE, o, &values[off], array);
5312       } break;
5313     case ADD_ALL_VALUES:
5314       for (p = 0, off = 0; p < numPoints*2; p += 2, off += dof) {
5315         PetscInt o = points[p+1];
5316         ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5317         updatePoint_private(section, points[p], dof, add,    PETSC_TRUE,  o, &values[off], array);
5318       } break;
5319     default:
5320       SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %D", mode);
5321     }
5322   }
5323   /* Cleanup points */
5324   if (!clPoints) {ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);}
5325   else           {ierr = ISRestoreIndices(clPoints, &clp);CHKERRQ(ierr);}
5326   /* Cleanup array */
5327   ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
5328   PetscFunctionReturn(0);
5329 }
5330 
5331 #undef __FUNCT__
5332 #define __FUNCT__ "DMPlexPrintMatSetValues"
5333 PetscErrorCode DMPlexPrintMatSetValues(PetscViewer viewer, Mat A, PetscInt point, PetscInt numRIndices, const PetscInt rindices[], PetscInt numCIndices, const PetscInt cindices[], const PetscScalar values[])
5334 {
5335   PetscMPIInt    rank;
5336   PetscInt       i, j;
5337   PetscErrorCode ierr;
5338 
5339   PetscFunctionBegin;
5340   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRQ(ierr);
5341   ierr = PetscViewerASCIIPrintf(viewer, "[%D]mat for sieve point %D\n", rank, point);CHKERRQ(ierr);
5342   for (i = 0; i < numRIndices; i++) {ierr = PetscViewerASCIIPrintf(viewer, "[%D]mat row indices[%D] = %D\n", rank, i, rindices[i]);CHKERRQ(ierr);}
5343   for (i = 0; i < numCIndices; i++) {ierr = PetscViewerASCIIPrintf(viewer, "[%D]mat col indices[%D] = %D\n", rank, i, cindices[i]);CHKERRQ(ierr);}
5344   numCIndices = numCIndices ? numCIndices : numRIndices;
5345   for (i = 0; i < numRIndices; i++) {
5346     ierr = PetscViewerASCIIPrintf(viewer, "[%D]", rank);CHKERRQ(ierr);
5347     for (j = 0; j < numCIndices; j++) {
5348 #if defined(PETSC_USE_COMPLEX)
5349       ierr = PetscViewerASCIIPrintf(viewer, " (%g,%g)", (double)PetscRealPart(values[i*numCIndices+j]), (double)PetscImaginaryPart(values[i*numCIndices+j]));CHKERRQ(ierr);
5350 #else
5351       ierr = PetscViewerASCIIPrintf(viewer, " %g", (double)values[i*numCIndices+j]);CHKERRQ(ierr);
5352 #endif
5353     }
5354     ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
5355   }
5356   PetscFunctionReturn(0);
5357 }
5358 
5359 #undef __FUNCT__
5360 #define __FUNCT__ "indicesPoint_private"
5361 /* . off - The global offset of this point */
5362 PetscErrorCode indicesPoint_private(PetscSection section, PetscInt point, PetscInt off, PetscInt *loff, PetscBool setBC, PetscInt orientation, PetscInt indices[])
5363 {
5364   PetscInt        dof;    /* The number of unknowns on this point */
5365   PetscInt        cdof;   /* The number of constraints on this point */
5366   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
5367   PetscInt        cind = 0, k;
5368   PetscErrorCode  ierr;
5369 
5370   PetscFunctionBegin;
5371   ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
5372   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
5373   if (!cdof || setBC) {
5374     if (orientation >= 0) {
5375       for (k = 0; k < dof; ++k) indices[*loff+k] = off+k;
5376     } else {
5377       for (k = 0; k < dof; ++k) indices[*loff+dof-k-1] = off+k;
5378     }
5379   } else {
5380     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
5381     if (orientation >= 0) {
5382       for (k = 0; k < dof; ++k) {
5383         if ((cind < cdof) && (k == cdofs[cind])) {
5384           /* Insert check for returning constrained indices */
5385           indices[*loff+k] = -(off+k+1);
5386           ++cind;
5387         } else {
5388           indices[*loff+k] = off+k-cind;
5389         }
5390       }
5391     } else {
5392       for (k = 0; k < dof; ++k) {
5393         if ((cind < cdof) && (k == cdofs[cind])) {
5394           /* Insert check for returning constrained indices */
5395           indices[*loff+dof-k-1] = -(off+k+1);
5396           ++cind;
5397         } else {
5398           indices[*loff+dof-k-1] = off+k-cind;
5399         }
5400       }
5401     }
5402   }
5403   *loff += dof;
5404   PetscFunctionReturn(0);
5405 }
5406 
5407 #undef __FUNCT__
5408 #define __FUNCT__ "indicesPointFields_private"
5409 /* . off - The global offset of this point */
5410 PetscErrorCode indicesPointFields_private(PetscSection section, PetscInt point, PetscInt off, PetscInt foffs[], PetscBool setBC, PetscInt orientation, PetscInt indices[])
5411 {
5412   PetscInt       numFields, foff, f;
5413   PetscErrorCode ierr;
5414 
5415   PetscFunctionBegin;
5416   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5417   for (f = 0, foff = 0; f < numFields; ++f) {
5418     PetscInt        fdof, fcomp, cfdof;
5419     const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
5420     PetscInt        cind = 0, k, c;
5421 
5422     ierr = PetscSectionGetFieldComponents(section, f, &fcomp);CHKERRQ(ierr);
5423     ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
5424     ierr = PetscSectionGetFieldConstraintDof(section, point, f, &cfdof);CHKERRQ(ierr);
5425     if (!cfdof || setBC) {
5426       if (orientation >= 0) {
5427         for (k = 0; k < fdof; ++k) indices[foffs[f]+k] = off+foff+k;
5428       } else {
5429         for (k = fdof/fcomp-1; k >= 0; --k) {
5430           for (c = 0; c < fcomp; ++c) {
5431             indices[foffs[f]+k*fcomp+c] = off+foff+(fdof/fcomp-1-k)*fcomp+c;
5432           }
5433         }
5434       }
5435     } else {
5436       ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
5437       if (orientation >= 0) {
5438         for (k = 0; k < fdof; ++k) {
5439           if ((cind < cfdof) && (k == fcdofs[cind])) {
5440             indices[foffs[f]+k] = -(off+foff+k+1);
5441             ++cind;
5442           } else {
5443             indices[foffs[f]+k] = off+foff+k-cind;
5444           }
5445         }
5446       } else {
5447         for (k = fdof/fcomp-1; k >= 0; --k) {
5448           for (c = 0; c < fcomp; ++c) {
5449             if ((cind < cfdof) && ((fdof/fcomp-1-k)*fcomp+c == fcdofs[cind])) {
5450               indices[foffs[f]+k*fcomp+c] = -(off+foff+(fdof/fcomp-1-k)*fcomp+c+1);
5451               ++cind;
5452             } else {
5453               indices[foffs[f]+k*fcomp+c] = off+foff+(fdof/fcomp-1-k)*fcomp+c-cind;
5454             }
5455           }
5456         }
5457       }
5458     }
5459     foff     += fdof - cfdof;
5460     foffs[f] += fdof;
5461   }
5462   PetscFunctionReturn(0);
5463 }
5464 
5465 #undef __FUNCT__
5466 #define __FUNCT__ "DMPlexMatSetClosure"
5467 /*@C
5468   DMPlexMatSetClosure - Set an array of the values on the closure of 'point'
5469 
5470   Not collective
5471 
5472   Input Parameters:
5473 + dm - The DM
5474 . section - The section describing the layout in v, or NULL to use the default section
5475 . globalSection - The section describing the layout in v, or NULL to use the default global section
5476 . A - The matrix
5477 . point - The sieve point in the DM
5478 . values - The array of values
5479 - mode - The insert mode, where INSERT_ALL_VALUES and ADD_ALL_VALUES also overwrite boundary conditions
5480 
5481   Fortran Notes:
5482   This routine is only available in Fortran 90, and you must include petsc.h90 in your code.
5483 
5484   Level: intermediate
5485 
5486 .seealso DMPlexVecGetClosure(), DMPlexVecSetClosure()
5487 @*/
5488 PetscErrorCode DMPlexMatSetClosure(DM dm, PetscSection section, PetscSection globalSection, Mat A, PetscInt point, const PetscScalar values[], InsertMode mode)
5489 {
5490   DM_Plex        *mesh   = (DM_Plex*) dm->data;
5491   PetscSection    clSection;
5492   IS              clPoints;
5493   PetscInt       *points = NULL;
5494   const PetscInt *clp;
5495   PetscInt       *indices;
5496   PetscInt        offsets[32];
5497   PetscInt        numFields, numPoints, numIndices, dof, off, globalOff, pStart, pEnd, p, q, f;
5498   PetscErrorCode  ierr;
5499 
5500   PetscFunctionBegin;
5501   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5502   if (!section) {ierr = DMGetDefaultSection(dm, &section);CHKERRQ(ierr);}
5503   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
5504   if (!globalSection) {ierr = DMGetDefaultGlobalSection(dm, &globalSection);CHKERRQ(ierr);}
5505   PetscValidHeaderSpecific(globalSection, PETSC_SECTION_CLASSID, 3);
5506   PetscValidHeaderSpecific(A, MAT_CLASSID, 4);
5507   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5508   if (numFields > 31) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", numFields);
5509   ierr = PetscMemzero(offsets, 32 * sizeof(PetscInt));CHKERRQ(ierr);
5510   ierr = PetscSectionGetClosureIndex(section, (PetscObject) dm, &clSection, &clPoints);CHKERRQ(ierr);
5511   if (!clPoints) {
5512     ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
5513     /* Compress out points not in the section */
5514     ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
5515     for (p = 0, q = 0; p < numPoints*2; p += 2) {
5516       if ((points[p] >= pStart) && (points[p] < pEnd)) {
5517         points[q*2]   = points[p];
5518         points[q*2+1] = points[p+1];
5519         ++q;
5520       }
5521     }
5522     numPoints = q;
5523   } else {
5524     PetscInt dof, off;
5525 
5526     ierr = PetscSectionGetDof(clSection, point, &dof);CHKERRQ(ierr);
5527     numPoints = dof/2;
5528     ierr = PetscSectionGetOffset(clSection, point, &off);CHKERRQ(ierr);
5529     ierr = ISGetIndices(clPoints, &clp);CHKERRQ(ierr);
5530     points = (PetscInt *) &clp[off];
5531   }
5532   for (p = 0, numIndices = 0; p < numPoints*2; p += 2) {
5533     PetscInt fdof;
5534 
5535     ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5536     for (f = 0; f < numFields; ++f) {
5537       ierr          = PetscSectionGetFieldDof(section, points[p], f, &fdof);CHKERRQ(ierr);
5538       offsets[f+1] += fdof;
5539     }
5540     numIndices += dof;
5541   }
5542   for (f = 1; f < numFields; ++f) offsets[f+1] += offsets[f];
5543 
5544   if (numFields && offsets[numFields] != numIndices) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid size for closure %d should be %d", offsets[numFields], numIndices);
5545   ierr = DMGetWorkArray(dm, numIndices, PETSC_INT, &indices);CHKERRQ(ierr);
5546   if (numFields) {
5547     for (p = 0; p < numPoints*2; p += 2) {
5548       PetscInt o = points[p+1];
5549       ierr = PetscSectionGetOffset(globalSection, points[p], &globalOff);CHKERRQ(ierr);
5550       indicesPointFields_private(section, points[p], globalOff < 0 ? -(globalOff+1) : globalOff, offsets, PETSC_FALSE, o, indices);
5551     }
5552   } else {
5553     for (p = 0, off = 0; p < numPoints*2; p += 2) {
5554       PetscInt o = points[p+1];
5555       ierr = PetscSectionGetOffset(globalSection, points[p], &globalOff);CHKERRQ(ierr);
5556       indicesPoint_private(section, points[p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, o, indices);
5557     }
5558   }
5559   if (mesh->printSetValues) {ierr = DMPlexPrintMatSetValues(PETSC_VIEWER_STDOUT_SELF, A, point, numIndices, indices, 0, NULL, values);CHKERRQ(ierr);}
5560   ierr = MatSetValues(A, numIndices, indices, numIndices, indices, values, mode);
5561   if (ierr) {
5562     PetscMPIInt    rank;
5563     PetscErrorCode ierr2;
5564 
5565     ierr2 = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRQ(ierr2);
5566     ierr2 = (*PetscErrorPrintf)("[%D]ERROR in DMPlexMatSetClosure\n", rank);CHKERRQ(ierr2);
5567     ierr2 = DMPlexPrintMatSetValues(PETSC_VIEWER_STDERR_SELF, A, point, numIndices, indices, 0, NULL, values);CHKERRQ(ierr2);
5568     ierr2 = DMRestoreWorkArray(dm, numIndices, PETSC_INT, &indices);CHKERRQ(ierr2);
5569     CHKERRQ(ierr);
5570   }
5571   if (!clPoints) {
5572     ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
5573   } else {
5574     ierr = ISRestoreIndices(clPoints, &clp);CHKERRQ(ierr);
5575   }
5576   ierr = DMRestoreWorkArray(dm, numIndices, PETSC_INT, &indices);CHKERRQ(ierr);
5577   PetscFunctionReturn(0);
5578 }
5579 
5580 #undef __FUNCT__
5581 #define __FUNCT__ "DMPlexMatSetClosureRefined"
5582 PetscErrorCode DMPlexMatSetClosureRefined(DM dmf, PetscSection fsection, PetscSection globalFSection, DM dmc, PetscSection csection, PetscSection globalCSection, Mat A, PetscInt point, const PetscScalar values[], InsertMode mode)
5583 {
5584   DM_Plex        *mesh   = (DM_Plex*) dmf->data;
5585   PetscInt       *fpoints = NULL, *ftotpoints = NULL;
5586   PetscInt       *cpoints = NULL;
5587   PetscInt       *findices, *cindices;
5588   PetscInt        foffsets[32], coffsets[32];
5589   CellRefiner     cellRefiner;
5590   PetscInt        numFields, numSubcells, maxFPoints, numFPoints, numCPoints, numFIndices, numCIndices, dof, off, globalOff, pStart, pEnd, p, q, r, s, f;
5591   PetscErrorCode  ierr;
5592 
5593   PetscFunctionBegin;
5594   PetscValidHeaderSpecific(dmf, DM_CLASSID, 1);
5595   PetscValidHeaderSpecific(dmc, DM_CLASSID, 4);
5596   if (!fsection) {ierr = DMGetDefaultSection(dmf, &fsection);CHKERRQ(ierr);}
5597   PetscValidHeaderSpecific(fsection, PETSC_SECTION_CLASSID, 2);
5598   if (!csection) {ierr = DMGetDefaultSection(dmc, &csection);CHKERRQ(ierr);}
5599   PetscValidHeaderSpecific(csection, PETSC_SECTION_CLASSID, 5);
5600   if (!globalFSection) {ierr = DMGetDefaultGlobalSection(dmf, &globalFSection);CHKERRQ(ierr);}
5601   PetscValidHeaderSpecific(globalFSection, PETSC_SECTION_CLASSID, 3);
5602   if (!globalCSection) {ierr = DMGetDefaultGlobalSection(dmc, &globalCSection);CHKERRQ(ierr);}
5603   PetscValidHeaderSpecific(globalCSection, PETSC_SECTION_CLASSID, 6);
5604   PetscValidHeaderSpecific(A, MAT_CLASSID, 7);
5605   ierr = PetscSectionGetNumFields(fsection, &numFields);CHKERRQ(ierr);
5606   if (numFields > 31) SETERRQ1(PetscObjectComm((PetscObject)dmf), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", numFields);
5607   ierr = PetscMemzero(foffsets, 32 * sizeof(PetscInt));CHKERRQ(ierr);
5608   ierr = PetscMemzero(coffsets, 32 * sizeof(PetscInt));CHKERRQ(ierr);
5609   /* Column indices */
5610   ierr = DMPlexGetTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
5611   maxFPoints = numCPoints;
5612   /* Compress out points not in the section */
5613   /*   TODO: Squeeze out points with 0 dof as well */
5614   ierr = PetscSectionGetChart(csection, &pStart, &pEnd);CHKERRQ(ierr);
5615   for (p = 0, q = 0; p < numCPoints*2; p += 2) {
5616     if ((cpoints[p] >= pStart) && (cpoints[p] < pEnd)) {
5617       cpoints[q*2]   = cpoints[p];
5618       cpoints[q*2+1] = cpoints[p+1];
5619       ++q;
5620     }
5621   }
5622   numCPoints = q;
5623   for (p = 0, numCIndices = 0; p < numCPoints*2; p += 2) {
5624     PetscInt fdof;
5625 
5626     ierr = PetscSectionGetDof(csection, cpoints[p], &dof);CHKERRQ(ierr);
5627     if (!dof) continue;
5628     for (f = 0; f < numFields; ++f) {
5629       ierr           = PetscSectionGetFieldDof(csection, cpoints[p], f, &fdof);CHKERRQ(ierr);
5630       coffsets[f+1] += fdof;
5631     }
5632     numCIndices += dof;
5633   }
5634   for (f = 1; f < numFields; ++f) coffsets[f+1] += coffsets[f];
5635   /* Row indices */
5636   ierr = DMPlexGetCellRefiner_Internal(dmc, &cellRefiner);CHKERRQ(ierr);
5637   ierr = CellRefinerGetAffineTransforms_Internal(cellRefiner, &numSubcells, NULL, NULL, NULL);CHKERRQ(ierr);
5638   ierr = DMGetWorkArray(dmf, maxFPoints*2*numSubcells, PETSC_INT, &ftotpoints);CHKERRQ(ierr);
5639   for (r = 0, q = 0; r < numSubcells; ++r) {
5640     /* TODO Map from coarse to fine cells */
5641     ierr = DMPlexGetTransitiveClosure(dmf, point*numSubcells + r, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
5642     /* Compress out points not in the section */
5643     ierr = PetscSectionGetChart(fsection, &pStart, &pEnd);CHKERRQ(ierr);
5644     for (p = 0; p < numFPoints*2; p += 2) {
5645       if ((fpoints[p] >= pStart) && (fpoints[p] < pEnd)) {
5646         ierr = PetscSectionGetDof(fsection, fpoints[p], &dof);CHKERRQ(ierr);
5647         if (!dof) continue;
5648         for (s = 0; s < q; ++s) if (fpoints[p] == ftotpoints[s*2]) break;
5649         if (s < q) continue;
5650         ftotpoints[q*2]   = fpoints[p];
5651         ftotpoints[q*2+1] = fpoints[p+1];
5652         ++q;
5653       }
5654     }
5655     ierr = DMPlexRestoreTransitiveClosure(dmf, point, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
5656   }
5657   numFPoints = q;
5658   for (p = 0, numFIndices = 0; p < numFPoints*2; p += 2) {
5659     PetscInt fdof;
5660 
5661     ierr = PetscSectionGetDof(fsection, ftotpoints[p], &dof);CHKERRQ(ierr);
5662     if (!dof) continue;
5663     for (f = 0; f < numFields; ++f) {
5664       ierr           = PetscSectionGetFieldDof(fsection, ftotpoints[p], f, &fdof);CHKERRQ(ierr);
5665       foffsets[f+1] += fdof;
5666     }
5667     numFIndices += dof;
5668   }
5669   for (f = 1; f < numFields; ++f) foffsets[f+1] += foffsets[f];
5670 
5671   if (numFields && foffsets[numFields] != numFIndices) SETERRQ2(PetscObjectComm((PetscObject)dmf), PETSC_ERR_PLIB, "Invalid size for closure %d should be %d", foffsets[numFields], numFIndices);
5672   if (numFields && coffsets[numFields] != numCIndices) SETERRQ2(PetscObjectComm((PetscObject)dmc), PETSC_ERR_PLIB, "Invalid size for closure %d should be %d", coffsets[numFields], numCIndices);
5673   ierr = DMGetWorkArray(dmf, numFIndices, PETSC_INT, &findices);CHKERRQ(ierr);
5674   ierr = DMGetWorkArray(dmc, numCIndices, PETSC_INT, &cindices);CHKERRQ(ierr);
5675   if (numFields) {
5676     for (p = 0; p < numFPoints*2; p += 2) {
5677       PetscInt o = ftotpoints[p+1];
5678       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[p], &globalOff);CHKERRQ(ierr);
5679       indicesPointFields_private(fsection, ftotpoints[p], globalOff < 0 ? -(globalOff+1) : globalOff, foffsets, PETSC_FALSE, o, findices);
5680     }
5681     for (p = 0; p < numCPoints*2; p += 2) {
5682       PetscInt o = cpoints[p+1];
5683       ierr = PetscSectionGetOffset(globalCSection, cpoints[p], &globalOff);CHKERRQ(ierr);
5684       indicesPointFields_private(csection, cpoints[p], globalOff < 0 ? -(globalOff+1) : globalOff, coffsets, PETSC_FALSE, o, cindices);
5685     }
5686   } else {
5687     for (p = 0, off = 0; p < numFPoints*2; p += 2) {
5688       PetscInt o = ftotpoints[p+1];
5689       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[p], &globalOff);CHKERRQ(ierr);
5690       indicesPoint_private(fsection, ftotpoints[p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, o, findices);
5691     }
5692     for (p = 0, off = 0; p < numCPoints*2; p += 2) {
5693       PetscInt o = cpoints[p+1];
5694       ierr = PetscSectionGetOffset(globalCSection, cpoints[p], &globalOff);CHKERRQ(ierr);
5695       indicesPoint_private(csection, cpoints[p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, o, cindices);
5696     }
5697   }
5698   if (mesh->printSetValues) {ierr = DMPlexPrintMatSetValues(PETSC_VIEWER_STDOUT_SELF, A, point, numFIndices, findices, numCIndices, cindices, values);CHKERRQ(ierr);}
5699   ierr = MatSetValues(A, numFIndices, findices, numCIndices, cindices, values, mode);
5700   if (ierr) {
5701     PetscMPIInt    rank;
5702     PetscErrorCode ierr2;
5703 
5704     ierr2 = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRQ(ierr2);
5705     ierr2 = (*PetscErrorPrintf)("[%D]ERROR in DMPlexMatSetClosure\n", rank);CHKERRQ(ierr2);
5706     ierr2 = DMPlexPrintMatSetValues(PETSC_VIEWER_STDERR_SELF, A, point, numFIndices, findices, numCIndices, cindices, values);CHKERRQ(ierr2);
5707     ierr2 = DMRestoreWorkArray(dmf, numFIndices, PETSC_INT, &findices);CHKERRQ(ierr2);
5708     ierr2 = DMRestoreWorkArray(dmc, numCIndices, PETSC_INT, &cindices);CHKERRQ(ierr2);
5709     CHKERRQ(ierr);
5710   }
5711   ierr = DMRestoreWorkArray(dmf, numCPoints*2*4, PETSC_INT, &ftotpoints);CHKERRQ(ierr);
5712   ierr = DMPlexRestoreTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
5713   ierr = DMRestoreWorkArray(dmf, numFIndices, PETSC_INT, &findices);CHKERRQ(ierr);
5714   ierr = DMRestoreWorkArray(dmc, numCIndices, PETSC_INT, &cindices);CHKERRQ(ierr);
5715   PetscFunctionReturn(0);
5716 }
5717 
5718 #undef __FUNCT__
5719 #define __FUNCT__ "DMPlexGetHybridBounds"
5720 /*@
5721   DMPlexGetHybridBounds - Get the first mesh point of each dimension which is a hybrid
5722 
5723   Input Parameter:
5724 . dm - The DMPlex object
5725 
5726   Output Parameters:
5727 + cMax - The first hybrid cell
5728 . cMax - The first hybrid face
5729 . cMax - The first hybrid edge
5730 - cMax - The first hybrid vertex
5731 
5732   Level: developer
5733 
5734 .seealso DMPlexCreateHybridMesh()
5735 @*/
5736 PetscErrorCode DMPlexGetHybridBounds(DM dm, PetscInt *cMax, PetscInt *fMax, PetscInt *eMax, PetscInt *vMax)
5737 {
5738   DM_Plex       *mesh = (DM_Plex*) dm->data;
5739   PetscInt       dim;
5740   PetscErrorCode ierr;
5741 
5742   PetscFunctionBegin;
5743   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5744   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
5745   if (cMax) *cMax = mesh->hybridPointMax[dim];
5746   if (fMax) *fMax = mesh->hybridPointMax[dim-1];
5747   if (eMax) *eMax = mesh->hybridPointMax[1];
5748   if (vMax) *vMax = mesh->hybridPointMax[0];
5749   PetscFunctionReturn(0);
5750 }
5751 
5752 #undef __FUNCT__
5753 #define __FUNCT__ "DMPlexSetHybridBounds"
5754 PetscErrorCode DMPlexSetHybridBounds(DM dm, PetscInt cMax, PetscInt fMax, PetscInt eMax, PetscInt vMax)
5755 {
5756   DM_Plex       *mesh = (DM_Plex*) dm->data;
5757   PetscInt       dim;
5758   PetscErrorCode ierr;
5759 
5760   PetscFunctionBegin;
5761   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5762   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
5763   if (cMax >= 0) mesh->hybridPointMax[dim]   = cMax;
5764   if (fMax >= 0) mesh->hybridPointMax[dim-1] = fMax;
5765   if (eMax >= 0) mesh->hybridPointMax[1]     = eMax;
5766   if (vMax >= 0) mesh->hybridPointMax[0]     = vMax;
5767   PetscFunctionReturn(0);
5768 }
5769 
5770 #undef __FUNCT__
5771 #define __FUNCT__ "DMPlexGetVTKCellHeight"
5772 PetscErrorCode DMPlexGetVTKCellHeight(DM dm, PetscInt *cellHeight)
5773 {
5774   DM_Plex *mesh = (DM_Plex*) dm->data;
5775 
5776   PetscFunctionBegin;
5777   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5778   PetscValidPointer(cellHeight, 2);
5779   *cellHeight = mesh->vtkCellHeight;
5780   PetscFunctionReturn(0);
5781 }
5782 
5783 #undef __FUNCT__
5784 #define __FUNCT__ "DMPlexSetVTKCellHeight"
5785 PetscErrorCode DMPlexSetVTKCellHeight(DM dm, PetscInt cellHeight)
5786 {
5787   DM_Plex *mesh = (DM_Plex*) dm->data;
5788 
5789   PetscFunctionBegin;
5790   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5791   mesh->vtkCellHeight = cellHeight;
5792   PetscFunctionReturn(0);
5793 }
5794 
5795 #undef __FUNCT__
5796 #define __FUNCT__ "DMPlexCreateNumbering_Private"
5797 /* We can easily have a form that takes an IS instead */
5798 PetscErrorCode DMPlexCreateNumbering_Private(DM dm, PetscInt pStart, PetscInt pEnd, PetscSF sf, IS *numbering)
5799 {
5800   PetscSection   section, globalSection;
5801   PetscInt      *numbers, p;
5802   PetscErrorCode ierr;
5803 
5804   PetscFunctionBegin;
5805   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &section);CHKERRQ(ierr);
5806   ierr = PetscSectionSetChart(section, pStart, pEnd);CHKERRQ(ierr);
5807   for (p = pStart; p < pEnd; ++p) {
5808     ierr = PetscSectionSetDof(section, p, 1);CHKERRQ(ierr);
5809   }
5810   ierr = PetscSectionSetUp(section);CHKERRQ(ierr);
5811   ierr = PetscSectionCreateGlobalSection(section, sf, PETSC_FALSE, &globalSection);CHKERRQ(ierr);
5812   ierr = PetscMalloc1((pEnd - pStart), &numbers);CHKERRQ(ierr);
5813   for (p = pStart; p < pEnd; ++p) {
5814     ierr = PetscSectionGetOffset(globalSection, p, &numbers[p-pStart]);CHKERRQ(ierr);
5815   }
5816   ierr = ISCreateGeneral(PetscObjectComm((PetscObject)dm), pEnd - pStart, numbers, PETSC_OWN_POINTER, numbering);CHKERRQ(ierr);
5817   ierr = PetscSectionDestroy(&section);CHKERRQ(ierr);
5818   ierr = PetscSectionDestroy(&globalSection);CHKERRQ(ierr);
5819   PetscFunctionReturn(0);
5820 }
5821 
5822 #undef __FUNCT__
5823 #define __FUNCT__ "DMPlexGetCellNumbering"
5824 PetscErrorCode DMPlexGetCellNumbering(DM dm, IS *globalCellNumbers)
5825 {
5826   DM_Plex       *mesh = (DM_Plex*) dm->data;
5827   PetscInt       cellHeight, cStart, cEnd, cMax;
5828   PetscErrorCode ierr;
5829 
5830   PetscFunctionBegin;
5831   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5832   if (!mesh->globalCellNumbers) {
5833     ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
5834     ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
5835     ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
5836     if (cMax >= 0) cEnd = PetscMin(cEnd, cMax);
5837     ierr = DMPlexCreateNumbering_Private(dm, cStart, cEnd, dm->sf, &mesh->globalCellNumbers);CHKERRQ(ierr);
5838   }
5839   *globalCellNumbers = mesh->globalCellNumbers;
5840   PetscFunctionReturn(0);
5841 }
5842 
5843 #undef __FUNCT__
5844 #define __FUNCT__ "DMPlexGetVertexNumbering"
5845 PetscErrorCode DMPlexGetVertexNumbering(DM dm, IS *globalVertexNumbers)
5846 {
5847   DM_Plex       *mesh = (DM_Plex*) dm->data;
5848   PetscInt       vStart, vEnd, vMax;
5849   PetscErrorCode ierr;
5850 
5851   PetscFunctionBegin;
5852   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5853   if (!mesh->globalVertexNumbers) {
5854     ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
5855     ierr = DMPlexGetHybridBounds(dm, NULL, NULL, NULL, &vMax);CHKERRQ(ierr);
5856     if (vMax >= 0) vEnd = PetscMin(vEnd, vMax);
5857     ierr = DMPlexCreateNumbering_Private(dm, vStart, vEnd, dm->sf, &mesh->globalVertexNumbers);CHKERRQ(ierr);
5858   }
5859   *globalVertexNumbers = mesh->globalVertexNumbers;
5860   PetscFunctionReturn(0);
5861 }
5862 
5863 
5864 #undef __FUNCT__
5865 #define __FUNCT__ "PetscSectionCreateGlobalSectionLabel"
5866 /*@C
5867   PetscSectionCreateGlobalSectionLabel - Create a section describing the global field layout using
5868   the local section and an SF describing the section point overlap.
5869 
5870   Input Parameters:
5871   + s - The PetscSection for the local field layout
5872   . sf - The SF describing parallel layout of the section points
5873   . includeConstraints - By default this is PETSC_FALSE, meaning that the global field vector will not possess constrained dofs
5874   . label - The label specifying the points
5875   - labelValue - The label stratum specifying the points
5876 
5877   Output Parameter:
5878   . gsection - The PetscSection for the global field layout
5879 
5880   Note: This gives negative sizes and offsets to points not owned by this process
5881 
5882   Level: developer
5883 
5884 .seealso: PetscSectionCreate()
5885 @*/
5886 PetscErrorCode PetscSectionCreateGlobalSectionLabel(PetscSection s, PetscSF sf, PetscBool includeConstraints, DMLabel label, PetscInt labelValue, PetscSection *gsection)
5887 {
5888   PetscInt      *neg = NULL, *tmpOff = NULL;
5889   PetscInt       pStart, pEnd, p, dof, cdof, off, globalOff = 0, nroots;
5890   PetscErrorCode ierr;
5891 
5892   PetscFunctionBegin;
5893   ierr = PetscSectionCreate(s->atlasLayout.comm, gsection);CHKERRQ(ierr);
5894   ierr = PetscSectionGetChart(s, &pStart, &pEnd);CHKERRQ(ierr);
5895   ierr = PetscSectionSetChart(*gsection, pStart, pEnd);CHKERRQ(ierr);
5896   ierr = PetscSFGetGraph(sf, &nroots, NULL, NULL, NULL);CHKERRQ(ierr);
5897   if (nroots >= 0) {
5898     if (nroots < pEnd-pStart) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "PetscSF nroots %d < %d section size", nroots, pEnd-pStart);
5899     ierr = PetscCalloc1(nroots, &neg);CHKERRQ(ierr);
5900     if (nroots > pEnd-pStart) {
5901       ierr = PetscCalloc1(nroots, &tmpOff);CHKERRQ(ierr);
5902     } else {
5903       tmpOff = &(*gsection)->atlasDof[-pStart];
5904     }
5905   }
5906   /* Mark ghost points with negative dof */
5907   for (p = pStart; p < pEnd; ++p) {
5908     PetscInt value;
5909 
5910     ierr = DMLabelGetValue(label, p, &value);CHKERRQ(ierr);
5911     if (value != labelValue) continue;
5912     ierr = PetscSectionGetDof(s, p, &dof);CHKERRQ(ierr);
5913     ierr = PetscSectionSetDof(*gsection, p, dof);CHKERRQ(ierr);
5914     ierr = PetscSectionGetConstraintDof(s, p, &cdof);CHKERRQ(ierr);
5915     if (!includeConstraints && cdof > 0) {ierr = PetscSectionSetConstraintDof(*gsection, p, cdof);CHKERRQ(ierr);}
5916     if (neg) neg[p] = -(dof+1);
5917   }
5918   ierr = PetscSectionSetUpBC(*gsection);CHKERRQ(ierr);
5919   if (nroots >= 0) {
5920     ierr = PetscSFBcastBegin(sf, MPIU_INT, neg, tmpOff);CHKERRQ(ierr);
5921     ierr = PetscSFBcastEnd(sf, MPIU_INT, neg, tmpOff);CHKERRQ(ierr);
5922     if (nroots > pEnd-pStart) {
5923       for (p = pStart; p < pEnd; ++p) {if (tmpOff[p] < 0) (*gsection)->atlasDof[p-pStart] = tmpOff[p];}
5924     }
5925   }
5926   /* Calculate new sizes, get proccess offset, and calculate point offsets */
5927   for (p = 0, off = 0; p < pEnd-pStart; ++p) {
5928     cdof = (!includeConstraints && s->bc) ? s->bc->atlasDof[p] : 0;
5929     (*gsection)->atlasOff[p] = off;
5930     off += (*gsection)->atlasDof[p] > 0 ? (*gsection)->atlasDof[p]-cdof : 0;
5931   }
5932   ierr       = MPI_Scan(&off, &globalOff, 1, MPIU_INT, MPI_SUM, s->atlasLayout.comm);CHKERRQ(ierr);
5933   globalOff -= off;
5934   for (p = 0, off = 0; p < pEnd-pStart; ++p) {
5935     (*gsection)->atlasOff[p] += globalOff;
5936     if (neg) neg[p] = -((*gsection)->atlasOff[p]+1);
5937   }
5938   /* Put in negative offsets for ghost points */
5939   if (nroots >= 0) {
5940     ierr = PetscSFBcastBegin(sf, MPIU_INT, neg, tmpOff);CHKERRQ(ierr);
5941     ierr = PetscSFBcastEnd(sf, MPIU_INT, neg, tmpOff);CHKERRQ(ierr);
5942     if (nroots > pEnd-pStart) {
5943       for (p = pStart; p < pEnd; ++p) {if (tmpOff[p] < 0) (*gsection)->atlasOff[p-pStart] = tmpOff[p];}
5944     }
5945   }
5946   if (nroots >= 0 && nroots > pEnd-pStart) {ierr = PetscFree(tmpOff);CHKERRQ(ierr);}
5947   ierr = PetscFree(neg);CHKERRQ(ierr);
5948   PetscFunctionReturn(0);
5949 }
5950 
5951 #undef __FUNCT__
5952 #define __FUNCT__ "DMPlexCheckSymmetry"
5953 /*@
5954   DMPlexCheckSymmetry - Check that the adjacency information in the mesh is symmetric.
5955 
5956   Input Parameters:
5957   + dm - The DMPlex object
5958 
5959   Note: This is a useful diagnostic when creating meshes programmatically.
5960 
5961   Level: developer
5962 
5963 .seealso: DMCreate(), DMCheckSkeleton(), DMCheckFaces()
5964 @*/
5965 PetscErrorCode DMPlexCheckSymmetry(DM dm)
5966 {
5967   PetscSection    coneSection, supportSection;
5968   const PetscInt *cone, *support;
5969   PetscInt        coneSize, c, supportSize, s;
5970   PetscInt        pStart, pEnd, p, csize, ssize;
5971   PetscErrorCode  ierr;
5972 
5973   PetscFunctionBegin;
5974   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5975   ierr = DMPlexGetConeSection(dm, &coneSection);CHKERRQ(ierr);
5976   ierr = DMPlexGetSupportSection(dm, &supportSection);CHKERRQ(ierr);
5977   /* Check that point p is found in the support of its cone points, and vice versa */
5978   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
5979   for (p = pStart; p < pEnd; ++p) {
5980     ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
5981     ierr = DMPlexGetCone(dm, p, &cone);CHKERRQ(ierr);
5982     for (c = 0; c < coneSize; ++c) {
5983       PetscBool dup = PETSC_FALSE;
5984       PetscInt  d;
5985       for (d = c-1; d >= 0; --d) {
5986         if (cone[c] == cone[d]) {dup = PETSC_TRUE; break;}
5987       }
5988       ierr = DMPlexGetSupportSize(dm, cone[c], &supportSize);CHKERRQ(ierr);
5989       ierr = DMPlexGetSupport(dm, cone[c], &support);CHKERRQ(ierr);
5990       for (s = 0; s < supportSize; ++s) {
5991         if (support[s] == p) break;
5992       }
5993       if ((s >= supportSize) || (dup && (support[s+1] != p))) {
5994         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %d cone: ", p);
5995         for (s = 0; s < coneSize; ++s) {
5996           ierr = PetscPrintf(PETSC_COMM_SELF, "%d, ", cone[s]);
5997         }
5998         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");
5999         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %d support: ", cone[c]);
6000         for (s = 0; s < supportSize; ++s) {
6001           ierr = PetscPrintf(PETSC_COMM_SELF, "%d, ", support[s]);
6002         }
6003         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");
6004         if (dup) {
6005           SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %d not repeatedly found in support of repeated cone point %d", p, cone[c]);
6006         } else {
6007           SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %d not found in support of cone point %d", p, cone[c]);
6008         }
6009       }
6010     }
6011     ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
6012     ierr = DMPlexGetSupport(dm, p, &support);CHKERRQ(ierr);
6013     for (s = 0; s < supportSize; ++s) {
6014       ierr = DMPlexGetConeSize(dm, support[s], &coneSize);CHKERRQ(ierr);
6015       ierr = DMPlexGetCone(dm, support[s], &cone);CHKERRQ(ierr);
6016       for (c = 0; c < coneSize; ++c) {
6017         if (cone[c] == p) break;
6018       }
6019       if (c >= coneSize) {
6020         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %d support: ", p);
6021         for (c = 0; c < supportSize; ++c) {
6022           ierr = PetscPrintf(PETSC_COMM_SELF, "%d, ", support[c]);
6023         }
6024         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");
6025         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %d cone: ", support[s]);
6026         for (c = 0; c < coneSize; ++c) {
6027           ierr = PetscPrintf(PETSC_COMM_SELF, "%d, ", cone[c]);
6028         }
6029         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");
6030         SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %d not found in cone of support point %d", p, support[s]);
6031       }
6032     }
6033   }
6034   ierr = PetscSectionGetStorageSize(coneSection, &csize);CHKERRQ(ierr);
6035   ierr = PetscSectionGetStorageSize(supportSection, &ssize);CHKERRQ(ierr);
6036   if (csize != ssize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "Total cone size %d != Total support size %d", csize, ssize);
6037   PetscFunctionReturn(0);
6038 }
6039 
6040 #undef __FUNCT__
6041 #define __FUNCT__ "DMPlexCheckSkeleton"
6042 /*@
6043   DMPlexCheckSkeleton - Check that each cell has the correct number of vertices
6044 
6045   Input Parameters:
6046 + dm - The DMPlex object
6047 . isSimplex - Are the cells simplices or tensor products
6048 - cellHeight - Normally 0
6049 
6050   Note: This is a useful diagnostic when creating meshes programmatically.
6051 
6052   Level: developer
6053 
6054 .seealso: DMCreate(), DMCheckSymmetry(), DMCheckFaces()
6055 @*/
6056 PetscErrorCode DMPlexCheckSkeleton(DM dm, PetscBool isSimplex, PetscInt cellHeight)
6057 {
6058   PetscInt       dim, numCorners, numHybridCorners, vStart, vEnd, cStart, cEnd, cMax, c;
6059   PetscErrorCode ierr;
6060 
6061   PetscFunctionBegin;
6062   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6063   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
6064   switch (dim) {
6065   case 1: numCorners = isSimplex ? 2 : 2; numHybridCorners = isSimplex ? 2 : 2; break;
6066   case 2: numCorners = isSimplex ? 3 : 4; numHybridCorners = isSimplex ? 4 : 4; break;
6067   case 3: numCorners = isSimplex ? 4 : 8; numHybridCorners = isSimplex ? 6 : 8; break;
6068   default:
6069     SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_OUTOFRANGE, "Cannot handle meshes of dimension %d", dim);
6070   }
6071   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
6072   ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
6073   ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
6074   cMax = cMax >= 0 ? cMax : cEnd;
6075   for (c = cStart; c < cMax; ++c) {
6076     PetscInt *closure = NULL, closureSize, cl, coneSize = 0;
6077 
6078     ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6079     for (cl = 0; cl < closureSize*2; cl += 2) {
6080       const PetscInt p = closure[cl];
6081       if ((p >= vStart) && (p < vEnd)) ++coneSize;
6082     }
6083     ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6084     if (coneSize != numCorners) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %d has  %d vertices != %d", c, coneSize, numCorners);
6085   }
6086   for (c = cMax; c < cEnd; ++c) {
6087     PetscInt *closure = NULL, closureSize, cl, coneSize = 0;
6088 
6089     ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6090     for (cl = 0; cl < closureSize*2; cl += 2) {
6091       const PetscInt p = closure[cl];
6092       if ((p >= vStart) && (p < vEnd)) ++coneSize;
6093     }
6094     ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6095     if (coneSize > numHybridCorners) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Hybrid cell %d has  %d vertices > %d", c, coneSize, numHybridCorners);
6096   }
6097   PetscFunctionReturn(0);
6098 }
6099 
6100 #undef __FUNCT__
6101 #define __FUNCT__ "DMPlexCheckFaces"
6102 /*@
6103   DMPlexCheckFaces - Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type
6104 
6105   Input Parameters:
6106 + dm - The DMPlex object
6107 . isSimplex - Are the cells simplices or tensor products
6108 - cellHeight - Normally 0
6109 
6110   Note: This is a useful diagnostic when creating meshes programmatically.
6111 
6112   Level: developer
6113 
6114 .seealso: DMCreate(), DMCheckSymmetry(), DMCheckSkeleton()
6115 @*/
6116 PetscErrorCode DMPlexCheckFaces(DM dm, PetscBool isSimplex, PetscInt cellHeight)
6117 {
6118   PetscInt       pMax[4];
6119   PetscInt       dim, vStart, vEnd, cStart, cEnd, c, h;
6120   PetscErrorCode ierr;
6121 
6122   PetscFunctionBegin;
6123   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6124   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
6125   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
6126   ierr = DMPlexGetHybridBounds(dm, &pMax[dim], &pMax[dim-1], &pMax[1], &pMax[0]);CHKERRQ(ierr);
6127   for (h = cellHeight; h < dim; ++h) {
6128     ierr = DMPlexGetHeightStratum(dm, h, &cStart, &cEnd);CHKERRQ(ierr);
6129     for (c = cStart; c < cEnd; ++c) {
6130       const PetscInt *cone, *ornt, *faces;
6131       PetscInt        numFaces, faceSize, coneSize,f;
6132       PetscInt       *closure = NULL, closureSize, cl, numCorners = 0;
6133 
6134       if (pMax[dim-h] >= 0 && c >= pMax[dim-h]) continue;
6135       ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
6136       ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
6137       ierr = DMPlexGetConeOrientation(dm, c, &ornt);CHKERRQ(ierr);
6138       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6139       for (cl = 0; cl < closureSize*2; cl += 2) {
6140         const PetscInt p = closure[cl];
6141         if ((p >= vStart) && (p < vEnd)) closure[numCorners++] = p;
6142       }
6143       ierr = DMPlexGetRawFaces_Internal(dm, dim-h, numCorners, closure, &numFaces, &faceSize, &faces);CHKERRQ(ierr);
6144       if (coneSize != numFaces) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %d has %d faces but should have %d", c, coneSize, numFaces);
6145       for (f = 0; f < numFaces; ++f) {
6146         PetscInt *fclosure = NULL, fclosureSize, cl, fnumCorners = 0, v;
6147 
6148         ierr = DMPlexGetTransitiveClosure_Internal(dm, cone[f], ornt[f], PETSC_TRUE, &fclosureSize, &fclosure);CHKERRQ(ierr);
6149         for (cl = 0; cl < fclosureSize*2; cl += 2) {
6150           const PetscInt p = fclosure[cl];
6151           if ((p >= vStart) && (p < vEnd)) fclosure[fnumCorners++] = p;
6152         }
6153         if (fnumCorners != faceSize) SETERRQ5(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %d (%d) of cell %d has %d vertices but should have %d", cone[f], f, c, fnumCorners, faceSize);
6154         for (v = 0; v < fnumCorners; ++v) {
6155           if (fclosure[v] != faces[f*faceSize+v]) SETERRQ6(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %d (%d) of cell %d vertex %d, %d != %d", cone[f], f, c, v, fclosure[v], faces[f*faceSize+v]);
6156         }
6157         ierr = DMPlexRestoreTransitiveClosure(dm, cone[f], PETSC_TRUE, &fclosureSize, &fclosure);CHKERRQ(ierr);
6158       }
6159       ierr = DMPlexRestoreFaces_Internal(dm, dim, c, &numFaces, &faceSize, &faces);CHKERRQ(ierr);
6160       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6161     }
6162   }
6163   PetscFunctionReturn(0);
6164 }
6165 
6166 #undef __FUNCT__
6167 #define __FUNCT__ "DMCreateInterpolation_Plex"
6168 /* Pointwise interpolation
6169      Just code FEM for now
6170      u^f = I u^c
6171      sum_k u^f_k phi^f_k = I sum_j u^c_j phi^c_j
6172      u^f_i = sum_j psi^f_i I phi^c_j u^c_j
6173      I_{ij} = psi^f_i phi^c_j
6174 */
6175 PetscErrorCode DMCreateInterpolation_Plex(DM dmCoarse, DM dmFine, Mat *interpolation, Vec *scaling)
6176 {
6177   PetscSection   gsc, gsf;
6178   PetscInt       m, n;
6179   void          *ctx;
6180   PetscErrorCode ierr;
6181 
6182   PetscFunctionBegin;
6183   /*
6184   Loop over coarse cells
6185     Loop over coarse basis functions
6186       Loop over fine cells in coarse cell
6187         Loop over fine dual basis functions
6188           Evaluate coarse basis on fine dual basis quad points
6189           Sum
6190           Update local element matrix
6191     Accumulate to interpolation matrix
6192 
6193    Can extend PetscFEIntegrateJacobian_Basic() to do a specialized cell loop
6194   */
6195   ierr = DMGetDefaultGlobalSection(dmFine, &gsf);CHKERRQ(ierr);
6196   ierr = PetscSectionGetConstrainedStorageSize(gsf, &m);CHKERRQ(ierr);
6197   ierr = DMGetDefaultGlobalSection(dmCoarse, &gsc);CHKERRQ(ierr);
6198   ierr = PetscSectionGetConstrainedStorageSize(gsc, &n);CHKERRQ(ierr);
6199   /* We need to preallocate properly */
6200   ierr = MatCreate(PetscObjectComm((PetscObject) dmCoarse), interpolation);CHKERRQ(ierr);
6201   ierr = MatSetSizes(*interpolation, m, n, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
6202   ierr = MatSetType(*interpolation, dmCoarse->mattype);CHKERRQ(ierr);
6203   ierr = MatSetUp(*interpolation);CHKERRQ(ierr);
6204   ierr = MatSetFromOptions(*interpolation);CHKERRQ(ierr);
6205   ierr = MatSetOption(*interpolation, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_FALSE);CHKERRQ(ierr);
6206   ierr = DMGetApplicationContext(dmFine, &ctx);CHKERRQ(ierr);
6207   ierr = DMPlexComputeInterpolatorFEM(dmCoarse, dmFine, *interpolation, ctx);CHKERRQ(ierr);
6208   /* Use naive scaling */
6209   ierr = DMCreateInterpolationScale(dmCoarse, dmFine, *interpolation, scaling);CHKERRQ(ierr);
6210   PetscFunctionReturn(0);
6211 }
6212 
6213 #undef __FUNCT__
6214 #define __FUNCT__ "DMCreateInjection_Plex"
6215 PetscErrorCode DMCreateInjection_Plex(DM dmCoarse, DM dmFine, VecScatter *ctx)
6216 {
6217   Vec             cv,  fv;
6218   IS              cis, fis, fpointIS;
6219   PetscSection    sc, gsc, gsf;
6220   const PetscInt *fpoints;
6221   PetscInt       *cindices, *findices;
6222   PetscInt        cpStart, cpEnd, m, off, cp;
6223   PetscErrorCode  ierr;
6224 
6225   PetscFunctionBegin;
6226   ierr = DMGetDefaultGlobalSection(dmFine, &gsf);CHKERRQ(ierr);
6227   ierr = DMGetGlobalVector(dmFine, &fv);CHKERRQ(ierr);
6228   ierr = DMGetDefaultSection(dmCoarse, &sc);CHKERRQ(ierr);
6229   ierr = DMGetDefaultGlobalSection(dmCoarse, &gsc);CHKERRQ(ierr);
6230   ierr = DMGetGlobalVector(dmCoarse, &cv);CHKERRQ(ierr);
6231   ierr = DMPlexCreateCoarsePointIS(dmCoarse, &fpointIS);CHKERRQ(ierr);
6232   ierr = PetscSectionGetConstrainedStorageSize(gsc, &m);CHKERRQ(ierr);
6233   ierr = PetscMalloc2(m,&cindices,m,&findices);CHKERRQ(ierr);
6234   ierr = PetscSectionGetChart(gsc, &cpStart, &cpEnd);CHKERRQ(ierr);
6235   ierr = ISGetIndices(fpointIS, &fpoints);CHKERRQ(ierr);
6236   for (cp = cpStart, off = 0; cp < cpEnd; ++cp) {
6237     const PetscInt *cdofsC = NULL;
6238     PetscInt        fp     = fpoints[cp-cpStart], dofC, cdofC, dofF, offC, offF, d, e;
6239 
6240     ierr = PetscSectionGetDof(gsc, cp, &dofC);CHKERRQ(ierr);
6241     if (dofC <= 0) continue;
6242     ierr = PetscSectionGetConstraintDof(sc, cp, &cdofC);CHKERRQ(ierr);
6243     ierr = PetscSectionGetDof(gsf, fp, &dofF);CHKERRQ(ierr);
6244     ierr = PetscSectionGetOffset(gsc, cp, &offC);CHKERRQ(ierr);
6245     ierr = PetscSectionGetOffset(gsf, fp, &offF);CHKERRQ(ierr);
6246     if (cdofC) {ierr = PetscSectionGetConstraintIndices(sc, cp, &cdofsC);CHKERRQ(ierr);}
6247     if (dofC != dofF) SETERRQ4(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Point %d (%d) has %d coarse dofs != %d fine dofs", cp, fp, dofC, dofF);
6248     if (offC < 0 || offF < 0) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Coarse point %d has invalid offset %d (%d)", cp, offC, offF);
6249     for (d = 0, e = 0; d < dofC; ++d) {
6250       if (cdofsC && cdofsC[e] == d) {++e; continue;}
6251       cindices[off+d-e] = offC+d; findices[off+d-e] = offF+d;
6252     }
6253     if (e != cdofC) SETERRQ4(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Point %d (%d) has invalid number of constraints %d != %d", cp, fp, e, cdofC);
6254     off += dofC-cdofC;
6255   }
6256   ierr = ISRestoreIndices(fpointIS, &fpoints);CHKERRQ(ierr);
6257   if (off != m) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of coarse dofs %d != %d", off, m);
6258   ierr = ISCreateGeneral(PETSC_COMM_SELF, m, cindices, PETSC_OWN_POINTER, &cis);CHKERRQ(ierr);
6259   ierr = ISCreateGeneral(PETSC_COMM_SELF, m, findices, PETSC_OWN_POINTER, &fis);CHKERRQ(ierr);
6260   ierr = VecScatterCreate(cv, cis, fv, fis, ctx);CHKERRQ(ierr);
6261   ierr = ISDestroy(&cis);CHKERRQ(ierr);
6262   ierr = ISDestroy(&fis);CHKERRQ(ierr);
6263   ierr = DMRestoreGlobalVector(dmFine, &fv);CHKERRQ(ierr);
6264   ierr = DMRestoreGlobalVector(dmCoarse, &cv);CHKERRQ(ierr);
6265   ierr = ISDestroy(&fpointIS);CHKERRQ(ierr);
6266   PetscFunctionReturn(0);
6267 }
6268 
6269 #undef __FUNCT__
6270 #define __FUNCT__ "DMCreateDefaultSection_Plex"
6271 /* Pointwise interpolation
6272      Just code FEM for now
6273      u^f = I u^c
6274      sum_k u^f_k phi^f_k = I sum_l u^c_l phi^c_l
6275      u^f_i = sum_l int psi^f_i I phi^c_l u^c_l
6276      I_{ij} = int psi^f_i phi^c_j
6277 */
6278 PetscErrorCode DMCreateDefaultSection_Plex(DM dm)
6279 {
6280   PetscSection   section;
6281   IS            *bcPoints;
6282   PetscInt      *bcFields, *numComp, *numDof;
6283   PetscInt       depth, dim, numBd, numBC = 0, numFields, bd, bc, f;
6284   PetscErrorCode ierr;
6285 
6286   PetscFunctionBegin;
6287   /* Handle boundary conditions */
6288   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
6289   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
6290   ierr = DMPlexGetNumBoundary(dm, &numBd);CHKERRQ(ierr);
6291   for (bd = 0; bd < numBd; ++bd) {
6292     PetscBool isEssential;
6293     ierr = DMPlexGetBoundary(dm, bd, &isEssential, NULL, NULL, NULL, NULL, NULL, NULL);CHKERRQ(ierr);
6294     if (isEssential) ++numBC;
6295   }
6296   ierr = PetscMalloc2(numBC,&bcFields,numBC,&bcPoints);CHKERRQ(ierr);
6297   for (bd = 0, bc = 0; bd < numBd; ++bd) {
6298     const char     *bdLabel;
6299     DMLabel         label;
6300     const PetscInt *values;
6301     PetscInt        field, numValues;
6302     PetscBool       isEssential, has;
6303 
6304     ierr = DMPlexGetBoundary(dm, bd, &isEssential, &bdLabel, &field, NULL, &numValues, &values, NULL);CHKERRQ(ierr);
6305     if (numValues != 1) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Bug me and I will fix this");
6306     ierr = DMPlexHasLabel(dm, bdLabel, &has);CHKERRQ(ierr);
6307     if (!has) {
6308       ierr = DMPlexCreateLabel(dm, bdLabel);CHKERRQ(ierr);
6309       ierr = DMPlexGetLabel(dm, bdLabel, &label);CHKERRQ(ierr);
6310       ierr = DMPlexMarkBoundaryFaces(dm, label);CHKERRQ(ierr);
6311     }
6312     ierr = DMPlexGetLabel(dm, bdLabel, &label);CHKERRQ(ierr);
6313     ierr = DMPlexLabelComplete(dm, label);CHKERRQ(ierr);
6314     if (isEssential) {
6315       bcFields[bc] = field;
6316       ierr = DMPlexGetStratumIS(dm, bdLabel, values[0], &bcPoints[bc++]);CHKERRQ(ierr);
6317     }
6318   }
6319   /* Handle discretization */
6320   ierr = DMGetNumFields(dm, &numFields);CHKERRQ(ierr);
6321   ierr = PetscMalloc2(numFields,&numComp,numFields*(dim+1),&numDof);CHKERRQ(ierr);
6322   for (f = 0; f < numFields; ++f) {
6323     PetscFE         fe;
6324     const PetscInt *numFieldDof;
6325     PetscInt        d;
6326 
6327     ierr = DMGetField(dm, f, (PetscObject *) &fe);CHKERRQ(ierr);
6328     ierr = PetscFEGetNumComponents(fe, &numComp[f]);CHKERRQ(ierr);
6329     ierr = PetscFEGetNumDof(fe, &numFieldDof);CHKERRQ(ierr);
6330     for (d = 0; d < dim+1; ++d) numDof[f*(dim+1)+d] = numFieldDof[d];
6331   }
6332   for (f = 0; f < numFields; ++f) {
6333     PetscInt d;
6334     for (d = 1; d < dim; ++d) {
6335       if ((numDof[f*(dim+1)+d] > 0) && (depth < dim)) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Mesh must be interpolated when unknowns are specified on edges or faces.");
6336     }
6337   }
6338   ierr = DMPlexCreateSection(dm, dim, numFields, numComp, numDof, numBC, bcFields, bcPoints, &section);CHKERRQ(ierr);
6339   for (f = 0; f < numFields; ++f) {
6340     PetscFE     fe;
6341     const char *name;
6342 
6343     ierr = DMGetField(dm, f, (PetscObject *) &fe);CHKERRQ(ierr);
6344     ierr = PetscObjectGetName((PetscObject) fe, &name);CHKERRQ(ierr);
6345     ierr = PetscSectionSetFieldName(section, f, name);CHKERRQ(ierr);
6346   }
6347   ierr = DMSetDefaultSection(dm, section);CHKERRQ(ierr);
6348   ierr = PetscSectionDestroy(&section);CHKERRQ(ierr);
6349   for (bc = 0; bc < numBC; ++bc) {ierr = ISDestroy(&bcPoints[bc]);CHKERRQ(ierr);}
6350   ierr = PetscFree2(bcFields,bcPoints);CHKERRQ(ierr);
6351   ierr = PetscFree2(numComp,numDof);CHKERRQ(ierr);
6352   PetscFunctionReturn(0);
6353 }
6354 
6355 #undef __FUNCT__
6356 #define __FUNCT__ "DMPlexGetCoarseDM"
6357 /*@
6358   DMPlexGetCoarseDM - Get the coarse mesh from which this was obtained by refinement
6359 
6360   Input Parameter:
6361 . dm - The DMPlex object
6362 
6363   Output Parameter:
6364 . cdm - The coarse DM
6365 
6366   Level: intermediate
6367 
6368 .seealso: DMPlexSetCoarseDM()
6369 @*/
6370 PetscErrorCode DMPlexGetCoarseDM(DM dm, DM *cdm)
6371 {
6372   PetscFunctionBegin;
6373   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6374   PetscValidPointer(cdm, 2);
6375   *cdm = ((DM_Plex *) dm->data)->coarseMesh;
6376   PetscFunctionReturn(0);
6377 }
6378 
6379 #undef __FUNCT__
6380 #define __FUNCT__ "DMPlexSetCoarseDM"
6381 /*@
6382   DMPlexSetCoarseDM - Set the coarse mesh from which this was obtained by refinement
6383 
6384   Input Parameters:
6385 + dm - The DMPlex object
6386 - cdm - The coarse DM
6387 
6388   Level: intermediate
6389 
6390 .seealso: DMPlexGetCoarseDM()
6391 @*/
6392 PetscErrorCode DMPlexSetCoarseDM(DM dm, DM cdm)
6393 {
6394   DM_Plex       *mesh;
6395   PetscErrorCode ierr;
6396 
6397   PetscFunctionBegin;
6398   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6399   if (cdm) PetscValidHeaderSpecific(cdm, DM_CLASSID, 2);
6400   mesh = (DM_Plex *) dm->data;
6401   ierr = DMDestroy(&mesh->coarseMesh);CHKERRQ(ierr);
6402   mesh->coarseMesh = cdm;
6403   ierr = PetscObjectReference((PetscObject) mesh->coarseMesh);CHKERRQ(ierr);
6404   PetscFunctionReturn(0);
6405 }
6406