xref: /petsc/src/dm/impls/plex/plex.c (revision 0c8c14c502e79aaeb7ec26e1cd1e476322d9010e)
1 #include <petsc-private/dmpleximpl.h>   /*I      "petscdmplex.h"   I*/
2 #include <../src/sys/utils/hash.h>
3 #include <petsc-private/isimpl.h>
4 #include <petscsf.h>
5 #include <petscviewerhdf5.h>
6 
7 /* Logging support */
8 PetscLogEvent DMPLEX_Interpolate, DMPLEX_Partition, DMPLEX_Distribute, DMPLEX_DistributeCones, DMPLEX_DistributeLabels, DMPLEX_DistributeSF, DMPLEX_DistributeField, DMPLEX_DistributeData, DMPLEX_Stratify, DMPLEX_Preallocate, DMPLEX_ResidualFEM, DMPLEX_JacobianFEM;
9 
10 PETSC_EXTERN PetscErrorCode VecView_Seq(Vec, PetscViewer);
11 PETSC_EXTERN PetscErrorCode VecView_MPI(Vec, PetscViewer);
12 PETSC_EXTERN PetscErrorCode VecLoad_Default(Vec, PetscViewer);
13 PETSC_EXTERN PetscErrorCode DMTSGetTimeStepNumber(DM,PetscInt*);
14 
15 #undef __FUNCT__
16 #define __FUNCT__ "GetFieldType_Static"
17 static PetscErrorCode GetFieldType_Static(DM dm, PetscSection section, PetscInt field, PetscInt *sStart, PetscInt *sEnd, PetscViewerVTKFieldType *ft)
18 {
19   PetscInt       dim, pStart, pEnd, vStart, vEnd, cStart, cEnd, vdof = 0, cdof = 0;
20   PetscErrorCode ierr;
21 
22   PetscFunctionBegin;
23   *ft  = PETSC_VTK_POINT_FIELD;
24   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
25   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
26   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
27   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
28   if (field >= 0) {
29     if ((vStart >= pStart) && (vStart < pEnd)) {ierr = PetscSectionGetFieldDof(section, vStart, field, &vdof);CHKERRQ(ierr);}
30     if ((cStart >= pStart) && (cStart < pEnd)) {ierr = PetscSectionGetFieldDof(section, cStart, field, &cdof);CHKERRQ(ierr);}
31   } else {
32     if ((vStart >= pStart) && (vStart < pEnd)) {ierr = PetscSectionGetDof(section, vStart, &vdof);CHKERRQ(ierr);}
33     if ((cStart >= pStart) && (cStart < pEnd)) {ierr = PetscSectionGetDof(section, cStart, &cdof);CHKERRQ(ierr);}
34   }
35   if (vdof) {
36     *sStart = vStart;
37     *sEnd   = vEnd;
38     if (vdof == dim) *ft = PETSC_VTK_POINT_VECTOR_FIELD;
39     else             *ft = PETSC_VTK_POINT_FIELD;
40   } else if (cdof) {
41     *sStart = cStart;
42     *sEnd   = cEnd;
43     if (cdof == dim) *ft = PETSC_VTK_CELL_VECTOR_FIELD;
44     else             *ft = PETSC_VTK_CELL_FIELD;
45   } else SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Could not classify input Vec for VTK");
46   PetscFunctionReturn(0);
47 }
48 
49 #if defined(PETSC_HAVE_HDF5)
50 #undef __FUNCT__
51 #define __FUNCT__ "GetField_Static"
52 static PetscErrorCode GetField_Static(DM dm, PetscSection section, PetscSection sectionGlobal, Vec v, PetscInt field, PetscInt pStart, PetscInt pEnd, IS *is, Vec *subv)
53 {
54   PetscInt      *subIndices;
55   PetscInt       Nc, subSize = 0, subOff = 0, p;
56   PetscErrorCode ierr;
57 
58   PetscFunctionBegin;
59   ierr = PetscSectionGetFieldComponents(section, field, &Nc);CHKERRQ(ierr);
60   for (p = pStart; p < pEnd; ++p) {
61     PetscInt gdof, fdof = 0;
62 
63     ierr = PetscSectionGetDof(sectionGlobal, p, &gdof);CHKERRQ(ierr);
64     if (gdof > 0) {ierr = PetscSectionGetFieldDof(section, p, field, &fdof);CHKERRQ(ierr);}
65     subSize += fdof;
66   }
67   ierr = PetscMalloc1(subSize, &subIndices);CHKERRQ(ierr);
68   for (p = pStart; p < pEnd; ++p) {
69     PetscInt gdof, goff;
70 
71     ierr = PetscSectionGetDof(sectionGlobal, p, &gdof);CHKERRQ(ierr);
72     if (gdof > 0) {
73       PetscInt fdof, fc, f2, poff = 0;
74 
75       ierr = PetscSectionGetOffset(sectionGlobal, p, &goff);CHKERRQ(ierr);
76       /* Can get rid of this loop by storing field information in the global section */
77       for (f2 = 0; f2 < field; ++f2) {
78         ierr  = PetscSectionGetFieldDof(section, p, f2, &fdof);CHKERRQ(ierr);
79         poff += fdof;
80       }
81       ierr = PetscSectionGetFieldDof(section, p, field, &fdof);CHKERRQ(ierr);
82       for (fc = 0; fc < fdof; ++fc, ++subOff) subIndices[subOff] = goff+poff+fc;
83     }
84   }
85   ierr = ISCreateGeneral(PetscObjectComm((PetscObject) dm), subSize, subIndices, PETSC_OWN_POINTER, is);CHKERRQ(ierr);
86   ierr = VecGetSubVector(v, *is, subv);CHKERRQ(ierr);
87   ierr = VecSetBlockSize(*subv, Nc);CHKERRQ(ierr);
88   PetscFunctionReturn(0);
89 }
90 
91 #undef __FUNCT__
92 #define __FUNCT__ "RestoreField_Static"
93 static PetscErrorCode RestoreField_Static(DM dm, PetscSection section, PetscSection sectionGlobal, Vec v, PetscInt field, PetscInt pStart, PetscInt pEnd, IS *is, Vec *subv)
94 {
95   PetscErrorCode ierr;
96 
97   PetscFunctionBegin;
98   ierr = VecRestoreSubVector(v, *is, subv);CHKERRQ(ierr);
99   ierr = ISDestroy(is);CHKERRQ(ierr);
100   PetscFunctionReturn(0);
101 }
102 
103 #undef __FUNCT__
104 #define __FUNCT__ "DMSequenceView_HDF5"
105 static PetscErrorCode DMSequenceView_HDF5(DM dm, const char *seqname, PetscInt seqnum, PetscScalar value, PetscViewer viewer)
106 {
107   Vec            stamp;
108   PetscMPIInt    rank;
109   PetscErrorCode ierr;
110 
111   PetscFunctionBegin;
112   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) viewer), &rank);CHKERRQ(ierr);
113   ierr = VecCreateMPI(PetscObjectComm((PetscObject) viewer), rank ? 0 : 1, 1, &stamp);CHKERRQ(ierr);
114   ierr = VecSetBlockSize(stamp, 1);CHKERRQ(ierr);
115   ierr = PetscObjectSetName((PetscObject) stamp, seqname);CHKERRQ(ierr);
116   if (!rank) {
117     PetscReal timeScale;
118     PetscBool istime;
119 
120     ierr = PetscStrncmp(seqname, "time", 5, &istime);CHKERRQ(ierr);
121     if (istime) {ierr = DMPlexGetScale(dm, PETSC_UNIT_TIME, &timeScale);CHKERRQ(ierr); value *= timeScale;}
122     ierr = VecSetValue(stamp, 0, value, INSERT_VALUES);CHKERRQ(ierr);
123   }
124   ierr = VecAssemblyBegin(stamp);CHKERRQ(ierr);
125   ierr = VecAssemblyEnd(stamp);CHKERRQ(ierr);
126   ierr = PetscViewerHDF5PushGroup(viewer, "/");CHKERRQ(ierr);
127   ierr = PetscViewerHDF5SetTimestep(viewer, seqnum);CHKERRQ(ierr);
128   ierr = VecView(stamp, viewer);CHKERRQ(ierr);
129   ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
130   ierr = VecDestroy(&stamp);CHKERRQ(ierr);
131   PetscFunctionReturn(0);
132 }
133 
134 #undef __FUNCT__
135 #define __FUNCT__ "VecView_Plex_Local_HDF5"
136 static PetscErrorCode VecView_Plex_Local_HDF5(Vec v, PetscViewer viewer)
137 {
138   DM                      dm;
139   DM                      dmBC;
140   PetscSection            section, sectionGlobal;
141   Vec                     gv;
142   const char             *name;
143   PetscViewerVTKFieldType ft;
144   PetscViewerFormat       format;
145   PetscInt                seqnum;
146   PetscBool               isseq;
147   PetscErrorCode          ierr;
148 
149   PetscFunctionBegin;
150   ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
151   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
152   ierr = DMGetDefaultSection(dm, &section);CHKERRQ(ierr);
153   ierr = DMGetOutputSequenceNumber(dm, &seqnum);CHKERRQ(ierr);
154   ierr = PetscViewerHDF5SetTimestep(viewer, seqnum);CHKERRQ(ierr);
155   ierr = DMSequenceView_HDF5(dm, "time", seqnum, (PetscScalar) seqnum, viewer);CHKERRQ(ierr);
156   ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
157   ierr = DMGetOutputDM(dm, &dmBC);CHKERRQ(ierr);
158   ierr = DMGetDefaultGlobalSection(dmBC, &sectionGlobal);CHKERRQ(ierr);
159   ierr = DMGetGlobalVector(dmBC, &gv);CHKERRQ(ierr);
160   ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
161   ierr = PetscObjectSetName((PetscObject) gv, name);CHKERRQ(ierr);
162   ierr = DMLocalToGlobalBegin(dmBC, v, INSERT_VALUES, gv);CHKERRQ(ierr);
163   ierr = DMLocalToGlobalEnd(dmBC, v, INSERT_VALUES, gv);CHKERRQ(ierr);
164   ierr = PetscObjectTypeCompare((PetscObject) gv, VECSEQ, &isseq);CHKERRQ(ierr);
165   if (isseq) {ierr = VecView_Seq(gv, viewer);CHKERRQ(ierr);}
166   else       {ierr = VecView_MPI(gv, viewer);CHKERRQ(ierr);}
167   if (format == PETSC_VIEWER_HDF5_VIZ) {
168     /* Output visualization representation */
169     PetscInt numFields, f;
170 
171     ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
172     for (f = 0; f < numFields; ++f) {
173       Vec         subv;
174       IS          is;
175       const char *fname, *fgroup;
176       char        group[PETSC_MAX_PATH_LEN];
177       PetscInt    pStart, pEnd;
178       PetscBool   flag;
179 
180       ierr = GetFieldType_Static(dm, section, f, &pStart, &pEnd, &ft);CHKERRQ(ierr);
181       fgroup = (ft == PETSC_VTK_POINT_VECTOR_FIELD) || (ft == PETSC_VTK_POINT_FIELD) ? "/vertex_fields" : "/cell_fields";
182       ierr = PetscSectionGetFieldName(section, f, &fname);CHKERRQ(ierr);
183       ierr = PetscViewerHDF5PushGroup(viewer, fgroup);CHKERRQ(ierr);
184       ierr = GetField_Static(dmBC, section, sectionGlobal, gv, f, pStart, pEnd, &is, &subv);CHKERRQ(ierr);
185       ierr = PetscObjectSetName((PetscObject) subv, fname);CHKERRQ(ierr);
186       if (isseq) {ierr = VecView_Seq(subv, viewer);CHKERRQ(ierr);}
187       else       {ierr = VecView_MPI(subv, viewer);CHKERRQ(ierr);}
188       ierr = RestoreField_Static(dmBC, section, sectionGlobal, gv, f, pStart, pEnd, &is, &subv);CHKERRQ(ierr);
189       ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
190       ierr = PetscSNPrintf(group, PETSC_MAX_PATH_LEN, "%s/%s", fgroup, fname);CHKERRQ(ierr);
191       ierr = PetscViewerHDF5HasAttribute(viewer, group, "vector_field_type", &flag);CHKERRQ(ierr);
192       if (!flag) {
193         if ((ft == PETSC_VTK_POINT_VECTOR_FIELD) || (ft == PETSC_VTK_CELL_VECTOR_FIELD)) {
194           ierr = PetscViewerHDF5WriteAttribute(viewer, group, "vector_field_type", PETSC_STRING, "vector");CHKERRQ(ierr);
195         } else {
196           ierr = PetscViewerHDF5WriteAttribute(viewer, group, "vector_field_type", PETSC_STRING, "scalar");CHKERRQ(ierr);
197         }
198       }
199     }
200   }
201   ierr = DMRestoreGlobalVector(dmBC, &gv);CHKERRQ(ierr);
202   PetscFunctionReturn(0);
203 }
204 #endif
205 
206 #undef __FUNCT__
207 #define __FUNCT__ "VecView_Plex_Local"
208 PetscErrorCode VecView_Plex_Local(Vec v, PetscViewer viewer)
209 {
210   DM             dm;
211   PetscBool      isvtk, ishdf5, isseq;
212   PetscErrorCode ierr;
213 
214   PetscFunctionBegin;
215   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
216   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
217   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,  &isvtk);CHKERRQ(ierr);
218   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
219   ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
220   if (isvtk || ishdf5) {
221     PetscInt  numFields;
222     PetscBool fem = PETSC_FALSE;
223 
224     ierr = DMGetNumFields(dm, &numFields);CHKERRQ(ierr);
225     if (numFields) {
226       PetscObject fe;
227 
228       ierr = DMGetField(dm, 0, &fe);CHKERRQ(ierr);
229       if (fe->classid == PETSCFE_CLASSID) fem = PETSC_TRUE;
230     }
231     if (fem) {
232       ierr = DMPlexInsertBoundaryValuesFEM(dm, v);CHKERRQ(ierr);
233     } else {
234       /* TODO Fix the time, and add FVM objects */
235       ierr = DMPlexInsertBoundaryValuesFVM(dm, 0.0, v);CHKERRQ(ierr);
236     }
237   }
238   if (isvtk) {
239     PetscSection            section;
240     PetscViewerVTKFieldType ft;
241     PetscInt                pStart, pEnd;
242 
243     ierr = DMGetDefaultSection(dm, &section);CHKERRQ(ierr);
244     ierr = GetFieldType_Static(dm, section, PETSC_DETERMINE, &pStart, &pEnd, &ft);CHKERRQ(ierr);
245     ierr = PetscObjectReference((PetscObject) dm);CHKERRQ(ierr); /* viewer drops reference */
246     ierr = PetscObjectReference((PetscObject) v);CHKERRQ(ierr);  /* viewer drops reference */
247     ierr = PetscViewerVTKAddField(viewer, (PetscObject) dm, DMPlexVTKWriteAll, ft, (PetscObject) v);CHKERRQ(ierr);
248   } else if (ishdf5) {
249 #if defined(PETSC_HAVE_HDF5)
250     ierr = VecView_Plex_Local_HDF5(v, viewer);CHKERRQ(ierr);
251 #else
252     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
253 #endif
254   } else {
255     if (isseq) {ierr = VecView_Seq(v, viewer);CHKERRQ(ierr);}
256     else       {ierr = VecView_MPI(v, viewer);CHKERRQ(ierr);}
257   }
258   PetscFunctionReturn(0);
259 }
260 
261 #if defined(PETSC_HAVE_HDF5)
262 #undef __FUNCT__
263 #define __FUNCT__ "VecView_Plex_HDF5"
264 static PetscErrorCode VecView_Plex_HDF5(Vec v, PetscViewer viewer)
265 {
266   DM             dm;
267   Vec            locv;
268   const char    *name;
269   PetscErrorCode ierr;
270 
271   PetscFunctionBegin;
272   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
273   ierr = DMGetLocalVector(dm, &locv);CHKERRQ(ierr);
274   ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
275   ierr = PetscObjectSetName((PetscObject) locv, name);CHKERRQ(ierr);
276   ierr = DMGlobalToLocalBegin(dm, v, INSERT_VALUES, locv);CHKERRQ(ierr);
277   ierr = DMGlobalToLocalEnd(dm, v, INSERT_VALUES, locv);CHKERRQ(ierr);
278   ierr = PetscViewerHDF5PushGroup(viewer, "/fields");CHKERRQ(ierr);
279   ierr = PetscViewerPushFormat(viewer, PETSC_VIEWER_HDF5_VIZ);CHKERRQ(ierr);
280   ierr = VecView_Plex_Local(locv, viewer);CHKERRQ(ierr);
281   ierr = PetscViewerPopFormat(viewer);CHKERRQ(ierr);
282   ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
283   ierr = DMRestoreLocalVector(dm, &locv);CHKERRQ(ierr);
284   PetscFunctionReturn(0);
285 }
286 #endif
287 
288 #undef __FUNCT__
289 #define __FUNCT__ "VecView_Plex"
290 PetscErrorCode VecView_Plex(Vec v, PetscViewer viewer)
291 {
292   DM             dm;
293   PetscBool      isvtk, ishdf5, isseq;
294   PetscErrorCode ierr;
295 
296   PetscFunctionBegin;
297   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
298   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
299   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,  &isvtk);CHKERRQ(ierr);
300   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
301   ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
302   if (isvtk) {
303     Vec         locv;
304     const char *name;
305 
306     ierr = DMGetLocalVector(dm, &locv);CHKERRQ(ierr);
307     ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
308     ierr = PetscObjectSetName((PetscObject) locv, name);CHKERRQ(ierr);
309     ierr = DMGlobalToLocalBegin(dm, v, INSERT_VALUES, locv);CHKERRQ(ierr);
310     ierr = DMGlobalToLocalEnd(dm, v, INSERT_VALUES, locv);CHKERRQ(ierr);
311     ierr = VecView_Plex_Local(locv, viewer);CHKERRQ(ierr);
312     ierr = DMRestoreLocalVector(dm, &locv);CHKERRQ(ierr);
313   } else if (ishdf5) {
314 #if defined(PETSC_HAVE_HDF5)
315     ierr = VecView_Plex_HDF5(v, viewer);CHKERRQ(ierr);
316 #else
317     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
318 #endif
319   } else {
320     if (isseq) {ierr = VecView_Seq(v, viewer);CHKERRQ(ierr);}
321     else       {ierr = VecView_MPI(v, viewer);CHKERRQ(ierr);}
322   }
323   PetscFunctionReturn(0);
324 }
325 
326 #undef __FUNCT__
327 #define __FUNCT__ "VecLoad_Plex_Local"
328 PetscErrorCode VecLoad_Plex_Local(Vec v, PetscViewer viewer)
329 {
330   DM             dm;
331   PetscBool      ishdf5;
332   PetscErrorCode ierr;
333 
334   PetscFunctionBegin;
335   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
336   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
337   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
338   if (ishdf5) {
339     DM          dmBC;
340     Vec         gv;
341     const char *name;
342 
343     ierr = DMGetOutputDM(dm, &dmBC);CHKERRQ(ierr);
344     ierr = DMGetGlobalVector(dmBC, &gv);CHKERRQ(ierr);
345     ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
346     ierr = PetscObjectSetName((PetscObject) gv, name);CHKERRQ(ierr);
347     ierr = VecLoad_Default(gv, viewer);CHKERRQ(ierr);
348     ierr = DMGlobalToLocalBegin(dmBC, gv, INSERT_VALUES, v);CHKERRQ(ierr);
349     ierr = DMGlobalToLocalEnd(dmBC, gv, INSERT_VALUES, v);CHKERRQ(ierr);
350     ierr = DMRestoreGlobalVector(dmBC, &gv);CHKERRQ(ierr);
351   } else {
352     ierr = VecLoad_Default(v, viewer);CHKERRQ(ierr);
353   }
354   PetscFunctionReturn(0);
355 }
356 
357 #if defined(PETSC_HAVE_HDF5)
358 #undef __FUNCT__
359 #define __FUNCT__ "VecLoad_Plex_HDF5"
360 static PetscErrorCode VecLoad_Plex_HDF5(Vec v, PetscViewer viewer)
361 {
362   DM             dm;
363   Vec            locv;
364   const char    *name;
365   PetscInt       seqnum;
366   PetscErrorCode ierr;
367 
368   PetscFunctionBegin;
369   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
370   ierr = DMGetLocalVector(dm, &locv);CHKERRQ(ierr);
371   ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
372   ierr = PetscObjectSetName((PetscObject) locv, name);CHKERRQ(ierr);
373   ierr = DMGetOutputSequenceNumber(dm, &seqnum);CHKERRQ(ierr);
374   ierr = PetscViewerHDF5SetTimestep(viewer, seqnum);CHKERRQ(ierr);
375   ierr = PetscViewerHDF5PushGroup(viewer, "/fields");CHKERRQ(ierr);
376   ierr = VecLoad_Plex_Local(locv, viewer);CHKERRQ(ierr);
377   ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
378   ierr = DMLocalToGlobalBegin(dm, locv, INSERT_VALUES, v);CHKERRQ(ierr);
379   ierr = DMLocalToGlobalEnd(dm, locv, INSERT_VALUES, v);CHKERRQ(ierr);
380   ierr = DMRestoreLocalVector(dm, &locv);CHKERRQ(ierr);
381   PetscFunctionReturn(0);
382 }
383 #endif
384 
385 #undef __FUNCT__
386 #define __FUNCT__ "VecLoad_Plex"
387 PetscErrorCode VecLoad_Plex(Vec v, PetscViewer viewer)
388 {
389   DM             dm;
390   PetscBool      ishdf5;
391   PetscErrorCode ierr;
392 
393   PetscFunctionBegin;
394   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
395   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
396   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
397   if (ishdf5) {
398 #if defined(PETSC_HAVE_HDF5)
399     ierr = VecLoad_Plex_HDF5(v, viewer);CHKERRQ(ierr);
400 #else
401     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
402 #endif
403   } else {
404     ierr = VecLoad_Default(v, viewer);CHKERRQ(ierr);
405   }
406   PetscFunctionReturn(0);
407 }
408 
409 #undef __FUNCT__
410 #define __FUNCT__ "DMPlexView_Ascii"
411 PetscErrorCode DMPlexView_Ascii(DM dm, PetscViewer viewer)
412 {
413   DM_Plex          *mesh = (DM_Plex*) dm->data;
414   DM                cdm;
415   DMLabel           markers;
416   PetscSection      coordSection;
417   Vec               coordinates;
418   PetscViewerFormat format;
419   PetscErrorCode    ierr;
420 
421   PetscFunctionBegin;
422   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
423   ierr = DMGetDefaultSection(cdm, &coordSection);CHKERRQ(ierr);
424   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
425   ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
426   if (format == PETSC_VIEWER_ASCII_INFO_DETAIL) {
427     const char *name;
428     PetscInt    maxConeSize, maxSupportSize;
429     PetscInt    pStart, pEnd, p;
430     PetscMPIInt rank, size;
431 
432     ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRQ(ierr);
433     ierr = MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size);CHKERRQ(ierr);
434     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
435     ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
436     ierr = DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize);CHKERRQ(ierr);
437     ierr = PetscViewerASCIISynchronizedAllow(viewer, PETSC_TRUE);CHKERRQ(ierr);
438     ierr = PetscViewerASCIIPrintf(viewer, "Mesh '%s':\n", name);CHKERRQ(ierr);
439     ierr = PetscViewerASCIISynchronizedPrintf(viewer, "Max sizes cone: %D support: %D\n", maxConeSize, maxSupportSize);CHKERRQ(ierr);
440     ierr = PetscViewerASCIIPrintf(viewer, "orientation is missing\n", name);CHKERRQ(ierr);
441     ierr = PetscViewerASCIIPrintf(viewer, "cap --> base:\n", name);CHKERRQ(ierr);
442     for (p = pStart; p < pEnd; ++p) {
443       PetscInt dof, off, s;
444 
445       ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
446       ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
447       for (s = off; s < off+dof; ++s) {
448         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%D]: %D ----> %D\n", rank, p, mesh->supports[s]);CHKERRQ(ierr);
449       }
450     }
451     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
452     ierr = PetscViewerASCIIPrintf(viewer, "base <-- cap:\n", name);CHKERRQ(ierr);
453     for (p = pStart; p < pEnd; ++p) {
454       PetscInt dof, off, c;
455 
456       ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
457       ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
458       for (c = off; c < off+dof; ++c) {
459         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%D]: %D <---- %D (%D)\n", rank, p, mesh->cones[c], mesh->coneOrientations[c]);CHKERRQ(ierr);
460       }
461     }
462     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
463     ierr = PetscSectionGetChart(coordSection, &pStart, NULL);CHKERRQ(ierr);
464     if (pStart >= 0) {ierr = PetscSectionVecView(coordSection, coordinates, viewer);CHKERRQ(ierr);}
465     ierr = DMPlexGetLabel(dm, "marker", &markers);CHKERRQ(ierr);
466     ierr = DMLabelView(markers,viewer);CHKERRQ(ierr);
467     if (size > 1) {
468       PetscSF sf;
469 
470       ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr);
471       ierr = PetscSFView(sf, viewer);CHKERRQ(ierr);
472     }
473     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
474   } else if (format == PETSC_VIEWER_ASCII_LATEX) {
475     const char  *name;
476     const char  *colors[3] = {"red", "blue", "green"};
477     const int    numColors  = 3;
478     PetscReal    scale      = 2.0;
479     PetscScalar *coords;
480     PetscInt     depth, cStart, cEnd, c, vStart, vEnd, v, eStart = 0, eEnd = 0, e, p;
481     PetscMPIInt  rank, size;
482 
483     ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
484     ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRQ(ierr);
485     ierr = MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size);CHKERRQ(ierr);
486     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
487     ierr = PetscViewerASCIISynchronizedAllow(viewer, PETSC_TRUE);CHKERRQ(ierr);
488     ierr = PetscViewerASCIIPrintf(viewer, "\
489 \\documentclass[crop,multi=false]{standalone}\n\n\
490 \\usepackage{tikz}\n\
491 \\usepackage{pgflibraryshapes}\n\
492 \\usetikzlibrary{backgrounds}\n\
493 \\usetikzlibrary{arrows}\n\
494 \\begin{document}\n\
495 \\section{%s}\n\
496 \\begin{center}\n", name, 8.0/scale);CHKERRQ(ierr);
497     ierr = PetscViewerASCIIPrintf(viewer, "Mesh for process ");CHKERRQ(ierr);
498     for (p = 0; p < size; ++p) {
499       if (p > 0 && p == size-1) {
500         ierr = PetscViewerASCIIPrintf(viewer, ", and ", colors[p%numColors], p);CHKERRQ(ierr);
501       } else if (p > 0) {
502         ierr = PetscViewerASCIIPrintf(viewer, ", ", colors[p%numColors], p);CHKERRQ(ierr);
503       }
504       ierr = PetscViewerASCIIPrintf(viewer, "{\\textcolor{%s}%D}", colors[p%numColors], p);CHKERRQ(ierr);
505     }
506     ierr = PetscViewerASCIIPrintf(viewer, ".\n\n\n\
507 \\begin{tikzpicture}[scale = %g,font=\\fontsize{8}{8}\\selectfont]\n");CHKERRQ(ierr);
508     /* Plot vertices */
509     ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
510     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
511     ierr = PetscViewerASCIIPrintf(viewer, "\\path\n");CHKERRQ(ierr);
512     for (v = vStart; v < vEnd; ++v) {
513       PetscInt off, dof, d;
514 
515       ierr = PetscSectionGetDof(coordSection, v, &dof);CHKERRQ(ierr);
516       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
517       ierr = PetscViewerASCIISynchronizedPrintf(viewer, "(");CHKERRQ(ierr);
518       for (d = 0; d < dof; ++d) {
519         if (d > 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, ",");CHKERRQ(ierr);}
520         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "%g", (double)(scale*PetscRealPart(coords[off+d])));CHKERRQ(ierr);
521       }
522       ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%D) [draw,shape=circle,color=%s] {%D} --\n", v, rank, colors[rank%numColors], v);CHKERRQ(ierr);
523     }
524     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
525     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
526     ierr = PetscViewerASCIIPrintf(viewer, "(0,0);\n");CHKERRQ(ierr);
527     /* Plot edges */
528     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
529     ierr = PetscViewerASCIIPrintf(viewer, "\\path\n");CHKERRQ(ierr);
530     if (depth > 1) {ierr = DMPlexGetDepthStratum(dm, 1, &eStart, &eEnd);CHKERRQ(ierr);}
531     for (e = eStart; e < eEnd; ++e) {
532       const PetscInt *cone;
533       PetscInt        coneSize, offA, offB, dof, d;
534 
535       ierr = DMPlexGetConeSize(dm, e, &coneSize);CHKERRQ(ierr);
536       if (coneSize != 2) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Edge %d cone should have two vertices, not %d", e, coneSize);
537       ierr = DMPlexGetCone(dm, e, &cone);CHKERRQ(ierr);
538       ierr = PetscSectionGetDof(coordSection, cone[0], &dof);CHKERRQ(ierr);
539       ierr = PetscSectionGetOffset(coordSection, cone[0], &offA);CHKERRQ(ierr);
540       ierr = PetscSectionGetOffset(coordSection, cone[1], &offB);CHKERRQ(ierr);
541       ierr = PetscViewerASCIISynchronizedPrintf(viewer, "(");CHKERRQ(ierr);
542       for (d = 0; d < dof; ++d) {
543         if (d > 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, ",");CHKERRQ(ierr);}
544         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "%g", (double)(scale*0.5*PetscRealPart(coords[offA+d]+coords[offB+d])));CHKERRQ(ierr);
545       }
546       ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%D) [draw,shape=circle,color=%s] {%D} --\n", e, rank, colors[rank%numColors], e);CHKERRQ(ierr);
547     }
548     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
549     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
550     ierr = PetscViewerASCIIPrintf(viewer, "(0,0);\n");CHKERRQ(ierr);
551     /* Plot cells */
552     ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
553     for (c = cStart; c < cEnd; ++c) {
554       PetscInt *closure = NULL;
555       PetscInt  closureSize, firstPoint = -1;
556 
557       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
558       ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\draw[color=%s] ", colors[rank%numColors]);CHKERRQ(ierr);
559       for (p = 0; p < closureSize*2; p += 2) {
560         const PetscInt point = closure[p];
561 
562         if ((point < vStart) || (point >= vEnd)) continue;
563         if (firstPoint >= 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- ");CHKERRQ(ierr);}
564         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "(%D_%D)", point, rank);CHKERRQ(ierr);
565         if (firstPoint < 0) firstPoint = point;
566       }
567       /* Why doesn't this work? ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- cycle;\n");CHKERRQ(ierr); */
568       ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- (%D_%D);\n", firstPoint, rank);CHKERRQ(ierr);
569       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
570     }
571     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
572     ierr = PetscViewerASCIIPrintf(viewer, "\\end{tikzpicture}\n\\end{center}\n");CHKERRQ(ierr);
573     ierr = PetscViewerASCIIPrintf(viewer, "\\end{document}\n", name);CHKERRQ(ierr);
574   } else {
575     MPI_Comm    comm;
576     PetscInt   *sizes, *hybsizes;
577     PetscInt    locDepth, depth, dim, d, pMax[4];
578     PetscInt    pStart, pEnd, p;
579     PetscInt    numLabels, l;
580     const char *name;
581     PetscMPIInt size;
582 
583     ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
584     ierr = MPI_Comm_size(comm, &size);CHKERRQ(ierr);
585     ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
586     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
587     if (name) {ierr = PetscViewerASCIIPrintf(viewer, "%s in %D dimensions:\n", name, dim);CHKERRQ(ierr);}
588     else      {ierr = PetscViewerASCIIPrintf(viewer, "Mesh in %D dimensions:\n", dim);CHKERRQ(ierr);}
589     ierr = DMPlexGetDepth(dm, &locDepth);CHKERRQ(ierr);
590     ierr = MPI_Allreduce(&locDepth, &depth, 1, MPIU_INT, MPI_MAX, comm);CHKERRQ(ierr);
591     ierr = DMPlexGetHybridBounds(dm, &pMax[depth], &pMax[depth-1], &pMax[1], &pMax[0]);CHKERRQ(ierr);
592     ierr = PetscMalloc2(size,&sizes,size,&hybsizes);CHKERRQ(ierr);
593     if (depth == 1) {
594       ierr = DMPlexGetDepthStratum(dm, 0, &pStart, &pEnd);CHKERRQ(ierr);
595       pEnd = pEnd - pStart;
596       ierr = MPI_Gather(&pEnd, 1, MPIU_INT, sizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
597       ierr = PetscViewerASCIIPrintf(viewer, "  %D-cells:", 0);CHKERRQ(ierr);
598       for (p = 0; p < size; ++p) {ierr = PetscViewerASCIIPrintf(viewer, " %D", sizes[p]);CHKERRQ(ierr);}
599       ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
600       ierr = DMPlexGetHeightStratum(dm, 0, &pStart, &pEnd);CHKERRQ(ierr);
601       pEnd = pEnd - pStart;
602       ierr = MPI_Gather(&pEnd, 1, MPIU_INT, sizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
603       ierr = PetscViewerASCIIPrintf(viewer, "  %D-cells:", dim);CHKERRQ(ierr);
604       for (p = 0; p < size; ++p) {ierr = PetscViewerASCIIPrintf(viewer, " %D", sizes[p]);CHKERRQ(ierr);}
605       ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
606     } else {
607       for (d = 0; d <= dim; d++) {
608         ierr = DMPlexGetDepthStratum(dm, d, &pStart, &pEnd);CHKERRQ(ierr);
609         pEnd    -= pStart;
610         pMax[d] -= pStart;
611         ierr = MPI_Gather(&pEnd, 1, MPIU_INT, sizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
612         ierr = MPI_Gather(&pMax[d], 1, MPIU_INT, hybsizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
613         ierr = PetscViewerASCIIPrintf(viewer, "  %D-cells:", d);CHKERRQ(ierr);
614         for (p = 0; p < size; ++p) {
615           if (hybsizes[p] >= 0) {ierr = PetscViewerASCIIPrintf(viewer, " %D (%D)", sizes[p], sizes[p] - hybsizes[p]);CHKERRQ(ierr);}
616           else                  {ierr = PetscViewerASCIIPrintf(viewer, " %D", sizes[p]);CHKERRQ(ierr);}
617         }
618         ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
619       }
620     }
621     ierr = PetscFree2(sizes,hybsizes);CHKERRQ(ierr);
622     ierr = DMPlexGetNumLabels(dm, &numLabels);CHKERRQ(ierr);
623     if (numLabels) {ierr = PetscViewerASCIIPrintf(viewer, "Labels:\n");CHKERRQ(ierr);}
624     for (l = 0; l < numLabels; ++l) {
625       DMLabel         label;
626       const char     *name;
627       IS              valueIS;
628       const PetscInt *values;
629       PetscInt        numValues, v;
630 
631       ierr = DMPlexGetLabelName(dm, l, &name);CHKERRQ(ierr);
632       ierr = DMPlexGetLabel(dm, name, &label);CHKERRQ(ierr);
633       ierr = DMLabelGetNumValues(label, &numValues);CHKERRQ(ierr);
634       ierr = PetscViewerASCIIPrintf(viewer, "  %s: %d strata of sizes (", name, numValues);CHKERRQ(ierr);
635       ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
636       ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
637       for (v = 0; v < numValues; ++v) {
638         PetscInt size;
639 
640         ierr = DMLabelGetStratumSize(label, values[v], &size);CHKERRQ(ierr);
641         if (v > 0) {ierr = PetscViewerASCIIPrintf(viewer, ", ");CHKERRQ(ierr);}
642         ierr = PetscViewerASCIIPrintf(viewer, "%d", size);CHKERRQ(ierr);
643       }
644       ierr = PetscViewerASCIIPrintf(viewer, ")\n");CHKERRQ(ierr);
645       ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
646       ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
647     }
648   }
649   PetscFunctionReturn(0);
650 }
651 
652 #if defined(PETSC_HAVE_HDF5)
653 #undef __FUNCT__
654 #define __FUNCT__ "DMPlexView_HDF5"
655 /* We only write cells and vertices. Does this screw up parallel reading? */
656 static PetscErrorCode DMPlexView_HDF5(DM dm, PetscViewer viewer)
657 {
658   DM              cdm;
659   Vec             coordinates, newcoords;
660   Vec             coneVec, cellVec;
661   IS              globalVertexNumbers;
662   const PetscInt *gvertex;
663   PetscScalar    *sizes, *vertices;
664   PetscReal       lengthScale;
665   const char     *label   = NULL;
666   PetscInt        labelId = 0, dim;
667   char            group[PETSC_MAX_PATH_LEN];
668   PetscInt        vStart, vEnd, v, cellHeight, cStart, cEnd, cMax, cell, conesSize = 0, numCornersLocal = 0, numCorners, numLabels, l;
669   hid_t           fileId, groupId;
670   herr_t          status;
671   PetscErrorCode  ierr;
672 
673   PetscFunctionBegin;
674   /* Write coordinates */
675   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
676   ierr = DMPlexGetScale(dm, PETSC_UNIT_LENGTH, &lengthScale);CHKERRQ(ierr);
677   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
678   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
679   ierr = VecDuplicate(coordinates, &newcoords);CHKERRQ(ierr);
680   ierr = PetscObjectSetName((PetscObject) newcoords, "vertices");CHKERRQ(ierr);
681   ierr = VecCopy(coordinates, newcoords);CHKERRQ(ierr);
682   ierr = VecScale(newcoords, lengthScale);CHKERRQ(ierr);
683   /* Use the local version to bypass the default group setting */
684   ierr = PetscViewerHDF5PushGroup(viewer, "/geometry");CHKERRQ(ierr);
685   ierr = VecView(newcoords, viewer);CHKERRQ(ierr);
686   ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
687   ierr = VecDestroy(&newcoords);CHKERRQ(ierr);
688   /* Write toplogy */
689   ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
690   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
691   ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
692   ierr = DMPlexGetHybridBounds(dm, &cMax, PETSC_NULL, PETSC_NULL, PETSC_NULL);CHKERRQ(ierr);
693   if (cMax >= 0) cEnd = PetscMin(cEnd, cMax);
694 
695   ierr = VecCreate(PetscObjectComm((PetscObject) dm), &coneVec);CHKERRQ(ierr);
696   ierr = VecSetSizes(coneVec, cEnd-cStart, PETSC_DETERMINE);CHKERRQ(ierr);
697   ierr = VecSetBlockSize(coneVec, 1);CHKERRQ(ierr);
698   ierr = VecSetFromOptions(coneVec);CHKERRQ(ierr);
699   ierr = PetscObjectSetName((PetscObject) coneVec, "coneSize");CHKERRQ(ierr);
700   ierr = VecGetArray(coneVec, &sizes);CHKERRQ(ierr);
701   for (cell = cStart; cell < cEnd; ++cell) {
702     PetscInt *closure = NULL;
703     PetscInt  closureSize, v, Nc = 0;
704 
705     if (label) {
706       PetscInt value;
707       ierr = DMPlexGetLabelValue(dm, label, cell, &value);CHKERRQ(ierr);
708       if (value == labelId) continue;
709     }
710     ierr = DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
711     for (v = 0; v < closureSize*2; v += 2) {
712       if ((closure[v] >= vStart) && (closure[v] < vEnd)) ++Nc;
713     }
714     ierr = DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
715     conesSize += Nc;
716     if (!numCornersLocal)           numCornersLocal = Nc;
717     else if (numCornersLocal != Nc) numCornersLocal = 1;
718   }
719   ierr = VecRestoreArray(coneVec, &sizes);CHKERRQ(ierr);
720   ierr = MPI_Allreduce(&numCornersLocal, &numCorners, 1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject) dm));CHKERRQ(ierr);
721   if (numCornersLocal && numCornersLocal != numCorners) numCorners = 1;
722 
723   ierr = DMPlexGetVertexNumbering(dm, &globalVertexNumbers);CHKERRQ(ierr);
724   ierr = ISGetIndices(globalVertexNumbers, &gvertex);CHKERRQ(ierr);
725   ierr = VecCreate(PetscObjectComm((PetscObject) dm), &cellVec);CHKERRQ(ierr);
726   ierr = VecSetSizes(cellVec, conesSize, PETSC_DETERMINE);CHKERRQ(ierr);
727   ierr = VecSetBlockSize(cellVec, numCorners);CHKERRQ(ierr);
728   ierr = VecSetFromOptions(cellVec);CHKERRQ(ierr);
729   ierr = PetscObjectSetName((PetscObject) cellVec, "cells");CHKERRQ(ierr);
730   ierr = VecGetArray(cellVec, &vertices);CHKERRQ(ierr);
731   for (cell = cStart, v = 0; cell < cEnd; ++cell) {
732     PetscInt *closure = NULL;
733     PetscInt  closureSize, Nc = 0, p;
734 
735     if (label) {
736       PetscInt value;
737       ierr = DMPlexGetLabelValue(dm, label, cell, &value);CHKERRQ(ierr);
738       if (value == labelId) continue;
739     }
740     ierr = DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
741     for (p = 0; p < closureSize*2; p += 2) {
742       if ((closure[p] >= vStart) && (closure[p] < vEnd)) {
743         closure[Nc++] = closure[p];
744         }
745     }
746     ierr = DMPlexInvertCell(dim, Nc, closure);CHKERRQ(ierr);
747     for (p = 0; p < Nc; ++p) {
748       const PetscInt gv = gvertex[closure[p] - vStart];
749       vertices[v++] = gv < 0 ? -(gv+1) : gv;
750     }
751     ierr = DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
752   }
753   if (v != conesSize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_LIB, "Total number of cell vertices %d != %d", v, conesSize);
754   ierr = VecRestoreArray(cellVec, &vertices);CHKERRQ(ierr);
755   ierr = PetscViewerHDF5PushGroup(viewer, "/topology");CHKERRQ(ierr);
756   ierr = VecView(cellVec, viewer);CHKERRQ(ierr);
757   if (numCorners == 1) {
758     ierr = VecView(coneVec, viewer);CHKERRQ(ierr);
759   } else {
760     ierr = PetscViewerHDF5WriteAttribute(viewer, "/topology/cells", "cell_corners", PETSC_INT, (void *) &numCorners);CHKERRQ(ierr);
761   }
762   ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
763   ierr = VecDestroy(&cellVec);CHKERRQ(ierr);
764   ierr = VecDestroy(&coneVec);CHKERRQ(ierr);
765   ierr = ISRestoreIndices(globalVertexNumbers, &gvertex);CHKERRQ(ierr);
766 
767   ierr = PetscViewerHDF5WriteAttribute(viewer, "/topology/cells", "cell_dim", PETSC_INT, (void *) &dim);CHKERRQ(ierr);
768   /* Write Labels*/
769   ierr = PetscViewerHDF5PushGroup(viewer, "/labels");CHKERRQ(ierr);
770   ierr = PetscViewerHDF5OpenGroup(viewer, &fileId, &groupId);CHKERRQ(ierr);
771   if (groupId != fileId) {status = H5Gclose(groupId);CHKERRQ(status);}
772   ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
773   ierr = DMPlexGetNumLabels(dm, &numLabels);CHKERRQ(ierr);
774   for (l = 0; l < numLabels; ++l) {
775     DMLabel         label;
776     const char     *name;
777     IS              valueIS;
778     const PetscInt *values;
779     PetscInt        numValues, v;
780     PetscBool       isDepth;
781 
782     ierr = DMPlexGetLabelName(dm, l, &name);CHKERRQ(ierr);
783     ierr = PetscStrncmp(name, "depth", 10, &isDepth);CHKERRQ(ierr);
784     if (isDepth) continue;
785     ierr = DMPlexGetLabel(dm, name, &label);CHKERRQ(ierr);
786     ierr = DMLabelGetNumValues(label, &numValues);CHKERRQ(ierr);
787     ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
788     ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
789     ierr = PetscSNPrintf(group, PETSC_MAX_PATH_LEN, "/labels/%s", name);CHKERRQ(ierr);
790     ierr = PetscViewerHDF5PushGroup(viewer, group);CHKERRQ(ierr);
791     ierr = PetscViewerHDF5OpenGroup(viewer, &fileId, &groupId);CHKERRQ(ierr);
792     if (groupId != fileId) {status = H5Gclose(groupId);CHKERRQ(status);}
793     ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
794     /* TODO: Need to actually loop over the union of label values, ISAllGather() */
795     for (v = 0; v < numValues; ++v) {
796       IS   stratumIS;
797 
798       ierr = PetscSNPrintf(group, PETSC_MAX_PATH_LEN, "/labels/%s/%d", name, values[v]);CHKERRQ(ierr);
799       ierr = DMLabelGetStratumIS(label, values[v], &stratumIS);CHKERRQ(ierr);
800       /* TODO: Need to globalize point names and remove unowned points */
801       ierr = PetscViewerHDF5PushGroup(viewer, group);CHKERRQ(ierr);
802       ierr = ISView(stratumIS, viewer);CHKERRQ(ierr);
803       ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
804       ierr = ISDestroy(&stratumIS);CHKERRQ(ierr);
805     }
806     ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
807     ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
808   }
809   PetscFunctionReturn(0);
810 }
811 #endif
812 
813 #undef __FUNCT__
814 #define __FUNCT__ "DMView_Plex"
815 PetscErrorCode DMView_Plex(DM dm, PetscViewer viewer)
816 {
817   PetscBool      iascii, ishdf5;
818   PetscErrorCode ierr;
819 
820   PetscFunctionBegin;
821   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
822   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
823   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERASCII, &iascii);CHKERRQ(ierr);
824   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,  &ishdf5);CHKERRQ(ierr);
825   if (iascii) {
826     ierr = DMPlexView_Ascii(dm, viewer);CHKERRQ(ierr);
827   } else if (ishdf5) {
828 #if defined(PETSC_HAVE_HDF5)
829     ierr = DMPlexView_HDF5(dm, viewer);CHKERRQ(ierr);
830 #else
831     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
832 #endif
833   }
834   PetscFunctionReturn(0);
835 }
836 
837 #if defined(PETSC_HAVE_HDF5)
838 typedef struct {
839   DM          dm;
840   PetscViewer viewer;
841   DMLabel     label;
842 } LabelCtx;
843 
844 static herr_t ReadLabelStratumHDF5_Static(hid_t g_id, const char *name, const H5L_info_t *info, void *op_data)
845 {
846   PetscViewer     viewer = ((LabelCtx *) op_data)->viewer;
847   DMLabel         label  = ((LabelCtx *) op_data)->label;
848   IS              stratumIS;
849   const PetscInt *ind;
850   PetscInt        value, N, i;
851   const char     *lname;
852   char            group[PETSC_MAX_PATH_LEN];
853   PetscErrorCode  ierr;
854 
855   ierr = PetscOptionsStringToInt(name, &value);
856   ierr = ISCreate(PetscObjectComm((PetscObject) viewer), &stratumIS);
857   ierr = PetscObjectSetName((PetscObject) stratumIS, "indices");
858   ierr = DMLabelGetName(label, &lname);
859   ierr = PetscSNPrintf(group, PETSC_MAX_PATH_LEN, "/labels/%s/%s", lname, name);CHKERRQ(ierr);
860   ierr = PetscViewerHDF5PushGroup(viewer, group);CHKERRQ(ierr);
861   ierr = ISLoad(stratumIS, viewer);
862   ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
863   ierr = ISGetSize(stratumIS, &N);
864   ierr = ISGetIndices(stratumIS, &ind);
865   for (i = 0; i < N; ++i) {ierr = DMLabelSetValue(label, ind[i], value);}
866   ierr = ISRestoreIndices(stratumIS, &ind);
867   ierr = ISDestroy(&stratumIS);
868   return 0;
869 }
870 
871 static herr_t ReadLabelHDF5_Static(hid_t g_id, const char *name, const H5L_info_t *info, void *op_data)
872 {
873   DM             dm = ((LabelCtx *) op_data)->dm;
874   hsize_t        idx;
875   herr_t         status;
876   PetscErrorCode ierr;
877 
878   ierr = DMPlexCreateLabel(dm, name); if (ierr) return (herr_t) ierr;
879   ierr = DMPlexGetLabel(dm, name, &((LabelCtx *) op_data)->label); if (ierr) return (herr_t) ierr;
880   status = H5Literate_by_name(g_id, name, H5_INDEX_NAME, H5_ITER_NATIVE, &idx, ReadLabelStratumHDF5_Static, op_data, 0);
881   return status;
882 }
883 
884 #undef __FUNCT__
885 #define __FUNCT__ "DMPlexLoad_HDF5"
886 /* The first version will read everything onto proc 0, letting the user distribute
887    The next will create a naive partition, and then rebalance after reading
888 */
889 static PetscErrorCode DMPlexLoad_HDF5(DM dm, PetscViewer viewer)
890 {
891   LabelCtx        ctx;
892   PetscSection    coordSection;
893   Vec             coordinates;
894   Vec             cellVec;
895   PetscScalar    *cells;
896   PetscReal       lengthScale;
897   PetscInt       *cone;
898   PetscInt        dim, spatialDim, numVertices, v, numCorners, numCells, cell, c;
899   hid_t           fileId, groupId;
900   hsize_t         idx = 0;
901   herr_t          status;
902   PetscErrorCode  ierr;
903 
904   PetscFunctionBegin;
905   /* Read geometry */
906   ierr = PetscViewerHDF5PushGroup(viewer, "/geometry");CHKERRQ(ierr);
907   ierr = VecCreate(PetscObjectComm((PetscObject) dm), &coordinates);CHKERRQ(ierr);
908   ierr = PetscObjectSetName((PetscObject) coordinates, "vertices");CHKERRQ(ierr);
909   ierr = VecLoad(coordinates, viewer);CHKERRQ(ierr);
910   ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
911   ierr = DMPlexGetScale(dm, PETSC_UNIT_LENGTH, &lengthScale);CHKERRQ(ierr);
912   ierr = VecScale(coordinates, 1.0/lengthScale);CHKERRQ(ierr);
913   ierr = VecGetSize(coordinates, &numVertices);CHKERRQ(ierr);
914   ierr = VecGetBlockSize(coordinates, &spatialDim);CHKERRQ(ierr);
915   numVertices /= spatialDim;
916   /* Read toplogy */
917   ierr = PetscViewerHDF5ReadAttribute(viewer, "/topology/cells", "cell_dim", PETSC_INT, (void *) &dim);CHKERRQ(ierr);
918   ierr = DMPlexSetDimension(dm, dim);CHKERRQ(ierr);
919   /*   TODO Check for coneSize vector rather than this attribute */
920   ierr = PetscViewerHDF5ReadAttribute(viewer, "/topology/cells", "cell_corners", PETSC_INT, (void *) &numCorners);CHKERRQ(ierr);
921   ierr = PetscViewerHDF5PushGroup(viewer, "/topology");CHKERRQ(ierr);
922   ierr = VecCreate(PetscObjectComm((PetscObject) dm), &cellVec);CHKERRQ(ierr);
923   ierr = VecSetBlockSize(cellVec, numCorners);CHKERRQ(ierr);
924   ierr = PetscObjectSetName((PetscObject) cellVec, "cells");CHKERRQ(ierr);
925   ierr = VecLoad(cellVec, viewer);CHKERRQ(ierr);
926   ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
927   ierr = VecGetSize(cellVec, &numCells);CHKERRQ(ierr);
928   numCells /= numCorners;
929   /* Create Plex */
930   ierr = DMPlexSetChart(dm, 0, numCells+numVertices);CHKERRQ(ierr);
931   for (cell = 0; cell < numCells; ++cell) {ierr = DMPlexSetConeSize(dm, cell, numCorners);CHKERRQ(ierr);}
932   ierr = DMSetUp(dm);CHKERRQ(ierr);
933   ierr = PetscMalloc1(numCorners,&cone);CHKERRQ(ierr);
934   ierr = VecGetArray(cellVec, &cells);CHKERRQ(ierr);
935   for (cell = 0; cell < numCells; ++cell) {
936     for (c = 0; c < numCorners; ++c) {cone[c] = numCells + cells[cell*numCorners+c];}
937     ierr = DMPlexSetCone(dm, cell, cone);CHKERRQ(ierr);
938   }
939   ierr = VecRestoreArray(cellVec, &cells);CHKERRQ(ierr);
940   ierr = PetscFree(cone);CHKERRQ(ierr);
941   ierr = VecDestroy(&cellVec);CHKERRQ(ierr);
942   ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
943   ierr = DMPlexStratify(dm);CHKERRQ(ierr);
944   /* Create coordinates */
945   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
946   ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
947   ierr = PetscSectionSetFieldComponents(coordSection, 0, spatialDim);CHKERRQ(ierr);
948   ierr = PetscSectionSetChart(coordSection, numCells, numCells+numVertices);CHKERRQ(ierr);
949   for (v = numCells; v < numCells+numVertices; ++v) {
950     ierr = PetscSectionSetDof(coordSection, v, spatialDim);CHKERRQ(ierr);
951     ierr = PetscSectionSetFieldDof(coordSection, v, 0, spatialDim);CHKERRQ(ierr);
952   }
953   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
954   ierr = DMSetCoordinates(dm, coordinates);CHKERRQ(ierr);
955   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
956   /* Read Labels*/
957   ctx.dm     = dm;
958   ctx.viewer = viewer;
959   ierr = PetscViewerHDF5PushGroup(viewer, "/labels");CHKERRQ(ierr);
960   ierr = PetscViewerHDF5OpenGroup(viewer, &fileId, &groupId);CHKERRQ(ierr);
961   status = H5Literate(groupId, H5_INDEX_NAME, H5_ITER_NATIVE, &idx, ReadLabelHDF5_Static, &ctx);CHKERRQ(status);
962   status = H5Gclose(groupId);CHKERRQ(status);
963   ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
964   PetscFunctionReturn(0);
965 }
966 #endif
967 
968 #undef __FUNCT__
969 #define __FUNCT__ "DMLoad_Plex"
970 PetscErrorCode DMLoad_Plex(DM dm, PetscViewer viewer)
971 {
972   PetscBool      isbinary, ishdf5;
973   PetscErrorCode ierr;
974 
975   PetscFunctionBegin;
976   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
977   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
978   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERBINARY, &isbinary);CHKERRQ(ierr);
979   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,   &ishdf5);CHKERRQ(ierr);
980   if (isbinary) {SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Do not yet support binary viewers");}
981   else if (ishdf5) {
982 #if defined(PETSC_HAVE_HDF5)
983     DM odm;
984     ierr = DMCreate(PetscObjectComm((PetscObject) dm), &odm);CHKERRQ(ierr);
985     ierr = DMSetType(odm, DMPLEX);CHKERRQ(ierr);
986     ierr = DMPlexLoad_HDF5(odm, viewer);CHKERRQ(ierr);
987     ierr = DMPlexInterpolate(odm, &dm);CHKERRQ(ierr);
988     ierr = DMPlexCopyCoordinates(odm, dm);CHKERRQ(ierr);
989     ierr = DMPlexCopyLabels(odm, dm);CHKERRQ(ierr);
990     ierr = DMDestroy(&odm);CHKERRQ(ierr);
991 #else
992     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
993 #endif
994   }
995   PetscFunctionReturn(0);
996 }
997 
998 #undef __FUNCT__
999 #define __FUNCT__ "BoundaryDestroy"
1000 static PetscErrorCode BoundaryDestroy(DMBoundary *boundary)
1001 {
1002   DMBoundary     b, next;
1003   PetscErrorCode ierr;
1004 
1005   PetscFunctionBeginUser;
1006   if (!boundary) PetscFunctionReturn(0);
1007   b = *boundary;
1008   *boundary = NULL;
1009   for (; b; b = next) {
1010     next = b->next;
1011     ierr = PetscFree(b->ids);CHKERRQ(ierr);
1012     ierr = PetscFree(b->name);CHKERRQ(ierr);
1013     ierr = PetscFree(b);CHKERRQ(ierr);
1014   }
1015   PetscFunctionReturn(0);
1016 }
1017 
1018 #undef __FUNCT__
1019 #define __FUNCT__ "DMDestroy_Plex"
1020 PetscErrorCode DMDestroy_Plex(DM dm)
1021 {
1022   DM_Plex       *mesh = (DM_Plex*) dm->data;
1023   DMLabel        next  = mesh->labels;
1024   PetscErrorCode ierr;
1025 
1026   PetscFunctionBegin;
1027   if (--mesh->refct > 0) PetscFunctionReturn(0);
1028   ierr = PetscSectionDestroy(&mesh->coneSection);CHKERRQ(ierr);
1029   ierr = PetscFree(mesh->cones);CHKERRQ(ierr);
1030   ierr = PetscFree(mesh->coneOrientations);CHKERRQ(ierr);
1031   ierr = PetscSectionDestroy(&mesh->supportSection);CHKERRQ(ierr);
1032   ierr = PetscFree(mesh->supports);CHKERRQ(ierr);
1033   ierr = PetscFree(mesh->facesTmp);CHKERRQ(ierr);
1034   while (next) {
1035     DMLabel tmp = next->next;
1036 
1037     ierr = DMLabelDestroy(&next);CHKERRQ(ierr);
1038     next = tmp;
1039   }
1040   ierr = DMDestroy(&mesh->coarseMesh);CHKERRQ(ierr);
1041   ierr = DMLabelDestroy(&mesh->subpointMap);CHKERRQ(ierr);
1042   ierr = ISDestroy(&mesh->globalVertexNumbers);CHKERRQ(ierr);
1043   ierr = ISDestroy(&mesh->globalCellNumbers);CHKERRQ(ierr);
1044   ierr = BoundaryDestroy(&mesh->boundary);CHKERRQ(ierr);
1045   /* This was originally freed in DMDestroy(), but that prevents reference counting of backend objects */
1046   ierr = PetscFree(mesh);CHKERRQ(ierr);
1047   PetscFunctionReturn(0);
1048 }
1049 
1050 #undef __FUNCT__
1051 #define __FUNCT__ "DMCreateMatrix_Plex"
1052 PetscErrorCode DMCreateMatrix_Plex(DM dm, Mat *J)
1053 {
1054   PetscSection   section, sectionGlobal;
1055   PetscInt       bs = -1;
1056   PetscInt       localSize;
1057   PetscBool      isShell, isBlock, isSeqBlock, isMPIBlock, isSymBlock, isSymSeqBlock, isSymMPIBlock;
1058   PetscErrorCode ierr;
1059   MatType        mtype;
1060 
1061   PetscFunctionBegin;
1062   ierr = MatInitializePackage();CHKERRQ(ierr);
1063   mtype = dm->mattype;
1064   ierr = DMGetDefaultSection(dm, &section);CHKERRQ(ierr);
1065   ierr = DMGetDefaultGlobalSection(dm, &sectionGlobal);CHKERRQ(ierr);
1066   /* ierr = PetscSectionGetStorageSize(sectionGlobal, &localSize);CHKERRQ(ierr); */
1067   ierr = PetscSectionGetConstrainedStorageSize(sectionGlobal, &localSize);CHKERRQ(ierr);
1068   ierr = MatCreate(PetscObjectComm((PetscObject)dm), J);CHKERRQ(ierr);
1069   ierr = MatSetSizes(*J, localSize, localSize, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
1070   ierr = MatSetType(*J, mtype);CHKERRQ(ierr);
1071   ierr = MatSetFromOptions(*J);CHKERRQ(ierr);
1072   ierr = PetscStrcmp(mtype, MATSHELL, &isShell);CHKERRQ(ierr);
1073   ierr = PetscStrcmp(mtype, MATBAIJ, &isBlock);CHKERRQ(ierr);
1074   ierr = PetscStrcmp(mtype, MATSEQBAIJ, &isSeqBlock);CHKERRQ(ierr);
1075   ierr = PetscStrcmp(mtype, MATMPIBAIJ, &isMPIBlock);CHKERRQ(ierr);
1076   ierr = PetscStrcmp(mtype, MATSBAIJ, &isSymBlock);CHKERRQ(ierr);
1077   ierr = PetscStrcmp(mtype, MATSEQSBAIJ, &isSymSeqBlock);CHKERRQ(ierr);
1078   ierr = PetscStrcmp(mtype, MATMPISBAIJ, &isSymMPIBlock);CHKERRQ(ierr);
1079   if (!isShell) {
1080     PetscBool fillMatrix = (PetscBool) !dm->prealloc_only;
1081     PetscInt *dnz, *onz, *dnzu, *onzu, bsLocal, bsMax, bsMin;
1082 
1083     if (bs < 0) {
1084       if (isBlock || isSeqBlock || isMPIBlock || isSymBlock || isSymSeqBlock || isSymMPIBlock) {
1085         PetscInt pStart, pEnd, p, dof, cdof;
1086 
1087         ierr = PetscSectionGetChart(sectionGlobal, &pStart, &pEnd);CHKERRQ(ierr);
1088         for (p = pStart; p < pEnd; ++p) {
1089           ierr = PetscSectionGetDof(sectionGlobal, p, &dof);CHKERRQ(ierr);
1090           ierr = PetscSectionGetConstraintDof(sectionGlobal, p, &cdof);CHKERRQ(ierr);
1091           if (dof-cdof) {
1092             if (bs < 0) {
1093               bs = dof-cdof;
1094             } else if (bs != dof-cdof) {
1095               /* Layout does not admit a pointwise block size */
1096               bs = 1;
1097               break;
1098             }
1099           }
1100         }
1101         /* Must have same blocksize on all procs (some might have no points) */
1102         bsLocal = bs;
1103         ierr = MPI_Allreduce(&bsLocal, &bsMax, 1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
1104         bsLocal = bs < 0 ? bsMax : bs;
1105         ierr = MPI_Allreduce(&bsLocal, &bsMin, 1, MPIU_INT, MPI_MIN, PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
1106         if (bsMin != bsMax) {
1107           bs = 1;
1108         } else {
1109           bs = bsMax;
1110         }
1111       } else {
1112         bs = 1;
1113       }
1114     }
1115     ierr = PetscCalloc4(localSize/bs, &dnz, localSize/bs, &onz, localSize/bs, &dnzu, localSize/bs, &onzu);CHKERRQ(ierr);
1116     ierr = DMPlexPreallocateOperator(dm, bs, section, sectionGlobal, dnz, onz, dnzu, onzu, *J, fillMatrix);CHKERRQ(ierr);
1117     ierr = PetscFree4(dnz, onz, dnzu, onzu);CHKERRQ(ierr);
1118   }
1119   PetscFunctionReturn(0);
1120 }
1121 
1122 #undef __FUNCT__
1123 #define __FUNCT__ "DMPlexGetDimension"
1124 /*@
1125   DMPlexGetDimension - Return the topological mesh dimension
1126 
1127   Not collective
1128 
1129   Input Parameter:
1130 . mesh - The DMPlex
1131 
1132   Output Parameter:
1133 . dim - The topological mesh dimension
1134 
1135   Level: beginner
1136 
1137 .seealso: DMPlexCreate()
1138 @*/
1139 PetscErrorCode DMPlexGetDimension(DM dm, PetscInt *dim)
1140 {
1141   DM_Plex *mesh = (DM_Plex*) dm->data;
1142 
1143   PetscFunctionBegin;
1144   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1145   PetscValidPointer(dim, 2);
1146   *dim = mesh->dim;
1147   PetscFunctionReturn(0);
1148 }
1149 
1150 #undef __FUNCT__
1151 #define __FUNCT__ "DMPlexSetDimension"
1152 /*@
1153   DMPlexSetDimension - Set the topological mesh dimension
1154 
1155   Collective on mesh
1156 
1157   Input Parameters:
1158 + mesh - The DMPlex
1159 - dim - The topological mesh dimension
1160 
1161   Level: beginner
1162 
1163 .seealso: DMPlexCreate()
1164 @*/
1165 PetscErrorCode DMPlexSetDimension(DM dm, PetscInt dim)
1166 {
1167   DM_Plex *mesh = (DM_Plex*) dm->data;
1168 
1169   PetscFunctionBegin;
1170   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1171   PetscValidLogicalCollectiveInt(dm, dim, 2);
1172   mesh->dim = dim;
1173   PetscFunctionReturn(0);
1174 }
1175 
1176 #undef __FUNCT__
1177 #define __FUNCT__ "DMPlexGetChart"
1178 /*@
1179   DMPlexGetChart - Return the interval for all mesh points [pStart, pEnd)
1180 
1181   Not collective
1182 
1183   Input Parameter:
1184 . mesh - The DMPlex
1185 
1186   Output Parameters:
1187 + pStart - The first mesh point
1188 - pEnd   - The upper bound for mesh points
1189 
1190   Level: beginner
1191 
1192 .seealso: DMPlexCreate(), DMPlexSetChart()
1193 @*/
1194 PetscErrorCode DMPlexGetChart(DM dm, PetscInt *pStart, PetscInt *pEnd)
1195 {
1196   DM_Plex       *mesh = (DM_Plex*) dm->data;
1197   PetscErrorCode ierr;
1198 
1199   PetscFunctionBegin;
1200   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1201   ierr = PetscSectionGetChart(mesh->coneSection, pStart, pEnd);CHKERRQ(ierr);
1202   PetscFunctionReturn(0);
1203 }
1204 
1205 #undef __FUNCT__
1206 #define __FUNCT__ "DMPlexSetChart"
1207 /*@
1208   DMPlexSetChart - Set the interval for all mesh points [pStart, pEnd)
1209 
1210   Not collective
1211 
1212   Input Parameters:
1213 + mesh - The DMPlex
1214 . pStart - The first mesh point
1215 - pEnd   - The upper bound for mesh points
1216 
1217   Output Parameters:
1218 
1219   Level: beginner
1220 
1221 .seealso: DMPlexCreate(), DMPlexGetChart()
1222 @*/
1223 PetscErrorCode DMPlexSetChart(DM dm, PetscInt pStart, PetscInt pEnd)
1224 {
1225   DM_Plex       *mesh = (DM_Plex*) dm->data;
1226   PetscErrorCode ierr;
1227 
1228   PetscFunctionBegin;
1229   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1230   ierr = PetscSectionSetChart(mesh->coneSection, pStart, pEnd);CHKERRQ(ierr);
1231   ierr = PetscSectionSetChart(mesh->supportSection, pStart, pEnd);CHKERRQ(ierr);
1232   PetscFunctionReturn(0);
1233 }
1234 
1235 #undef __FUNCT__
1236 #define __FUNCT__ "DMPlexGetConeSize"
1237 /*@
1238   DMPlexGetConeSize - Return the number of in-edges for this point in the Sieve DAG
1239 
1240   Not collective
1241 
1242   Input Parameters:
1243 + mesh - The DMPlex
1244 - p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1245 
1246   Output Parameter:
1247 . size - The cone size for point p
1248 
1249   Level: beginner
1250 
1251 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart()
1252 @*/
1253 PetscErrorCode DMPlexGetConeSize(DM dm, PetscInt p, PetscInt *size)
1254 {
1255   DM_Plex       *mesh = (DM_Plex*) dm->data;
1256   PetscErrorCode ierr;
1257 
1258   PetscFunctionBegin;
1259   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1260   PetscValidPointer(size, 3);
1261   ierr = PetscSectionGetDof(mesh->coneSection, p, size);CHKERRQ(ierr);
1262   PetscFunctionReturn(0);
1263 }
1264 
1265 #undef __FUNCT__
1266 #define __FUNCT__ "DMPlexSetConeSize"
1267 /*@
1268   DMPlexSetConeSize - Set the number of in-edges for this point in the Sieve DAG
1269 
1270   Not collective
1271 
1272   Input Parameters:
1273 + mesh - The DMPlex
1274 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1275 - size - The cone size for point p
1276 
1277   Output Parameter:
1278 
1279   Note:
1280   This should be called after DMPlexSetChart().
1281 
1282   Level: beginner
1283 
1284 .seealso: DMPlexCreate(), DMPlexGetConeSize(), DMPlexSetChart()
1285 @*/
1286 PetscErrorCode DMPlexSetConeSize(DM dm, PetscInt p, PetscInt size)
1287 {
1288   DM_Plex       *mesh = (DM_Plex*) dm->data;
1289   PetscErrorCode ierr;
1290 
1291   PetscFunctionBegin;
1292   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1293   ierr = PetscSectionSetDof(mesh->coneSection, p, size);CHKERRQ(ierr);
1294 
1295   mesh->maxConeSize = PetscMax(mesh->maxConeSize, size);
1296   PetscFunctionReturn(0);
1297 }
1298 
1299 #undef __FUNCT__
1300 #define __FUNCT__ "DMPlexGetCone"
1301 /*@C
1302   DMPlexGetCone - Return the points on the in-edges for this point in the Sieve DAG
1303 
1304   Not collective
1305 
1306   Input Parameters:
1307 + mesh - The DMPlex
1308 - p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1309 
1310   Output Parameter:
1311 . cone - An array of points which are on the in-edges for point p
1312 
1313   Level: beginner
1314 
1315   Fortran Notes:
1316   Since it returns an array, this routine is only available in Fortran 90, and you must
1317   include petsc.h90 in your code.
1318 
1319   You must also call DMPlexRestoreCone() after you finish using the returned array.
1320 
1321 .seealso: DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart()
1322 @*/
1323 PetscErrorCode DMPlexGetCone(DM dm, PetscInt p, const PetscInt *cone[])
1324 {
1325   DM_Plex       *mesh = (DM_Plex*) dm->data;
1326   PetscInt       off;
1327   PetscErrorCode ierr;
1328 
1329   PetscFunctionBegin;
1330   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1331   PetscValidPointer(cone, 3);
1332   ierr  = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1333   *cone = &mesh->cones[off];
1334   PetscFunctionReturn(0);
1335 }
1336 
1337 #undef __FUNCT__
1338 #define __FUNCT__ "DMPlexSetCone"
1339 /*@
1340   DMPlexSetCone - Set the points on the in-edges for this point in the Sieve DAG
1341 
1342   Not collective
1343 
1344   Input Parameters:
1345 + mesh - The DMPlex
1346 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1347 - cone - An array of points which are on the in-edges for point p
1348 
1349   Output Parameter:
1350 
1351   Note:
1352   This should be called after all calls to DMPlexSetConeSize() and DMSetUp().
1353 
1354   Level: beginner
1355 
1356 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
1357 @*/
1358 PetscErrorCode DMPlexSetCone(DM dm, PetscInt p, const PetscInt cone[])
1359 {
1360   DM_Plex       *mesh = (DM_Plex*) dm->data;
1361   PetscInt       pStart, pEnd;
1362   PetscInt       dof, off, c;
1363   PetscErrorCode ierr;
1364 
1365   PetscFunctionBegin;
1366   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1367   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
1368   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
1369   if (dof) PetscValidPointer(cone, 3);
1370   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1371   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
1372   for (c = 0; c < dof; ++c) {
1373     if ((cone[c] < pStart) || (cone[c] >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone point %D is not in the valid range [%D, %D)", cone[c], pStart, pEnd);
1374     mesh->cones[off+c] = cone[c];
1375   }
1376   PetscFunctionReturn(0);
1377 }
1378 
1379 #undef __FUNCT__
1380 #define __FUNCT__ "DMPlexGetConeOrientation"
1381 /*@C
1382   DMPlexGetConeOrientation - Return the orientations on the in-edges for this point in the Sieve DAG
1383 
1384   Not collective
1385 
1386   Input Parameters:
1387 + mesh - The DMPlex
1388 - p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1389 
1390   Output Parameter:
1391 . coneOrientation - An array of orientations which are on the in-edges for point p. An orientation is an
1392                     integer giving the prescription for cone traversal. If it is negative, the cone is
1393                     traversed in the opposite direction. Its value 'o', or if negative '-(o+1)', gives
1394                     the index of the cone point on which to start.
1395 
1396   Level: beginner
1397 
1398   Fortran Notes:
1399   Since it returns an array, this routine is only available in Fortran 90, and you must
1400   include petsc.h90 in your code.
1401 
1402   You must also call DMPlexRestoreConeOrientation() after you finish using the returned array.
1403 
1404 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetCone(), DMPlexSetChart()
1405 @*/
1406 PetscErrorCode DMPlexGetConeOrientation(DM dm, PetscInt p, const PetscInt *coneOrientation[])
1407 {
1408   DM_Plex       *mesh = (DM_Plex*) dm->data;
1409   PetscInt       off;
1410   PetscErrorCode ierr;
1411 
1412   PetscFunctionBegin;
1413   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1414 #if defined(PETSC_USE_DEBUG)
1415   {
1416     PetscInt dof;
1417     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
1418     if (dof) PetscValidPointer(coneOrientation, 3);
1419   }
1420 #endif
1421   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1422 
1423   *coneOrientation = &mesh->coneOrientations[off];
1424   PetscFunctionReturn(0);
1425 }
1426 
1427 #undef __FUNCT__
1428 #define __FUNCT__ "DMPlexSetConeOrientation"
1429 /*@
1430   DMPlexSetConeOrientation - Set the orientations on the in-edges for this point in the Sieve DAG
1431 
1432   Not collective
1433 
1434   Input Parameters:
1435 + mesh - The DMPlex
1436 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1437 - coneOrientation - An array of orientations which are on the in-edges for point p. An orientation is an
1438                     integer giving the prescription for cone traversal. If it is negative, the cone is
1439                     traversed in the opposite direction. Its value 'o', or if negative '-(o+1)', gives
1440                     the index of the cone point on which to start.
1441 
1442   Output Parameter:
1443 
1444   Note:
1445   This should be called after all calls to DMPlexSetConeSize() and DMSetUp().
1446 
1447   Level: beginner
1448 
1449 .seealso: DMPlexCreate(), DMPlexGetConeOrientation(), DMPlexSetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
1450 @*/
1451 PetscErrorCode DMPlexSetConeOrientation(DM dm, PetscInt p, const PetscInt coneOrientation[])
1452 {
1453   DM_Plex       *mesh = (DM_Plex*) dm->data;
1454   PetscInt       pStart, pEnd;
1455   PetscInt       dof, off, c;
1456   PetscErrorCode ierr;
1457 
1458   PetscFunctionBegin;
1459   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1460   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
1461   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
1462   if (dof) PetscValidPointer(coneOrientation, 3);
1463   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1464   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
1465   for (c = 0; c < dof; ++c) {
1466     PetscInt cdof, o = coneOrientation[c];
1467 
1468     ierr = PetscSectionGetDof(mesh->coneSection, mesh->cones[off+c], &cdof);CHKERRQ(ierr);
1469     if (o && ((o < -(cdof+1)) || (o >= cdof))) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone orientation %D is not in the valid range [%D. %D)", o, -(cdof+1), cdof);
1470     mesh->coneOrientations[off+c] = o;
1471   }
1472   PetscFunctionReturn(0);
1473 }
1474 
1475 #undef __FUNCT__
1476 #define __FUNCT__ "DMPlexInsertCone"
1477 PetscErrorCode DMPlexInsertCone(DM dm, PetscInt p, PetscInt conePos, PetscInt conePoint)
1478 {
1479   DM_Plex       *mesh = (DM_Plex*) dm->data;
1480   PetscInt       pStart, pEnd;
1481   PetscInt       dof, off;
1482   PetscErrorCode ierr;
1483 
1484   PetscFunctionBegin;
1485   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1486   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
1487   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
1488   if ((conePoint < pStart) || (conePoint >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone point %D is not in the valid range [%D, %D)", conePoint, pStart, pEnd);
1489   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
1490   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1491   if ((conePos < 0) || (conePos >= dof)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone position %D of point %D is not in the valid range [0, %D)", conePos, p, dof);
1492   mesh->cones[off+conePos] = conePoint;
1493   PetscFunctionReturn(0);
1494 }
1495 
1496 #undef __FUNCT__
1497 #define __FUNCT__ "DMPlexInsertConeOrientation"
1498 PetscErrorCode DMPlexInsertConeOrientation(DM dm, PetscInt p, PetscInt conePos, PetscInt coneOrientation)
1499 {
1500   DM_Plex       *mesh = (DM_Plex*) dm->data;
1501   PetscInt       pStart, pEnd;
1502   PetscInt       dof, off;
1503   PetscErrorCode ierr;
1504 
1505   PetscFunctionBegin;
1506   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1507   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
1508   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
1509   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
1510   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1511   if ((conePos < 0) || (conePos >= dof)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone position %D of point %D is not in the valid range [0, %D)", conePos, p, dof);
1512   mesh->coneOrientations[off+conePos] = coneOrientation;
1513   PetscFunctionReturn(0);
1514 }
1515 
1516 #undef __FUNCT__
1517 #define __FUNCT__ "DMPlexGetSupportSize"
1518 /*@
1519   DMPlexGetSupportSize - Return the number of out-edges for this point in the Sieve DAG
1520 
1521   Not collective
1522 
1523   Input Parameters:
1524 + mesh - The DMPlex
1525 - p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1526 
1527   Output Parameter:
1528 . size - The support size for point p
1529 
1530   Level: beginner
1531 
1532 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart(), DMPlexGetConeSize()
1533 @*/
1534 PetscErrorCode DMPlexGetSupportSize(DM dm, PetscInt p, PetscInt *size)
1535 {
1536   DM_Plex       *mesh = (DM_Plex*) dm->data;
1537   PetscErrorCode ierr;
1538 
1539   PetscFunctionBegin;
1540   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1541   PetscValidPointer(size, 3);
1542   ierr = PetscSectionGetDof(mesh->supportSection, p, size);CHKERRQ(ierr);
1543   PetscFunctionReturn(0);
1544 }
1545 
1546 #undef __FUNCT__
1547 #define __FUNCT__ "DMPlexSetSupportSize"
1548 /*@
1549   DMPlexSetSupportSize - Set the number of out-edges for this point in the Sieve DAG
1550 
1551   Not collective
1552 
1553   Input Parameters:
1554 + mesh - The DMPlex
1555 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1556 - size - The support size for point p
1557 
1558   Output Parameter:
1559 
1560   Note:
1561   This should be called after DMPlexSetChart().
1562 
1563   Level: beginner
1564 
1565 .seealso: DMPlexCreate(), DMPlexGetSupportSize(), DMPlexSetChart()
1566 @*/
1567 PetscErrorCode DMPlexSetSupportSize(DM dm, PetscInt p, PetscInt size)
1568 {
1569   DM_Plex       *mesh = (DM_Plex*) dm->data;
1570   PetscErrorCode ierr;
1571 
1572   PetscFunctionBegin;
1573   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1574   ierr = PetscSectionSetDof(mesh->supportSection, p, size);CHKERRQ(ierr);
1575 
1576   mesh->maxSupportSize = PetscMax(mesh->maxSupportSize, size);
1577   PetscFunctionReturn(0);
1578 }
1579 
1580 #undef __FUNCT__
1581 #define __FUNCT__ "DMPlexGetSupport"
1582 /*@C
1583   DMPlexGetSupport - Return the points on the out-edges for this point in the Sieve DAG
1584 
1585   Not collective
1586 
1587   Input Parameters:
1588 + mesh - The DMPlex
1589 - p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1590 
1591   Output Parameter:
1592 . support - An array of points which are on the out-edges for point p
1593 
1594   Level: beginner
1595 
1596   Fortran Notes:
1597   Since it returns an array, this routine is only available in Fortran 90, and you must
1598   include petsc.h90 in your code.
1599 
1600   You must also call DMPlexRestoreSupport() after you finish using the returned array.
1601 
1602 .seealso: DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
1603 @*/
1604 PetscErrorCode DMPlexGetSupport(DM dm, PetscInt p, const PetscInt *support[])
1605 {
1606   DM_Plex       *mesh = (DM_Plex*) dm->data;
1607   PetscInt       off;
1608   PetscErrorCode ierr;
1609 
1610   PetscFunctionBegin;
1611   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1612   PetscValidPointer(support, 3);
1613   ierr     = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
1614   *support = &mesh->supports[off];
1615   PetscFunctionReturn(0);
1616 }
1617 
1618 #undef __FUNCT__
1619 #define __FUNCT__ "DMPlexSetSupport"
1620 /*@
1621   DMPlexSetSupport - Set the points on the out-edges for this point in the Sieve DAG
1622 
1623   Not collective
1624 
1625   Input Parameters:
1626 + mesh - The DMPlex
1627 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1628 - support - An array of points which are on the in-edges for point p
1629 
1630   Output Parameter:
1631 
1632   Note:
1633   This should be called after all calls to DMPlexSetSupportSize() and DMSetUp().
1634 
1635   Level: beginner
1636 
1637 .seealso: DMPlexCreate(), DMPlexGetSupport(), DMPlexSetChart(), DMPlexSetSupportSize(), DMSetUp()
1638 @*/
1639 PetscErrorCode DMPlexSetSupport(DM dm, PetscInt p, const PetscInt support[])
1640 {
1641   DM_Plex       *mesh = (DM_Plex*) dm->data;
1642   PetscInt       pStart, pEnd;
1643   PetscInt       dof, off, c;
1644   PetscErrorCode ierr;
1645 
1646   PetscFunctionBegin;
1647   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1648   ierr = PetscSectionGetChart(mesh->supportSection, &pStart, &pEnd);CHKERRQ(ierr);
1649   ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
1650   if (dof) PetscValidPointer(support, 3);
1651   ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
1652   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
1653   for (c = 0; c < dof; ++c) {
1654     if ((support[c] < pStart) || (support[c] >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support point %D is not in the valid range [%D, %D)", support[c], pStart, pEnd);
1655     mesh->supports[off+c] = support[c];
1656   }
1657   PetscFunctionReturn(0);
1658 }
1659 
1660 #undef __FUNCT__
1661 #define __FUNCT__ "DMPlexInsertSupport"
1662 PetscErrorCode DMPlexInsertSupport(DM dm, PetscInt p, PetscInt supportPos, PetscInt supportPoint)
1663 {
1664   DM_Plex       *mesh = (DM_Plex*) dm->data;
1665   PetscInt       pStart, pEnd;
1666   PetscInt       dof, off;
1667   PetscErrorCode ierr;
1668 
1669   PetscFunctionBegin;
1670   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1671   ierr = PetscSectionGetChart(mesh->supportSection, &pStart, &pEnd);CHKERRQ(ierr);
1672   ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
1673   ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
1674   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
1675   if ((supportPoint < pStart) || (supportPoint >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support point %D is not in the valid range [%D, %D)", supportPoint, pStart, pEnd);
1676   if (supportPos >= dof) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support position %D of point %D is not in the valid range [0, %D)", supportPos, p, dof);
1677   mesh->supports[off+supportPos] = supportPoint;
1678   PetscFunctionReturn(0);
1679 }
1680 
1681 #undef __FUNCT__
1682 #define __FUNCT__ "DMPlexGetTransitiveClosure"
1683 /*@C
1684   DMPlexGetTransitiveClosure - Return the points on the transitive closure of the in-edges or out-edges for this point in the Sieve DAG
1685 
1686   Not collective
1687 
1688   Input Parameters:
1689 + mesh - The DMPlex
1690 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1691 . useCone - PETSC_TRUE for in-edges,  otherwise use out-edges
1692 - points - If points is NULL on input, internal storage will be returned, otherwise the provided array is used
1693 
1694   Output Parameters:
1695 + numPoints - The number of points in the closure, so points[] is of size 2*numPoints
1696 - points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...]
1697 
1698   Note:
1699   If using internal storage (points is NULL on input), each call overwrites the last output.
1700 
1701   Fortran Notes:
1702   Since it returns an array, this routine is only available in Fortran 90, and you must
1703   include petsc.h90 in your code.
1704 
1705   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
1706 
1707   Level: beginner
1708 
1709 .seealso: DMPlexRestoreTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
1710 @*/
1711 PetscErrorCode DMPlexGetTransitiveClosure(DM dm, PetscInt p, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
1712 {
1713   DM_Plex        *mesh = (DM_Plex*) dm->data;
1714   PetscInt       *closure, *fifo;
1715   const PetscInt *tmp = NULL, *tmpO = NULL;
1716   PetscInt        tmpSize, t;
1717   PetscInt        depth       = 0, maxSize;
1718   PetscInt        closureSize = 2, fifoSize = 0, fifoStart = 0;
1719   PetscErrorCode  ierr;
1720 
1721   PetscFunctionBegin;
1722   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1723   ierr    = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
1724   /* This is only 1-level */
1725   if (useCone) {
1726     ierr = DMPlexGetConeSize(dm, p, &tmpSize);CHKERRQ(ierr);
1727     ierr = DMPlexGetCone(dm, p, &tmp);CHKERRQ(ierr);
1728     ierr = DMPlexGetConeOrientation(dm, p, &tmpO);CHKERRQ(ierr);
1729   } else {
1730     ierr = DMPlexGetSupportSize(dm, p, &tmpSize);CHKERRQ(ierr);
1731     ierr = DMPlexGetSupport(dm, p, &tmp);CHKERRQ(ierr);
1732   }
1733   if (depth == 1) {
1734     if (*points) {
1735       closure = *points;
1736     } else {
1737       maxSize = 2*(PetscMax(mesh->maxConeSize, mesh->maxSupportSize)+1);
1738       ierr = DMGetWorkArray(dm, maxSize, PETSC_INT, &closure);CHKERRQ(ierr);
1739     }
1740     closure[0] = p; closure[1] = 0;
1741     for (t = 0; t < tmpSize; ++t, closureSize += 2) {
1742       closure[closureSize]   = tmp[t];
1743       closure[closureSize+1] = tmpO ? tmpO[t] : 0;
1744     }
1745     if (numPoints) *numPoints = closureSize/2;
1746     if (points)    *points    = closure;
1747     PetscFunctionReturn(0);
1748   }
1749   maxSize = 2*PetscMax(PetscMax(PetscPowInt(mesh->maxConeSize,depth+1),PetscPowInt(mesh->maxSupportSize,depth+1)),depth+1);
1750   ierr    = DMGetWorkArray(dm, maxSize, PETSC_INT, &fifo);CHKERRQ(ierr);
1751   if (*points) {
1752     closure = *points;
1753   } else {
1754     ierr = DMGetWorkArray(dm, maxSize, PETSC_INT, &closure);CHKERRQ(ierr);
1755   }
1756   closure[0] = p; closure[1] = 0;
1757   for (t = 0; t < tmpSize; ++t, closureSize += 2, fifoSize += 2) {
1758     const PetscInt cp = tmp[t];
1759     const PetscInt co = tmpO ? tmpO[t] : 0;
1760 
1761     closure[closureSize]   = cp;
1762     closure[closureSize+1] = co;
1763     fifo[fifoSize]         = cp;
1764     fifo[fifoSize+1]       = co;
1765   }
1766   /* Should kick out early when depth is reached, rather than checking all vertices for empty cones */
1767   while (fifoSize - fifoStart) {
1768     const PetscInt q   = fifo[fifoStart];
1769     const PetscInt o   = fifo[fifoStart+1];
1770     const PetscInt rev = o >= 0 ? 0 : 1;
1771     const PetscInt off = rev ? -(o+1) : o;
1772 
1773     if (useCone) {
1774       ierr = DMPlexGetConeSize(dm, q, &tmpSize);CHKERRQ(ierr);
1775       ierr = DMPlexGetCone(dm, q, &tmp);CHKERRQ(ierr);
1776       ierr = DMPlexGetConeOrientation(dm, q, &tmpO);CHKERRQ(ierr);
1777     } else {
1778       ierr = DMPlexGetSupportSize(dm, q, &tmpSize);CHKERRQ(ierr);
1779       ierr = DMPlexGetSupport(dm, q, &tmp);CHKERRQ(ierr);
1780       tmpO = NULL;
1781     }
1782     for (t = 0; t < tmpSize; ++t) {
1783       const PetscInt i  = ((rev ? tmpSize-t : t) + off)%tmpSize;
1784       const PetscInt cp = tmp[i];
1785       /* Must propogate orientation: When we reverse orientation, we both reverse the direction of iteration and start at the other end of the chain. */
1786       /* HACK: It is worse to get the size here, than to change the interpretation of -(*+1)
1787        const PetscInt co = tmpO ? (rev ? -(tmpO[i]+1) : tmpO[i]) : 0; */
1788       PetscInt       co = tmpO ? tmpO[i] : 0;
1789       PetscInt       c;
1790 
1791       if (rev) {
1792         PetscInt childSize, coff;
1793         ierr = DMPlexGetConeSize(dm, cp, &childSize);CHKERRQ(ierr);
1794         coff = tmpO[i] < 0 ? -(tmpO[i]+1) : tmpO[i];
1795         co   = childSize ? -(((coff+childSize-1)%childSize)+1) : 0;
1796       }
1797       /* Check for duplicate */
1798       for (c = 0; c < closureSize; c += 2) {
1799         if (closure[c] == cp) break;
1800       }
1801       if (c == closureSize) {
1802         closure[closureSize]   = cp;
1803         closure[closureSize+1] = co;
1804         fifo[fifoSize]         = cp;
1805         fifo[fifoSize+1]       = co;
1806         closureSize           += 2;
1807         fifoSize              += 2;
1808       }
1809     }
1810     fifoStart += 2;
1811   }
1812   if (numPoints) *numPoints = closureSize/2;
1813   if (points)    *points    = closure;
1814   ierr = DMRestoreWorkArray(dm, maxSize, PETSC_INT, &fifo);CHKERRQ(ierr);
1815   PetscFunctionReturn(0);
1816 }
1817 
1818 #undef __FUNCT__
1819 #define __FUNCT__ "DMPlexGetTransitiveClosure_Internal"
1820 /*@C
1821   DMPlexGetTransitiveClosure_Internal - Return the points on the transitive closure of the in-edges or out-edges for this point in the Sieve DAG with a specified initial orientation
1822 
1823   Not collective
1824 
1825   Input Parameters:
1826 + mesh - The DMPlex
1827 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1828 . orientation - The orientation of the point
1829 . useCone - PETSC_TRUE for in-edges,  otherwise use out-edges
1830 - points - If points is NULL on input, internal storage will be returned, otherwise the provided array is used
1831 
1832   Output Parameters:
1833 + numPoints - The number of points in the closure, so points[] is of size 2*numPoints
1834 - points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...]
1835 
1836   Note:
1837   If using internal storage (points is NULL on input), each call overwrites the last output.
1838 
1839   Fortran Notes:
1840   Since it returns an array, this routine is only available in Fortran 90, and you must
1841   include petsc.h90 in your code.
1842 
1843   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
1844 
1845   Level: beginner
1846 
1847 .seealso: DMPlexRestoreTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
1848 @*/
1849 PetscErrorCode DMPlexGetTransitiveClosure_Internal(DM dm, PetscInt p, PetscInt ornt, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
1850 {
1851   DM_Plex        *mesh = (DM_Plex*) dm->data;
1852   PetscInt       *closure, *fifo;
1853   const PetscInt *tmp = NULL, *tmpO = NULL;
1854   PetscInt        tmpSize, t;
1855   PetscInt        depth       = 0, maxSize;
1856   PetscInt        closureSize = 2, fifoSize = 0, fifoStart = 0;
1857   PetscErrorCode  ierr;
1858 
1859   PetscFunctionBegin;
1860   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1861   ierr    = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
1862   /* This is only 1-level */
1863   if (useCone) {
1864     ierr = DMPlexGetConeSize(dm, p, &tmpSize);CHKERRQ(ierr);
1865     ierr = DMPlexGetCone(dm, p, &tmp);CHKERRQ(ierr);
1866     ierr = DMPlexGetConeOrientation(dm, p, &tmpO);CHKERRQ(ierr);
1867   } else {
1868     ierr = DMPlexGetSupportSize(dm, p, &tmpSize);CHKERRQ(ierr);
1869     ierr = DMPlexGetSupport(dm, p, &tmp);CHKERRQ(ierr);
1870   }
1871   if (depth == 1) {
1872     if (*points) {
1873       closure = *points;
1874     } else {
1875       maxSize = 2*(PetscMax(mesh->maxConeSize, mesh->maxSupportSize)+1);
1876       ierr = DMGetWorkArray(dm, maxSize, PETSC_INT, &closure);CHKERRQ(ierr);
1877     }
1878     closure[0] = p; closure[1] = ornt;
1879     for (t = 0; t < tmpSize; ++t, closureSize += 2) {
1880       const PetscInt i = ornt >= 0 ? (t+ornt)%tmpSize : (-(ornt+1) + tmpSize-t)%tmpSize;
1881       closure[closureSize]   = tmp[i];
1882       closure[closureSize+1] = tmpO ? tmpO[i] : 0;
1883     }
1884     if (numPoints) *numPoints = closureSize/2;
1885     if (points)    *points    = closure;
1886     PetscFunctionReturn(0);
1887   }
1888   maxSize = 2*PetscMax(PetscMax(PetscPowInt(mesh->maxConeSize,depth+1),PetscPowInt(mesh->maxSupportSize,depth+1)),depth+1);
1889   ierr    = DMGetWorkArray(dm, maxSize, PETSC_INT, &fifo);CHKERRQ(ierr);
1890   if (*points) {
1891     closure = *points;
1892   } else {
1893     ierr = DMGetWorkArray(dm, maxSize, PETSC_INT, &closure);CHKERRQ(ierr);
1894   }
1895   closure[0] = p; closure[1] = ornt;
1896   for (t = 0; t < tmpSize; ++t, closureSize += 2, fifoSize += 2) {
1897     const PetscInt i  = ornt >= 0 ? (t+ornt)%tmpSize : (-(ornt+1) + tmpSize-t)%tmpSize;
1898     const PetscInt cp = tmp[i];
1899     PetscInt       co = tmpO ? tmpO[i] : 0;
1900 
1901     if (ornt < 0) {
1902       PetscInt childSize, coff;
1903       ierr = DMPlexGetConeSize(dm, cp, &childSize);CHKERRQ(ierr);
1904       coff = tmpO[i] < 0 ? -(tmpO[i]+1) : tmpO[i];
1905       co   = childSize ? -(((coff+childSize-1)%childSize)+1) : 0;
1906     }
1907     closure[closureSize]   = cp;
1908     closure[closureSize+1] = co;
1909     fifo[fifoSize]         = cp;
1910     fifo[fifoSize+1]       = co;
1911   }
1912   /* Should kick out early when depth is reached, rather than checking all vertices for empty cones */
1913   while (fifoSize - fifoStart) {
1914     const PetscInt q   = fifo[fifoStart];
1915     const PetscInt o   = fifo[fifoStart+1];
1916     const PetscInt rev = o >= 0 ? 0 : 1;
1917     const PetscInt off = rev ? -(o+1) : o;
1918 
1919     if (useCone) {
1920       ierr = DMPlexGetConeSize(dm, q, &tmpSize);CHKERRQ(ierr);
1921       ierr = DMPlexGetCone(dm, q, &tmp);CHKERRQ(ierr);
1922       ierr = DMPlexGetConeOrientation(dm, q, &tmpO);CHKERRQ(ierr);
1923     } else {
1924       ierr = DMPlexGetSupportSize(dm, q, &tmpSize);CHKERRQ(ierr);
1925       ierr = DMPlexGetSupport(dm, q, &tmp);CHKERRQ(ierr);
1926       tmpO = NULL;
1927     }
1928     for (t = 0; t < tmpSize; ++t) {
1929       const PetscInt i  = ((rev ? tmpSize-t : t) + off)%tmpSize;
1930       const PetscInt cp = tmp[i];
1931       /* Must propogate orientation: When we reverse orientation, we both reverse the direction of iteration and start at the other end of the chain. */
1932       /* HACK: It is worse to get the size here, than to change the interpretation of -(*+1)
1933        const PetscInt co = tmpO ? (rev ? -(tmpO[i]+1) : tmpO[i]) : 0; */
1934       PetscInt       co = tmpO ? tmpO[i] : 0;
1935       PetscInt       c;
1936 
1937       if (rev) {
1938         PetscInt childSize, coff;
1939         ierr = DMPlexGetConeSize(dm, cp, &childSize);CHKERRQ(ierr);
1940         coff = tmpO[i] < 0 ? -(tmpO[i]+1) : tmpO[i];
1941         co   = childSize ? -(((coff+childSize-1)%childSize)+1) : 0;
1942       }
1943       /* Check for duplicate */
1944       for (c = 0; c < closureSize; c += 2) {
1945         if (closure[c] == cp) break;
1946       }
1947       if (c == closureSize) {
1948         closure[closureSize]   = cp;
1949         closure[closureSize+1] = co;
1950         fifo[fifoSize]         = cp;
1951         fifo[fifoSize+1]       = co;
1952         closureSize           += 2;
1953         fifoSize              += 2;
1954       }
1955     }
1956     fifoStart += 2;
1957   }
1958   if (numPoints) *numPoints = closureSize/2;
1959   if (points)    *points    = closure;
1960   ierr = DMRestoreWorkArray(dm, maxSize, PETSC_INT, &fifo);CHKERRQ(ierr);
1961   PetscFunctionReturn(0);
1962 }
1963 
1964 #undef __FUNCT__
1965 #define __FUNCT__ "DMPlexRestoreTransitiveClosure"
1966 /*@C
1967   DMPlexRestoreTransitiveClosure - Restore the array of points on the transitive closure of the in-edges or out-edges for this point in the Sieve DAG
1968 
1969   Not collective
1970 
1971   Input Parameters:
1972 + mesh - The DMPlex
1973 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1974 . useCone - PETSC_TRUE for in-edges,  otherwise use out-edges
1975 . numPoints - The number of points in the closure, so points[] is of size 2*numPoints, zeroed on exit
1976 - points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...], zeroed on exit
1977 
1978   Note:
1979   If not using internal storage (points is not NULL on input), this call is unnecessary
1980 
1981   Fortran Notes:
1982   Since it returns an array, this routine is only available in Fortran 90, and you must
1983   include petsc.h90 in your code.
1984 
1985   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
1986 
1987   Level: beginner
1988 
1989 .seealso: DMPlexGetTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
1990 @*/
1991 PetscErrorCode DMPlexRestoreTransitiveClosure(DM dm, PetscInt p, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
1992 {
1993   PetscErrorCode ierr;
1994 
1995   PetscFunctionBegin;
1996   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1997   if (numPoints) PetscValidIntPointer(numPoints,4);
1998   if (points) PetscValidPointer(points,5);
1999   ierr = DMRestoreWorkArray(dm, 0, PETSC_INT, points);CHKERRQ(ierr);
2000   if (numPoints) *numPoints = 0;
2001   PetscFunctionReturn(0);
2002 }
2003 
2004 #undef __FUNCT__
2005 #define __FUNCT__ "DMPlexGetMaxSizes"
2006 /*@
2007   DMPlexGetMaxSizes - Return the maximum number of in-edges (cone) and out-edges (support) for any point in the Sieve DAG
2008 
2009   Not collective
2010 
2011   Input Parameter:
2012 . mesh - The DMPlex
2013 
2014   Output Parameters:
2015 + maxConeSize - The maximum number of in-edges
2016 - maxSupportSize - The maximum number of out-edges
2017 
2018   Level: beginner
2019 
2020 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart()
2021 @*/
2022 PetscErrorCode DMPlexGetMaxSizes(DM dm, PetscInt *maxConeSize, PetscInt *maxSupportSize)
2023 {
2024   DM_Plex *mesh = (DM_Plex*) dm->data;
2025 
2026   PetscFunctionBegin;
2027   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2028   if (maxConeSize)    *maxConeSize    = mesh->maxConeSize;
2029   if (maxSupportSize) *maxSupportSize = mesh->maxSupportSize;
2030   PetscFunctionReturn(0);
2031 }
2032 
2033 #undef __FUNCT__
2034 #define __FUNCT__ "DMSetUp_Plex"
2035 PetscErrorCode DMSetUp_Plex(DM dm)
2036 {
2037   DM_Plex       *mesh = (DM_Plex*) dm->data;
2038   PetscInt       size;
2039   PetscErrorCode ierr;
2040 
2041   PetscFunctionBegin;
2042   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2043   ierr = PetscSectionSetUp(mesh->coneSection);CHKERRQ(ierr);
2044   ierr = PetscSectionGetStorageSize(mesh->coneSection, &size);CHKERRQ(ierr);
2045   ierr = PetscMalloc1(size, &mesh->cones);CHKERRQ(ierr);
2046   ierr = PetscCalloc1(size, &mesh->coneOrientations);CHKERRQ(ierr);
2047   if (mesh->maxSupportSize) {
2048     ierr = PetscSectionSetUp(mesh->supportSection);CHKERRQ(ierr);
2049     ierr = PetscSectionGetStorageSize(mesh->supportSection, &size);CHKERRQ(ierr);
2050     ierr = PetscMalloc1(size, &mesh->supports);CHKERRQ(ierr);
2051   }
2052   PetscFunctionReturn(0);
2053 }
2054 
2055 #undef __FUNCT__
2056 #define __FUNCT__ "DMCreateSubDM_Plex"
2057 PetscErrorCode DMCreateSubDM_Plex(DM dm, PetscInt numFields, PetscInt fields[], IS *is, DM *subdm)
2058 {
2059   PetscErrorCode ierr;
2060 
2061   PetscFunctionBegin;
2062   if (subdm) {ierr = DMClone(dm, subdm);CHKERRQ(ierr);}
2063   ierr = DMCreateSubDM_Section_Private(dm, numFields, fields, is, subdm);CHKERRQ(ierr);
2064   PetscFunctionReturn(0);
2065 }
2066 
2067 #undef __FUNCT__
2068 #define __FUNCT__ "DMPlexSymmetrize"
2069 /*@
2070   DMPlexSymmetrize - Creates support (out-edge) information from cone (in-edge) inoformation
2071 
2072   Not collective
2073 
2074   Input Parameter:
2075 . mesh - The DMPlex
2076 
2077   Output Parameter:
2078 
2079   Note:
2080   This should be called after all calls to DMPlexSetCone()
2081 
2082   Level: beginner
2083 
2084 .seealso: DMPlexCreate(), DMPlexSetChart(), DMPlexSetConeSize(), DMPlexSetCone()
2085 @*/
2086 PetscErrorCode DMPlexSymmetrize(DM dm)
2087 {
2088   DM_Plex       *mesh = (DM_Plex*) dm->data;
2089   PetscInt      *offsets;
2090   PetscInt       supportSize;
2091   PetscInt       pStart, pEnd, p;
2092   PetscErrorCode ierr;
2093 
2094   PetscFunctionBegin;
2095   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2096   if (mesh->supports) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONGSTATE, "Supports were already setup in this DMPlex");
2097   /* Calculate support sizes */
2098   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
2099   for (p = pStart; p < pEnd; ++p) {
2100     PetscInt dof, off, c;
2101 
2102     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2103     ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2104     for (c = off; c < off+dof; ++c) {
2105       ierr = PetscSectionAddDof(mesh->supportSection, mesh->cones[c], 1);CHKERRQ(ierr);
2106     }
2107   }
2108   for (p = pStart; p < pEnd; ++p) {
2109     PetscInt dof;
2110 
2111     ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
2112 
2113     mesh->maxSupportSize = PetscMax(mesh->maxSupportSize, dof);
2114   }
2115   ierr = PetscSectionSetUp(mesh->supportSection);CHKERRQ(ierr);
2116   /* Calculate supports */
2117   ierr = PetscSectionGetStorageSize(mesh->supportSection, &supportSize);CHKERRQ(ierr);
2118   ierr = PetscMalloc1(supportSize, &mesh->supports);CHKERRQ(ierr);
2119   ierr = PetscCalloc1(pEnd - pStart, &offsets);CHKERRQ(ierr);
2120   for (p = pStart; p < pEnd; ++p) {
2121     PetscInt dof, off, c;
2122 
2123     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2124     ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2125     for (c = off; c < off+dof; ++c) {
2126       const PetscInt q = mesh->cones[c];
2127       PetscInt       offS;
2128 
2129       ierr = PetscSectionGetOffset(mesh->supportSection, q, &offS);CHKERRQ(ierr);
2130 
2131       mesh->supports[offS+offsets[q]] = p;
2132       ++offsets[q];
2133     }
2134   }
2135   ierr = PetscFree(offsets);CHKERRQ(ierr);
2136   PetscFunctionReturn(0);
2137 }
2138 
2139 #undef __FUNCT__
2140 #define __FUNCT__ "DMPlexStratify"
2141 /*@
2142   DMPlexStratify - The Sieve DAG for most topologies is a graded poset (http://en.wikipedia.org/wiki/Graded_poset), and
2143   can be illustrated by Hasse Diagram (a http://en.wikipedia.org/wiki/Hasse_diagram). The strata group all points of the
2144   same grade, and this function calculates the strata. This grade can be seen as the height (or depth) of the point in
2145   the DAG.
2146 
2147   Not collective
2148 
2149   Input Parameter:
2150 . mesh - The DMPlex
2151 
2152   Output Parameter:
2153 
2154   Notes:
2155   The normal association for the point grade is element dimension (or co-dimension). For instance, all vertices would
2156   have depth 0, and all edges depth 1. Likewise, all cells heights would have height 0, and all faces height 1.
2157 
2158   This should be called after all calls to DMPlexSymmetrize()
2159 
2160   Level: beginner
2161 
2162 .seealso: DMPlexCreate(), DMPlexSymmetrize()
2163 @*/
2164 PetscErrorCode DMPlexStratify(DM dm)
2165 {
2166   DMLabel        label;
2167   PetscInt       pStart, pEnd, p;
2168   PetscInt       numRoots = 0, numLeaves = 0;
2169   PetscErrorCode ierr;
2170 
2171   PetscFunctionBegin;
2172   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2173   ierr = PetscLogEventBegin(DMPLEX_Stratify,dm,0,0,0);CHKERRQ(ierr);
2174   /* Calculate depth */
2175   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
2176   ierr = DMPlexCreateLabel(dm, "depth");CHKERRQ(ierr);
2177   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
2178   /* Initialize roots and count leaves */
2179   for (p = pStart; p < pEnd; ++p) {
2180     PetscInt coneSize, supportSize;
2181 
2182     ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
2183     ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
2184     if (!coneSize && supportSize) {
2185       ++numRoots;
2186       ierr = DMLabelSetValue(label, p, 0);CHKERRQ(ierr);
2187     } else if (!supportSize && coneSize) {
2188       ++numLeaves;
2189     } else if (!supportSize && !coneSize) {
2190       /* Isolated points */
2191       ierr = DMLabelSetValue(label, p, 0);CHKERRQ(ierr);
2192     }
2193   }
2194   if (numRoots + numLeaves == (pEnd - pStart)) {
2195     for (p = pStart; p < pEnd; ++p) {
2196       PetscInt coneSize, supportSize;
2197 
2198       ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
2199       ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
2200       if (!supportSize && coneSize) {
2201         ierr = DMLabelSetValue(label, p, 1);CHKERRQ(ierr);
2202       }
2203     }
2204   } else {
2205     IS       pointIS;
2206     PetscInt numPoints = 0, level = 0;
2207 
2208     ierr = DMLabelGetStratumIS(label, level, &pointIS);CHKERRQ(ierr);
2209     if (pointIS) {ierr = ISGetLocalSize(pointIS, &numPoints);CHKERRQ(ierr);}
2210     while (numPoints) {
2211       const PetscInt *points;
2212       const PetscInt  newLevel = level+1;
2213 
2214       ierr = ISGetIndices(pointIS, &points);CHKERRQ(ierr);
2215       for (p = 0; p < numPoints; ++p) {
2216         const PetscInt  point = points[p];
2217         const PetscInt *support;
2218         PetscInt        supportSize, s;
2219 
2220         ierr = DMPlexGetSupportSize(dm, point, &supportSize);CHKERRQ(ierr);
2221         ierr = DMPlexGetSupport(dm, point, &support);CHKERRQ(ierr);
2222         for (s = 0; s < supportSize; ++s) {
2223           ierr = DMLabelSetValue(label, support[s], newLevel);CHKERRQ(ierr);
2224         }
2225       }
2226       ierr = ISRestoreIndices(pointIS, &points);CHKERRQ(ierr);
2227       ++level;
2228       ierr = ISDestroy(&pointIS);CHKERRQ(ierr);
2229       ierr = DMLabelGetStratumIS(label, level, &pointIS);CHKERRQ(ierr);
2230       if (pointIS) {ierr = ISGetLocalSize(pointIS, &numPoints);CHKERRQ(ierr);}
2231       else         {numPoints = 0;}
2232     }
2233     ierr = ISDestroy(&pointIS);CHKERRQ(ierr);
2234   }
2235   ierr = PetscLogEventEnd(DMPLEX_Stratify,dm,0,0,0);CHKERRQ(ierr);
2236   PetscFunctionReturn(0);
2237 }
2238 
2239 #undef __FUNCT__
2240 #define __FUNCT__ "DMPlexGetJoin"
2241 /*@C
2242   DMPlexGetJoin - Get an array for the join of the set of points
2243 
2244   Not Collective
2245 
2246   Input Parameters:
2247 + dm - The DMPlex object
2248 . numPoints - The number of input points for the join
2249 - points - The input points
2250 
2251   Output Parameters:
2252 + numCoveredPoints - The number of points in the join
2253 - coveredPoints - The points in the join
2254 
2255   Level: intermediate
2256 
2257   Note: Currently, this is restricted to a single level join
2258 
2259   Fortran Notes:
2260   Since it returns an array, this routine is only available in Fortran 90, and you must
2261   include petsc.h90 in your code.
2262 
2263   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2264 
2265 .keywords: mesh
2266 .seealso: DMPlexRestoreJoin(), DMPlexGetMeet()
2267 @*/
2268 PetscErrorCode DMPlexGetJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
2269 {
2270   DM_Plex       *mesh = (DM_Plex*) dm->data;
2271   PetscInt      *join[2];
2272   PetscInt       joinSize, i = 0;
2273   PetscInt       dof, off, p, c, m;
2274   PetscErrorCode ierr;
2275 
2276   PetscFunctionBegin;
2277   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2278   PetscValidPointer(points, 2);
2279   PetscValidPointer(numCoveredPoints, 3);
2280   PetscValidPointer(coveredPoints, 4);
2281   ierr = DMGetWorkArray(dm, mesh->maxSupportSize, PETSC_INT, &join[0]);CHKERRQ(ierr);
2282   ierr = DMGetWorkArray(dm, mesh->maxSupportSize, PETSC_INT, &join[1]);CHKERRQ(ierr);
2283   /* Copy in support of first point */
2284   ierr = PetscSectionGetDof(mesh->supportSection, points[0], &dof);CHKERRQ(ierr);
2285   ierr = PetscSectionGetOffset(mesh->supportSection, points[0], &off);CHKERRQ(ierr);
2286   for (joinSize = 0; joinSize < dof; ++joinSize) {
2287     join[i][joinSize] = mesh->supports[off+joinSize];
2288   }
2289   /* Check each successive support */
2290   for (p = 1; p < numPoints; ++p) {
2291     PetscInt newJoinSize = 0;
2292 
2293     ierr = PetscSectionGetDof(mesh->supportSection, points[p], &dof);CHKERRQ(ierr);
2294     ierr = PetscSectionGetOffset(mesh->supportSection, points[p], &off);CHKERRQ(ierr);
2295     for (c = 0; c < dof; ++c) {
2296       const PetscInt point = mesh->supports[off+c];
2297 
2298       for (m = 0; m < joinSize; ++m) {
2299         if (point == join[i][m]) {
2300           join[1-i][newJoinSize++] = point;
2301           break;
2302         }
2303       }
2304     }
2305     joinSize = newJoinSize;
2306     i        = 1-i;
2307   }
2308   *numCoveredPoints = joinSize;
2309   *coveredPoints    = join[i];
2310   ierr              = DMRestoreWorkArray(dm, mesh->maxSupportSize, PETSC_INT, &join[1-i]);CHKERRQ(ierr);
2311   PetscFunctionReturn(0);
2312 }
2313 
2314 #undef __FUNCT__
2315 #define __FUNCT__ "DMPlexRestoreJoin"
2316 /*@C
2317   DMPlexRestoreJoin - Restore an array for the join of the set of points
2318 
2319   Not Collective
2320 
2321   Input Parameters:
2322 + dm - The DMPlex object
2323 . numPoints - The number of input points for the join
2324 - points - The input points
2325 
2326   Output Parameters:
2327 + numCoveredPoints - The number of points in the join
2328 - coveredPoints - The points in the join
2329 
2330   Fortran Notes:
2331   Since it returns an array, this routine is only available in Fortran 90, and you must
2332   include petsc.h90 in your code.
2333 
2334   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2335 
2336   Level: intermediate
2337 
2338 .keywords: mesh
2339 .seealso: DMPlexGetJoin(), DMPlexGetFullJoin(), DMPlexGetMeet()
2340 @*/
2341 PetscErrorCode DMPlexRestoreJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
2342 {
2343   PetscErrorCode ierr;
2344 
2345   PetscFunctionBegin;
2346   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2347   if (points) PetscValidIntPointer(points,3);
2348   if (numCoveredPoints) PetscValidIntPointer(numCoveredPoints,4);
2349   PetscValidPointer(coveredPoints, 5);
2350   ierr = DMRestoreWorkArray(dm, 0, PETSC_INT, (void*) coveredPoints);CHKERRQ(ierr);
2351   if (numCoveredPoints) *numCoveredPoints = 0;
2352   PetscFunctionReturn(0);
2353 }
2354 
2355 #undef __FUNCT__
2356 #define __FUNCT__ "DMPlexGetFullJoin"
2357 /*@C
2358   DMPlexGetFullJoin - Get an array for the join of the set of points
2359 
2360   Not Collective
2361 
2362   Input Parameters:
2363 + dm - The DMPlex object
2364 . numPoints - The number of input points for the join
2365 - points - The input points
2366 
2367   Output Parameters:
2368 + numCoveredPoints - The number of points in the join
2369 - coveredPoints - The points in the join
2370 
2371   Fortran Notes:
2372   Since it returns an array, this routine is only available in Fortran 90, and you must
2373   include petsc.h90 in your code.
2374 
2375   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2376 
2377   Level: intermediate
2378 
2379 .keywords: mesh
2380 .seealso: DMPlexGetJoin(), DMPlexRestoreJoin(), DMPlexGetMeet()
2381 @*/
2382 PetscErrorCode DMPlexGetFullJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
2383 {
2384   DM_Plex       *mesh = (DM_Plex*) dm->data;
2385   PetscInt      *offsets, **closures;
2386   PetscInt      *join[2];
2387   PetscInt       depth = 0, maxSize, joinSize = 0, i = 0;
2388   PetscInt       p, d, c, m;
2389   PetscErrorCode ierr;
2390 
2391   PetscFunctionBegin;
2392   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2393   PetscValidPointer(points, 2);
2394   PetscValidPointer(numCoveredPoints, 3);
2395   PetscValidPointer(coveredPoints, 4);
2396 
2397   ierr    = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
2398   ierr    = PetscCalloc1(numPoints, &closures);CHKERRQ(ierr);
2399   ierr    = DMGetWorkArray(dm, numPoints*(depth+2), PETSC_INT, &offsets);CHKERRQ(ierr);
2400   maxSize = PetscPowInt(mesh->maxSupportSize,depth+1);
2401   ierr    = DMGetWorkArray(dm, maxSize, PETSC_INT, &join[0]);CHKERRQ(ierr);
2402   ierr    = DMGetWorkArray(dm, maxSize, PETSC_INT, &join[1]);CHKERRQ(ierr);
2403 
2404   for (p = 0; p < numPoints; ++p) {
2405     PetscInt closureSize;
2406 
2407     ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_FALSE, &closureSize, &closures[p]);CHKERRQ(ierr);
2408 
2409     offsets[p*(depth+2)+0] = 0;
2410     for (d = 0; d < depth+1; ++d) {
2411       PetscInt pStart, pEnd, i;
2412 
2413       ierr = DMPlexGetDepthStratum(dm, d, &pStart, &pEnd);CHKERRQ(ierr);
2414       for (i = offsets[p*(depth+2)+d]; i < closureSize; ++i) {
2415         if ((pStart > closures[p][i*2]) || (pEnd <= closures[p][i*2])) {
2416           offsets[p*(depth+2)+d+1] = i;
2417           break;
2418         }
2419       }
2420       if (i == closureSize) offsets[p*(depth+2)+d+1] = i;
2421     }
2422     if (offsets[p*(depth+2)+depth+1] != closureSize) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Total size of closure %D should be %D", offsets[p*(depth+2)+depth+1], closureSize);
2423   }
2424   for (d = 0; d < depth+1; ++d) {
2425     PetscInt dof;
2426 
2427     /* Copy in support of first point */
2428     dof = offsets[d+1] - offsets[d];
2429     for (joinSize = 0; joinSize < dof; ++joinSize) {
2430       join[i][joinSize] = closures[0][(offsets[d]+joinSize)*2];
2431     }
2432     /* Check each successive cone */
2433     for (p = 1; p < numPoints && joinSize; ++p) {
2434       PetscInt newJoinSize = 0;
2435 
2436       dof = offsets[p*(depth+2)+d+1] - offsets[p*(depth+2)+d];
2437       for (c = 0; c < dof; ++c) {
2438         const PetscInt point = closures[p][(offsets[p*(depth+2)+d]+c)*2];
2439 
2440         for (m = 0; m < joinSize; ++m) {
2441           if (point == join[i][m]) {
2442             join[1-i][newJoinSize++] = point;
2443             break;
2444           }
2445         }
2446       }
2447       joinSize = newJoinSize;
2448       i        = 1-i;
2449     }
2450     if (joinSize) break;
2451   }
2452   *numCoveredPoints = joinSize;
2453   *coveredPoints    = join[i];
2454   for (p = 0; p < numPoints; ++p) {
2455     ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_FALSE, NULL, &closures[p]);CHKERRQ(ierr);
2456   }
2457   ierr = PetscFree(closures);CHKERRQ(ierr);
2458   ierr = DMRestoreWorkArray(dm, numPoints*(depth+2), PETSC_INT, &offsets);CHKERRQ(ierr);
2459   ierr = DMRestoreWorkArray(dm, mesh->maxSupportSize, PETSC_INT, &join[1-i]);CHKERRQ(ierr);
2460   PetscFunctionReturn(0);
2461 }
2462 
2463 #undef __FUNCT__
2464 #define __FUNCT__ "DMPlexGetMeet"
2465 /*@C
2466   DMPlexGetMeet - Get an array for the meet of the set of points
2467 
2468   Not Collective
2469 
2470   Input Parameters:
2471 + dm - The DMPlex object
2472 . numPoints - The number of input points for the meet
2473 - points - The input points
2474 
2475   Output Parameters:
2476 + numCoveredPoints - The number of points in the meet
2477 - coveredPoints - The points in the meet
2478 
2479   Level: intermediate
2480 
2481   Note: Currently, this is restricted to a single level meet
2482 
2483   Fortran Notes:
2484   Since it returns an array, this routine is only available in Fortran 90, and you must
2485   include petsc.h90 in your code.
2486 
2487   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2488 
2489 .keywords: mesh
2490 .seealso: DMPlexRestoreMeet(), DMPlexGetJoin()
2491 @*/
2492 PetscErrorCode DMPlexGetMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveringPoints, const PetscInt **coveringPoints)
2493 {
2494   DM_Plex       *mesh = (DM_Plex*) dm->data;
2495   PetscInt      *meet[2];
2496   PetscInt       meetSize, i = 0;
2497   PetscInt       dof, off, p, c, m;
2498   PetscErrorCode ierr;
2499 
2500   PetscFunctionBegin;
2501   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2502   PetscValidPointer(points, 2);
2503   PetscValidPointer(numCoveringPoints, 3);
2504   PetscValidPointer(coveringPoints, 4);
2505   ierr = DMGetWorkArray(dm, mesh->maxConeSize, PETSC_INT, &meet[0]);CHKERRQ(ierr);
2506   ierr = DMGetWorkArray(dm, mesh->maxConeSize, PETSC_INT, &meet[1]);CHKERRQ(ierr);
2507   /* Copy in cone of first point */
2508   ierr = PetscSectionGetDof(mesh->coneSection, points[0], &dof);CHKERRQ(ierr);
2509   ierr = PetscSectionGetOffset(mesh->coneSection, points[0], &off);CHKERRQ(ierr);
2510   for (meetSize = 0; meetSize < dof; ++meetSize) {
2511     meet[i][meetSize] = mesh->cones[off+meetSize];
2512   }
2513   /* Check each successive cone */
2514   for (p = 1; p < numPoints; ++p) {
2515     PetscInt newMeetSize = 0;
2516 
2517     ierr = PetscSectionGetDof(mesh->coneSection, points[p], &dof);CHKERRQ(ierr);
2518     ierr = PetscSectionGetOffset(mesh->coneSection, points[p], &off);CHKERRQ(ierr);
2519     for (c = 0; c < dof; ++c) {
2520       const PetscInt point = mesh->cones[off+c];
2521 
2522       for (m = 0; m < meetSize; ++m) {
2523         if (point == meet[i][m]) {
2524           meet[1-i][newMeetSize++] = point;
2525           break;
2526         }
2527       }
2528     }
2529     meetSize = newMeetSize;
2530     i        = 1-i;
2531   }
2532   *numCoveringPoints = meetSize;
2533   *coveringPoints    = meet[i];
2534   ierr               = DMRestoreWorkArray(dm, mesh->maxConeSize, PETSC_INT, &meet[1-i]);CHKERRQ(ierr);
2535   PetscFunctionReturn(0);
2536 }
2537 
2538 #undef __FUNCT__
2539 #define __FUNCT__ "DMPlexRestoreMeet"
2540 /*@C
2541   DMPlexRestoreMeet - Restore an array for the meet of the set of points
2542 
2543   Not Collective
2544 
2545   Input Parameters:
2546 + dm - The DMPlex object
2547 . numPoints - The number of input points for the meet
2548 - points - The input points
2549 
2550   Output Parameters:
2551 + numCoveredPoints - The number of points in the meet
2552 - coveredPoints - The points in the meet
2553 
2554   Level: intermediate
2555 
2556   Fortran Notes:
2557   Since it returns an array, this routine is only available in Fortran 90, and you must
2558   include petsc.h90 in your code.
2559 
2560   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2561 
2562 .keywords: mesh
2563 .seealso: DMPlexGetMeet(), DMPlexGetFullMeet(), DMPlexGetJoin()
2564 @*/
2565 PetscErrorCode DMPlexRestoreMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
2566 {
2567   PetscErrorCode ierr;
2568 
2569   PetscFunctionBegin;
2570   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2571   if (points) PetscValidIntPointer(points,3);
2572   if (numCoveredPoints) PetscValidIntPointer(numCoveredPoints,4);
2573   PetscValidPointer(coveredPoints,5);
2574   ierr = DMRestoreWorkArray(dm, 0, PETSC_INT, (void*) coveredPoints);CHKERRQ(ierr);
2575   if (numCoveredPoints) *numCoveredPoints = 0;
2576   PetscFunctionReturn(0);
2577 }
2578 
2579 #undef __FUNCT__
2580 #define __FUNCT__ "DMPlexGetFullMeet"
2581 /*@C
2582   DMPlexGetFullMeet - Get an array for the meet of the set of points
2583 
2584   Not Collective
2585 
2586   Input Parameters:
2587 + dm - The DMPlex object
2588 . numPoints - The number of input points for the meet
2589 - points - The input points
2590 
2591   Output Parameters:
2592 + numCoveredPoints - The number of points in the meet
2593 - coveredPoints - The points in the meet
2594 
2595   Level: intermediate
2596 
2597   Fortran Notes:
2598   Since it returns an array, this routine is only available in Fortran 90, and you must
2599   include petsc.h90 in your code.
2600 
2601   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2602 
2603 .keywords: mesh
2604 .seealso: DMPlexGetMeet(), DMPlexRestoreMeet(), DMPlexGetJoin()
2605 @*/
2606 PetscErrorCode DMPlexGetFullMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
2607 {
2608   DM_Plex       *mesh = (DM_Plex*) dm->data;
2609   PetscInt      *offsets, **closures;
2610   PetscInt      *meet[2];
2611   PetscInt       height = 0, maxSize, meetSize = 0, i = 0;
2612   PetscInt       p, h, c, m;
2613   PetscErrorCode ierr;
2614 
2615   PetscFunctionBegin;
2616   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2617   PetscValidPointer(points, 2);
2618   PetscValidPointer(numCoveredPoints, 3);
2619   PetscValidPointer(coveredPoints, 4);
2620 
2621   ierr    = DMPlexGetDepth(dm, &height);CHKERRQ(ierr);
2622   ierr    = PetscMalloc1(numPoints, &closures);CHKERRQ(ierr);
2623   ierr    = DMGetWorkArray(dm, numPoints*(height+2), PETSC_INT, &offsets);CHKERRQ(ierr);
2624   maxSize = PetscPowInt(mesh->maxConeSize,height+1);
2625   ierr    = DMGetWorkArray(dm, maxSize, PETSC_INT, &meet[0]);CHKERRQ(ierr);
2626   ierr    = DMGetWorkArray(dm, maxSize, PETSC_INT, &meet[1]);CHKERRQ(ierr);
2627 
2628   for (p = 0; p < numPoints; ++p) {
2629     PetscInt closureSize;
2630 
2631     ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_TRUE, &closureSize, &closures[p]);CHKERRQ(ierr);
2632 
2633     offsets[p*(height+2)+0] = 0;
2634     for (h = 0; h < height+1; ++h) {
2635       PetscInt pStart, pEnd, i;
2636 
2637       ierr = DMPlexGetHeightStratum(dm, h, &pStart, &pEnd);CHKERRQ(ierr);
2638       for (i = offsets[p*(height+2)+h]; i < closureSize; ++i) {
2639         if ((pStart > closures[p][i*2]) || (pEnd <= closures[p][i*2])) {
2640           offsets[p*(height+2)+h+1] = i;
2641           break;
2642         }
2643       }
2644       if (i == closureSize) offsets[p*(height+2)+h+1] = i;
2645     }
2646     if (offsets[p*(height+2)+height+1] != closureSize) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Total size of closure %D should be %D", offsets[p*(height+2)+height+1], closureSize);
2647   }
2648   for (h = 0; h < height+1; ++h) {
2649     PetscInt dof;
2650 
2651     /* Copy in cone of first point */
2652     dof = offsets[h+1] - offsets[h];
2653     for (meetSize = 0; meetSize < dof; ++meetSize) {
2654       meet[i][meetSize] = closures[0][(offsets[h]+meetSize)*2];
2655     }
2656     /* Check each successive cone */
2657     for (p = 1; p < numPoints && meetSize; ++p) {
2658       PetscInt newMeetSize = 0;
2659 
2660       dof = offsets[p*(height+2)+h+1] - offsets[p*(height+2)+h];
2661       for (c = 0; c < dof; ++c) {
2662         const PetscInt point = closures[p][(offsets[p*(height+2)+h]+c)*2];
2663 
2664         for (m = 0; m < meetSize; ++m) {
2665           if (point == meet[i][m]) {
2666             meet[1-i][newMeetSize++] = point;
2667             break;
2668           }
2669         }
2670       }
2671       meetSize = newMeetSize;
2672       i        = 1-i;
2673     }
2674     if (meetSize) break;
2675   }
2676   *numCoveredPoints = meetSize;
2677   *coveredPoints    = meet[i];
2678   for (p = 0; p < numPoints; ++p) {
2679     ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_TRUE, NULL, &closures[p]);CHKERRQ(ierr);
2680   }
2681   ierr = PetscFree(closures);CHKERRQ(ierr);
2682   ierr = DMRestoreWorkArray(dm, numPoints*(height+2), PETSC_INT, &offsets);CHKERRQ(ierr);
2683   ierr = DMRestoreWorkArray(dm, mesh->maxConeSize, PETSC_INT, &meet[1-i]);CHKERRQ(ierr);
2684   PetscFunctionReturn(0);
2685 }
2686 
2687 #undef __FUNCT__
2688 #define __FUNCT__ "DMPlexEqual"
2689 /*@C
2690   DMPlexEqual - Determine if two DMs have the same topology
2691 
2692   Not Collective
2693 
2694   Input Parameters:
2695 + dmA - A DMPlex object
2696 - dmB - A DMPlex object
2697 
2698   Output Parameters:
2699 . equal - PETSC_TRUE if the topologies are identical
2700 
2701   Level: intermediate
2702 
2703   Notes:
2704   We are not solving graph isomorphism, so we do not permutation.
2705 
2706 .keywords: mesh
2707 .seealso: DMPlexGetCone()
2708 @*/
2709 PetscErrorCode DMPlexEqual(DM dmA, DM dmB, PetscBool *equal)
2710 {
2711   PetscInt       depth, depthB, pStart, pEnd, pStartB, pEndB, p;
2712   PetscErrorCode ierr;
2713 
2714   PetscFunctionBegin;
2715   PetscValidHeaderSpecific(dmA, DM_CLASSID, 1);
2716   PetscValidHeaderSpecific(dmB, DM_CLASSID, 2);
2717   PetscValidPointer(equal, 3);
2718 
2719   *equal = PETSC_FALSE;
2720   ierr = DMPlexGetDepth(dmA, &depth);CHKERRQ(ierr);
2721   ierr = DMPlexGetDepth(dmB, &depthB);CHKERRQ(ierr);
2722   if (depth != depthB) PetscFunctionReturn(0);
2723   ierr = DMPlexGetChart(dmA, &pStart,  &pEnd);CHKERRQ(ierr);
2724   ierr = DMPlexGetChart(dmB, &pStartB, &pEndB);CHKERRQ(ierr);
2725   if ((pStart != pStartB) || (pEnd != pEndB)) PetscFunctionReturn(0);
2726   for (p = pStart; p < pEnd; ++p) {
2727     const PetscInt *cone, *coneB, *ornt, *orntB, *support, *supportB;
2728     PetscInt        coneSize, coneSizeB, c, supportSize, supportSizeB, s;
2729 
2730     ierr = DMPlexGetConeSize(dmA, p, &coneSize);CHKERRQ(ierr);
2731     ierr = DMPlexGetCone(dmA, p, &cone);CHKERRQ(ierr);
2732     ierr = DMPlexGetConeOrientation(dmA, p, &ornt);CHKERRQ(ierr);
2733     ierr = DMPlexGetConeSize(dmB, p, &coneSizeB);CHKERRQ(ierr);
2734     ierr = DMPlexGetCone(dmB, p, &coneB);CHKERRQ(ierr);
2735     ierr = DMPlexGetConeOrientation(dmB, p, &orntB);CHKERRQ(ierr);
2736     if (coneSize != coneSizeB) PetscFunctionReturn(0);
2737     for (c = 0; c < coneSize; ++c) {
2738       if (cone[c] != coneB[c]) PetscFunctionReturn(0);
2739       if (ornt[c] != orntB[c]) PetscFunctionReturn(0);
2740     }
2741     ierr = DMPlexGetSupportSize(dmA, p, &supportSize);CHKERRQ(ierr);
2742     ierr = DMPlexGetSupport(dmA, p, &support);CHKERRQ(ierr);
2743     ierr = DMPlexGetSupportSize(dmB, p, &supportSizeB);CHKERRQ(ierr);
2744     ierr = DMPlexGetSupport(dmB, p, &supportB);CHKERRQ(ierr);
2745     if (supportSize != supportSizeB) PetscFunctionReturn(0);
2746     for (s = 0; s < supportSize; ++s) {
2747       if (support[s] != supportB[s]) PetscFunctionReturn(0);
2748     }
2749   }
2750   *equal = PETSC_TRUE;
2751   PetscFunctionReturn(0);
2752 }
2753 
2754 #undef __FUNCT__
2755 #define __FUNCT__ "DMPlexGetNumFaceVertices"
2756 PetscErrorCode DMPlexGetNumFaceVertices(DM dm, PetscInt cellDim, PetscInt numCorners, PetscInt *numFaceVertices)
2757 {
2758   MPI_Comm       comm;
2759   PetscErrorCode ierr;
2760 
2761   PetscFunctionBegin;
2762   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
2763   PetscValidPointer(numFaceVertices,3);
2764   switch (cellDim) {
2765   case 0:
2766     *numFaceVertices = 0;
2767     break;
2768   case 1:
2769     *numFaceVertices = 1;
2770     break;
2771   case 2:
2772     switch (numCorners) {
2773     case 3: /* triangle */
2774       *numFaceVertices = 2; /* Edge has 2 vertices */
2775       break;
2776     case 4: /* quadrilateral */
2777       *numFaceVertices = 2; /* Edge has 2 vertices */
2778       break;
2779     case 6: /* quadratic triangle, tri and quad cohesive Lagrange cells */
2780       *numFaceVertices = 3; /* Edge has 3 vertices */
2781       break;
2782     case 9: /* quadratic quadrilateral, quadratic quad cohesive Lagrange cells */
2783       *numFaceVertices = 3; /* Edge has 3 vertices */
2784       break;
2785     default:
2786       SETERRQ2(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid number of face corners %d for dimension %d", numCorners, cellDim);
2787     }
2788     break;
2789   case 3:
2790     switch (numCorners) {
2791     case 4: /* tetradehdron */
2792       *numFaceVertices = 3; /* Face has 3 vertices */
2793       break;
2794     case 6: /* tet cohesive cells */
2795       *numFaceVertices = 4; /* Face has 4 vertices */
2796       break;
2797     case 8: /* hexahedron */
2798       *numFaceVertices = 4; /* Face has 4 vertices */
2799       break;
2800     case 9: /* tet cohesive Lagrange cells */
2801       *numFaceVertices = 6; /* Face has 6 vertices */
2802       break;
2803     case 10: /* quadratic tetrahedron */
2804       *numFaceVertices = 6; /* Face has 6 vertices */
2805       break;
2806     case 12: /* hex cohesive Lagrange cells */
2807       *numFaceVertices = 6; /* Face has 6 vertices */
2808       break;
2809     case 18: /* quadratic tet cohesive Lagrange cells */
2810       *numFaceVertices = 6; /* Face has 6 vertices */
2811       break;
2812     case 27: /* quadratic hexahedron, quadratic hex cohesive Lagrange cells */
2813       *numFaceVertices = 9; /* Face has 9 vertices */
2814       break;
2815     default:
2816       SETERRQ2(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid number of face corners %d for dimension %d", numCorners, cellDim);
2817     }
2818     break;
2819   default:
2820     SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid cell dimension %d", cellDim);
2821   }
2822   PetscFunctionReturn(0);
2823 }
2824 
2825 #undef __FUNCT__
2826 #define __FUNCT__ "DMPlexOrient"
2827 /* Trys to give the mesh a consistent orientation */
2828 PetscErrorCode DMPlexOrient(DM dm)
2829 {
2830   PetscBT        seenCells, flippedCells, seenFaces;
2831   PetscInt      *faceFIFO, fTop, fBottom;
2832   PetscInt       dim, h, cStart, cEnd, c, fStart, fEnd, face, maxConeSize, *revcone, *revconeO;
2833   PetscErrorCode ierr;
2834 
2835   PetscFunctionBegin;
2836   /* Truth Table
2837      mismatch    flips   do action   mismatch   flipA ^ flipB   action
2838          F       0 flips     no         F             F           F
2839          F       1 flip      yes        F             T           T
2840          F       2 flips     no         T             F           T
2841          T       0 flips     yes        T             T           F
2842          T       1 flip      no
2843          T       2 flips     yes
2844   */
2845   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
2846   ierr = DMPlexGetVTKCellHeight(dm, &h);CHKERRQ(ierr);
2847   ierr = DMPlexGetHeightStratum(dm, h,   &cStart, &cEnd);CHKERRQ(ierr);
2848   ierr = DMPlexGetHeightStratum(dm, h+1, &fStart, &fEnd);CHKERRQ(ierr);
2849   ierr = PetscBTCreate(cEnd - cStart, &seenCells);CHKERRQ(ierr);
2850   ierr = PetscBTMemzero(cEnd - cStart, seenCells);CHKERRQ(ierr);
2851   ierr = PetscBTCreate(cEnd - cStart, &flippedCells);CHKERRQ(ierr);
2852   ierr = PetscBTMemzero(cEnd - cStart, flippedCells);CHKERRQ(ierr);
2853   ierr = PetscBTCreate(fEnd - fStart, &seenFaces);CHKERRQ(ierr);
2854   ierr = PetscBTMemzero(fEnd - fStart, seenFaces);CHKERRQ(ierr);
2855   ierr = PetscMalloc1((fEnd - fStart), &faceFIFO);CHKERRQ(ierr);
2856   fTop = fBottom = 0;
2857   /* Initialize FIFO with first cell */
2858   if (cEnd > cStart) {
2859     const PetscInt *cone;
2860     PetscInt        coneSize;
2861 
2862     ierr = DMPlexGetConeSize(dm, cStart, &coneSize);CHKERRQ(ierr);
2863     ierr = DMPlexGetCone(dm, cStart, &cone);CHKERRQ(ierr);
2864     for (c = 0; c < coneSize; ++c) {
2865       faceFIFO[fBottom++] = cone[c];
2866       ierr = PetscBTSet(seenFaces, cone[c]-fStart);CHKERRQ(ierr);
2867     }
2868   }
2869   /* Consider each face in FIFO */
2870   while (fTop < fBottom) {
2871     const PetscInt *support, *coneA, *coneB, *coneOA, *coneOB;
2872     PetscInt        supportSize, coneSizeA, coneSizeB, posA = -1, posB = -1;
2873     PetscInt        seenA, flippedA, seenB, flippedB, mismatch;
2874 
2875     face = faceFIFO[fTop++];
2876     ierr = DMPlexGetSupportSize(dm, face, &supportSize);CHKERRQ(ierr);
2877     ierr = DMPlexGetSupport(dm, face, &support);CHKERRQ(ierr);
2878     if (supportSize < 2) continue;
2879     if (supportSize != 2) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Faces should separate only two cells, not %d", supportSize);
2880     seenA    = PetscBTLookup(seenCells,    support[0]-cStart);
2881     flippedA = PetscBTLookup(flippedCells, support[0]-cStart) ? 1 : 0;
2882     seenB    = PetscBTLookup(seenCells,    support[1]-cStart);
2883     flippedB = PetscBTLookup(flippedCells, support[1]-cStart) ? 1 : 0;
2884 
2885     ierr = DMPlexGetConeSize(dm, support[0], &coneSizeA);CHKERRQ(ierr);
2886     ierr = DMPlexGetConeSize(dm, support[1], &coneSizeB);CHKERRQ(ierr);
2887     ierr = DMPlexGetCone(dm, support[0], &coneA);CHKERRQ(ierr);
2888     ierr = DMPlexGetCone(dm, support[1], &coneB);CHKERRQ(ierr);
2889     ierr = DMPlexGetConeOrientation(dm, support[0], &coneOA);CHKERRQ(ierr);
2890     ierr = DMPlexGetConeOrientation(dm, support[1], &coneOB);CHKERRQ(ierr);
2891     for (c = 0; c < coneSizeA; ++c) {
2892       if (!PetscBTLookup(seenFaces, coneA[c]-fStart)) {
2893         faceFIFO[fBottom++] = coneA[c];
2894         ierr = PetscBTSet(seenFaces, coneA[c]-fStart);CHKERRQ(ierr);
2895       }
2896       if (coneA[c] == face) posA = c;
2897       if (fBottom > fEnd-fStart) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Face %d was pushed exceeding capacity %d > %d", coneA[c], fBottom, fEnd-fStart);
2898     }
2899     if (posA < 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %d could not be located in cell %d", face, support[0]);
2900     for (c = 0; c < coneSizeB; ++c) {
2901       if (!PetscBTLookup(seenFaces, coneB[c]-fStart)) {
2902         faceFIFO[fBottom++] = coneB[c];
2903         ierr = PetscBTSet(seenFaces, coneB[c]-fStart);CHKERRQ(ierr);
2904       }
2905       if (coneB[c] == face) posB = c;
2906       if (fBottom > fEnd-fStart) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Face %d was pushed exceeding capacity %d > %d", coneA[c], fBottom, fEnd-fStart);
2907     }
2908     if (posB < 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %d could not be located in cell %d", face, support[1]);
2909 
2910     if (dim == 1) {
2911       mismatch = posA == posB;
2912     } else {
2913       mismatch = coneOA[posA] == coneOB[posB];
2914     }
2915 
2916     if (mismatch ^ (flippedA ^ flippedB)) {
2917       if (seenA && seenB) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Previously seen cells %d and %d do not match: Fault mesh is non-orientable", support[0], support[1]);
2918       if (!seenA && !flippedA) {
2919         ierr = PetscBTSet(flippedCells, support[0]-cStart);CHKERRQ(ierr);
2920       } else if (!seenB && !flippedB) {
2921         ierr = PetscBTSet(flippedCells, support[1]-cStart);CHKERRQ(ierr);
2922       } else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Inconsistent mesh orientation: Fault mesh is non-orientable");
2923     } else if (mismatch && flippedA && flippedB) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Attempt to flip already flipped cell: Fault mesh is non-orientable");
2924     ierr = PetscBTSet(seenCells, support[0]-cStart);CHKERRQ(ierr);
2925     ierr = PetscBTSet(seenCells, support[1]-cStart);CHKERRQ(ierr);
2926   }
2927   /* Now all subdomains are oriented, but we need a consistent parallel orientation */
2928   {
2929     /* Find a representative face (edge) separating pairs of procs */
2930     PetscSF            sf;
2931     const PetscInt    *lpoints;
2932     const PetscSFNode *rpoints;
2933     PetscInt          *neighbors, *nranks;
2934     PetscInt           numLeaves, numRoots, numNeighbors = 0, l, n;
2935 
2936     ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr);
2937     ierr = PetscSFGetGraph(sf, &numRoots, &numLeaves, &lpoints, &rpoints);CHKERRQ(ierr);
2938     if (numLeaves >= 0) {
2939       const PetscInt *cone, *ornt, *support;
2940       PetscInt        coneSize, supportSize;
2941       int            *rornt, *lornt; /* PetscSF cannot handle smaller than int */
2942       PetscBool      *match, flipped = PETSC_FALSE;
2943 
2944       ierr = PetscMalloc1(numLeaves,&neighbors);CHKERRQ(ierr);
2945       /* I know this is p^2 time in general, but for bounded degree its alright */
2946       for (l = 0; l < numLeaves; ++l) {
2947         const PetscInt face = lpoints[l];
2948         if ((face >= fStart) && (face < fEnd)) {
2949           const PetscInt rank = rpoints[l].rank;
2950           for (n = 0; n < numNeighbors; ++n) if (rank == rpoints[neighbors[n]].rank) break;
2951           if (n >= numNeighbors) {
2952             PetscInt supportSize;
2953             ierr = DMPlexGetSupportSize(dm, face, &supportSize);CHKERRQ(ierr);
2954             if (supportSize != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Boundary faces should see one cell, not %d", supportSize);
2955             neighbors[numNeighbors++] = l;
2956           }
2957         }
2958       }
2959       ierr = PetscCalloc4(numNeighbors,&match,numNeighbors,&nranks,numRoots,&rornt,numRoots,&lornt);CHKERRQ(ierr);
2960       for (face = fStart; face < fEnd; ++face) {
2961         ierr = DMPlexGetSupportSize(dm, face, &supportSize);CHKERRQ(ierr);
2962         if (supportSize != 1) continue;
2963         ierr = DMPlexGetSupport(dm, face, &support);CHKERRQ(ierr);
2964 
2965         ierr = DMPlexGetCone(dm, support[0], &cone);CHKERRQ(ierr);
2966         ierr = DMPlexGetConeSize(dm, support[0], &coneSize);CHKERRQ(ierr);
2967         ierr = DMPlexGetConeOrientation(dm, support[0], &ornt);CHKERRQ(ierr);
2968         for (c = 0; c < coneSize; ++c) if (cone[c] == face) break;
2969         if (dim == 1) {
2970           /* Use cone position instead, shifted to -1 or 1 */
2971           rornt[face] = c*2-1;
2972         } else {
2973           if (PetscBTLookup(flippedCells, support[0]-cStart)) rornt[face] = ornt[c] < 0 ? -1 :  1;
2974           else                                                rornt[face] = ornt[c] < 0 ?  1 : -1;
2975         }
2976       }
2977       /* Mark each edge with match or nomatch */
2978       ierr = PetscSFBcastBegin(sf, MPI_INT, rornt, lornt);CHKERRQ(ierr);
2979       ierr = PetscSFBcastEnd(sf, MPI_INT, rornt, lornt);CHKERRQ(ierr);
2980       for (n = 0; n < numNeighbors; ++n) {
2981         const PetscInt face = lpoints[neighbors[n]];
2982 
2983         if (rornt[face]*lornt[face] < 0) match[n] = PETSC_TRUE;
2984         else                             match[n] = PETSC_FALSE;
2985         nranks[n] = rpoints[neighbors[n]].rank;
2986       }
2987       /* Collect the graph on 0 */
2988       {
2989         MPI_Comm     comm = PetscObjectComm((PetscObject) sf);
2990         PetscBT      seenProcs, flippedProcs;
2991         PetscInt    *procFIFO, pTop, pBottom;
2992         PetscInt    *adj = NULL;
2993         PetscBool   *val = NULL;
2994         PetscMPIInt *recvcounts = NULL, *displs = NULL, p;
2995         PetscMPIInt  N = numNeighbors, numProcs = 0, rank;
2996         PetscInt     debug = 0;
2997 
2998         ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
2999         if (!rank) {ierr = MPI_Comm_size(comm, &numProcs);CHKERRQ(ierr);}
3000         ierr = PetscCalloc2(numProcs,&recvcounts,numProcs+1,&displs);CHKERRQ(ierr);
3001         ierr = MPI_Gather(&N, 1, MPI_INT, recvcounts, 1, MPI_INT, 0, comm);CHKERRQ(ierr);
3002         for (p = 0; p < numProcs; ++p) {
3003           displs[p+1] = displs[p] + recvcounts[p];
3004         }
3005         if (!rank) {ierr = PetscMalloc2(displs[numProcs],&adj,displs[numProcs],&val);CHKERRQ(ierr);}
3006         ierr = MPI_Gatherv(nranks, numNeighbors, MPIU_INT, adj, recvcounts, displs, MPIU_INT, 0, comm);CHKERRQ(ierr);
3007         ierr = MPI_Gatherv(match, numNeighbors, MPIU_BOOL, val, recvcounts, displs, MPIU_BOOL, 0, comm);CHKERRQ(ierr);
3008         if (debug) {
3009           for (p = 0; p < numProcs; ++p) {
3010             ierr = PetscPrintf(comm, "Proc %d:\n", p);
3011             for (n = 0; n < recvcounts[p]; ++n) {
3012               ierr = PetscPrintf(comm, "  edge %d (%d):\n", adj[displs[p]+n], val[displs[p]+n]);
3013             }
3014           }
3015         }
3016         ierr = PetscBTCreate(numProcs, &seenProcs);CHKERRQ(ierr);
3017         ierr = PetscBTMemzero(numProcs, seenProcs);CHKERRQ(ierr);
3018         ierr = PetscBTCreate(numProcs, &flippedProcs);CHKERRQ(ierr);
3019         ierr = PetscBTMemzero(numProcs, flippedProcs);CHKERRQ(ierr);
3020         ierr = PetscMalloc1(numProcs,&procFIFO);CHKERRQ(ierr);
3021         pTop = pBottom = 0;
3022         for (p = 0; p < numProcs; ++p) {
3023           if (PetscBTLookup(seenProcs, p)) continue;
3024           /* Initialize FIFO with next proc */
3025           procFIFO[pBottom++] = p;
3026           ierr = PetscBTSet(seenProcs, p);CHKERRQ(ierr);
3027           /* Consider each proc in FIFO */
3028           while (pTop < pBottom) {
3029             PetscInt proc, nproc, seen, flippedA, flippedB, mismatch;
3030 
3031             proc     = procFIFO[pTop++];
3032             flippedA = PetscBTLookup(flippedProcs, proc) ? 1 : 0;
3033             /* Loop over neighboring procs */
3034             for (n = 0; n < recvcounts[proc]; ++n) {
3035               nproc    = adj[displs[proc]+n];
3036               mismatch = val[displs[proc]+n] ? 0 : 1;
3037               seen     = PetscBTLookup(seenProcs, nproc);
3038               flippedB = PetscBTLookup(flippedProcs, nproc) ? 1 : 0;
3039 
3040               if (mismatch ^ (flippedA ^ flippedB)) {
3041                 if (seen) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Previously seen procs %d and %d do not match: Fault mesh is non-orientable", proc, nproc);
3042                 if (!flippedB) {
3043                   ierr = PetscBTSet(flippedProcs, nproc);CHKERRQ(ierr);
3044               } else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Inconsistent mesh orientation: Fault mesh is non-orientable");
3045               } else if (mismatch && flippedA && flippedB) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Attempt to flip already flipped cell: Fault mesh is non-orientable");
3046               if (!seen) {
3047                 procFIFO[pBottom++] = nproc;
3048                 ierr = PetscBTSet(seenProcs, nproc);CHKERRQ(ierr);
3049               }
3050             }
3051           }
3052         }
3053         ierr = PetscFree(procFIFO);CHKERRQ(ierr);
3054 
3055         ierr = PetscFree2(recvcounts,displs);CHKERRQ(ierr);
3056         ierr = PetscFree2(adj,val);CHKERRQ(ierr);
3057         {
3058           PetscBool *flips;
3059 
3060           ierr = PetscMalloc1(numProcs,&flips);CHKERRQ(ierr);
3061           for (p = 0; p < numProcs; ++p) {
3062             flips[p] = PetscBTLookup(flippedProcs, p) ? PETSC_TRUE : PETSC_FALSE;
3063             if (debug && flips[p]) {ierr = PetscPrintf(comm, "Flipping Proc %d:\n", p);}
3064           }
3065           ierr = MPI_Scatter(flips, 1, MPIU_BOOL, &flipped, 1, MPIU_BOOL, 0, comm);CHKERRQ(ierr);
3066           ierr = PetscFree(flips);CHKERRQ(ierr);
3067         }
3068         ierr = PetscBTDestroy(&seenProcs);CHKERRQ(ierr);
3069         ierr = PetscBTDestroy(&flippedProcs);CHKERRQ(ierr);
3070       }
3071       ierr = PetscFree4(match,nranks,rornt,lornt);CHKERRQ(ierr);
3072       ierr = PetscFree(neighbors);CHKERRQ(ierr);
3073       if (flipped) {for (c = cStart; c < cEnd; ++c) {ierr = PetscBTNegate(flippedCells, c-cStart);CHKERRQ(ierr);}}
3074     }
3075   }
3076   /* Reverse flipped cells in the mesh */
3077   ierr = DMPlexGetMaxSizes(dm, &maxConeSize, NULL);CHKERRQ(ierr);
3078   ierr = DMGetWorkArray(dm, maxConeSize, PETSC_INT, &revcone);CHKERRQ(ierr);
3079   ierr = DMGetWorkArray(dm, maxConeSize, PETSC_INT, &revconeO);CHKERRQ(ierr);
3080   for (c = cStart; c < cEnd; ++c) {
3081     const PetscInt *cone, *coneO, *support;
3082     PetscInt        coneSize, supportSize, faceSize, cp, sp;
3083 
3084     if (!PetscBTLookup(flippedCells, c-cStart)) continue;
3085     ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
3086     ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
3087     ierr = DMPlexGetConeOrientation(dm, c, &coneO);CHKERRQ(ierr);
3088     for (cp = 0; cp < coneSize; ++cp) {
3089       const PetscInt rcp = coneSize-cp-1;
3090 
3091       ierr = DMPlexGetConeSize(dm, cone[rcp], &faceSize);CHKERRQ(ierr);
3092       revcone[cp]  = cone[rcp];
3093       revconeO[cp] = coneO[rcp] >= 0 ? -(faceSize-coneO[rcp]) : faceSize+coneO[rcp];
3094     }
3095     ierr = DMPlexSetCone(dm, c, revcone);CHKERRQ(ierr);
3096     ierr = DMPlexSetConeOrientation(dm, c, revconeO);CHKERRQ(ierr);
3097     /* Reverse orientations of support */
3098     faceSize = coneSize;
3099     ierr = DMPlexGetSupportSize(dm, c, &supportSize);CHKERRQ(ierr);
3100     ierr = DMPlexGetSupport(dm, c, &support);CHKERRQ(ierr);
3101     for (sp = 0; sp < supportSize; ++sp) {
3102       ierr = DMPlexGetConeSize(dm, support[sp], &coneSize);CHKERRQ(ierr);
3103       ierr = DMPlexGetCone(dm, support[sp], &cone);CHKERRQ(ierr);
3104       ierr = DMPlexGetConeOrientation(dm, support[sp], &coneO);CHKERRQ(ierr);
3105       for (cp = 0; cp < coneSize; ++cp) {
3106         if (cone[cp] != c) continue;
3107         ierr = DMPlexInsertConeOrientation(dm, support[sp], cp, coneO[cp] >= 0 ? -(faceSize-coneO[cp]) : faceSize+coneO[cp]);CHKERRQ(ierr);
3108       }
3109     }
3110   }
3111   ierr = DMRestoreWorkArray(dm, maxConeSize, PETSC_INT, &revcone);CHKERRQ(ierr);
3112   ierr = DMRestoreWorkArray(dm, maxConeSize, PETSC_INT, &revconeO);CHKERRQ(ierr);
3113   ierr = PetscBTDestroy(&seenCells);CHKERRQ(ierr);
3114   ierr = PetscBTDestroy(&flippedCells);CHKERRQ(ierr);
3115   ierr = PetscBTDestroy(&seenFaces);CHKERRQ(ierr);
3116   ierr = PetscFree(faceFIFO);CHKERRQ(ierr);
3117   PetscFunctionReturn(0);
3118 }
3119 
3120 #undef __FUNCT__
3121 #define __FUNCT__ "DMPlexInvertCell"
3122 /*@C
3123   DMPlexInvertCell - This flips tetrahedron and hexahedron orientation since Plex stores them internally with outward normals. Other cells are left untouched.
3124 
3125   Input Parameters:
3126 + numCorners - The number of vertices in a cell
3127 - cone - The incoming cone
3128 
3129   Output Parameter:
3130 . cone - The inverted cone (in-place)
3131 
3132   Level: developer
3133 
3134 .seealso: DMPlexGenerate()
3135 @*/
3136 PetscErrorCode DMPlexInvertCell(PetscInt dim, PetscInt numCorners, int cone[])
3137 {
3138   int tmpc;
3139 
3140   PetscFunctionBegin;
3141   if (dim != 3) PetscFunctionReturn(0);
3142   switch (numCorners) {
3143   case 4:
3144     tmpc    = cone[0];
3145     cone[0] = cone[1];
3146     cone[1] = tmpc;
3147     break;
3148   case 8:
3149     tmpc    = cone[1];
3150     cone[1] = cone[3];
3151     cone[3] = tmpc;
3152     break;
3153   default: break;
3154   }
3155   PetscFunctionReturn(0);
3156 }
3157 
3158 #undef __FUNCT__
3159 #define __FUNCT__ "DMPlexInvertCells_Internal"
3160 /* This is to fix the tetrahedron orientation from TetGen */
3161 PETSC_UNUSED static PetscErrorCode DMPlexInvertCells_Internal(PetscInt dim, PetscInt numCells, PetscInt numCorners, int cells[])
3162 {
3163   PetscInt       bound = numCells*numCorners, coff;
3164   PetscErrorCode ierr;
3165 
3166   PetscFunctionBegin;
3167   for (coff = 0; coff < bound; coff += numCorners) {
3168     ierr = DMPlexInvertCell(dim, numCorners, &cells[coff]);CHKERRQ(ierr);
3169   }
3170   PetscFunctionReturn(0);
3171 }
3172 
3173 #if defined(PETSC_HAVE_TRIANGLE)
3174 #include <triangle.h>
3175 
3176 #undef __FUNCT__
3177 #define __FUNCT__ "InitInput_Triangle"
3178 PetscErrorCode InitInput_Triangle(struct triangulateio *inputCtx)
3179 {
3180   PetscFunctionBegin;
3181   inputCtx->numberofpoints             = 0;
3182   inputCtx->numberofpointattributes    = 0;
3183   inputCtx->pointlist                  = NULL;
3184   inputCtx->pointattributelist         = NULL;
3185   inputCtx->pointmarkerlist            = NULL;
3186   inputCtx->numberofsegments           = 0;
3187   inputCtx->segmentlist                = NULL;
3188   inputCtx->segmentmarkerlist          = NULL;
3189   inputCtx->numberoftriangleattributes = 0;
3190   inputCtx->trianglelist               = NULL;
3191   inputCtx->numberofholes              = 0;
3192   inputCtx->holelist                   = NULL;
3193   inputCtx->numberofregions            = 0;
3194   inputCtx->regionlist                 = NULL;
3195   PetscFunctionReturn(0);
3196 }
3197 
3198 #undef __FUNCT__
3199 #define __FUNCT__ "InitOutput_Triangle"
3200 PetscErrorCode InitOutput_Triangle(struct triangulateio *outputCtx)
3201 {
3202   PetscFunctionBegin;
3203   outputCtx->numberofpoints        = 0;
3204   outputCtx->pointlist             = NULL;
3205   outputCtx->pointattributelist    = NULL;
3206   outputCtx->pointmarkerlist       = NULL;
3207   outputCtx->numberoftriangles     = 0;
3208   outputCtx->trianglelist          = NULL;
3209   outputCtx->triangleattributelist = NULL;
3210   outputCtx->neighborlist          = NULL;
3211   outputCtx->segmentlist           = NULL;
3212   outputCtx->segmentmarkerlist     = NULL;
3213   outputCtx->numberofedges         = 0;
3214   outputCtx->edgelist              = NULL;
3215   outputCtx->edgemarkerlist        = NULL;
3216   PetscFunctionReturn(0);
3217 }
3218 
3219 #undef __FUNCT__
3220 #define __FUNCT__ "FiniOutput_Triangle"
3221 PetscErrorCode FiniOutput_Triangle(struct triangulateio *outputCtx)
3222 {
3223   PetscFunctionBegin;
3224   free(outputCtx->pointlist);
3225   free(outputCtx->pointmarkerlist);
3226   free(outputCtx->segmentlist);
3227   free(outputCtx->segmentmarkerlist);
3228   free(outputCtx->edgelist);
3229   free(outputCtx->edgemarkerlist);
3230   free(outputCtx->trianglelist);
3231   free(outputCtx->neighborlist);
3232   PetscFunctionReturn(0);
3233 }
3234 
3235 #undef __FUNCT__
3236 #define __FUNCT__ "DMPlexGenerate_Triangle"
3237 PetscErrorCode DMPlexGenerate_Triangle(DM boundary, PetscBool interpolate, DM *dm)
3238 {
3239   MPI_Comm             comm;
3240   PetscInt             dim              = 2;
3241   const PetscBool      createConvexHull = PETSC_FALSE;
3242   const PetscBool      constrained      = PETSC_FALSE;
3243   struct triangulateio in;
3244   struct triangulateio out;
3245   PetscInt             vStart, vEnd, v, eStart, eEnd, e;
3246   PetscMPIInt          rank;
3247   PetscErrorCode       ierr;
3248 
3249   PetscFunctionBegin;
3250   ierr = PetscObjectGetComm((PetscObject)boundary,&comm);CHKERRQ(ierr);
3251   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
3252   ierr = InitInput_Triangle(&in);CHKERRQ(ierr);
3253   ierr = InitOutput_Triangle(&out);CHKERRQ(ierr);
3254   ierr = DMPlexGetDepthStratum(boundary, 0, &vStart, &vEnd);CHKERRQ(ierr);
3255 
3256   in.numberofpoints = vEnd - vStart;
3257   if (in.numberofpoints > 0) {
3258     PetscSection coordSection;
3259     Vec          coordinates;
3260     PetscScalar *array;
3261 
3262     ierr = PetscMalloc1(in.numberofpoints*dim, &in.pointlist);CHKERRQ(ierr);
3263     ierr = PetscMalloc1(in.numberofpoints, &in.pointmarkerlist);CHKERRQ(ierr);
3264     ierr = DMGetCoordinatesLocal(boundary, &coordinates);CHKERRQ(ierr);
3265     ierr = DMGetCoordinateSection(boundary, &coordSection);CHKERRQ(ierr);
3266     ierr = VecGetArray(coordinates, &array);CHKERRQ(ierr);
3267     for (v = vStart; v < vEnd; ++v) {
3268       const PetscInt idx = v - vStart;
3269       PetscInt       off, d;
3270 
3271       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
3272       for (d = 0; d < dim; ++d) {
3273         in.pointlist[idx*dim + d] = PetscRealPart(array[off+d]);
3274       }
3275       ierr = DMPlexGetLabelValue(boundary, "marker", v, &in.pointmarkerlist[idx]);CHKERRQ(ierr);
3276     }
3277     ierr = VecRestoreArray(coordinates, &array);CHKERRQ(ierr);
3278   }
3279   ierr  = DMPlexGetHeightStratum(boundary, 0, &eStart, &eEnd);CHKERRQ(ierr);
3280   in.numberofsegments = eEnd - eStart;
3281   if (in.numberofsegments > 0) {
3282     ierr = PetscMalloc1(in.numberofsegments*2, &in.segmentlist);CHKERRQ(ierr);
3283     ierr = PetscMalloc1(in.numberofsegments, &in.segmentmarkerlist);CHKERRQ(ierr);
3284     for (e = eStart; e < eEnd; ++e) {
3285       const PetscInt  idx = e - eStart;
3286       const PetscInt *cone;
3287 
3288       ierr = DMPlexGetCone(boundary, e, &cone);CHKERRQ(ierr);
3289 
3290       in.segmentlist[idx*2+0] = cone[0] - vStart;
3291       in.segmentlist[idx*2+1] = cone[1] - vStart;
3292 
3293       ierr = DMPlexGetLabelValue(boundary, "marker", e, &in.segmentmarkerlist[idx]);CHKERRQ(ierr);
3294     }
3295   }
3296 #if 0 /* Do not currently support holes */
3297   PetscReal *holeCoords;
3298   PetscInt   h, d;
3299 
3300   ierr = DMPlexGetHoles(boundary, &in.numberofholes, &holeCords);CHKERRQ(ierr);
3301   if (in.numberofholes > 0) {
3302     ierr = PetscMalloc1(in.numberofholes*dim, &in.holelist);CHKERRQ(ierr);
3303     for (h = 0; h < in.numberofholes; ++h) {
3304       for (d = 0; d < dim; ++d) {
3305         in.holelist[h*dim+d] = holeCoords[h*dim+d];
3306       }
3307     }
3308   }
3309 #endif
3310   if (!rank) {
3311     char args[32];
3312 
3313     /* Take away 'Q' for verbose output */
3314     ierr = PetscStrcpy(args, "pqezQ");CHKERRQ(ierr);
3315     if (createConvexHull) {
3316       ierr = PetscStrcat(args, "c");CHKERRQ(ierr);
3317     }
3318     if (constrained) {
3319       ierr = PetscStrcpy(args, "zepDQ");CHKERRQ(ierr);
3320     }
3321     triangulate(args, &in, &out, NULL);
3322   }
3323   ierr = PetscFree(in.pointlist);CHKERRQ(ierr);
3324   ierr = PetscFree(in.pointmarkerlist);CHKERRQ(ierr);
3325   ierr = PetscFree(in.segmentlist);CHKERRQ(ierr);
3326   ierr = PetscFree(in.segmentmarkerlist);CHKERRQ(ierr);
3327   ierr = PetscFree(in.holelist);CHKERRQ(ierr);
3328 
3329   {
3330     const PetscInt numCorners  = 3;
3331     const PetscInt numCells    = out.numberoftriangles;
3332     const PetscInt numVertices = out.numberofpoints;
3333     const int     *cells      = out.trianglelist;
3334     const double  *meshCoords = out.pointlist;
3335 
3336     ierr = DMPlexCreateFromCellList(comm, dim, numCells, numVertices, numCorners, interpolate, cells, dim, meshCoords, dm);CHKERRQ(ierr);
3337     /* Set labels */
3338     for (v = 0; v < numVertices; ++v) {
3339       if (out.pointmarkerlist[v]) {
3340         ierr = DMPlexSetLabelValue(*dm, "marker", v+numCells, out.pointmarkerlist[v]);CHKERRQ(ierr);
3341       }
3342     }
3343     if (interpolate) {
3344       for (e = 0; e < out.numberofedges; e++) {
3345         if (out.edgemarkerlist[e]) {
3346           const PetscInt  vertices[2] = {out.edgelist[e*2+0]+numCells, out.edgelist[e*2+1]+numCells};
3347           const PetscInt *edges;
3348           PetscInt        numEdges;
3349 
3350           ierr = DMPlexGetJoin(*dm, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3351           if (numEdges != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Two vertices must cover only one edge, not %D", numEdges);
3352           ierr = DMPlexSetLabelValue(*dm, "marker", edges[0], out.edgemarkerlist[e]);CHKERRQ(ierr);
3353           ierr = DMPlexRestoreJoin(*dm, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3354         }
3355       }
3356     }
3357     ierr = DMPlexSetRefinementUniform(*dm, PETSC_FALSE);CHKERRQ(ierr);
3358   }
3359 #if 0 /* Do not currently support holes */
3360   ierr = DMPlexCopyHoles(*dm, boundary);CHKERRQ(ierr);
3361 #endif
3362   ierr = FiniOutput_Triangle(&out);CHKERRQ(ierr);
3363   PetscFunctionReturn(0);
3364 }
3365 
3366 #undef __FUNCT__
3367 #define __FUNCT__ "DMPlexRefine_Triangle"
3368 PetscErrorCode DMPlexRefine_Triangle(DM dm, double *maxVolumes, DM *dmRefined)
3369 {
3370   MPI_Comm             comm;
3371   PetscInt             dim  = 2;
3372   struct triangulateio in;
3373   struct triangulateio out;
3374   PetscInt             vStart, vEnd, v, cStart, cEnd, c, depth, depthGlobal;
3375   PetscMPIInt          rank;
3376   PetscErrorCode       ierr;
3377 
3378   PetscFunctionBegin;
3379   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
3380   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
3381   ierr = InitInput_Triangle(&in);CHKERRQ(ierr);
3382   ierr = InitOutput_Triangle(&out);CHKERRQ(ierr);
3383   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3384   ierr = MPI_Allreduce(&depth, &depthGlobal, 1, MPIU_INT, MPI_MAX, comm);CHKERRQ(ierr);
3385   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
3386 
3387   in.numberofpoints = vEnd - vStart;
3388   if (in.numberofpoints > 0) {
3389     PetscSection coordSection;
3390     Vec          coordinates;
3391     PetscScalar *array;
3392 
3393     ierr = PetscMalloc1(in.numberofpoints*dim, &in.pointlist);CHKERRQ(ierr);
3394     ierr = PetscMalloc1(in.numberofpoints, &in.pointmarkerlist);CHKERRQ(ierr);
3395     ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
3396     ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
3397     ierr = VecGetArray(coordinates, &array);CHKERRQ(ierr);
3398     for (v = vStart; v < vEnd; ++v) {
3399       const PetscInt idx = v - vStart;
3400       PetscInt       off, d;
3401 
3402       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
3403       for (d = 0; d < dim; ++d) {
3404         in.pointlist[idx*dim + d] = PetscRealPart(array[off+d]);
3405       }
3406       ierr = DMPlexGetLabelValue(dm, "marker", v, &in.pointmarkerlist[idx]);CHKERRQ(ierr);
3407     }
3408     ierr = VecRestoreArray(coordinates, &array);CHKERRQ(ierr);
3409   }
3410   ierr  = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
3411 
3412   in.numberofcorners   = 3;
3413   in.numberoftriangles = cEnd - cStart;
3414 
3415   in.trianglearealist  = (double*) maxVolumes;
3416   if (in.numberoftriangles > 0) {
3417     ierr = PetscMalloc1(in.numberoftriangles*in.numberofcorners, &in.trianglelist);CHKERRQ(ierr);
3418     for (c = cStart; c < cEnd; ++c) {
3419       const PetscInt idx      = c - cStart;
3420       PetscInt      *closure = NULL;
3421       PetscInt       closureSize;
3422 
3423       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
3424       if ((closureSize != 4) && (closureSize != 7)) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Mesh has cell which is not a triangle, %D vertices in closure", closureSize);
3425       for (v = 0; v < 3; ++v) {
3426         in.trianglelist[idx*in.numberofcorners + v] = closure[(v+closureSize-3)*2] - vStart;
3427       }
3428       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
3429     }
3430   }
3431   /* TODO: Segment markers are missing on input */
3432 #if 0 /* Do not currently support holes */
3433   PetscReal *holeCoords;
3434   PetscInt   h, d;
3435 
3436   ierr = DMPlexGetHoles(boundary, &in.numberofholes, &holeCords);CHKERRQ(ierr);
3437   if (in.numberofholes > 0) {
3438     ierr = PetscMalloc1(in.numberofholes*dim, &in.holelist);CHKERRQ(ierr);
3439     for (h = 0; h < in.numberofholes; ++h) {
3440       for (d = 0; d < dim; ++d) {
3441         in.holelist[h*dim+d] = holeCoords[h*dim+d];
3442       }
3443     }
3444   }
3445 #endif
3446   if (!rank) {
3447     char args[32];
3448 
3449     /* Take away 'Q' for verbose output */
3450     ierr = PetscStrcpy(args, "pqezQra");CHKERRQ(ierr);
3451     triangulate(args, &in, &out, NULL);
3452   }
3453   ierr = PetscFree(in.pointlist);CHKERRQ(ierr);
3454   ierr = PetscFree(in.pointmarkerlist);CHKERRQ(ierr);
3455   ierr = PetscFree(in.segmentlist);CHKERRQ(ierr);
3456   ierr = PetscFree(in.segmentmarkerlist);CHKERRQ(ierr);
3457   ierr = PetscFree(in.trianglelist);CHKERRQ(ierr);
3458 
3459   {
3460     const PetscInt numCorners  = 3;
3461     const PetscInt numCells    = out.numberoftriangles;
3462     const PetscInt numVertices = out.numberofpoints;
3463     const int     *cells      = out.trianglelist;
3464     const double  *meshCoords = out.pointlist;
3465     PetscBool      interpolate = depthGlobal > 1 ? PETSC_TRUE : PETSC_FALSE;
3466 
3467     ierr = DMPlexCreateFromCellList(comm, dim, numCells, numVertices, numCorners, interpolate, cells, dim, meshCoords, dmRefined);CHKERRQ(ierr);
3468     /* Set labels */
3469     for (v = 0; v < numVertices; ++v) {
3470       if (out.pointmarkerlist[v]) {
3471         ierr = DMPlexSetLabelValue(*dmRefined, "marker", v+numCells, out.pointmarkerlist[v]);CHKERRQ(ierr);
3472       }
3473     }
3474     if (interpolate) {
3475       PetscInt e;
3476 
3477       for (e = 0; e < out.numberofedges; e++) {
3478         if (out.edgemarkerlist[e]) {
3479           const PetscInt  vertices[2] = {out.edgelist[e*2+0]+numCells, out.edgelist[e*2+1]+numCells};
3480           const PetscInt *edges;
3481           PetscInt        numEdges;
3482 
3483           ierr = DMPlexGetJoin(*dmRefined, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3484           if (numEdges != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Two vertices must cover only one edge, not %D", numEdges);
3485           ierr = DMPlexSetLabelValue(*dmRefined, "marker", edges[0], out.edgemarkerlist[e]);CHKERRQ(ierr);
3486           ierr = DMPlexRestoreJoin(*dmRefined, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3487         }
3488       }
3489     }
3490     ierr = DMPlexSetRefinementUniform(*dmRefined, PETSC_FALSE);CHKERRQ(ierr);
3491   }
3492 #if 0 /* Do not currently support holes */
3493   ierr = DMPlexCopyHoles(*dm, boundary);CHKERRQ(ierr);
3494 #endif
3495   ierr = FiniOutput_Triangle(&out);CHKERRQ(ierr);
3496   PetscFunctionReturn(0);
3497 }
3498 #endif
3499 
3500 #if defined(PETSC_HAVE_TETGEN)
3501 #include <tetgen.h>
3502 #undef __FUNCT__
3503 #define __FUNCT__ "DMPlexGenerate_Tetgen"
3504 PetscErrorCode DMPlexGenerate_Tetgen(DM boundary, PetscBool interpolate, DM *dm)
3505 {
3506   MPI_Comm       comm;
3507   const PetscInt dim  = 3;
3508   ::tetgenio     in;
3509   ::tetgenio     out;
3510   PetscInt       vStart, vEnd, v, fStart, fEnd, f;
3511   PetscMPIInt    rank;
3512   PetscErrorCode ierr;
3513 
3514   PetscFunctionBegin;
3515   ierr              = PetscObjectGetComm((PetscObject)boundary,&comm);CHKERRQ(ierr);
3516   ierr              = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
3517   ierr              = DMPlexGetDepthStratum(boundary, 0, &vStart, &vEnd);CHKERRQ(ierr);
3518   in.numberofpoints = vEnd - vStart;
3519   if (in.numberofpoints > 0) {
3520     PetscSection coordSection;
3521     Vec          coordinates;
3522     PetscScalar *array;
3523 
3524     in.pointlist       = new double[in.numberofpoints*dim];
3525     in.pointmarkerlist = new int[in.numberofpoints];
3526 
3527     ierr = DMGetCoordinatesLocal(boundary, &coordinates);CHKERRQ(ierr);
3528     ierr = DMGetCoordinateSection(boundary, &coordSection);CHKERRQ(ierr);
3529     ierr = VecGetArray(coordinates, &array);CHKERRQ(ierr);
3530     for (v = vStart; v < vEnd; ++v) {
3531       const PetscInt idx = v - vStart;
3532       PetscInt       off, d;
3533 
3534       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
3535       for (d = 0; d < dim; ++d) in.pointlist[idx*dim + d] = array[off+d];
3536       ierr = DMPlexGetLabelValue(boundary, "marker", v, &in.pointmarkerlist[idx]);CHKERRQ(ierr);
3537     }
3538     ierr = VecRestoreArray(coordinates, &array);CHKERRQ(ierr);
3539   }
3540   ierr  = DMPlexGetHeightStratum(boundary, 0, &fStart, &fEnd);CHKERRQ(ierr);
3541 
3542   in.numberoffacets = fEnd - fStart;
3543   if (in.numberoffacets > 0) {
3544     in.facetlist       = new tetgenio::facet[in.numberoffacets];
3545     in.facetmarkerlist = new int[in.numberoffacets];
3546     for (f = fStart; f < fEnd; ++f) {
3547       const PetscInt idx     = f - fStart;
3548       PetscInt      *points = NULL, numPoints, p, numVertices = 0, v;
3549 
3550       in.facetlist[idx].numberofpolygons = 1;
3551       in.facetlist[idx].polygonlist      = new tetgenio::polygon[in.facetlist[idx].numberofpolygons];
3552       in.facetlist[idx].numberofholes    = 0;
3553       in.facetlist[idx].holelist         = NULL;
3554 
3555       ierr = DMPlexGetTransitiveClosure(boundary, f, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
3556       for (p = 0; p < numPoints*2; p += 2) {
3557         const PetscInt point = points[p];
3558         if ((point >= vStart) && (point < vEnd)) points[numVertices++] = point;
3559       }
3560 
3561       tetgenio::polygon *poly = in.facetlist[idx].polygonlist;
3562       poly->numberofvertices = numVertices;
3563       poly->vertexlist       = new int[poly->numberofvertices];
3564       for (v = 0; v < numVertices; ++v) {
3565         const PetscInt vIdx = points[v] - vStart;
3566         poly->vertexlist[v] = vIdx;
3567       }
3568       ierr = DMPlexGetLabelValue(boundary, "marker", f, &in.facetmarkerlist[idx]);CHKERRQ(ierr);
3569       ierr = DMPlexRestoreTransitiveClosure(boundary, f, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
3570     }
3571   }
3572   if (!rank) {
3573     char args[32];
3574 
3575     /* Take away 'Q' for verbose output */
3576     ierr = PetscStrcpy(args, "pqezQ");CHKERRQ(ierr);
3577     ::tetrahedralize(args, &in, &out);
3578   }
3579   {
3580     const PetscInt numCorners  = 4;
3581     const PetscInt numCells    = out.numberoftetrahedra;
3582     const PetscInt numVertices = out.numberofpoints;
3583     const double   *meshCoords = out.pointlist;
3584     int            *cells      = out.tetrahedronlist;
3585 
3586     ierr = DMPlexInvertCells_Internal(dim, numCells, numCorners, cells);CHKERRQ(ierr);
3587     ierr = DMPlexCreateFromCellList(comm, dim, numCells, numVertices, numCorners, interpolate, cells, dim, meshCoords, dm);CHKERRQ(ierr);
3588     /* Set labels */
3589     for (v = 0; v < numVertices; ++v) {
3590       if (out.pointmarkerlist[v]) {
3591         ierr = DMPlexSetLabelValue(*dm, "marker", v+numCells, out.pointmarkerlist[v]);CHKERRQ(ierr);
3592       }
3593     }
3594     if (interpolate) {
3595       PetscInt e;
3596 
3597       for (e = 0; e < out.numberofedges; e++) {
3598         if (out.edgemarkerlist[e]) {
3599           const PetscInt  vertices[2] = {out.edgelist[e*2+0]+numCells, out.edgelist[e*2+1]+numCells};
3600           const PetscInt *edges;
3601           PetscInt        numEdges;
3602 
3603           ierr = DMPlexGetJoin(*dm, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3604           if (numEdges != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Two vertices must cover only one edge, not %D", numEdges);
3605           ierr = DMPlexSetLabelValue(*dm, "marker", edges[0], out.edgemarkerlist[e]);CHKERRQ(ierr);
3606           ierr = DMPlexRestoreJoin(*dm, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3607         }
3608       }
3609       for (f = 0; f < out.numberoftrifaces; f++) {
3610         if (out.trifacemarkerlist[f]) {
3611           const PetscInt  vertices[3] = {out.trifacelist[f*3+0]+numCells, out.trifacelist[f*3+1]+numCells, out.trifacelist[f*3+2]+numCells};
3612           const PetscInt *faces;
3613           PetscInt        numFaces;
3614 
3615           ierr = DMPlexGetJoin(*dm, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
3616           if (numFaces != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Three vertices must cover only one face, not %D", numFaces);
3617           ierr = DMPlexSetLabelValue(*dm, "marker", faces[0], out.trifacemarkerlist[f]);CHKERRQ(ierr);
3618           ierr = DMPlexRestoreJoin(*dm, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
3619         }
3620       }
3621     }
3622     ierr = DMPlexSetRefinementUniform(*dm, PETSC_FALSE);CHKERRQ(ierr);
3623   }
3624   PetscFunctionReturn(0);
3625 }
3626 
3627 #undef __FUNCT__
3628 #define __FUNCT__ "DMPlexRefine_Tetgen"
3629 PetscErrorCode DMPlexRefine_Tetgen(DM dm, double *maxVolumes, DM *dmRefined)
3630 {
3631   MPI_Comm       comm;
3632   const PetscInt dim  = 3;
3633   ::tetgenio     in;
3634   ::tetgenio     out;
3635   PetscInt       vStart, vEnd, v, cStart, cEnd, c, depth, depthGlobal;
3636   PetscMPIInt    rank;
3637   PetscErrorCode ierr;
3638 
3639   PetscFunctionBegin;
3640   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
3641   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
3642   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3643   ierr = MPI_Allreduce(&depth, &depthGlobal, 1, MPIU_INT, MPI_MAX, comm);CHKERRQ(ierr);
3644   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
3645 
3646   in.numberofpoints = vEnd - vStart;
3647   if (in.numberofpoints > 0) {
3648     PetscSection coordSection;
3649     Vec          coordinates;
3650     PetscScalar *array;
3651 
3652     in.pointlist       = new double[in.numberofpoints*dim];
3653     in.pointmarkerlist = new int[in.numberofpoints];
3654 
3655     ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
3656     ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
3657     ierr = VecGetArray(coordinates, &array);CHKERRQ(ierr);
3658     for (v = vStart; v < vEnd; ++v) {
3659       const PetscInt idx = v - vStart;
3660       PetscInt       off, d;
3661 
3662       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
3663       for (d = 0; d < dim; ++d) in.pointlist[idx*dim + d] = array[off+d];
3664       ierr = DMPlexGetLabelValue(dm, "marker", v, &in.pointmarkerlist[idx]);CHKERRQ(ierr);
3665     }
3666     ierr = VecRestoreArray(coordinates, &array);CHKERRQ(ierr);
3667   }
3668   ierr  = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
3669 
3670   in.numberofcorners       = 4;
3671   in.numberoftetrahedra    = cEnd - cStart;
3672   in.tetrahedronvolumelist = (double*) maxVolumes;
3673   if (in.numberoftetrahedra > 0) {
3674     in.tetrahedronlist = new int[in.numberoftetrahedra*in.numberofcorners];
3675     for (c = cStart; c < cEnd; ++c) {
3676       const PetscInt idx      = c - cStart;
3677       PetscInt      *closure = NULL;
3678       PetscInt       closureSize;
3679 
3680       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
3681       if ((closureSize != 5) && (closureSize != 15)) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Mesh has cell which is not a tetrahedron, %D vertices in closure", closureSize);
3682       for (v = 0; v < 4; ++v) {
3683         in.tetrahedronlist[idx*in.numberofcorners + v] = closure[(v+closureSize-4)*2] - vStart;
3684       }
3685       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
3686     }
3687   }
3688   /* TODO: Put in boundary faces with markers */
3689   if (!rank) {
3690     char args[32];
3691 
3692     /* Take away 'Q' for verbose output */
3693     /*ierr = PetscStrcpy(args, "qezQra");CHKERRQ(ierr); */
3694     ierr = PetscStrcpy(args, "qezraVVVV");CHKERRQ(ierr);
3695     ::tetrahedralize(args, &in, &out);
3696   }
3697   in.tetrahedronvolumelist = NULL;
3698 
3699   {
3700     const PetscInt numCorners  = 4;
3701     const PetscInt numCells    = out.numberoftetrahedra;
3702     const PetscInt numVertices = out.numberofpoints;
3703     const double   *meshCoords = out.pointlist;
3704     int            *cells      = out.tetrahedronlist;
3705 
3706     PetscBool      interpolate = depthGlobal > 1 ? PETSC_TRUE : PETSC_FALSE;
3707 
3708     ierr = DMPlexInvertCells_Internal(dim, numCells, numCorners, cells);CHKERRQ(ierr);
3709     ierr = DMPlexCreateFromCellList(comm, dim, numCells, numVertices, numCorners, interpolate, cells, dim, meshCoords, dmRefined);CHKERRQ(ierr);
3710     /* Set labels */
3711     for (v = 0; v < numVertices; ++v) {
3712       if (out.pointmarkerlist[v]) {
3713         ierr = DMPlexSetLabelValue(*dmRefined, "marker", v+numCells, out.pointmarkerlist[v]);CHKERRQ(ierr);
3714       }
3715     }
3716     if (interpolate) {
3717       PetscInt e, f;
3718 
3719       for (e = 0; e < out.numberofedges; e++) {
3720         if (out.edgemarkerlist[e]) {
3721           const PetscInt  vertices[2] = {out.edgelist[e*2+0]+numCells, out.edgelist[e*2+1]+numCells};
3722           const PetscInt *edges;
3723           PetscInt        numEdges;
3724 
3725           ierr = DMPlexGetJoin(*dmRefined, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3726           if (numEdges != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Two vertices must cover only one edge, not %D", numEdges);
3727           ierr = DMPlexSetLabelValue(*dmRefined, "marker", edges[0], out.edgemarkerlist[e]);CHKERRQ(ierr);
3728           ierr = DMPlexRestoreJoin(*dmRefined, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3729         }
3730       }
3731       for (f = 0; f < out.numberoftrifaces; f++) {
3732         if (out.trifacemarkerlist[f]) {
3733           const PetscInt  vertices[3] = {out.trifacelist[f*3+0]+numCells, out.trifacelist[f*3+1]+numCells, out.trifacelist[f*3+2]+numCells};
3734           const PetscInt *faces;
3735           PetscInt        numFaces;
3736 
3737           ierr = DMPlexGetJoin(*dmRefined, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
3738           if (numFaces != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Three vertices must cover only one face, not %D", numFaces);
3739           ierr = DMPlexSetLabelValue(*dmRefined, "marker", faces[0], out.trifacemarkerlist[f]);CHKERRQ(ierr);
3740           ierr = DMPlexRestoreJoin(*dmRefined, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
3741         }
3742       }
3743     }
3744     ierr = DMPlexSetRefinementUniform(*dmRefined, PETSC_FALSE);CHKERRQ(ierr);
3745   }
3746   PetscFunctionReturn(0);
3747 }
3748 #endif
3749 
3750 #if defined(PETSC_HAVE_CTETGEN)
3751 #include <ctetgen.h>
3752 
3753 #undef __FUNCT__
3754 #define __FUNCT__ "DMPlexGenerate_CTetgen"
3755 PetscErrorCode DMPlexGenerate_CTetgen(DM boundary, PetscBool interpolate, DM *dm)
3756 {
3757   MPI_Comm       comm;
3758   const PetscInt dim  = 3;
3759   PLC           *in, *out;
3760   PetscInt       verbose = 0, vStart, vEnd, v, fStart, fEnd, f;
3761   PetscMPIInt    rank;
3762   PetscErrorCode ierr;
3763 
3764   PetscFunctionBegin;
3765   ierr = PetscObjectGetComm((PetscObject)boundary,&comm);CHKERRQ(ierr);
3766   ierr = PetscOptionsGetInt(((PetscObject) boundary)->prefix, "-ctetgen_verbose", &verbose, NULL);CHKERRQ(ierr);
3767   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
3768   ierr = DMPlexGetDepthStratum(boundary, 0, &vStart, &vEnd);CHKERRQ(ierr);
3769   ierr = PLCCreate(&in);CHKERRQ(ierr);
3770   ierr = PLCCreate(&out);CHKERRQ(ierr);
3771 
3772   in->numberofpoints = vEnd - vStart;
3773   if (in->numberofpoints > 0) {
3774     PetscSection coordSection;
3775     Vec          coordinates;
3776     PetscScalar *array;
3777 
3778     ierr = PetscMalloc1(in->numberofpoints*dim, &in->pointlist);CHKERRQ(ierr);
3779     ierr = PetscMalloc1(in->numberofpoints,       &in->pointmarkerlist);CHKERRQ(ierr);
3780     ierr = DMGetCoordinatesLocal(boundary, &coordinates);CHKERRQ(ierr);
3781     ierr = DMGetCoordinateSection(boundary, &coordSection);CHKERRQ(ierr);
3782     ierr = VecGetArray(coordinates, &array);CHKERRQ(ierr);
3783     for (v = vStart; v < vEnd; ++v) {
3784       const PetscInt idx = v - vStart;
3785       PetscInt       off, d, m;
3786 
3787       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
3788       for (d = 0; d < dim; ++d) {
3789         in->pointlist[idx*dim + d] = PetscRealPart(array[off+d]);
3790       }
3791       ierr = DMPlexGetLabelValue(boundary, "marker", v, &m);CHKERRQ(ierr);
3792 
3793       in->pointmarkerlist[idx] = (int) m;
3794     }
3795     ierr = VecRestoreArray(coordinates, &array);CHKERRQ(ierr);
3796   }
3797   ierr  = DMPlexGetHeightStratum(boundary, 0, &fStart, &fEnd);CHKERRQ(ierr);
3798 
3799   in->numberoffacets = fEnd - fStart;
3800   if (in->numberoffacets > 0) {
3801     ierr = PetscMalloc1(in->numberoffacets, &in->facetlist);CHKERRQ(ierr);
3802     ierr = PetscMalloc1(in->numberoffacets,   &in->facetmarkerlist);CHKERRQ(ierr);
3803     for (f = fStart; f < fEnd; ++f) {
3804       const PetscInt idx     = f - fStart;
3805       PetscInt      *points = NULL, numPoints, p, numVertices = 0, v, m;
3806       polygon       *poly;
3807 
3808       in->facetlist[idx].numberofpolygons = 1;
3809 
3810       ierr = PetscMalloc1(in->facetlist[idx].numberofpolygons, &in->facetlist[idx].polygonlist);CHKERRQ(ierr);
3811 
3812       in->facetlist[idx].numberofholes    = 0;
3813       in->facetlist[idx].holelist         = NULL;
3814 
3815       ierr = DMPlexGetTransitiveClosure(boundary, f, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
3816       for (p = 0; p < numPoints*2; p += 2) {
3817         const PetscInt point = points[p];
3818         if ((point >= vStart) && (point < vEnd)) points[numVertices++] = point;
3819       }
3820 
3821       poly                   = in->facetlist[idx].polygonlist;
3822       poly->numberofvertices = numVertices;
3823       ierr                   = PetscMalloc1(poly->numberofvertices, &poly->vertexlist);CHKERRQ(ierr);
3824       for (v = 0; v < numVertices; ++v) {
3825         const PetscInt vIdx = points[v] - vStart;
3826         poly->vertexlist[v] = vIdx;
3827       }
3828       ierr                     = DMPlexGetLabelValue(boundary, "marker", f, &m);CHKERRQ(ierr);
3829       in->facetmarkerlist[idx] = (int) m;
3830       ierr                     = DMPlexRestoreTransitiveClosure(boundary, f, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
3831     }
3832   }
3833   if (!rank) {
3834     TetGenOpts t;
3835 
3836     ierr        = TetGenOptsInitialize(&t);CHKERRQ(ierr);
3837     t.in        = boundary; /* Should go away */
3838     t.plc       = 1;
3839     t.quality   = 1;
3840     t.edgesout  = 1;
3841     t.zeroindex = 1;
3842     t.quiet     = 1;
3843     t.verbose   = verbose;
3844     ierr        = TetGenCheckOpts(&t);CHKERRQ(ierr);
3845     ierr        = TetGenTetrahedralize(&t, in, out);CHKERRQ(ierr);
3846   }
3847   {
3848     const PetscInt numCorners  = 4;
3849     const PetscInt numCells    = out->numberoftetrahedra;
3850     const PetscInt numVertices = out->numberofpoints;
3851     const double   *meshCoords = out->pointlist;
3852     int            *cells      = out->tetrahedronlist;
3853 
3854     ierr = DMPlexInvertCells_Internal(dim, numCells, numCorners, cells);CHKERRQ(ierr);
3855     ierr = DMPlexCreateFromCellList(comm, dim, numCells, numVertices, numCorners, interpolate, cells, dim, meshCoords, dm);CHKERRQ(ierr);
3856     /* Set labels */
3857     for (v = 0; v < numVertices; ++v) {
3858       if (out->pointmarkerlist[v]) {
3859         ierr = DMPlexSetLabelValue(*dm, "marker", v+numCells, out->pointmarkerlist[v]);CHKERRQ(ierr);
3860       }
3861     }
3862     if (interpolate) {
3863       PetscInt e;
3864 
3865       for (e = 0; e < out->numberofedges; e++) {
3866         if (out->edgemarkerlist[e]) {
3867           const PetscInt  vertices[2] = {out->edgelist[e*2+0]+numCells, out->edgelist[e*2+1]+numCells};
3868           const PetscInt *edges;
3869           PetscInt        numEdges;
3870 
3871           ierr = DMPlexGetJoin(*dm, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3872           if (numEdges != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Two vertices must cover only one edge, not %D", numEdges);
3873           ierr = DMPlexSetLabelValue(*dm, "marker", edges[0], out->edgemarkerlist[e]);CHKERRQ(ierr);
3874           ierr = DMPlexRestoreJoin(*dm, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3875         }
3876       }
3877       for (f = 0; f < out->numberoftrifaces; f++) {
3878         if (out->trifacemarkerlist[f]) {
3879           const PetscInt  vertices[3] = {out->trifacelist[f*3+0]+numCells, out->trifacelist[f*3+1]+numCells, out->trifacelist[f*3+2]+numCells};
3880           const PetscInt *faces;
3881           PetscInt        numFaces;
3882 
3883           ierr = DMPlexGetFullJoin(*dm, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
3884           if (numFaces != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Three vertices must cover only one face, not %D", numFaces);
3885           ierr = DMPlexSetLabelValue(*dm, "marker", faces[0], out->trifacemarkerlist[f]);CHKERRQ(ierr);
3886           ierr = DMPlexRestoreJoin(*dm, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
3887         }
3888       }
3889     }
3890     ierr = DMPlexSetRefinementUniform(*dm, PETSC_FALSE);CHKERRQ(ierr);
3891   }
3892 
3893   ierr = PLCDestroy(&in);CHKERRQ(ierr);
3894   ierr = PLCDestroy(&out);CHKERRQ(ierr);
3895   PetscFunctionReturn(0);
3896 }
3897 
3898 #undef __FUNCT__
3899 #define __FUNCT__ "DMPlexRefine_CTetgen"
3900 PetscErrorCode DMPlexRefine_CTetgen(DM dm, PetscReal *maxVolumes, DM *dmRefined)
3901 {
3902   MPI_Comm       comm;
3903   const PetscInt dim  = 3;
3904   PLC           *in, *out;
3905   PetscInt       verbose = 0, vStart, vEnd, v, cStart, cEnd, c, depth, depthGlobal;
3906   PetscMPIInt    rank;
3907   PetscErrorCode ierr;
3908 
3909   PetscFunctionBegin;
3910   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
3911   ierr = PetscOptionsGetInt(((PetscObject) dm)->prefix, "-ctetgen_verbose", &verbose, NULL);CHKERRQ(ierr);
3912   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
3913   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3914   ierr = MPI_Allreduce(&depth, &depthGlobal, 1, MPIU_INT, MPI_MAX, comm);CHKERRQ(ierr);
3915   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
3916   ierr = PLCCreate(&in);CHKERRQ(ierr);
3917   ierr = PLCCreate(&out);CHKERRQ(ierr);
3918 
3919   in->numberofpoints = vEnd - vStart;
3920   if (in->numberofpoints > 0) {
3921     PetscSection coordSection;
3922     Vec          coordinates;
3923     PetscScalar *array;
3924 
3925     ierr = PetscMalloc1(in->numberofpoints*dim, &in->pointlist);CHKERRQ(ierr);
3926     ierr = PetscMalloc1(in->numberofpoints,       &in->pointmarkerlist);CHKERRQ(ierr);
3927     ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
3928     ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
3929     ierr = VecGetArray(coordinates, &array);CHKERRQ(ierr);
3930     for (v = vStart; v < vEnd; ++v) {
3931       const PetscInt idx = v - vStart;
3932       PetscInt       off, d, m;
3933 
3934       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
3935       for (d = 0; d < dim; ++d) {
3936         in->pointlist[idx*dim + d] = PetscRealPart(array[off+d]);
3937       }
3938       ierr = DMPlexGetLabelValue(dm, "marker", v, &m);CHKERRQ(ierr);
3939 
3940       in->pointmarkerlist[idx] = (int) m;
3941     }
3942     ierr = VecRestoreArray(coordinates, &array);CHKERRQ(ierr);
3943   }
3944   ierr  = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
3945 
3946   in->numberofcorners       = 4;
3947   in->numberoftetrahedra    = cEnd - cStart;
3948   in->tetrahedronvolumelist = maxVolumes;
3949   if (in->numberoftetrahedra > 0) {
3950     ierr = PetscMalloc1(in->numberoftetrahedra*in->numberofcorners, &in->tetrahedronlist);CHKERRQ(ierr);
3951     for (c = cStart; c < cEnd; ++c) {
3952       const PetscInt idx      = c - cStart;
3953       PetscInt      *closure = NULL;
3954       PetscInt       closureSize;
3955 
3956       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
3957       if ((closureSize != 5) && (closureSize != 15)) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Mesh has cell which is not a tetrahedron, %D vertices in closure", closureSize);
3958       for (v = 0; v < 4; ++v) {
3959         in->tetrahedronlist[idx*in->numberofcorners + v] = closure[(v+closureSize-4)*2] - vStart;
3960       }
3961       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
3962     }
3963   }
3964   if (!rank) {
3965     TetGenOpts t;
3966 
3967     ierr = TetGenOptsInitialize(&t);CHKERRQ(ierr);
3968 
3969     t.in        = dm; /* Should go away */
3970     t.refine    = 1;
3971     t.varvolume = 1;
3972     t.quality   = 1;
3973     t.edgesout  = 1;
3974     t.zeroindex = 1;
3975     t.quiet     = 1;
3976     t.verbose   = verbose; /* Change this */
3977 
3978     ierr = TetGenCheckOpts(&t);CHKERRQ(ierr);
3979     ierr = TetGenTetrahedralize(&t, in, out);CHKERRQ(ierr);
3980   }
3981   {
3982     const PetscInt numCorners  = 4;
3983     const PetscInt numCells    = out->numberoftetrahedra;
3984     const PetscInt numVertices = out->numberofpoints;
3985     const double   *meshCoords = out->pointlist;
3986     int            *cells      = out->tetrahedronlist;
3987     PetscBool      interpolate = depthGlobal > 1 ? PETSC_TRUE : PETSC_FALSE;
3988 
3989     ierr = DMPlexInvertCells_Internal(dim, numCells, numCorners, cells);CHKERRQ(ierr);
3990     ierr = DMPlexCreateFromCellList(comm, dim, numCells, numVertices, numCorners, interpolate, cells, dim, meshCoords, dmRefined);CHKERRQ(ierr);
3991     /* Set labels */
3992     for (v = 0; v < numVertices; ++v) {
3993       if (out->pointmarkerlist[v]) {
3994         ierr = DMPlexSetLabelValue(*dmRefined, "marker", v+numCells, out->pointmarkerlist[v]);CHKERRQ(ierr);
3995       }
3996     }
3997     if (interpolate) {
3998       PetscInt e, f;
3999 
4000       for (e = 0; e < out->numberofedges; e++) {
4001         if (out->edgemarkerlist[e]) {
4002           const PetscInt  vertices[2] = {out->edgelist[e*2+0]+numCells, out->edgelist[e*2+1]+numCells};
4003           const PetscInt *edges;
4004           PetscInt        numEdges;
4005 
4006           ierr = DMPlexGetJoin(*dmRefined, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
4007           if (numEdges != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Two vertices must cover only one edge, not %D", numEdges);
4008           ierr = DMPlexSetLabelValue(*dmRefined, "marker", edges[0], out->edgemarkerlist[e]);CHKERRQ(ierr);
4009           ierr = DMPlexRestoreJoin(*dmRefined, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
4010         }
4011       }
4012       for (f = 0; f < out->numberoftrifaces; f++) {
4013         if (out->trifacemarkerlist[f]) {
4014           const PetscInt  vertices[3] = {out->trifacelist[f*3+0]+numCells, out->trifacelist[f*3+1]+numCells, out->trifacelist[f*3+2]+numCells};
4015           const PetscInt *faces;
4016           PetscInt        numFaces;
4017 
4018           ierr = DMPlexGetFullJoin(*dmRefined, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
4019           if (numFaces != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Three vertices must cover only one face, not %D", numFaces);
4020           ierr = DMPlexSetLabelValue(*dmRefined, "marker", faces[0], out->trifacemarkerlist[f]);CHKERRQ(ierr);
4021           ierr = DMPlexRestoreJoin(*dmRefined, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
4022         }
4023       }
4024     }
4025     ierr = DMPlexSetRefinementUniform(*dmRefined, PETSC_FALSE);CHKERRQ(ierr);
4026   }
4027   ierr = PLCDestroy(&in);CHKERRQ(ierr);
4028   ierr = PLCDestroy(&out);CHKERRQ(ierr);
4029   PetscFunctionReturn(0);
4030 }
4031 #endif
4032 
4033 #undef __FUNCT__
4034 #define __FUNCT__ "DMPlexGenerate"
4035 /*@C
4036   DMPlexGenerate - Generates a mesh.
4037 
4038   Not Collective
4039 
4040   Input Parameters:
4041 + boundary - The DMPlex boundary object
4042 . name - The mesh generation package name
4043 - interpolate - Flag to create intermediate mesh elements
4044 
4045   Output Parameter:
4046 . mesh - The DMPlex object
4047 
4048   Level: intermediate
4049 
4050 .keywords: mesh, elements
4051 .seealso: DMPlexCreate(), DMRefine()
4052 @*/
4053 PetscErrorCode DMPlexGenerate(DM boundary, const char name[], PetscBool interpolate, DM *mesh)
4054 {
4055   PetscInt       dim;
4056   char           genname[1024];
4057   PetscBool      isTriangle = PETSC_FALSE, isTetgen = PETSC_FALSE, isCTetgen = PETSC_FALSE, flg;
4058   PetscErrorCode ierr;
4059 
4060   PetscFunctionBegin;
4061   PetscValidHeaderSpecific(boundary, DM_CLASSID, 1);
4062   PetscValidLogicalCollectiveBool(boundary, interpolate, 2);
4063   ierr = DMPlexGetDimension(boundary, &dim);CHKERRQ(ierr);
4064   ierr = PetscOptionsGetString(((PetscObject) boundary)->prefix, "-dm_plex_generator", genname, 1024, &flg);CHKERRQ(ierr);
4065   if (flg) name = genname;
4066   if (name) {
4067     ierr = PetscStrcmp(name, "triangle", &isTriangle);CHKERRQ(ierr);
4068     ierr = PetscStrcmp(name, "tetgen",   &isTetgen);CHKERRQ(ierr);
4069     ierr = PetscStrcmp(name, "ctetgen",  &isCTetgen);CHKERRQ(ierr);
4070   }
4071   switch (dim) {
4072   case 1:
4073     if (!name || isTriangle) {
4074 #if defined(PETSC_HAVE_TRIANGLE)
4075       ierr = DMPlexGenerate_Triangle(boundary, interpolate, mesh);CHKERRQ(ierr);
4076 #else
4077       SETERRQ(PetscObjectComm((PetscObject)boundary), PETSC_ERR_SUP, "Mesh generation needs external package support.\nPlease reconfigure with --download-triangle.");
4078 #endif
4079     } else SETERRQ1(PetscObjectComm((PetscObject)boundary), PETSC_ERR_SUP, "Unknown 2D mesh generation package %s", name);
4080     break;
4081   case 2:
4082     if (!name || isCTetgen) {
4083 #if defined(PETSC_HAVE_CTETGEN)
4084       ierr = DMPlexGenerate_CTetgen(boundary, interpolate, mesh);CHKERRQ(ierr);
4085 #else
4086       SETERRQ(PetscObjectComm((PetscObject)boundary), PETSC_ERR_SUP, "CTetgen needs external package support.\nPlease reconfigure with --download-ctetgen.");
4087 #endif
4088     } else if (isTetgen) {
4089 #if defined(PETSC_HAVE_TETGEN)
4090       ierr = DMPlexGenerate_Tetgen(boundary, interpolate, mesh);CHKERRQ(ierr);
4091 #else
4092       SETERRQ(PetscObjectComm((PetscObject)boundary), PETSC_ERR_SUP, "Tetgen needs external package support.\nPlease reconfigure with --with-c-language=cxx --download-tetgen.");
4093 #endif
4094     } else SETERRQ1(PetscObjectComm((PetscObject)boundary), PETSC_ERR_SUP, "Unknown 3D mesh generation package %s", name);
4095     break;
4096   default:
4097     SETERRQ1(PetscObjectComm((PetscObject)boundary), PETSC_ERR_SUP, "Mesh generation for a dimension %d boundary is not supported.", dim);
4098   }
4099   PetscFunctionReturn(0);
4100 }
4101 
4102 #undef __FUNCT__
4103 #define __FUNCT__ "DMRefine_Plex"
4104 PetscErrorCode DMRefine_Plex(DM dm, MPI_Comm comm, DM *dmRefined)
4105 {
4106   PetscReal      refinementLimit;
4107   PetscInt       dim, cStart, cEnd;
4108   char           genname[1024], *name = NULL;
4109   PetscBool      isUniform, isTriangle = PETSC_FALSE, isTetgen = PETSC_FALSE, isCTetgen = PETSC_FALSE, flg;
4110   PetscErrorCode ierr;
4111 
4112   PetscFunctionBegin;
4113   ierr = DMPlexGetRefinementUniform(dm, &isUniform);CHKERRQ(ierr);
4114   if (isUniform) {
4115     CellRefiner cellRefiner;
4116 
4117     ierr = DMPlexGetCellRefiner_Internal(dm, &cellRefiner);CHKERRQ(ierr);
4118     ierr = DMPlexRefineUniform_Internal(dm, cellRefiner, dmRefined);CHKERRQ(ierr);
4119     PetscFunctionReturn(0);
4120   }
4121   ierr = DMPlexGetRefinementLimit(dm, &refinementLimit);CHKERRQ(ierr);
4122   if (refinementLimit == 0.0) PetscFunctionReturn(0);
4123   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
4124   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
4125   ierr = PetscOptionsGetString(((PetscObject) dm)->prefix, "-dm_plex_generator", genname, 1024, &flg);CHKERRQ(ierr);
4126   if (flg) name = genname;
4127   if (name) {
4128     ierr = PetscStrcmp(name, "triangle", &isTriangle);CHKERRQ(ierr);
4129     ierr = PetscStrcmp(name, "tetgen",   &isTetgen);CHKERRQ(ierr);
4130     ierr = PetscStrcmp(name, "ctetgen",  &isCTetgen);CHKERRQ(ierr);
4131   }
4132   switch (dim) {
4133   case 2:
4134     if (!name || isTriangle) {
4135 #if defined(PETSC_HAVE_TRIANGLE)
4136       double  *maxVolumes;
4137       PetscInt c;
4138 
4139       ierr = PetscMalloc1((cEnd - cStart), &maxVolumes);CHKERRQ(ierr);
4140       for (c = 0; c < cEnd-cStart; ++c) maxVolumes[c] = refinementLimit;
4141       ierr = DMPlexRefine_Triangle(dm, maxVolumes, dmRefined);CHKERRQ(ierr);
4142       ierr = PetscFree(maxVolumes);CHKERRQ(ierr);
4143 #else
4144       SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Mesh refinement needs external package support.\nPlease reconfigure with --download-triangle.");
4145 #endif
4146     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Unknown 2D mesh generation package %s", name);
4147     break;
4148   case 3:
4149     if (!name || isCTetgen) {
4150 #if defined(PETSC_HAVE_CTETGEN)
4151       PetscReal *maxVolumes;
4152       PetscInt   c;
4153 
4154       ierr = PetscMalloc1((cEnd - cStart), &maxVolumes);CHKERRQ(ierr);
4155       for (c = 0; c < cEnd-cStart; ++c) maxVolumes[c] = refinementLimit;
4156       ierr = DMPlexRefine_CTetgen(dm, maxVolumes, dmRefined);CHKERRQ(ierr);
4157 #else
4158       SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "CTetgen needs external package support.\nPlease reconfigure with --download-ctetgen.");
4159 #endif
4160     } else if (isTetgen) {
4161 #if defined(PETSC_HAVE_TETGEN)
4162       double  *maxVolumes;
4163       PetscInt c;
4164 
4165       ierr = PetscMalloc1((cEnd - cStart), &maxVolumes);CHKERRQ(ierr);
4166       for (c = 0; c < cEnd-cStart; ++c) maxVolumes[c] = refinementLimit;
4167       ierr = DMPlexRefine_Tetgen(dm, maxVolumes, dmRefined);CHKERRQ(ierr);
4168 #else
4169       SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Tetgen needs external package support.\nPlease reconfigure with --with-c-language=cxx --download-tetgen.");
4170 #endif
4171     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Unknown 3D mesh generation package %s", name);
4172     break;
4173   default:
4174     SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Mesh refinement in dimension %d is not supported.", dim);
4175   }
4176   PetscFunctionReturn(0);
4177 }
4178 
4179 #undef __FUNCT__
4180 #define __FUNCT__ "DMRefineHierarchy_Plex"
4181 PetscErrorCode DMRefineHierarchy_Plex(DM dm, PetscInt nlevels, DM dmRefined[])
4182 {
4183   DM             cdm = dm;
4184   PetscInt       r;
4185   PetscBool      isUniform;
4186   PetscErrorCode ierr;
4187 
4188   PetscFunctionBegin;
4189   ierr = DMPlexGetRefinementUniform(dm, &isUniform);CHKERRQ(ierr);
4190   if (!isUniform) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Non-uniform refinement is incompatible with the hierarchy");
4191   for (r = 0; r < nlevels; ++r) {
4192     CellRefiner cellRefiner;
4193 
4194     ierr = DMPlexGetCellRefiner_Internal(cdm, &cellRefiner);CHKERRQ(ierr);
4195     ierr = DMPlexRefineUniform_Internal(cdm, cellRefiner, &dmRefined[r]);CHKERRQ(ierr);
4196     ierr = DMPlexSetCoarseDM(dmRefined[r], cdm);CHKERRQ(ierr);
4197     cdm  = dmRefined[r];
4198   }
4199   PetscFunctionReturn(0);
4200 }
4201 
4202 #undef __FUNCT__
4203 #define __FUNCT__ "DMCoarsen_Plex"
4204 PetscErrorCode DMCoarsen_Plex(DM dm, MPI_Comm comm, DM *dmCoarsened)
4205 {
4206   DM_Plex       *mesh = (DM_Plex*) dm->data;
4207   PetscErrorCode ierr;
4208 
4209   PetscFunctionBegin;
4210   ierr = PetscObjectReference((PetscObject) mesh->coarseMesh);CHKERRQ(ierr);
4211   *dmCoarsened = mesh->coarseMesh;
4212   PetscFunctionReturn(0);
4213 }
4214 
4215 #undef __FUNCT__
4216 #define __FUNCT__ "DMPlexLocalizeCoordinate_Internal"
4217 PetscErrorCode DMPlexLocalizeCoordinate_Internal(DM dm, PetscInt dim, const PetscScalar anchor[], const PetscScalar in[], PetscScalar out[])
4218 {
4219   PetscInt d;
4220 
4221   PetscFunctionBegin;
4222   if (!dm->maxCell) {
4223     for (d = 0; d < dim; ++d) out[d] = in[d];
4224   } else {
4225     for (d = 0; d < dim; ++d) {
4226       if (PetscAbsScalar(anchor[d] - in[d]) > dm->maxCell[d]) {
4227         out[d] = PetscRealPart(anchor[d]) > PetscRealPart(in[d]) ? dm->L[d] + in[d] : in[d] - dm->L[d];
4228       } else {
4229         out[d] = in[d];
4230       }
4231     }
4232   }
4233   PetscFunctionReturn(0);
4234 }
4235 
4236 #undef __FUNCT__
4237 #define __FUNCT__ "DMPlexLocalizeAddCoordinate_Internal"
4238 PetscErrorCode DMPlexLocalizeAddCoordinate_Internal(DM dm, PetscInt dim, const PetscScalar anchor[], const PetscScalar in[], PetscScalar out[])
4239 {
4240   PetscInt d;
4241 
4242   PetscFunctionBegin;
4243   if (!dm->maxCell) {
4244     for (d = 0; d < dim; ++d) out[d] += in[d];
4245   } else {
4246     for (d = 0; d < dim; ++d) {
4247       if (PetscAbsScalar(anchor[d] - in[d]) > dm->maxCell[d]) {
4248         out[d] += PetscRealPart(anchor[d]) > PetscRealPart(in[d]) ? dm->L[d] + in[d] : in[d] - dm->L[d];
4249       } else {
4250         out[d] += in[d];
4251       }
4252     }
4253   }
4254   PetscFunctionReturn(0);
4255 }
4256 
4257 #undef __FUNCT__
4258 #define __FUNCT__ "DMPlexLocalizeCoordinates"
4259 PetscErrorCode DMPlexLocalizeCoordinates(DM dm)
4260 {
4261   PetscSection   coordSection, cSection;
4262   Vec            coordinates,  cVec;
4263   PetscScalar   *coords, *coords2, *anchor;
4264   PetscInt       cStart, cEnd, c, vStart, vEnd, v, dof, d, off, off2, bs, coordSize;
4265   PetscErrorCode ierr;
4266 
4267   PetscFunctionBegin;
4268   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4269   if (!dm->maxCell) PetscFunctionReturn(0);
4270   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
4271   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
4272   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
4273   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
4274   ierr = PetscSectionCreate(PetscObjectComm((PetscObject) dm), &cSection);CHKERRQ(ierr);
4275   ierr = PetscSectionSetChart(cSection, cStart, vEnd);CHKERRQ(ierr);
4276   for (v = vStart; v < vEnd; ++v) {
4277     ierr = PetscSectionGetDof(coordSection, v, &dof);CHKERRQ(ierr);
4278     ierr = PetscSectionSetDof(cSection,     v,  dof);CHKERRQ(ierr);
4279   }
4280   for (c = cStart; c < cEnd; ++c) {
4281     ierr = DMPlexVecGetClosure(dm, coordSection, coordinates, c, &dof, NULL);CHKERRQ(ierr);
4282     ierr = PetscSectionSetDof(cSection, c, dof);CHKERRQ(ierr);
4283   }
4284   ierr = PetscSectionSetUp(cSection);CHKERRQ(ierr);
4285   ierr = PetscSectionGetStorageSize(cSection, &coordSize);CHKERRQ(ierr);
4286   ierr = VecCreate(PetscObjectComm((PetscObject) dm), &cVec);CHKERRQ(ierr);
4287   ierr = VecGetBlockSize(coordinates, &bs);CHKERRQ(ierr);
4288   ierr = VecSetBlockSize(cVec,         bs);CHKERRQ(ierr);
4289   ierr = VecSetSizes(cVec, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
4290   ierr = VecSetType(cVec,VECSTANDARD);CHKERRQ(ierr);
4291   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
4292   ierr = VecGetArray(cVec,        &coords2);CHKERRQ(ierr);
4293   for (v = vStart; v < vEnd; ++v) {
4294     ierr = PetscSectionGetDof(coordSection, v, &dof);CHKERRQ(ierr);
4295     ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
4296     ierr = PetscSectionGetOffset(cSection,     v, &off2);CHKERRQ(ierr);
4297     for (d = 0; d < dof; ++d) coords2[off2+d] = coords[off+d];
4298   }
4299   ierr = DMGetWorkArray(dm, 3, PETSC_SCALAR, &anchor);CHKERRQ(ierr);
4300   for (c = cStart; c < cEnd; ++c) {
4301     PetscScalar *cellCoords = NULL;
4302     PetscInt     b;
4303 
4304     ierr = DMPlexVecGetClosure(dm, coordSection, coordinates, c, &dof, &cellCoords);CHKERRQ(ierr);
4305     ierr = PetscSectionGetOffset(cSection, c, &off2);CHKERRQ(ierr);
4306     for (b = 0; b < bs; ++b) anchor[b] = cellCoords[b];
4307     for (d = 0; d < dof/bs; ++d) {ierr = DMPlexLocalizeCoordinate_Internal(dm, bs, anchor, &cellCoords[d*bs], &coords2[off2+d*bs]);CHKERRQ(ierr);}
4308     ierr = DMPlexVecRestoreClosure(dm, coordSection, coordinates, c, &dof, &cellCoords);CHKERRQ(ierr);
4309   }
4310   ierr = DMRestoreWorkArray(dm, 3, PETSC_SCALAR, &anchor);CHKERRQ(ierr);
4311   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
4312   ierr = VecRestoreArray(cVec,        &coords2);CHKERRQ(ierr);
4313   ierr = DMSetCoordinateSection(dm, cSection);CHKERRQ(ierr);
4314   ierr = DMSetCoordinatesLocal(dm, cVec);CHKERRQ(ierr);
4315   ierr = VecDestroy(&cVec);CHKERRQ(ierr);
4316   ierr = PetscSectionDestroy(&cSection);CHKERRQ(ierr);
4317   PetscFunctionReturn(0);
4318 }
4319 
4320 #undef __FUNCT__
4321 #define __FUNCT__ "DMPlexGetDepthLabel"
4322 /*@
4323   DMPlexGetDepthLabel - Get the DMLabel recording the depth of each point
4324 
4325   Not Collective
4326 
4327   Input Parameter:
4328 . dm    - The DMPlex object
4329 
4330   Output Parameter:
4331 . depthLabel - The DMLabel recording point depth
4332 
4333   Level: developer
4334 
4335 .keywords: mesh, points
4336 .seealso: DMPlexGetDepth(), DMPlexGetHeightStratum(), DMPlexGetDepthStratum()
4337 @*/
4338 PetscErrorCode DMPlexGetDepthLabel(DM dm, DMLabel *depthLabel)
4339 {
4340   DM_Plex       *mesh = (DM_Plex*) dm->data;
4341   PetscErrorCode ierr;
4342 
4343   PetscFunctionBegin;
4344   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4345   PetscValidPointer(depthLabel, 2);
4346   if (!mesh->depthLabel) {ierr = DMPlexGetLabel(dm, "depth", &mesh->depthLabel);CHKERRQ(ierr);}
4347   *depthLabel = mesh->depthLabel;
4348   PetscFunctionReturn(0);
4349 }
4350 
4351 #undef __FUNCT__
4352 #define __FUNCT__ "DMPlexGetDepth"
4353 /*@
4354   DMPlexGetDepth - Get the depth of the DAG representing this mesh
4355 
4356   Not Collective
4357 
4358   Input Parameter:
4359 . dm    - The DMPlex object
4360 
4361   Output Parameter:
4362 . depth - The number of strata (breadth first levels) in the DAG
4363 
4364   Level: developer
4365 
4366 .keywords: mesh, points
4367 .seealso: DMPlexGetDepthLabel(), DMPlexGetHeightStratum(), DMPlexGetDepthStratum()
4368 @*/
4369 PetscErrorCode DMPlexGetDepth(DM dm, PetscInt *depth)
4370 {
4371   DMLabel        label;
4372   PetscInt       d = 0;
4373   PetscErrorCode ierr;
4374 
4375   PetscFunctionBegin;
4376   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4377   PetscValidPointer(depth, 2);
4378   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
4379   if (label) {ierr = DMLabelGetNumValues(label, &d);CHKERRQ(ierr);}
4380   *depth = d-1;
4381   PetscFunctionReturn(0);
4382 }
4383 
4384 #undef __FUNCT__
4385 #define __FUNCT__ "DMPlexGetDepthStratum"
4386 /*@
4387   DMPlexGetDepthStratum - Get the bounds [start, end) for all points at a certain depth.
4388 
4389   Not Collective
4390 
4391   Input Parameters:
4392 + dm           - The DMPlex object
4393 - stratumValue - The requested depth
4394 
4395   Output Parameters:
4396 + start - The first point at this depth
4397 - end   - One beyond the last point at this depth
4398 
4399   Level: developer
4400 
4401 .keywords: mesh, points
4402 .seealso: DMPlexGetHeightStratum(), DMPlexGetDepth()
4403 @*/
4404 PetscErrorCode DMPlexGetDepthStratum(DM dm, PetscInt stratumValue, PetscInt *start, PetscInt *end)
4405 {
4406   DMLabel        label;
4407   PetscInt       pStart, pEnd;
4408   PetscErrorCode ierr;
4409 
4410   PetscFunctionBegin;
4411   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4412   if (start) {PetscValidPointer(start, 3); *start = 0;}
4413   if (end)   {PetscValidPointer(end,   4); *end   = 0;}
4414   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
4415   if (pStart == pEnd) PetscFunctionReturn(0);
4416   if (stratumValue < 0) {
4417     if (start) *start = pStart;
4418     if (end)   *end   = pEnd;
4419     PetscFunctionReturn(0);
4420   }
4421   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
4422   if (!label) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "No label named depth was found");
4423   ierr = DMLabelGetStratumBounds(label, stratumValue, start, end);CHKERRQ(ierr);
4424   PetscFunctionReturn(0);
4425 }
4426 
4427 #undef __FUNCT__
4428 #define __FUNCT__ "DMPlexGetHeightStratum"
4429 /*@
4430   DMPlexGetHeightStratum - Get the bounds [start, end) for all points at a certain height.
4431 
4432   Not Collective
4433 
4434   Input Parameters:
4435 + dm           - The DMPlex object
4436 - stratumValue - The requested height
4437 
4438   Output Parameters:
4439 + start - The first point at this height
4440 - end   - One beyond the last point at this height
4441 
4442   Level: developer
4443 
4444 .keywords: mesh, points
4445 .seealso: DMPlexGetDepthStratum(), DMPlexGetDepth()
4446 @*/
4447 PetscErrorCode DMPlexGetHeightStratum(DM dm, PetscInt stratumValue, PetscInt *start, PetscInt *end)
4448 {
4449   DMLabel        label;
4450   PetscInt       depth, pStart, pEnd;
4451   PetscErrorCode ierr;
4452 
4453   PetscFunctionBegin;
4454   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4455   if (start) {PetscValidPointer(start, 3); *start = 0;}
4456   if (end)   {PetscValidPointer(end,   4); *end   = 0;}
4457   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
4458   if (pStart == pEnd) PetscFunctionReturn(0);
4459   if (stratumValue < 0) {
4460     if (start) *start = pStart;
4461     if (end)   *end   = pEnd;
4462     PetscFunctionReturn(0);
4463   }
4464   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
4465   if (!label) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "No label named depth was found");CHKERRQ(ierr);
4466   ierr = DMLabelGetNumValues(label, &depth);CHKERRQ(ierr);
4467   ierr = DMLabelGetStratumBounds(label, depth-1-stratumValue, start, end);CHKERRQ(ierr);
4468   PetscFunctionReturn(0);
4469 }
4470 
4471 #undef __FUNCT__
4472 #define __FUNCT__ "DMPlexCreateSectionInitial"
4473 /* Set the number of dof on each point and separate by fields */
4474 PetscErrorCode DMPlexCreateSectionInitial(DM dm, PetscInt dim, PetscInt numFields,const PetscInt numComp[],const PetscInt numDof[], PetscSection *section)
4475 {
4476   PetscInt      *numDofTot;
4477   PetscInt       depth, pStart = 0, pEnd = 0;
4478   PetscInt       p, d, dep, f;
4479   PetscErrorCode ierr;
4480 
4481   PetscFunctionBegin;
4482   ierr = PetscMalloc1((dim+1), &numDofTot);CHKERRQ(ierr);
4483   for (d = 0; d <= dim; ++d) {
4484     numDofTot[d] = 0;
4485     for (f = 0; f < numFields; ++f) numDofTot[d] += numDof[f*(dim+1)+d];
4486   }
4487   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), section);CHKERRQ(ierr);
4488   if (numFields > 0) {
4489     ierr = PetscSectionSetNumFields(*section, numFields);CHKERRQ(ierr);
4490     if (numComp) {
4491       for (f = 0; f < numFields; ++f) {
4492         ierr = PetscSectionSetFieldComponents(*section, f, numComp[f]);CHKERRQ(ierr);
4493       }
4494     }
4495   }
4496   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
4497   ierr = PetscSectionSetChart(*section, pStart, pEnd);CHKERRQ(ierr);
4498   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
4499   for (dep = 0; dep <= depth; ++dep) {
4500     d    = dim == depth ? dep : (!dep ? 0 : dim);
4501     ierr = DMPlexGetDepthStratum(dm, dep, &pStart, &pEnd);CHKERRQ(ierr);
4502     for (p = pStart; p < pEnd; ++p) {
4503       for (f = 0; f < numFields; ++f) {
4504         ierr = PetscSectionSetFieldDof(*section, p, f, numDof[f*(dim+1)+d]);CHKERRQ(ierr);
4505       }
4506       ierr = PetscSectionSetDof(*section, p, numDofTot[d]);CHKERRQ(ierr);
4507     }
4508   }
4509   ierr = PetscFree(numDofTot);CHKERRQ(ierr);
4510   PetscFunctionReturn(0);
4511 }
4512 
4513 #undef __FUNCT__
4514 #define __FUNCT__ "DMPlexCreateSectionBCDof"
4515 /* Set the number of dof on each point and separate by fields
4516    If constDof is PETSC_DETERMINE, constrain every dof on the point
4517 */
4518 PetscErrorCode DMPlexCreateSectionBCDof(DM dm, PetscInt numBC,const PetscInt bcField[],const IS bcPoints[], PetscInt constDof, PetscSection section)
4519 {
4520   PetscInt       numFields;
4521   PetscInt       bc;
4522   PetscErrorCode ierr;
4523 
4524   PetscFunctionBegin;
4525   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
4526   for (bc = 0; bc < numBC; ++bc) {
4527     PetscInt        field = 0;
4528     const PetscInt *idx;
4529     PetscInt        n, i;
4530 
4531     if (numFields) field = bcField[bc];
4532     ierr = ISGetLocalSize(bcPoints[bc], &n);CHKERRQ(ierr);
4533     ierr = ISGetIndices(bcPoints[bc], &idx);CHKERRQ(ierr);
4534     for (i = 0; i < n; ++i) {
4535       const PetscInt p        = idx[i];
4536       PetscInt       numConst = constDof;
4537 
4538       /* Constrain every dof on the point */
4539       if (numConst < 0) {
4540         if (numFields) {
4541           ierr = PetscSectionGetFieldDof(section, p, field, &numConst);CHKERRQ(ierr);
4542         } else {
4543           ierr = PetscSectionGetDof(section, p, &numConst);CHKERRQ(ierr);
4544         }
4545       }
4546       if (numFields) {
4547         ierr = PetscSectionAddFieldConstraintDof(section, p, field, numConst);CHKERRQ(ierr);
4548       }
4549       ierr = PetscSectionAddConstraintDof(section, p, numConst);CHKERRQ(ierr);
4550     }
4551     ierr = ISRestoreIndices(bcPoints[bc], &idx);CHKERRQ(ierr);
4552   }
4553   PetscFunctionReturn(0);
4554 }
4555 
4556 #undef __FUNCT__
4557 #define __FUNCT__ "DMPlexCreateSectionBCIndicesAll"
4558 /* Set the constrained indices on each point and separate by fields */
4559 PetscErrorCode DMPlexCreateSectionBCIndicesAll(DM dm, PetscSection section)
4560 {
4561   PetscInt      *maxConstraints;
4562   PetscInt       numFields, f, pStart = 0, pEnd = 0, p;
4563   PetscErrorCode ierr;
4564 
4565   PetscFunctionBegin;
4566   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
4567   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
4568   ierr = PetscMalloc1((numFields+1), &maxConstraints);CHKERRQ(ierr);
4569   for (f = 0; f <= numFields; ++f) maxConstraints[f] = 0;
4570   for (p = pStart; p < pEnd; ++p) {
4571     PetscInt cdof;
4572 
4573     if (numFields) {
4574       for (f = 0; f < numFields; ++f) {
4575         ierr              = PetscSectionGetFieldConstraintDof(section, p, f, &cdof);CHKERRQ(ierr);
4576         maxConstraints[f] = PetscMax(maxConstraints[f], cdof);
4577       }
4578     } else {
4579       ierr              = PetscSectionGetConstraintDof(section, p, &cdof);CHKERRQ(ierr);
4580       maxConstraints[0] = PetscMax(maxConstraints[0], cdof);
4581     }
4582   }
4583   for (f = 0; f < numFields; ++f) {
4584     maxConstraints[numFields] += maxConstraints[f];
4585   }
4586   if (maxConstraints[numFields]) {
4587     PetscInt *indices;
4588 
4589     ierr = PetscMalloc1(maxConstraints[numFields], &indices);CHKERRQ(ierr);
4590     for (p = pStart; p < pEnd; ++p) {
4591       PetscInt cdof, d;
4592 
4593       ierr = PetscSectionGetConstraintDof(section, p, &cdof);CHKERRQ(ierr);
4594       if (cdof) {
4595         if (cdof > maxConstraints[numFields]) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_LIB, "Likely memory corruption, point %D cDof %D > maxConstraints %D", p, cdof, maxConstraints[numFields]);
4596         if (numFields) {
4597           PetscInt numConst = 0, foff = 0;
4598 
4599           for (f = 0; f < numFields; ++f) {
4600             PetscInt cfdof, fdof;
4601 
4602             ierr = PetscSectionGetFieldDof(section, p, f, &fdof);CHKERRQ(ierr);
4603             ierr = PetscSectionGetFieldConstraintDof(section, p, f, &cfdof);CHKERRQ(ierr);
4604             /* Change constraint numbering from absolute local dof number to field relative local dof number */
4605             for (d = 0; d < cfdof; ++d) indices[numConst+d] = d;
4606             ierr = PetscSectionSetFieldConstraintIndices(section, p, f, &indices[numConst]);CHKERRQ(ierr);
4607             for (d = 0; d < cfdof; ++d) indices[numConst+d] += foff;
4608             numConst += cfdof;
4609             foff     += fdof;
4610           }
4611           if (cdof != numConst) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_LIB, "Total number of field constraints %D should be %D", numConst, cdof);
4612         } else {
4613           for (d = 0; d < cdof; ++d) indices[d] = d;
4614         }
4615         ierr = PetscSectionSetConstraintIndices(section, p, indices);CHKERRQ(ierr);
4616       }
4617     }
4618     ierr = PetscFree(indices);CHKERRQ(ierr);
4619   }
4620   ierr = PetscFree(maxConstraints);CHKERRQ(ierr);
4621   PetscFunctionReturn(0);
4622 }
4623 
4624 #undef __FUNCT__
4625 #define __FUNCT__ "DMPlexCreateSectionBCIndicesField"
4626 /* Set the constrained field indices on each point */
4627 PetscErrorCode DMPlexCreateSectionBCIndicesField(DM dm, PetscInt field, IS bcPoints, IS constraintIndices, PetscSection section)
4628 {
4629   const PetscInt *points, *indices;
4630   PetscInt        numFields, maxDof, numPoints, p, numConstraints;
4631   PetscErrorCode  ierr;
4632 
4633   PetscFunctionBegin;
4634   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
4635   if ((field < 0) || (field >= numFields)) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Section field %d should be in [%d, %d)", field, 0, numFields);
4636 
4637   ierr = ISGetLocalSize(bcPoints, &numPoints);CHKERRQ(ierr);
4638   ierr = ISGetIndices(bcPoints, &points);CHKERRQ(ierr);
4639   if (!constraintIndices) {
4640     PetscInt *idx, i;
4641 
4642     ierr = PetscSectionGetMaxDof(section, &maxDof);CHKERRQ(ierr);
4643     ierr = PetscMalloc1(maxDof, &idx);CHKERRQ(ierr);
4644     for (i = 0; i < maxDof; ++i) idx[i] = i;
4645     for (p = 0; p < numPoints; ++p) {
4646       ierr = PetscSectionSetFieldConstraintIndices(section, points[p], field, idx);CHKERRQ(ierr);
4647     }
4648     ierr = PetscFree(idx);CHKERRQ(ierr);
4649   } else {
4650     ierr = ISGetLocalSize(constraintIndices, &numConstraints);CHKERRQ(ierr);
4651     ierr = ISGetIndices(constraintIndices, &indices);CHKERRQ(ierr);
4652     for (p = 0; p < numPoints; ++p) {
4653       PetscInt fcdof;
4654 
4655       ierr = PetscSectionGetFieldConstraintDof(section, points[p], field, &fcdof);CHKERRQ(ierr);
4656       if (fcdof != numConstraints) SETERRQ4(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Section point %d field %d has %d constraints, but yo ugave %d indices", p, field, fcdof, numConstraints);
4657       ierr = PetscSectionSetFieldConstraintIndices(section, points[p], field, indices);CHKERRQ(ierr);
4658     }
4659     ierr = ISRestoreIndices(constraintIndices, &indices);CHKERRQ(ierr);
4660   }
4661   ierr = ISRestoreIndices(bcPoints, &points);CHKERRQ(ierr);
4662   PetscFunctionReturn(0);
4663 }
4664 
4665 #undef __FUNCT__
4666 #define __FUNCT__ "DMPlexCreateSectionBCIndices"
4667 /* Set the constrained indices on each point and separate by fields */
4668 PetscErrorCode DMPlexCreateSectionBCIndices(DM dm, PetscSection section)
4669 {
4670   PetscInt      *indices;
4671   PetscInt       numFields, maxDof, f, pStart = 0, pEnd = 0, p;
4672   PetscErrorCode ierr;
4673 
4674   PetscFunctionBegin;
4675   ierr = PetscSectionGetMaxDof(section, &maxDof);CHKERRQ(ierr);
4676   ierr = PetscMalloc1(maxDof, &indices);CHKERRQ(ierr);
4677   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
4678   if (!numFields) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "This function only works after users have set field constraint indices.");
4679   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
4680   for (p = pStart; p < pEnd; ++p) {
4681     PetscInt cdof, d;
4682 
4683     ierr = PetscSectionGetConstraintDof(section, p, &cdof);CHKERRQ(ierr);
4684     if (cdof) {
4685       PetscInt numConst = 0, foff = 0;
4686 
4687       for (f = 0; f < numFields; ++f) {
4688         const PetscInt *fcind;
4689         PetscInt        fdof, fcdof;
4690 
4691         ierr = PetscSectionGetFieldDof(section, p, f, &fdof);CHKERRQ(ierr);
4692         ierr = PetscSectionGetFieldConstraintDof(section, p, f, &fcdof);CHKERRQ(ierr);
4693         if (fcdof) {ierr = PetscSectionGetFieldConstraintIndices(section, p, f, &fcind);CHKERRQ(ierr);}
4694         /* Change constraint numbering from field relative local dof number to absolute local dof number */
4695         for (d = 0; d < fcdof; ++d) indices[numConst+d] = fcind[d]+foff;
4696         foff     += fdof;
4697         numConst += fcdof;
4698       }
4699       if (cdof != numConst) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_LIB, "Total number of field constraints %D should be %D", numConst, cdof);
4700       ierr = PetscSectionSetConstraintIndices(section, p, indices);CHKERRQ(ierr);
4701     }
4702   }
4703   ierr = PetscFree(indices);CHKERRQ(ierr);
4704   PetscFunctionReturn(0);
4705 }
4706 
4707 #undef __FUNCT__
4708 #define __FUNCT__ "DMPlexCreateSection"
4709 /*@C
4710   DMPlexCreateSection - Create a PetscSection based upon the dof layout specification provided.
4711 
4712   Not Collective
4713 
4714   Input Parameters:
4715 + dm        - The DMPlex object
4716 . dim       - The spatial dimension of the problem
4717 . numFields - The number of fields in the problem
4718 . numComp   - An array of size numFields that holds the number of components for each field
4719 . numDof    - An array of size numFields*(dim+1) which holds the number of dof for each field on a mesh piece of dimension d
4720 . numBC     - The number of boundary conditions
4721 . bcField   - An array of size numBC giving the field number for each boundry condition
4722 . bcPoints  - An array of size numBC giving an IS holding the sieve points to which each boundary condition applies
4723 - perm      - Optional permutation of the chart, or NULL
4724 
4725   Output Parameter:
4726 . section - The PetscSection object
4727 
4728   Notes: numDof[f*(dim+1)+d] gives the number of dof for field f on sieve points of dimension d. For instance, numDof[1] is the
4729   number of dof for field 0 on each edge.
4730 
4731   The chart permutation is the same one set using PetscSectionSetPermutation()
4732 
4733   Level: developer
4734 
4735   Fortran Notes:
4736   A Fortran 90 version is available as DMPlexCreateSectionF90()
4737 
4738 .keywords: mesh, elements
4739 .seealso: DMPlexCreate(), PetscSectionCreate(), PetscSectionSetPermutation()
4740 @*/
4741 PetscErrorCode DMPlexCreateSection(DM dm, PetscInt dim, PetscInt numFields,const PetscInt numComp[],const PetscInt numDof[], PetscInt numBC,const PetscInt bcField[],const IS bcPoints[], IS perm, PetscSection *section)
4742 {
4743   PetscErrorCode ierr;
4744 
4745   PetscFunctionBegin;
4746   ierr = DMPlexCreateSectionInitial(dm, dim, numFields, numComp, numDof, section);CHKERRQ(ierr);
4747   ierr = DMPlexCreateSectionBCDof(dm, numBC, bcField, bcPoints, PETSC_DETERMINE, *section);CHKERRQ(ierr);
4748   if (perm) {ierr = PetscSectionSetPermutation(*section, perm);CHKERRQ(ierr);}
4749   ierr = PetscSectionSetUp(*section);CHKERRQ(ierr);
4750   if (numBC) {ierr = DMPlexCreateSectionBCIndicesAll(dm, *section);CHKERRQ(ierr);}
4751   ierr = PetscSectionViewFromOptions(*section,NULL,"-section_view");CHKERRQ(ierr);
4752   PetscFunctionReturn(0);
4753 }
4754 
4755 #undef __FUNCT__
4756 #define __FUNCT__ "DMCreateCoordinateDM_Plex"
4757 PetscErrorCode DMCreateCoordinateDM_Plex(DM dm, DM *cdm)
4758 {
4759   PetscSection   section;
4760   PetscErrorCode ierr;
4761 
4762   PetscFunctionBegin;
4763   ierr = DMClone(dm, cdm);CHKERRQ(ierr);
4764   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &section);CHKERRQ(ierr);
4765   ierr = DMSetDefaultSection(*cdm, section);CHKERRQ(ierr);
4766   ierr = PetscSectionDestroy(&section);CHKERRQ(ierr);
4767   PetscFunctionReturn(0);
4768 }
4769 
4770 #undef __FUNCT__
4771 #define __FUNCT__ "DMPlexGetConeSection"
4772 PetscErrorCode DMPlexGetConeSection(DM dm, PetscSection *section)
4773 {
4774   DM_Plex *mesh = (DM_Plex*) dm->data;
4775 
4776   PetscFunctionBegin;
4777   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4778   if (section) *section = mesh->coneSection;
4779   PetscFunctionReturn(0);
4780 }
4781 
4782 #undef __FUNCT__
4783 #define __FUNCT__ "DMPlexGetSupportSection"
4784 PetscErrorCode DMPlexGetSupportSection(DM dm, PetscSection *section)
4785 {
4786   DM_Plex *mesh = (DM_Plex*) dm->data;
4787 
4788   PetscFunctionBegin;
4789   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4790   if (section) *section = mesh->supportSection;
4791   PetscFunctionReturn(0);
4792 }
4793 
4794 #undef __FUNCT__
4795 #define __FUNCT__ "DMPlexGetCones"
4796 PetscErrorCode DMPlexGetCones(DM dm, PetscInt *cones[])
4797 {
4798   DM_Plex *mesh = (DM_Plex*) dm->data;
4799 
4800   PetscFunctionBegin;
4801   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4802   if (cones) *cones = mesh->cones;
4803   PetscFunctionReturn(0);
4804 }
4805 
4806 #undef __FUNCT__
4807 #define __FUNCT__ "DMPlexGetConeOrientations"
4808 PetscErrorCode DMPlexGetConeOrientations(DM dm, PetscInt *coneOrientations[])
4809 {
4810   DM_Plex *mesh = (DM_Plex*) dm->data;
4811 
4812   PetscFunctionBegin;
4813   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4814   if (coneOrientations) *coneOrientations = mesh->coneOrientations;
4815   PetscFunctionReturn(0);
4816 }
4817 
4818 /******************************** FEM Support **********************************/
4819 
4820 #undef __FUNCT__
4821 #define __FUNCT__ "DMPlexVecGetClosure_Depth1_Static"
4822 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecGetClosure_Depth1_Static(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
4823 {
4824   PetscScalar    *array, *vArray;
4825   const PetscInt *cone, *coneO;
4826   PetscInt        pStart, pEnd, p, numPoints, size = 0, offset = 0;
4827   PetscErrorCode  ierr;
4828 
4829   PetscFunctionBeginHot;
4830   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
4831   ierr = DMPlexGetConeSize(dm, point, &numPoints);CHKERRQ(ierr);
4832   ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
4833   ierr = DMPlexGetConeOrientation(dm, point, &coneO);CHKERRQ(ierr);
4834   if (!values || !*values) {
4835     if ((point >= pStart) && (point < pEnd)) {
4836       PetscInt dof;
4837 
4838       ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4839       size += dof;
4840     }
4841     for (p = 0; p < numPoints; ++p) {
4842       const PetscInt cp = cone[p];
4843       PetscInt       dof;
4844 
4845       if ((cp < pStart) || (cp >= pEnd)) continue;
4846       ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
4847       size += dof;
4848     }
4849     if (!values) {
4850       if (csize) *csize = size;
4851       PetscFunctionReturn(0);
4852     }
4853     ierr = DMGetWorkArray(dm, size, PETSC_SCALAR, &array);CHKERRQ(ierr);
4854   } else {
4855     array = *values;
4856   }
4857   size = 0;
4858   ierr = VecGetArray(v, &vArray);CHKERRQ(ierr);
4859   if ((point >= pStart) && (point < pEnd)) {
4860     PetscInt     dof, off, d;
4861     PetscScalar *varr;
4862 
4863     ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4864     ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
4865     varr = &vArray[off];
4866     for (d = 0; d < dof; ++d, ++offset) {
4867       array[offset] = varr[d];
4868     }
4869     size += dof;
4870   }
4871   for (p = 0; p < numPoints; ++p) {
4872     const PetscInt cp = cone[p];
4873     PetscInt       o  = coneO[p];
4874     PetscInt       dof, off, d;
4875     PetscScalar   *varr;
4876 
4877     if ((cp < pStart) || (cp >= pEnd)) continue;
4878     ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
4879     ierr = PetscSectionGetOffset(section, cp, &off);CHKERRQ(ierr);
4880     varr = &vArray[off];
4881     if (o >= 0) {
4882       for (d = 0; d < dof; ++d, ++offset) {
4883         array[offset] = varr[d];
4884       }
4885     } else {
4886       for (d = dof-1; d >= 0; --d, ++offset) {
4887         array[offset] = varr[d];
4888       }
4889     }
4890     size += dof;
4891   }
4892   ierr = VecRestoreArray(v, &vArray);CHKERRQ(ierr);
4893   if (!*values) {
4894     if (csize) *csize = size;
4895     *values = array;
4896   } else {
4897     if (size > *csize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Size of input array %d < actual size %d", *csize, size);
4898     *csize = size;
4899   }
4900   PetscFunctionReturn(0);
4901 }
4902 
4903 #undef __FUNCT__
4904 #define __FUNCT__ "DMPlexVecGetClosure_Static"
4905 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecGetClosure_Static(PetscSection section, PetscInt numPoints, const PetscInt points[], const PetscScalar vArray[], PetscInt *size, PetscScalar array[])
4906 {
4907   PetscInt       offset = 0, p;
4908   PetscErrorCode ierr;
4909 
4910   PetscFunctionBeginHot;
4911   *size = 0;
4912   for (p = 0; p < numPoints*2; p += 2) {
4913     const PetscInt point = points[p];
4914     const PetscInt o     = points[p+1];
4915     PetscInt       dof, off, d;
4916     const PetscScalar *varr;
4917 
4918     ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4919     ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
4920     varr = &vArray[off];
4921     if (o >= 0) {
4922       for (d = 0; d < dof; ++d, ++offset)    array[offset] = varr[d];
4923     } else {
4924       for (d = dof-1; d >= 0; --d, ++offset) array[offset] = varr[d];
4925     }
4926   }
4927   *size = offset;
4928   PetscFunctionReturn(0);
4929 }
4930 
4931 #undef __FUNCT__
4932 #define __FUNCT__ "DMPlexVecGetClosure_Fields_Static"
4933 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecGetClosure_Fields_Static(PetscSection section, PetscInt numPoints, const PetscInt points[], PetscInt numFields, const PetscScalar vArray[], PetscInt *size, PetscScalar array[])
4934 {
4935   PetscInt       offset = 0, f;
4936   PetscErrorCode ierr;
4937 
4938   PetscFunctionBeginHot;
4939   *size = 0;
4940   for (f = 0; f < numFields; ++f) {
4941     PetscInt fcomp, p;
4942 
4943     ierr = PetscSectionGetFieldComponents(section, f, &fcomp);CHKERRQ(ierr);
4944     for (p = 0; p < numPoints*2; p += 2) {
4945       const PetscInt point = points[p];
4946       const PetscInt o     = points[p+1];
4947       PetscInt       fdof, foff, d, c;
4948       const PetscScalar *varr;
4949 
4950       ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
4951       ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
4952       varr = &vArray[foff];
4953       if (o >= 0) {
4954         for (d = 0; d < fdof; ++d, ++offset) array[offset] = varr[d];
4955       } else {
4956         for (d = fdof/fcomp-1; d >= 0; --d) {
4957           for (c = 0; c < fcomp; ++c, ++offset) {
4958             array[offset] = varr[d*fcomp+c];
4959           }
4960         }
4961       }
4962     }
4963   }
4964   *size = offset;
4965   PetscFunctionReturn(0);
4966 }
4967 
4968 #undef __FUNCT__
4969 #define __FUNCT__ "DMPlexVecGetClosure"
4970 /*@C
4971   DMPlexVecGetClosure - Get an array of the values on the closure of 'point'
4972 
4973   Not collective
4974 
4975   Input Parameters:
4976 + dm - The DM
4977 . section - The section describing the layout in v, or NULL to use the default section
4978 . v - The local vector
4979 - point - The sieve point in the DM
4980 
4981   Output Parameters:
4982 + csize - The number of values in the closure, or NULL
4983 - values - The array of values, which is a borrowed array and should not be freed
4984 
4985   Fortran Notes:
4986   Since it returns an array, this routine is only available in Fortran 90, and you must
4987   include petsc.h90 in your code.
4988 
4989   The csize argument is not present in the Fortran 90 binding since it is internal to the array.
4990 
4991   Level: intermediate
4992 
4993 .seealso DMPlexVecRestoreClosure(), DMPlexVecSetClosure(), DMPlexMatSetClosure()
4994 @*/
4995 PetscErrorCode DMPlexVecGetClosure(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
4996 {
4997   PetscSection    clSection;
4998   IS              clPoints;
4999   PetscScalar    *array, *vArray;
5000   PetscInt       *points = NULL;
5001   const PetscInt *clp;
5002   PetscInt        depth, numFields, numPoints, size;
5003   PetscErrorCode  ierr;
5004 
5005   PetscFunctionBeginHot;
5006   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5007   if (!section) {ierr = DMGetDefaultSection(dm, &section);CHKERRQ(ierr);}
5008   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
5009   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
5010   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
5011   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5012   if (depth == 1 && numFields < 2) {
5013     ierr = DMPlexVecGetClosure_Depth1_Static(dm, section, v, point, csize, values);CHKERRQ(ierr);
5014     PetscFunctionReturn(0);
5015   }
5016   /* Get points */
5017   ierr = PetscSectionGetClosureIndex(section, (PetscObject) dm, &clSection, &clPoints);CHKERRQ(ierr);
5018   if (!clPoints) {
5019     PetscInt pStart, pEnd, p, q;
5020 
5021     ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
5022     ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
5023     /* Compress out points not in the section */
5024     for (p = 0, q = 0; p < numPoints*2; p += 2) {
5025       if ((points[p] >= pStart) && (points[p] < pEnd)) {
5026         points[q*2]   = points[p];
5027         points[q*2+1] = points[p+1];
5028         ++q;
5029       }
5030     }
5031     numPoints = q;
5032   } else {
5033     PetscInt dof, off;
5034 
5035     ierr = PetscSectionGetDof(clSection, point, &dof);CHKERRQ(ierr);
5036     ierr = PetscSectionGetOffset(clSection, point, &off);CHKERRQ(ierr);
5037     ierr = ISGetIndices(clPoints, &clp);CHKERRQ(ierr);
5038     numPoints = dof/2;
5039     points    = (PetscInt *) &clp[off];
5040   }
5041   /* Get array */
5042   if (!values || !*values) {
5043     PetscInt asize = 0, dof, p;
5044 
5045     for (p = 0; p < numPoints*2; p += 2) {
5046       ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5047       asize += dof;
5048     }
5049     if (!values) {
5050       if (!clPoints) {ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);}
5051       else           {ierr = ISRestoreIndices(clPoints, &clp);CHKERRQ(ierr);}
5052       if (csize) *csize = asize;
5053       PetscFunctionReturn(0);
5054     }
5055     ierr = DMGetWorkArray(dm, asize, PETSC_SCALAR, &array);CHKERRQ(ierr);
5056   } else {
5057     array = *values;
5058   }
5059   ierr = VecGetArray(v, &vArray);CHKERRQ(ierr);
5060   /* Get values */
5061   if (numFields > 0) {ierr = DMPlexVecGetClosure_Fields_Static(section, numPoints, points, numFields, vArray, &size, array);CHKERRQ(ierr);}
5062   else               {ierr = DMPlexVecGetClosure_Static(section, numPoints, points, vArray, &size, array);CHKERRQ(ierr);}
5063   /* Cleanup points */
5064   if (!clPoints) {ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);}
5065   else           {ierr = ISRestoreIndices(clPoints, &clp);CHKERRQ(ierr);}
5066   /* Cleanup array */
5067   ierr = VecRestoreArray(v, &vArray);CHKERRQ(ierr);
5068   if (!*values) {
5069     if (csize) *csize = size;
5070     *values = array;
5071   } else {
5072     if (size > *csize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Size of input array %d < actual size %d", *csize, size);
5073     *csize = size;
5074   }
5075   PetscFunctionReturn(0);
5076 }
5077 
5078 #undef __FUNCT__
5079 #define __FUNCT__ "DMPlexVecRestoreClosure"
5080 /*@C
5081   DMPlexVecRestoreClosure - Restore the array of the values on the closure of 'point'
5082 
5083   Not collective
5084 
5085   Input Parameters:
5086 + dm - The DM
5087 . section - The section describing the layout in v, or NULL to use the default section
5088 . v - The local vector
5089 . point - The sieve point in the DM
5090 . csize - The number of values in the closure, or NULL
5091 - values - The array of values, which is a borrowed array and should not be freed
5092 
5093   Fortran Notes:
5094   Since it returns an array, this routine is only available in Fortran 90, and you must
5095   include petsc.h90 in your code.
5096 
5097   The csize argument is not present in the Fortran 90 binding since it is internal to the array.
5098 
5099   Level: intermediate
5100 
5101 .seealso DMPlexVecGetClosure(), DMPlexVecSetClosure(), DMPlexMatSetClosure()
5102 @*/
5103 PetscErrorCode DMPlexVecRestoreClosure(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
5104 {
5105   PetscInt       size = 0;
5106   PetscErrorCode ierr;
5107 
5108   PetscFunctionBegin;
5109   /* Should work without recalculating size */
5110   ierr = DMRestoreWorkArray(dm, size, PETSC_SCALAR, (void*) values);CHKERRQ(ierr);
5111   PetscFunctionReturn(0);
5112 }
5113 
5114 PETSC_STATIC_INLINE void add   (PetscScalar *x, PetscScalar y) {*x += y;}
5115 PETSC_STATIC_INLINE void insert(PetscScalar *x, PetscScalar y) {*x  = y;}
5116 
5117 #undef __FUNCT__
5118 #define __FUNCT__ "updatePoint_private"
5119 PETSC_STATIC_INLINE PetscErrorCode updatePoint_private(PetscSection section, PetscInt point, PetscInt dof, void (*fuse)(PetscScalar*, PetscScalar), PetscBool setBC, PetscInt orientation, const PetscScalar values[], PetscScalar array[])
5120 {
5121   PetscInt        cdof;   /* The number of constraints on this point */
5122   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
5123   PetscScalar    *a;
5124   PetscInt        off, cind = 0, k;
5125   PetscErrorCode  ierr;
5126 
5127   PetscFunctionBegin;
5128   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
5129   ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
5130   a    = &array[off];
5131   if (!cdof || setBC) {
5132     if (orientation >= 0) {
5133       for (k = 0; k < dof; ++k) {
5134         fuse(&a[k], values[k]);
5135       }
5136     } else {
5137       for (k = 0; k < dof; ++k) {
5138         fuse(&a[k], values[dof-k-1]);
5139       }
5140     }
5141   } else {
5142     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
5143     if (orientation >= 0) {
5144       for (k = 0; k < dof; ++k) {
5145         if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
5146         fuse(&a[k], values[k]);
5147       }
5148     } else {
5149       for (k = 0; k < dof; ++k) {
5150         if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
5151         fuse(&a[k], values[dof-k-1]);
5152       }
5153     }
5154   }
5155   PetscFunctionReturn(0);
5156 }
5157 
5158 #undef __FUNCT__
5159 #define __FUNCT__ "updatePointBC_private"
5160 PETSC_STATIC_INLINE PetscErrorCode updatePointBC_private(PetscSection section, PetscInt point, PetscInt dof, void (*fuse)(PetscScalar*, PetscScalar), PetscInt orientation, const PetscScalar values[], PetscScalar array[])
5161 {
5162   PetscInt        cdof;   /* The number of constraints on this point */
5163   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
5164   PetscScalar    *a;
5165   PetscInt        off, cind = 0, k;
5166   PetscErrorCode  ierr;
5167 
5168   PetscFunctionBegin;
5169   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
5170   ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
5171   a    = &array[off];
5172   if (cdof) {
5173     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
5174     if (orientation >= 0) {
5175       for (k = 0; k < dof; ++k) {
5176         if ((cind < cdof) && (k == cdofs[cind])) {
5177           fuse(&a[k], values[k]);
5178           ++cind;
5179         }
5180       }
5181     } else {
5182       for (k = 0; k < dof; ++k) {
5183         if ((cind < cdof) && (k == cdofs[cind])) {
5184           fuse(&a[k], values[dof-k-1]);
5185           ++cind;
5186         }
5187       }
5188     }
5189   }
5190   PetscFunctionReturn(0);
5191 }
5192 
5193 #undef __FUNCT__
5194 #define __FUNCT__ "updatePointFields_private"
5195 PETSC_STATIC_INLINE PetscErrorCode updatePointFields_private(PetscSection section, PetscInt point, PetscInt o, PetscInt f, PetscInt fcomp, void (*fuse)(PetscScalar*, PetscScalar), PetscBool setBC, const PetscScalar values[], PetscInt *offset, PetscScalar array[])
5196 {
5197   PetscScalar    *a;
5198   PetscInt        fdof, foff, fcdof, foffset = *offset;
5199   const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
5200   PetscInt        cind = 0, k, c;
5201   PetscErrorCode  ierr;
5202 
5203   PetscFunctionBegin;
5204   ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
5205   ierr = PetscSectionGetFieldConstraintDof(section, point, f, &fcdof);CHKERRQ(ierr);
5206   ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
5207   a    = &array[foff];
5208   if (!fcdof || setBC) {
5209     if (o >= 0) {
5210       for (k = 0; k < fdof; ++k) fuse(&a[k], values[foffset+k]);
5211     } else {
5212       for (k = fdof/fcomp-1; k >= 0; --k) {
5213         for (c = 0; c < fcomp; ++c) {
5214           fuse(&a[(fdof/fcomp-1-k)*fcomp+c], values[foffset+k*fcomp+c]);
5215         }
5216       }
5217     }
5218   } else {
5219     ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
5220     if (o >= 0) {
5221       for (k = 0; k < fdof; ++k) {
5222         if ((cind < fcdof) && (k == fcdofs[cind])) {++cind; continue;}
5223         fuse(&a[k], values[foffset+k]);
5224       }
5225     } else {
5226       for (k = fdof/fcomp-1; k >= 0; --k) {
5227         for (c = 0; c < fcomp; ++c) {
5228           if ((cind < fcdof) && (k*fcomp+c == fcdofs[cind])) {++cind; continue;}
5229           fuse(&a[(fdof/fcomp-1-k)*fcomp+c], values[foffset+k*fcomp+c]);
5230         }
5231       }
5232     }
5233   }
5234   *offset += fdof;
5235   PetscFunctionReturn(0);
5236 }
5237 
5238 #undef __FUNCT__
5239 #define __FUNCT__ "updatePointFieldsBC_private"
5240 PETSC_STATIC_INLINE PetscErrorCode updatePointFieldsBC_private(PetscSection section, PetscInt point, PetscInt o, PetscInt f, PetscInt fcomp, void (*fuse)(PetscScalar*, PetscScalar), const PetscScalar values[], PetscInt *offset, PetscScalar array[])
5241 {
5242   PetscScalar    *a;
5243   PetscInt        fdof, foff, fcdof, foffset = *offset;
5244   const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
5245   PetscInt        cind = 0, k, c;
5246   PetscErrorCode  ierr;
5247 
5248   PetscFunctionBegin;
5249   ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
5250   ierr = PetscSectionGetFieldConstraintDof(section, point, f, &fcdof);CHKERRQ(ierr);
5251   ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
5252   a    = &array[foff];
5253   if (fcdof) {
5254     ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
5255     if (o >= 0) {
5256       for (k = 0; k < fdof; ++k) {
5257         if ((cind < fcdof) && (k == fcdofs[cind])) {
5258           fuse(&a[k], values[foffset+k]);
5259           ++cind;
5260         }
5261       }
5262     } else {
5263       for (k = fdof/fcomp-1; k >= 0; --k) {
5264         for (c = 0; c < fcomp; ++c) {
5265           if ((cind < fcdof) && (k*fcomp+c == fcdofs[cind])) {
5266             fuse(&a[(fdof/fcomp-1-k)*fcomp+c], values[foffset+k*fcomp+c]);
5267             ++cind;
5268           }
5269         }
5270       }
5271     }
5272   }
5273   *offset += fdof;
5274   PetscFunctionReturn(0);
5275 }
5276 
5277 #undef __FUNCT__
5278 #define __FUNCT__ "DMPlexVecSetClosure_Static"
5279 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecSetClosure_Static(DM dm, PetscSection section, Vec v, PetscInt point, const PetscScalar values[], InsertMode mode)
5280 {
5281   PetscScalar    *array;
5282   const PetscInt *cone, *coneO;
5283   PetscInt        pStart, pEnd, p, numPoints, off, dof;
5284   PetscErrorCode  ierr;
5285 
5286   PetscFunctionBeginHot;
5287   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
5288   ierr = DMPlexGetConeSize(dm, point, &numPoints);CHKERRQ(ierr);
5289   ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
5290   ierr = DMPlexGetConeOrientation(dm, point, &coneO);CHKERRQ(ierr);
5291   ierr = VecGetArray(v, &array);CHKERRQ(ierr);
5292   for (p = 0, off = 0; p <= numPoints; ++p, off += dof) {
5293     const PetscInt cp = !p ? point : cone[p-1];
5294     const PetscInt o  = !p ? 0     : coneO[p-1];
5295 
5296     if ((cp < pStart) || (cp >= pEnd)) {dof = 0; continue;}
5297     ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
5298     /* ADD_VALUES */
5299     {
5300       const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
5301       PetscScalar    *a;
5302       PetscInt        cdof, coff, cind = 0, k;
5303 
5304       ierr = PetscSectionGetConstraintDof(section, cp, &cdof);CHKERRQ(ierr);
5305       ierr = PetscSectionGetOffset(section, cp, &coff);CHKERRQ(ierr);
5306       a    = &array[coff];
5307       if (!cdof) {
5308         if (o >= 0) {
5309           for (k = 0; k < dof; ++k) {
5310             a[k] += values[off+k];
5311           }
5312         } else {
5313           for (k = 0; k < dof; ++k) {
5314             a[k] += values[off+dof-k-1];
5315           }
5316         }
5317       } else {
5318         ierr = PetscSectionGetConstraintIndices(section, cp, &cdofs);CHKERRQ(ierr);
5319         if (o >= 0) {
5320           for (k = 0; k < dof; ++k) {
5321             if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
5322             a[k] += values[off+k];
5323           }
5324         } else {
5325           for (k = 0; k < dof; ++k) {
5326             if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
5327             a[k] += values[off+dof-k-1];
5328           }
5329         }
5330       }
5331     }
5332   }
5333   ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
5334   PetscFunctionReturn(0);
5335 }
5336 
5337 #undef __FUNCT__
5338 #define __FUNCT__ "DMPlexVecSetClosure"
5339 /*@C
5340   DMPlexVecSetClosure - Set an array of the values on the closure of 'point'
5341 
5342   Not collective
5343 
5344   Input Parameters:
5345 + dm - The DM
5346 . section - The section describing the layout in v, or NULL to use the default section
5347 . v - The local vector
5348 . point - The sieve point in the DM
5349 . values - The array of values
5350 - mode - The insert mode, where INSERT_ALL_VALUES and ADD_ALL_VALUES also overwrite boundary conditions
5351 
5352   Fortran Notes:
5353   This routine is only available in Fortran 90, and you must include petsc.h90 in your code.
5354 
5355   Level: intermediate
5356 
5357 .seealso DMPlexVecGetClosure(), DMPlexMatSetClosure()
5358 @*/
5359 PetscErrorCode DMPlexVecSetClosure(DM dm, PetscSection section, Vec v, PetscInt point, const PetscScalar values[], InsertMode mode)
5360 {
5361   PetscSection    clSection;
5362   IS              clPoints;
5363   PetscScalar    *array;
5364   PetscInt       *points = NULL;
5365   const PetscInt *clp;
5366   PetscInt        depth, numFields, numPoints, p;
5367   PetscErrorCode  ierr;
5368 
5369   PetscFunctionBeginHot;
5370   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5371   if (!section) {ierr = DMGetDefaultSection(dm, &section);CHKERRQ(ierr);}
5372   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
5373   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
5374   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
5375   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5376   if (depth == 1 && numFields < 2 && mode == ADD_VALUES) {
5377     ierr = DMPlexVecSetClosure_Static(dm, section, v, point, values, mode);CHKERRQ(ierr);
5378     PetscFunctionReturn(0);
5379   }
5380   /* Get points */
5381   ierr = PetscSectionGetClosureIndex(section, (PetscObject) dm, &clSection, &clPoints);CHKERRQ(ierr);
5382   if (!clPoints) {
5383     PetscInt pStart, pEnd, q;
5384 
5385     ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
5386     ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
5387     /* Compress out points not in the section */
5388     for (p = 0, q = 0; p < numPoints*2; p += 2) {
5389       if ((points[p] >= pStart) && (points[p] < pEnd)) {
5390         points[q*2]   = points[p];
5391         points[q*2+1] = points[p+1];
5392         ++q;
5393       }
5394     }
5395     numPoints = q;
5396   } else {
5397     PetscInt dof, off;
5398 
5399     ierr = PetscSectionGetDof(clSection, point, &dof);CHKERRQ(ierr);
5400     ierr = PetscSectionGetOffset(clSection, point, &off);CHKERRQ(ierr);
5401     ierr = ISGetIndices(clPoints, &clp);CHKERRQ(ierr);
5402     numPoints = dof/2;
5403     points    = (PetscInt *) &clp[off];
5404   }
5405   /* Get array */
5406   ierr = VecGetArray(v, &array);CHKERRQ(ierr);
5407   /* Get values */
5408   if (numFields > 0) {
5409     PetscInt offset = 0, fcomp, f;
5410     for (f = 0; f < numFields; ++f) {
5411       ierr = PetscSectionGetFieldComponents(section, f, &fcomp);CHKERRQ(ierr);
5412       switch (mode) {
5413       case INSERT_VALUES:
5414         for (p = 0; p < numPoints*2; p += 2) {
5415           const PetscInt point = points[p];
5416           const PetscInt o     = points[p+1];
5417           updatePointFields_private(section, point, o, f, fcomp, insert, PETSC_FALSE, values, &offset, array);
5418         } break;
5419       case INSERT_ALL_VALUES:
5420         for (p = 0; p < numPoints*2; p += 2) {
5421           const PetscInt point = points[p];
5422           const PetscInt o     = points[p+1];
5423           updatePointFields_private(section, point, o, f, fcomp, insert, PETSC_TRUE, values, &offset, array);
5424         } break;
5425       case INSERT_BC_VALUES:
5426         for (p = 0; p < numPoints*2; p += 2) {
5427           const PetscInt point = points[p];
5428           const PetscInt o     = points[p+1];
5429           updatePointFieldsBC_private(section, point, o, f, fcomp, insert, values, &offset, array);
5430         } break;
5431       case ADD_VALUES:
5432         for (p = 0; p < numPoints*2; p += 2) {
5433           const PetscInt point = points[p];
5434           const PetscInt o     = points[p+1];
5435           updatePointFields_private(section, point, o, f, fcomp, add, PETSC_FALSE, values, &offset, array);
5436         } break;
5437       case ADD_ALL_VALUES:
5438         for (p = 0; p < numPoints*2; p += 2) {
5439           const PetscInt point = points[p];
5440           const PetscInt o     = points[p+1];
5441           updatePointFields_private(section, point, o, f, fcomp, add, PETSC_TRUE, values, &offset, array);
5442         } break;
5443       default:
5444         SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %D", mode);
5445       }
5446     }
5447   } else {
5448     PetscInt dof, off;
5449 
5450     switch (mode) {
5451     case INSERT_VALUES:
5452       for (p = 0, off = 0; p < numPoints*2; p += 2, off += dof) {
5453         PetscInt o = points[p+1];
5454         ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5455         updatePoint_private(section, points[p], dof, insert, PETSC_FALSE, o, &values[off], array);
5456       } break;
5457     case INSERT_ALL_VALUES:
5458       for (p = 0, off = 0; p < numPoints*2; p += 2, off += dof) {
5459         PetscInt o = points[p+1];
5460         ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5461         updatePoint_private(section, points[p], dof, insert, PETSC_TRUE,  o, &values[off], array);
5462       } break;
5463     case INSERT_BC_VALUES:
5464       for (p = 0, off = 0; p < numPoints*2; p += 2, off += dof) {
5465         PetscInt o = points[p+1];
5466         ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5467         updatePointBC_private(section, points[p], dof, insert,  o, &values[off], array);
5468       } break;
5469     case ADD_VALUES:
5470       for (p = 0, off = 0; p < numPoints*2; p += 2, off += dof) {
5471         PetscInt o = points[p+1];
5472         ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5473         updatePoint_private(section, points[p], dof, add,    PETSC_FALSE, o, &values[off], array);
5474       } break;
5475     case ADD_ALL_VALUES:
5476       for (p = 0, off = 0; p < numPoints*2; p += 2, off += dof) {
5477         PetscInt o = points[p+1];
5478         ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5479         updatePoint_private(section, points[p], dof, add,    PETSC_TRUE,  o, &values[off], array);
5480       } break;
5481     default:
5482       SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %D", mode);
5483     }
5484   }
5485   /* Cleanup points */
5486   if (!clPoints) {ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);}
5487   else           {ierr = ISRestoreIndices(clPoints, &clp);CHKERRQ(ierr);}
5488   /* Cleanup array */
5489   ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
5490   PetscFunctionReturn(0);
5491 }
5492 
5493 #undef __FUNCT__
5494 #define __FUNCT__ "DMPlexPrintMatSetValues"
5495 PetscErrorCode DMPlexPrintMatSetValues(PetscViewer viewer, Mat A, PetscInt point, PetscInt numRIndices, const PetscInt rindices[], PetscInt numCIndices, const PetscInt cindices[], const PetscScalar values[])
5496 {
5497   PetscMPIInt    rank;
5498   PetscInt       i, j;
5499   PetscErrorCode ierr;
5500 
5501   PetscFunctionBegin;
5502   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRQ(ierr);
5503   ierr = PetscViewerASCIIPrintf(viewer, "[%D]mat for sieve point %D\n", rank, point);CHKERRQ(ierr);
5504   for (i = 0; i < numRIndices; i++) {ierr = PetscViewerASCIIPrintf(viewer, "[%D]mat row indices[%D] = %D\n", rank, i, rindices[i]);CHKERRQ(ierr);}
5505   for (i = 0; i < numCIndices; i++) {ierr = PetscViewerASCIIPrintf(viewer, "[%D]mat col indices[%D] = %D\n", rank, i, cindices[i]);CHKERRQ(ierr);}
5506   numCIndices = numCIndices ? numCIndices : numRIndices;
5507   for (i = 0; i < numRIndices; i++) {
5508     ierr = PetscViewerASCIIPrintf(viewer, "[%D]", rank);CHKERRQ(ierr);
5509     for (j = 0; j < numCIndices; j++) {
5510 #if defined(PETSC_USE_COMPLEX)
5511       ierr = PetscViewerASCIIPrintf(viewer, " (%g,%g)", (double)PetscRealPart(values[i*numCIndices+j]), (double)PetscImaginaryPart(values[i*numCIndices+j]));CHKERRQ(ierr);
5512 #else
5513       ierr = PetscViewerASCIIPrintf(viewer, " %g", (double)values[i*numCIndices+j]);CHKERRQ(ierr);
5514 #endif
5515     }
5516     ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
5517   }
5518   PetscFunctionReturn(0);
5519 }
5520 
5521 #undef __FUNCT__
5522 #define __FUNCT__ "indicesPoint_private"
5523 /* . off - The global offset of this point */
5524 PetscErrorCode indicesPoint_private(PetscSection section, PetscInt point, PetscInt off, PetscInt *loff, PetscBool setBC, PetscInt orientation, PetscInt indices[])
5525 {
5526   PetscInt        dof;    /* The number of unknowns on this point */
5527   PetscInt        cdof;   /* The number of constraints on this point */
5528   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
5529   PetscInt        cind = 0, k;
5530   PetscErrorCode  ierr;
5531 
5532   PetscFunctionBegin;
5533   ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
5534   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
5535   if (!cdof || setBC) {
5536     if (orientation >= 0) {
5537       for (k = 0; k < dof; ++k) indices[*loff+k] = off+k;
5538     } else {
5539       for (k = 0; k < dof; ++k) indices[*loff+dof-k-1] = off+k;
5540     }
5541   } else {
5542     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
5543     if (orientation >= 0) {
5544       for (k = 0; k < dof; ++k) {
5545         if ((cind < cdof) && (k == cdofs[cind])) {
5546           /* Insert check for returning constrained indices */
5547           indices[*loff+k] = -(off+k+1);
5548           ++cind;
5549         } else {
5550           indices[*loff+k] = off+k-cind;
5551         }
5552       }
5553     } else {
5554       for (k = 0; k < dof; ++k) {
5555         if ((cind < cdof) && (k == cdofs[cind])) {
5556           /* Insert check for returning constrained indices */
5557           indices[*loff+dof-k-1] = -(off+k+1);
5558           ++cind;
5559         } else {
5560           indices[*loff+dof-k-1] = off+k-cind;
5561         }
5562       }
5563     }
5564   }
5565   *loff += dof;
5566   PetscFunctionReturn(0);
5567 }
5568 
5569 #undef __FUNCT__
5570 #define __FUNCT__ "indicesPointFields_private"
5571 /* . off - The global offset of this point */
5572 PetscErrorCode indicesPointFields_private(PetscSection section, PetscInt point, PetscInt off, PetscInt foffs[], PetscBool setBC, PetscInt orientation, PetscInt indices[])
5573 {
5574   PetscInt       numFields, foff, f;
5575   PetscErrorCode ierr;
5576 
5577   PetscFunctionBegin;
5578   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5579   for (f = 0, foff = 0; f < numFields; ++f) {
5580     PetscInt        fdof, fcomp, cfdof;
5581     const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
5582     PetscInt        cind = 0, k, c;
5583 
5584     ierr = PetscSectionGetFieldComponents(section, f, &fcomp);CHKERRQ(ierr);
5585     ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
5586     ierr = PetscSectionGetFieldConstraintDof(section, point, f, &cfdof);CHKERRQ(ierr);
5587     if (!cfdof || setBC) {
5588       if (orientation >= 0) {
5589         for (k = 0; k < fdof; ++k) indices[foffs[f]+k] = off+foff+k;
5590       } else {
5591         for (k = fdof/fcomp-1; k >= 0; --k) {
5592           for (c = 0; c < fcomp; ++c) {
5593             indices[foffs[f]+k*fcomp+c] = off+foff+(fdof/fcomp-1-k)*fcomp+c;
5594           }
5595         }
5596       }
5597     } else {
5598       ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
5599       if (orientation >= 0) {
5600         for (k = 0; k < fdof; ++k) {
5601           if ((cind < cfdof) && (k == fcdofs[cind])) {
5602             indices[foffs[f]+k] = -(off+foff+k+1);
5603             ++cind;
5604           } else {
5605             indices[foffs[f]+k] = off+foff+k-cind;
5606           }
5607         }
5608       } else {
5609         for (k = fdof/fcomp-1; k >= 0; --k) {
5610           for (c = 0; c < fcomp; ++c) {
5611             if ((cind < cfdof) && ((fdof/fcomp-1-k)*fcomp+c == fcdofs[cind])) {
5612               indices[foffs[f]+k*fcomp+c] = -(off+foff+(fdof/fcomp-1-k)*fcomp+c+1);
5613               ++cind;
5614             } else {
5615               indices[foffs[f]+k*fcomp+c] = off+foff+(fdof/fcomp-1-k)*fcomp+c-cind;
5616             }
5617           }
5618         }
5619       }
5620     }
5621     foff     += fdof - cfdof;
5622     foffs[f] += fdof;
5623   }
5624   PetscFunctionReturn(0);
5625 }
5626 
5627 #undef __FUNCT__
5628 #define __FUNCT__ "DMPlexMatSetClosure"
5629 /*@C
5630   DMPlexMatSetClosure - Set an array of the values on the closure of 'point'
5631 
5632   Not collective
5633 
5634   Input Parameters:
5635 + dm - The DM
5636 . section - The section describing the layout in v, or NULL to use the default section
5637 . globalSection - The section describing the layout in v, or NULL to use the default global section
5638 . A - The matrix
5639 . point - The sieve point in the DM
5640 . values - The array of values
5641 - mode - The insert mode, where INSERT_ALL_VALUES and ADD_ALL_VALUES also overwrite boundary conditions
5642 
5643   Fortran Notes:
5644   This routine is only available in Fortran 90, and you must include petsc.h90 in your code.
5645 
5646   Level: intermediate
5647 
5648 .seealso DMPlexVecGetClosure(), DMPlexVecSetClosure()
5649 @*/
5650 PetscErrorCode DMPlexMatSetClosure(DM dm, PetscSection section, PetscSection globalSection, Mat A, PetscInt point, const PetscScalar values[], InsertMode mode)
5651 {
5652   DM_Plex        *mesh   = (DM_Plex*) dm->data;
5653   PetscSection    clSection;
5654   IS              clPoints;
5655   PetscInt       *points = NULL;
5656   const PetscInt *clp;
5657   PetscInt       *indices;
5658   PetscInt        offsets[32];
5659   PetscInt        numFields, numPoints, numIndices, dof, off, globalOff, pStart, pEnd, p, q, f;
5660   PetscErrorCode  ierr;
5661 
5662   PetscFunctionBegin;
5663   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5664   if (!section) {ierr = DMGetDefaultSection(dm, &section);CHKERRQ(ierr);}
5665   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
5666   if (!globalSection) {ierr = DMGetDefaultGlobalSection(dm, &globalSection);CHKERRQ(ierr);}
5667   PetscValidHeaderSpecific(globalSection, PETSC_SECTION_CLASSID, 3);
5668   PetscValidHeaderSpecific(A, MAT_CLASSID, 4);
5669   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5670   if (numFields > 31) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", numFields);
5671   ierr = PetscMemzero(offsets, 32 * sizeof(PetscInt));CHKERRQ(ierr);
5672   ierr = PetscSectionGetClosureIndex(section, (PetscObject) dm, &clSection, &clPoints);CHKERRQ(ierr);
5673   if (!clPoints) {
5674     ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
5675     /* Compress out points not in the section */
5676     ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
5677     for (p = 0, q = 0; p < numPoints*2; p += 2) {
5678       if ((points[p] >= pStart) && (points[p] < pEnd)) {
5679         points[q*2]   = points[p];
5680         points[q*2+1] = points[p+1];
5681         ++q;
5682       }
5683     }
5684     numPoints = q;
5685   } else {
5686     PetscInt dof, off;
5687 
5688     ierr = PetscSectionGetDof(clSection, point, &dof);CHKERRQ(ierr);
5689     numPoints = dof/2;
5690     ierr = PetscSectionGetOffset(clSection, point, &off);CHKERRQ(ierr);
5691     ierr = ISGetIndices(clPoints, &clp);CHKERRQ(ierr);
5692     points = (PetscInt *) &clp[off];
5693   }
5694   for (p = 0, numIndices = 0; p < numPoints*2; p += 2) {
5695     PetscInt fdof;
5696 
5697     ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5698     for (f = 0; f < numFields; ++f) {
5699       ierr          = PetscSectionGetFieldDof(section, points[p], f, &fdof);CHKERRQ(ierr);
5700       offsets[f+1] += fdof;
5701     }
5702     numIndices += dof;
5703   }
5704   for (f = 1; f < numFields; ++f) offsets[f+1] += offsets[f];
5705 
5706   if (numFields && offsets[numFields] != numIndices) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid size for closure %d should be %d", offsets[numFields], numIndices);
5707   ierr = DMGetWorkArray(dm, numIndices, PETSC_INT, &indices);CHKERRQ(ierr);
5708   if (numFields) {
5709     for (p = 0; p < numPoints*2; p += 2) {
5710       PetscInt o = points[p+1];
5711       ierr = PetscSectionGetOffset(globalSection, points[p], &globalOff);CHKERRQ(ierr);
5712       indicesPointFields_private(section, points[p], globalOff < 0 ? -(globalOff+1) : globalOff, offsets, PETSC_FALSE, o, indices);
5713     }
5714   } else {
5715     for (p = 0, off = 0; p < numPoints*2; p += 2) {
5716       PetscInt o = points[p+1];
5717       ierr = PetscSectionGetOffset(globalSection, points[p], &globalOff);CHKERRQ(ierr);
5718       indicesPoint_private(section, points[p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, o, indices);
5719     }
5720   }
5721   if (mesh->printSetValues) {ierr = DMPlexPrintMatSetValues(PETSC_VIEWER_STDOUT_SELF, A, point, numIndices, indices, 0, NULL, values);CHKERRQ(ierr);}
5722   ierr = MatSetValues(A, numIndices, indices, numIndices, indices, values, mode);
5723   if (ierr) {
5724     PetscMPIInt    rank;
5725     PetscErrorCode ierr2;
5726 
5727     ierr2 = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRQ(ierr2);
5728     ierr2 = (*PetscErrorPrintf)("[%D]ERROR in DMPlexMatSetClosure\n", rank);CHKERRQ(ierr2);
5729     ierr2 = DMPlexPrintMatSetValues(PETSC_VIEWER_STDERR_SELF, A, point, numIndices, indices, 0, NULL, values);CHKERRQ(ierr2);
5730     ierr2 = DMRestoreWorkArray(dm, numIndices, PETSC_INT, &indices);CHKERRQ(ierr2);
5731     CHKERRQ(ierr);
5732   }
5733   if (!clPoints) {
5734     ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
5735   } else {
5736     ierr = ISRestoreIndices(clPoints, &clp);CHKERRQ(ierr);
5737   }
5738   ierr = DMRestoreWorkArray(dm, numIndices, PETSC_INT, &indices);CHKERRQ(ierr);
5739   PetscFunctionReturn(0);
5740 }
5741 
5742 #undef __FUNCT__
5743 #define __FUNCT__ "DMPlexMatSetClosureRefined"
5744 PetscErrorCode DMPlexMatSetClosureRefined(DM dmf, PetscSection fsection, PetscSection globalFSection, DM dmc, PetscSection csection, PetscSection globalCSection, Mat A, PetscInt point, const PetscScalar values[], InsertMode mode)
5745 {
5746   DM_Plex        *mesh   = (DM_Plex*) dmf->data;
5747   PetscInt       *fpoints = NULL, *ftotpoints = NULL;
5748   PetscInt       *cpoints = NULL;
5749   PetscInt       *findices, *cindices;
5750   PetscInt        foffsets[32], coffsets[32];
5751   CellRefiner     cellRefiner;
5752   PetscInt        numFields, numSubcells, maxFPoints, numFPoints, numCPoints, numFIndices, numCIndices, dof, off, globalOff, pStart, pEnd, p, q, r, s, f;
5753   PetscErrorCode  ierr;
5754 
5755   PetscFunctionBegin;
5756   PetscValidHeaderSpecific(dmf, DM_CLASSID, 1);
5757   PetscValidHeaderSpecific(dmc, DM_CLASSID, 4);
5758   if (!fsection) {ierr = DMGetDefaultSection(dmf, &fsection);CHKERRQ(ierr);}
5759   PetscValidHeaderSpecific(fsection, PETSC_SECTION_CLASSID, 2);
5760   if (!csection) {ierr = DMGetDefaultSection(dmc, &csection);CHKERRQ(ierr);}
5761   PetscValidHeaderSpecific(csection, PETSC_SECTION_CLASSID, 5);
5762   if (!globalFSection) {ierr = DMGetDefaultGlobalSection(dmf, &globalFSection);CHKERRQ(ierr);}
5763   PetscValidHeaderSpecific(globalFSection, PETSC_SECTION_CLASSID, 3);
5764   if (!globalCSection) {ierr = DMGetDefaultGlobalSection(dmc, &globalCSection);CHKERRQ(ierr);}
5765   PetscValidHeaderSpecific(globalCSection, PETSC_SECTION_CLASSID, 6);
5766   PetscValidHeaderSpecific(A, MAT_CLASSID, 7);
5767   ierr = PetscSectionGetNumFields(fsection, &numFields);CHKERRQ(ierr);
5768   if (numFields > 31) SETERRQ1(PetscObjectComm((PetscObject)dmf), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", numFields);
5769   ierr = PetscMemzero(foffsets, 32 * sizeof(PetscInt));CHKERRQ(ierr);
5770   ierr = PetscMemzero(coffsets, 32 * sizeof(PetscInt));CHKERRQ(ierr);
5771   /* Column indices */
5772   ierr = DMPlexGetTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
5773   maxFPoints = numCPoints;
5774   /* Compress out points not in the section */
5775   /*   TODO: Squeeze out points with 0 dof as well */
5776   ierr = PetscSectionGetChart(csection, &pStart, &pEnd);CHKERRQ(ierr);
5777   for (p = 0, q = 0; p < numCPoints*2; p += 2) {
5778     if ((cpoints[p] >= pStart) && (cpoints[p] < pEnd)) {
5779       cpoints[q*2]   = cpoints[p];
5780       cpoints[q*2+1] = cpoints[p+1];
5781       ++q;
5782     }
5783   }
5784   numCPoints = q;
5785   for (p = 0, numCIndices = 0; p < numCPoints*2; p += 2) {
5786     PetscInt fdof;
5787 
5788     ierr = PetscSectionGetDof(csection, cpoints[p], &dof);CHKERRQ(ierr);
5789     if (!dof) continue;
5790     for (f = 0; f < numFields; ++f) {
5791       ierr           = PetscSectionGetFieldDof(csection, cpoints[p], f, &fdof);CHKERRQ(ierr);
5792       coffsets[f+1] += fdof;
5793     }
5794     numCIndices += dof;
5795   }
5796   for (f = 1; f < numFields; ++f) coffsets[f+1] += coffsets[f];
5797   /* Row indices */
5798   ierr = DMPlexGetCellRefiner_Internal(dmc, &cellRefiner);CHKERRQ(ierr);
5799   ierr = CellRefinerGetAffineTransforms_Internal(cellRefiner, &numSubcells, NULL, NULL, NULL);CHKERRQ(ierr);
5800   ierr = DMGetWorkArray(dmf, maxFPoints*2*numSubcells, PETSC_INT, &ftotpoints);CHKERRQ(ierr);
5801   for (r = 0, q = 0; r < numSubcells; ++r) {
5802     /* TODO Map from coarse to fine cells */
5803     ierr = DMPlexGetTransitiveClosure(dmf, point*numSubcells + r, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
5804     /* Compress out points not in the section */
5805     ierr = PetscSectionGetChart(fsection, &pStart, &pEnd);CHKERRQ(ierr);
5806     for (p = 0; p < numFPoints*2; p += 2) {
5807       if ((fpoints[p] >= pStart) && (fpoints[p] < pEnd)) {
5808         ierr = PetscSectionGetDof(fsection, fpoints[p], &dof);CHKERRQ(ierr);
5809         if (!dof) continue;
5810         for (s = 0; s < q; ++s) if (fpoints[p] == ftotpoints[s*2]) break;
5811         if (s < q) continue;
5812         ftotpoints[q*2]   = fpoints[p];
5813         ftotpoints[q*2+1] = fpoints[p+1];
5814         ++q;
5815       }
5816     }
5817     ierr = DMPlexRestoreTransitiveClosure(dmf, point, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
5818   }
5819   numFPoints = q;
5820   for (p = 0, numFIndices = 0; p < numFPoints*2; p += 2) {
5821     PetscInt fdof;
5822 
5823     ierr = PetscSectionGetDof(fsection, ftotpoints[p], &dof);CHKERRQ(ierr);
5824     if (!dof) continue;
5825     for (f = 0; f < numFields; ++f) {
5826       ierr           = PetscSectionGetFieldDof(fsection, ftotpoints[p], f, &fdof);CHKERRQ(ierr);
5827       foffsets[f+1] += fdof;
5828     }
5829     numFIndices += dof;
5830   }
5831   for (f = 1; f < numFields; ++f) foffsets[f+1] += foffsets[f];
5832 
5833   if (numFields && foffsets[numFields] != numFIndices) SETERRQ2(PetscObjectComm((PetscObject)dmf), PETSC_ERR_PLIB, "Invalid size for closure %d should be %d", foffsets[numFields], numFIndices);
5834   if (numFields && coffsets[numFields] != numCIndices) SETERRQ2(PetscObjectComm((PetscObject)dmc), PETSC_ERR_PLIB, "Invalid size for closure %d should be %d", coffsets[numFields], numCIndices);
5835   ierr = DMGetWorkArray(dmf, numFIndices, PETSC_INT, &findices);CHKERRQ(ierr);
5836   ierr = DMGetWorkArray(dmc, numCIndices, PETSC_INT, &cindices);CHKERRQ(ierr);
5837   if (numFields) {
5838     for (p = 0; p < numFPoints*2; p += 2) {
5839       PetscInt o = ftotpoints[p+1];
5840       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[p], &globalOff);CHKERRQ(ierr);
5841       indicesPointFields_private(fsection, ftotpoints[p], globalOff < 0 ? -(globalOff+1) : globalOff, foffsets, PETSC_FALSE, o, findices);
5842     }
5843     for (p = 0; p < numCPoints*2; p += 2) {
5844       PetscInt o = cpoints[p+1];
5845       ierr = PetscSectionGetOffset(globalCSection, cpoints[p], &globalOff);CHKERRQ(ierr);
5846       indicesPointFields_private(csection, cpoints[p], globalOff < 0 ? -(globalOff+1) : globalOff, coffsets, PETSC_FALSE, o, cindices);
5847     }
5848   } else {
5849     for (p = 0, off = 0; p < numFPoints*2; p += 2) {
5850       PetscInt o = ftotpoints[p+1];
5851       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[p], &globalOff);CHKERRQ(ierr);
5852       indicesPoint_private(fsection, ftotpoints[p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, o, findices);
5853     }
5854     for (p = 0, off = 0; p < numCPoints*2; p += 2) {
5855       PetscInt o = cpoints[p+1];
5856       ierr = PetscSectionGetOffset(globalCSection, cpoints[p], &globalOff);CHKERRQ(ierr);
5857       indicesPoint_private(csection, cpoints[p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, o, cindices);
5858     }
5859   }
5860   if (mesh->printSetValues) {ierr = DMPlexPrintMatSetValues(PETSC_VIEWER_STDOUT_SELF, A, point, numFIndices, findices, numCIndices, cindices, values);CHKERRQ(ierr);}
5861   ierr = MatSetValues(A, numFIndices, findices, numCIndices, cindices, values, mode);
5862   if (ierr) {
5863     PetscMPIInt    rank;
5864     PetscErrorCode ierr2;
5865 
5866     ierr2 = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRQ(ierr2);
5867     ierr2 = (*PetscErrorPrintf)("[%D]ERROR in DMPlexMatSetClosure\n", rank);CHKERRQ(ierr2);
5868     ierr2 = DMPlexPrintMatSetValues(PETSC_VIEWER_STDERR_SELF, A, point, numFIndices, findices, numCIndices, cindices, values);CHKERRQ(ierr2);
5869     ierr2 = DMRestoreWorkArray(dmf, numFIndices, PETSC_INT, &findices);CHKERRQ(ierr2);
5870     ierr2 = DMRestoreWorkArray(dmc, numCIndices, PETSC_INT, &cindices);CHKERRQ(ierr2);
5871     CHKERRQ(ierr);
5872   }
5873   ierr = DMRestoreWorkArray(dmf, numCPoints*2*4, PETSC_INT, &ftotpoints);CHKERRQ(ierr);
5874   ierr = DMPlexRestoreTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
5875   ierr = DMRestoreWorkArray(dmf, numFIndices, PETSC_INT, &findices);CHKERRQ(ierr);
5876   ierr = DMRestoreWorkArray(dmc, numCIndices, PETSC_INT, &cindices);CHKERRQ(ierr);
5877   PetscFunctionReturn(0);
5878 }
5879 
5880 #undef __FUNCT__
5881 #define __FUNCT__ "DMPlexGetHybridBounds"
5882 /*@
5883   DMPlexGetHybridBounds - Get the first mesh point of each dimension which is a hybrid
5884 
5885   Input Parameter:
5886 . dm - The DMPlex object
5887 
5888   Output Parameters:
5889 + cMax - The first hybrid cell
5890 . cMax - The first hybrid face
5891 . cMax - The first hybrid edge
5892 - cMax - The first hybrid vertex
5893 
5894   Level: developer
5895 
5896 .seealso DMPlexCreateHybridMesh()
5897 @*/
5898 PetscErrorCode DMPlexGetHybridBounds(DM dm, PetscInt *cMax, PetscInt *fMax, PetscInt *eMax, PetscInt *vMax)
5899 {
5900   DM_Plex       *mesh = (DM_Plex*) dm->data;
5901   PetscInt       dim;
5902   PetscErrorCode ierr;
5903 
5904   PetscFunctionBegin;
5905   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5906   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
5907   if (cMax) *cMax = mesh->hybridPointMax[dim];
5908   if (fMax) *fMax = mesh->hybridPointMax[dim-1];
5909   if (eMax) *eMax = mesh->hybridPointMax[1];
5910   if (vMax) *vMax = mesh->hybridPointMax[0];
5911   PetscFunctionReturn(0);
5912 }
5913 
5914 #undef __FUNCT__
5915 #define __FUNCT__ "DMPlexSetHybridBounds"
5916 PetscErrorCode DMPlexSetHybridBounds(DM dm, PetscInt cMax, PetscInt fMax, PetscInt eMax, PetscInt vMax)
5917 {
5918   DM_Plex       *mesh = (DM_Plex*) dm->data;
5919   PetscInt       dim;
5920   PetscErrorCode ierr;
5921 
5922   PetscFunctionBegin;
5923   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5924   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
5925   if (cMax >= 0) mesh->hybridPointMax[dim]   = cMax;
5926   if (fMax >= 0) mesh->hybridPointMax[dim-1] = fMax;
5927   if (eMax >= 0) mesh->hybridPointMax[1]     = eMax;
5928   if (vMax >= 0) mesh->hybridPointMax[0]     = vMax;
5929   PetscFunctionReturn(0);
5930 }
5931 
5932 #undef __FUNCT__
5933 #define __FUNCT__ "DMPlexGetVTKCellHeight"
5934 PetscErrorCode DMPlexGetVTKCellHeight(DM dm, PetscInt *cellHeight)
5935 {
5936   DM_Plex *mesh = (DM_Plex*) dm->data;
5937 
5938   PetscFunctionBegin;
5939   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5940   PetscValidPointer(cellHeight, 2);
5941   *cellHeight = mesh->vtkCellHeight;
5942   PetscFunctionReturn(0);
5943 }
5944 
5945 #undef __FUNCT__
5946 #define __FUNCT__ "DMPlexSetVTKCellHeight"
5947 PetscErrorCode DMPlexSetVTKCellHeight(DM dm, PetscInt cellHeight)
5948 {
5949   DM_Plex *mesh = (DM_Plex*) dm->data;
5950 
5951   PetscFunctionBegin;
5952   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5953   mesh->vtkCellHeight = cellHeight;
5954   PetscFunctionReturn(0);
5955 }
5956 
5957 #undef __FUNCT__
5958 #define __FUNCT__ "DMPlexCreateNumbering_Private"
5959 /* We can easily have a form that takes an IS instead */
5960 PetscErrorCode DMPlexCreateNumbering_Private(DM dm, PetscInt pStart, PetscInt pEnd, PetscSF sf, IS *numbering)
5961 {
5962   PetscSection   section, globalSection;
5963   PetscInt      *numbers, p;
5964   PetscErrorCode ierr;
5965 
5966   PetscFunctionBegin;
5967   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &section);CHKERRQ(ierr);
5968   ierr = PetscSectionSetChart(section, pStart, pEnd);CHKERRQ(ierr);
5969   for (p = pStart; p < pEnd; ++p) {
5970     ierr = PetscSectionSetDof(section, p, 1);CHKERRQ(ierr);
5971   }
5972   ierr = PetscSectionSetUp(section);CHKERRQ(ierr);
5973   ierr = PetscSectionCreateGlobalSection(section, sf, PETSC_FALSE, &globalSection);CHKERRQ(ierr);
5974   ierr = PetscMalloc1((pEnd - pStart), &numbers);CHKERRQ(ierr);
5975   for (p = pStart; p < pEnd; ++p) {
5976     ierr = PetscSectionGetOffset(globalSection, p, &numbers[p-pStart]);CHKERRQ(ierr);
5977   }
5978   ierr = ISCreateGeneral(PetscObjectComm((PetscObject)dm), pEnd - pStart, numbers, PETSC_OWN_POINTER, numbering);CHKERRQ(ierr);
5979   ierr = PetscSectionDestroy(&section);CHKERRQ(ierr);
5980   ierr = PetscSectionDestroy(&globalSection);CHKERRQ(ierr);
5981   PetscFunctionReturn(0);
5982 }
5983 
5984 #undef __FUNCT__
5985 #define __FUNCT__ "DMPlexGetCellNumbering"
5986 PetscErrorCode DMPlexGetCellNumbering(DM dm, IS *globalCellNumbers)
5987 {
5988   DM_Plex       *mesh = (DM_Plex*) dm->data;
5989   PetscInt       cellHeight, cStart, cEnd, cMax;
5990   PetscErrorCode ierr;
5991 
5992   PetscFunctionBegin;
5993   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5994   if (!mesh->globalCellNumbers) {
5995     ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
5996     ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
5997     ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
5998     if (cMax >= 0) cEnd = PetscMin(cEnd, cMax);
5999     ierr = DMPlexCreateNumbering_Private(dm, cStart, cEnd, dm->sf, &mesh->globalCellNumbers);CHKERRQ(ierr);
6000   }
6001   *globalCellNumbers = mesh->globalCellNumbers;
6002   PetscFunctionReturn(0);
6003 }
6004 
6005 #undef __FUNCT__
6006 #define __FUNCT__ "DMPlexGetVertexNumbering"
6007 PetscErrorCode DMPlexGetVertexNumbering(DM dm, IS *globalVertexNumbers)
6008 {
6009   DM_Plex       *mesh = (DM_Plex*) dm->data;
6010   PetscInt       vStart, vEnd, vMax;
6011   PetscErrorCode ierr;
6012 
6013   PetscFunctionBegin;
6014   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6015   if (!mesh->globalVertexNumbers) {
6016     ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
6017     ierr = DMPlexGetHybridBounds(dm, NULL, NULL, NULL, &vMax);CHKERRQ(ierr);
6018     if (vMax >= 0) vEnd = PetscMin(vEnd, vMax);
6019     ierr = DMPlexCreateNumbering_Private(dm, vStart, vEnd, dm->sf, &mesh->globalVertexNumbers);CHKERRQ(ierr);
6020   }
6021   *globalVertexNumbers = mesh->globalVertexNumbers;
6022   PetscFunctionReturn(0);
6023 }
6024 
6025 
6026 #undef __FUNCT__
6027 #define __FUNCT__ "PetscSectionCreateGlobalSectionLabel"
6028 /*@C
6029   PetscSectionCreateGlobalSectionLabel - Create a section describing the global field layout using
6030   the local section and an SF describing the section point overlap.
6031 
6032   Input Parameters:
6033   + s - The PetscSection for the local field layout
6034   . sf - The SF describing parallel layout of the section points
6035   . includeConstraints - By default this is PETSC_FALSE, meaning that the global field vector will not possess constrained dofs
6036   . label - The label specifying the points
6037   - labelValue - The label stratum specifying the points
6038 
6039   Output Parameter:
6040   . gsection - The PetscSection for the global field layout
6041 
6042   Note: This gives negative sizes and offsets to points not owned by this process
6043 
6044   Level: developer
6045 
6046 .seealso: PetscSectionCreate()
6047 @*/
6048 PetscErrorCode PetscSectionCreateGlobalSectionLabel(PetscSection s, PetscSF sf, PetscBool includeConstraints, DMLabel label, PetscInt labelValue, PetscSection *gsection)
6049 {
6050   PetscInt      *neg = NULL, *tmpOff = NULL;
6051   PetscInt       pStart, pEnd, p, dof, cdof, off, globalOff = 0, nroots;
6052   PetscErrorCode ierr;
6053 
6054   PetscFunctionBegin;
6055   ierr = PetscSectionCreate(PetscObjectComm((PetscObject) s), gsection);CHKERRQ(ierr);
6056   ierr = PetscSectionGetChart(s, &pStart, &pEnd);CHKERRQ(ierr);
6057   ierr = PetscSectionSetChart(*gsection, pStart, pEnd);CHKERRQ(ierr);
6058   ierr = PetscSFGetGraph(sf, &nroots, NULL, NULL, NULL);CHKERRQ(ierr);
6059   if (nroots >= 0) {
6060     if (nroots < pEnd-pStart) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "PetscSF nroots %d < %d section size", nroots, pEnd-pStart);
6061     ierr = PetscCalloc1(nroots, &neg);CHKERRQ(ierr);
6062     if (nroots > pEnd-pStart) {
6063       ierr = PetscCalloc1(nroots, &tmpOff);CHKERRQ(ierr);
6064     } else {
6065       tmpOff = &(*gsection)->atlasDof[-pStart];
6066     }
6067   }
6068   /* Mark ghost points with negative dof */
6069   for (p = pStart; p < pEnd; ++p) {
6070     PetscInt value;
6071 
6072     ierr = DMLabelGetValue(label, p, &value);CHKERRQ(ierr);
6073     if (value != labelValue) continue;
6074     ierr = PetscSectionGetDof(s, p, &dof);CHKERRQ(ierr);
6075     ierr = PetscSectionSetDof(*gsection, p, dof);CHKERRQ(ierr);
6076     ierr = PetscSectionGetConstraintDof(s, p, &cdof);CHKERRQ(ierr);
6077     if (!includeConstraints && cdof > 0) {ierr = PetscSectionSetConstraintDof(*gsection, p, cdof);CHKERRQ(ierr);}
6078     if (neg) neg[p] = -(dof+1);
6079   }
6080   ierr = PetscSectionSetUpBC(*gsection);CHKERRQ(ierr);
6081   if (nroots >= 0) {
6082     ierr = PetscSFBcastBegin(sf, MPIU_INT, neg, tmpOff);CHKERRQ(ierr);
6083     ierr = PetscSFBcastEnd(sf, MPIU_INT, neg, tmpOff);CHKERRQ(ierr);
6084     if (nroots > pEnd-pStart) {
6085       for (p = pStart; p < pEnd; ++p) {if (tmpOff[p] < 0) (*gsection)->atlasDof[p-pStart] = tmpOff[p];}
6086     }
6087   }
6088   /* Calculate new sizes, get proccess offset, and calculate point offsets */
6089   for (p = 0, off = 0; p < pEnd-pStart; ++p) {
6090     cdof = (!includeConstraints && s->bc) ? s->bc->atlasDof[p] : 0;
6091     (*gsection)->atlasOff[p] = off;
6092     off += (*gsection)->atlasDof[p] > 0 ? (*gsection)->atlasDof[p]-cdof : 0;
6093   }
6094   ierr       = MPI_Scan(&off, &globalOff, 1, MPIU_INT, MPI_SUM, PetscObjectComm((PetscObject) s));CHKERRQ(ierr);
6095   globalOff -= off;
6096   for (p = 0, off = 0; p < pEnd-pStart; ++p) {
6097     (*gsection)->atlasOff[p] += globalOff;
6098     if (neg) neg[p] = -((*gsection)->atlasOff[p]+1);
6099   }
6100   /* Put in negative offsets for ghost points */
6101   if (nroots >= 0) {
6102     ierr = PetscSFBcastBegin(sf, MPIU_INT, neg, tmpOff);CHKERRQ(ierr);
6103     ierr = PetscSFBcastEnd(sf, MPIU_INT, neg, tmpOff);CHKERRQ(ierr);
6104     if (nroots > pEnd-pStart) {
6105       for (p = pStart; p < pEnd; ++p) {if (tmpOff[p] < 0) (*gsection)->atlasOff[p-pStart] = tmpOff[p];}
6106     }
6107   }
6108   if (nroots >= 0 && nroots > pEnd-pStart) {ierr = PetscFree(tmpOff);CHKERRQ(ierr);}
6109   ierr = PetscFree(neg);CHKERRQ(ierr);
6110   PetscFunctionReturn(0);
6111 }
6112 
6113 #undef __FUNCT__
6114 #define __FUNCT__ "DMPlexCheckSymmetry"
6115 /*@
6116   DMPlexCheckSymmetry - Check that the adjacency information in the mesh is symmetric.
6117 
6118   Input Parameters:
6119   + dm - The DMPlex object
6120 
6121   Note: This is a useful diagnostic when creating meshes programmatically.
6122 
6123   Level: developer
6124 
6125 .seealso: DMCreate(), DMCheckSkeleton(), DMCheckFaces()
6126 @*/
6127 PetscErrorCode DMPlexCheckSymmetry(DM dm)
6128 {
6129   PetscSection    coneSection, supportSection;
6130   const PetscInt *cone, *support;
6131   PetscInt        coneSize, c, supportSize, s;
6132   PetscInt        pStart, pEnd, p, csize, ssize;
6133   PetscErrorCode  ierr;
6134 
6135   PetscFunctionBegin;
6136   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6137   ierr = DMPlexGetConeSection(dm, &coneSection);CHKERRQ(ierr);
6138   ierr = DMPlexGetSupportSection(dm, &supportSection);CHKERRQ(ierr);
6139   /* Check that point p is found in the support of its cone points, and vice versa */
6140   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
6141   for (p = pStart; p < pEnd; ++p) {
6142     ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
6143     ierr = DMPlexGetCone(dm, p, &cone);CHKERRQ(ierr);
6144     for (c = 0; c < coneSize; ++c) {
6145       PetscBool dup = PETSC_FALSE;
6146       PetscInt  d;
6147       for (d = c-1; d >= 0; --d) {
6148         if (cone[c] == cone[d]) {dup = PETSC_TRUE; break;}
6149       }
6150       ierr = DMPlexGetSupportSize(dm, cone[c], &supportSize);CHKERRQ(ierr);
6151       ierr = DMPlexGetSupport(dm, cone[c], &support);CHKERRQ(ierr);
6152       for (s = 0; s < supportSize; ++s) {
6153         if (support[s] == p) break;
6154       }
6155       if ((s >= supportSize) || (dup && (support[s+1] != p))) {
6156         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %d cone: ", p);
6157         for (s = 0; s < coneSize; ++s) {
6158           ierr = PetscPrintf(PETSC_COMM_SELF, "%d, ", cone[s]);
6159         }
6160         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");
6161         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %d support: ", cone[c]);
6162         for (s = 0; s < supportSize; ++s) {
6163           ierr = PetscPrintf(PETSC_COMM_SELF, "%d, ", support[s]);
6164         }
6165         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");
6166         if (dup) {
6167           SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %d not repeatedly found in support of repeated cone point %d", p, cone[c]);
6168         } else {
6169           SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %d not found in support of cone point %d", p, cone[c]);
6170         }
6171       }
6172     }
6173     ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
6174     ierr = DMPlexGetSupport(dm, p, &support);CHKERRQ(ierr);
6175     for (s = 0; s < supportSize; ++s) {
6176       ierr = DMPlexGetConeSize(dm, support[s], &coneSize);CHKERRQ(ierr);
6177       ierr = DMPlexGetCone(dm, support[s], &cone);CHKERRQ(ierr);
6178       for (c = 0; c < coneSize; ++c) {
6179         if (cone[c] == p) break;
6180       }
6181       if (c >= coneSize) {
6182         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %d support: ", p);
6183         for (c = 0; c < supportSize; ++c) {
6184           ierr = PetscPrintf(PETSC_COMM_SELF, "%d, ", support[c]);
6185         }
6186         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");
6187         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %d cone: ", support[s]);
6188         for (c = 0; c < coneSize; ++c) {
6189           ierr = PetscPrintf(PETSC_COMM_SELF, "%d, ", cone[c]);
6190         }
6191         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");
6192         SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %d not found in cone of support point %d", p, support[s]);
6193       }
6194     }
6195   }
6196   ierr = PetscSectionGetStorageSize(coneSection, &csize);CHKERRQ(ierr);
6197   ierr = PetscSectionGetStorageSize(supportSection, &ssize);CHKERRQ(ierr);
6198   if (csize != ssize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "Total cone size %d != Total support size %d", csize, ssize);
6199   PetscFunctionReturn(0);
6200 }
6201 
6202 #undef __FUNCT__
6203 #define __FUNCT__ "DMPlexCheckSkeleton"
6204 /*@
6205   DMPlexCheckSkeleton - Check that each cell has the correct number of vertices
6206 
6207   Input Parameters:
6208 + dm - The DMPlex object
6209 . isSimplex - Are the cells simplices or tensor products
6210 - cellHeight - Normally 0
6211 
6212   Note: This is a useful diagnostic when creating meshes programmatically.
6213 
6214   Level: developer
6215 
6216 .seealso: DMCreate(), DMCheckSymmetry(), DMCheckFaces()
6217 @*/
6218 PetscErrorCode DMPlexCheckSkeleton(DM dm, PetscBool isSimplex, PetscInt cellHeight)
6219 {
6220   PetscInt       dim, numCorners, numHybridCorners, vStart, vEnd, cStart, cEnd, cMax, c;
6221   PetscErrorCode ierr;
6222 
6223   PetscFunctionBegin;
6224   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6225   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
6226   switch (dim) {
6227   case 1: numCorners = isSimplex ? 2 : 2; numHybridCorners = isSimplex ? 2 : 2; break;
6228   case 2: numCorners = isSimplex ? 3 : 4; numHybridCorners = isSimplex ? 4 : 4; break;
6229   case 3: numCorners = isSimplex ? 4 : 8; numHybridCorners = isSimplex ? 6 : 8; break;
6230   default:
6231     SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_OUTOFRANGE, "Cannot handle meshes of dimension %d", dim);
6232   }
6233   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
6234   ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
6235   ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
6236   cMax = cMax >= 0 ? cMax : cEnd;
6237   for (c = cStart; c < cMax; ++c) {
6238     PetscInt *closure = NULL, closureSize, cl, coneSize = 0;
6239 
6240     ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6241     for (cl = 0; cl < closureSize*2; cl += 2) {
6242       const PetscInt p = closure[cl];
6243       if ((p >= vStart) && (p < vEnd)) ++coneSize;
6244     }
6245     ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6246     if (coneSize != numCorners) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %d has  %d vertices != %d", c, coneSize, numCorners);
6247   }
6248   for (c = cMax; c < cEnd; ++c) {
6249     PetscInt *closure = NULL, closureSize, cl, coneSize = 0;
6250 
6251     ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6252     for (cl = 0; cl < closureSize*2; cl += 2) {
6253       const PetscInt p = closure[cl];
6254       if ((p >= vStart) && (p < vEnd)) ++coneSize;
6255     }
6256     ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6257     if (coneSize > numHybridCorners) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Hybrid cell %d has  %d vertices > %d", c, coneSize, numHybridCorners);
6258   }
6259   PetscFunctionReturn(0);
6260 }
6261 
6262 #undef __FUNCT__
6263 #define __FUNCT__ "DMPlexCheckFaces"
6264 /*@
6265   DMPlexCheckFaces - Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type
6266 
6267   Input Parameters:
6268 + dm - The DMPlex object
6269 . isSimplex - Are the cells simplices or tensor products
6270 - cellHeight - Normally 0
6271 
6272   Note: This is a useful diagnostic when creating meshes programmatically.
6273 
6274   Level: developer
6275 
6276 .seealso: DMCreate(), DMCheckSymmetry(), DMCheckSkeleton()
6277 @*/
6278 PetscErrorCode DMPlexCheckFaces(DM dm, PetscBool isSimplex, PetscInt cellHeight)
6279 {
6280   PetscInt       pMax[4];
6281   PetscInt       dim, vStart, vEnd, cStart, cEnd, c, h;
6282   PetscErrorCode ierr;
6283 
6284   PetscFunctionBegin;
6285   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6286   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
6287   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
6288   ierr = DMPlexGetHybridBounds(dm, &pMax[dim], &pMax[dim-1], &pMax[1], &pMax[0]);CHKERRQ(ierr);
6289   for (h = cellHeight; h < dim; ++h) {
6290     ierr = DMPlexGetHeightStratum(dm, h, &cStart, &cEnd);CHKERRQ(ierr);
6291     for (c = cStart; c < cEnd; ++c) {
6292       const PetscInt *cone, *ornt, *faces;
6293       PetscInt        numFaces, faceSize, coneSize,f;
6294       PetscInt       *closure = NULL, closureSize, cl, numCorners = 0;
6295 
6296       if (pMax[dim-h] >= 0 && c >= pMax[dim-h]) continue;
6297       ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
6298       ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
6299       ierr = DMPlexGetConeOrientation(dm, c, &ornt);CHKERRQ(ierr);
6300       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6301       for (cl = 0; cl < closureSize*2; cl += 2) {
6302         const PetscInt p = closure[cl];
6303         if ((p >= vStart) && (p < vEnd)) closure[numCorners++] = p;
6304       }
6305       ierr = DMPlexGetRawFaces_Internal(dm, dim-h, numCorners, closure, &numFaces, &faceSize, &faces);CHKERRQ(ierr);
6306       if (coneSize != numFaces) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %d has %d faces but should have %d", c, coneSize, numFaces);
6307       for (f = 0; f < numFaces; ++f) {
6308         PetscInt *fclosure = NULL, fclosureSize, cl, fnumCorners = 0, v;
6309 
6310         ierr = DMPlexGetTransitiveClosure_Internal(dm, cone[f], ornt[f], PETSC_TRUE, &fclosureSize, &fclosure);CHKERRQ(ierr);
6311         for (cl = 0; cl < fclosureSize*2; cl += 2) {
6312           const PetscInt p = fclosure[cl];
6313           if ((p >= vStart) && (p < vEnd)) fclosure[fnumCorners++] = p;
6314         }
6315         if (fnumCorners != faceSize) SETERRQ5(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %d (%d) of cell %d has %d vertices but should have %d", cone[f], f, c, fnumCorners, faceSize);
6316         for (v = 0; v < fnumCorners; ++v) {
6317           if (fclosure[v] != faces[f*faceSize+v]) SETERRQ6(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %d (%d) of cell %d vertex %d, %d != %d", cone[f], f, c, v, fclosure[v], faces[f*faceSize+v]);
6318         }
6319         ierr = DMPlexRestoreTransitiveClosure(dm, cone[f], PETSC_TRUE, &fclosureSize, &fclosure);CHKERRQ(ierr);
6320       }
6321       ierr = DMPlexRestoreFaces_Internal(dm, dim, c, &numFaces, &faceSize, &faces);CHKERRQ(ierr);
6322       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6323     }
6324   }
6325   PetscFunctionReturn(0);
6326 }
6327 
6328 #undef __FUNCT__
6329 #define __FUNCT__ "DMCreateInterpolation_Plex"
6330 /* Pointwise interpolation
6331      Just code FEM for now
6332      u^f = I u^c
6333      sum_k u^f_k phi^f_k = I sum_j u^c_j phi^c_j
6334      u^f_i = sum_j psi^f_i I phi^c_j u^c_j
6335      I_{ij} = psi^f_i phi^c_j
6336 */
6337 PetscErrorCode DMCreateInterpolation_Plex(DM dmCoarse, DM dmFine, Mat *interpolation, Vec *scaling)
6338 {
6339   PetscSection   gsc, gsf;
6340   PetscInt       m, n;
6341   void          *ctx;
6342   PetscErrorCode ierr;
6343 
6344   PetscFunctionBegin;
6345   /*
6346   Loop over coarse cells
6347     Loop over coarse basis functions
6348       Loop over fine cells in coarse cell
6349         Loop over fine dual basis functions
6350           Evaluate coarse basis on fine dual basis quad points
6351           Sum
6352           Update local element matrix
6353     Accumulate to interpolation matrix
6354 
6355    Can extend PetscFEIntegrateJacobian_Basic() to do a specialized cell loop
6356   */
6357   ierr = DMGetDefaultGlobalSection(dmFine, &gsf);CHKERRQ(ierr);
6358   ierr = PetscSectionGetConstrainedStorageSize(gsf, &m);CHKERRQ(ierr);
6359   ierr = DMGetDefaultGlobalSection(dmCoarse, &gsc);CHKERRQ(ierr);
6360   ierr = PetscSectionGetConstrainedStorageSize(gsc, &n);CHKERRQ(ierr);
6361   /* We need to preallocate properly */
6362   ierr = MatCreate(PetscObjectComm((PetscObject) dmCoarse), interpolation);CHKERRQ(ierr);
6363   ierr = MatSetSizes(*interpolation, m, n, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
6364   ierr = MatSetType(*interpolation, dmCoarse->mattype);CHKERRQ(ierr);
6365   ierr = MatSetUp(*interpolation);CHKERRQ(ierr);
6366   ierr = MatSetFromOptions(*interpolation);CHKERRQ(ierr);
6367   ierr = MatSetOption(*interpolation, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_FALSE);CHKERRQ(ierr);
6368   ierr = DMGetApplicationContext(dmFine, &ctx);CHKERRQ(ierr);
6369   ierr = DMPlexComputeInterpolatorFEM(dmCoarse, dmFine, *interpolation, ctx);CHKERRQ(ierr);
6370   /* Use naive scaling */
6371   ierr = DMCreateInterpolationScale(dmCoarse, dmFine, *interpolation, scaling);CHKERRQ(ierr);
6372   PetscFunctionReturn(0);
6373 }
6374 
6375 #undef __FUNCT__
6376 #define __FUNCT__ "DMCreateInjection_Plex"
6377 PetscErrorCode DMCreateInjection_Plex(DM dmCoarse, DM dmFine, VecScatter *ctx)
6378 {
6379   Vec             cv,  fv;
6380   IS              cis, fis, fpointIS;
6381   PetscSection    sc, gsc, gsf;
6382   const PetscInt *fpoints;
6383   PetscInt       *cindices, *findices;
6384   PetscInt        cpStart, cpEnd, m, off, cp;
6385   PetscErrorCode  ierr;
6386 
6387   PetscFunctionBegin;
6388   ierr = DMGetDefaultGlobalSection(dmFine, &gsf);CHKERRQ(ierr);
6389   ierr = DMGetGlobalVector(dmFine, &fv);CHKERRQ(ierr);
6390   ierr = DMGetDefaultSection(dmCoarse, &sc);CHKERRQ(ierr);
6391   ierr = DMGetDefaultGlobalSection(dmCoarse, &gsc);CHKERRQ(ierr);
6392   ierr = DMGetGlobalVector(dmCoarse, &cv);CHKERRQ(ierr);
6393   ierr = DMPlexCreateCoarsePointIS(dmCoarse, &fpointIS);CHKERRQ(ierr);
6394   ierr = PetscSectionGetConstrainedStorageSize(gsc, &m);CHKERRQ(ierr);
6395   ierr = PetscMalloc2(m,&cindices,m,&findices);CHKERRQ(ierr);
6396   ierr = PetscSectionGetChart(gsc, &cpStart, &cpEnd);CHKERRQ(ierr);
6397   ierr = ISGetIndices(fpointIS, &fpoints);CHKERRQ(ierr);
6398   for (cp = cpStart, off = 0; cp < cpEnd; ++cp) {
6399     const PetscInt *cdofsC = NULL;
6400     PetscInt        fp     = fpoints[cp-cpStart], dofC, cdofC, dofF, offC, offF, d, e;
6401 
6402     ierr = PetscSectionGetDof(gsc, cp, &dofC);CHKERRQ(ierr);
6403     if (dofC <= 0) continue;
6404     ierr = PetscSectionGetConstraintDof(sc, cp, &cdofC);CHKERRQ(ierr);
6405     ierr = PetscSectionGetDof(gsf, fp, &dofF);CHKERRQ(ierr);
6406     ierr = PetscSectionGetOffset(gsc, cp, &offC);CHKERRQ(ierr);
6407     ierr = PetscSectionGetOffset(gsf, fp, &offF);CHKERRQ(ierr);
6408     if (cdofC) {ierr = PetscSectionGetConstraintIndices(sc, cp, &cdofsC);CHKERRQ(ierr);}
6409     if (dofC != dofF) SETERRQ4(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Point %d (%d) has %d coarse dofs != %d fine dofs", cp, fp, dofC, dofF);
6410     if (offC < 0 || offF < 0) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Coarse point %d has invalid offset %d (%d)", cp, offC, offF);
6411     for (d = 0, e = 0; d < dofC; ++d) {
6412       if (cdofsC && cdofsC[e] == d) {++e; continue;}
6413       cindices[off+d-e] = offC+d; findices[off+d-e] = offF+d;
6414     }
6415     if (e != cdofC) SETERRQ4(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Point %d (%d) has invalid number of constraints %d != %d", cp, fp, e, cdofC);
6416     off += dofC-cdofC;
6417   }
6418   ierr = ISRestoreIndices(fpointIS, &fpoints);CHKERRQ(ierr);
6419   if (off != m) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of coarse dofs %d != %d", off, m);
6420   ierr = ISCreateGeneral(PETSC_COMM_SELF, m, cindices, PETSC_OWN_POINTER, &cis);CHKERRQ(ierr);
6421   ierr = ISCreateGeneral(PETSC_COMM_SELF, m, findices, PETSC_OWN_POINTER, &fis);CHKERRQ(ierr);
6422   ierr = VecScatterCreate(cv, cis, fv, fis, ctx);CHKERRQ(ierr);
6423   ierr = ISDestroy(&cis);CHKERRQ(ierr);
6424   ierr = ISDestroy(&fis);CHKERRQ(ierr);
6425   ierr = DMRestoreGlobalVector(dmFine, &fv);CHKERRQ(ierr);
6426   ierr = DMRestoreGlobalVector(dmCoarse, &cv);CHKERRQ(ierr);
6427   ierr = ISDestroy(&fpointIS);CHKERRQ(ierr);
6428   PetscFunctionReturn(0);
6429 }
6430 
6431 #undef __FUNCT__
6432 #define __FUNCT__ "DMCreateDefaultSection_Plex"
6433 /* Pointwise interpolation
6434      Just code FEM for now
6435      u^f = I u^c
6436      sum_k u^f_k phi^f_k = I sum_l u^c_l phi^c_l
6437      u^f_i = sum_l int psi^f_i I phi^c_l u^c_l
6438      I_{ij} = int psi^f_i phi^c_j
6439 */
6440 PetscErrorCode DMCreateDefaultSection_Plex(DM dm)
6441 {
6442   PetscSection   section;
6443   IS            *bcPoints;
6444   PetscInt      *bcFields, *numComp, *numDof;
6445   PetscInt       depth, dim, numBd, numBC = 0, numFields, bd, bc, f;
6446   PetscErrorCode ierr;
6447 
6448   PetscFunctionBegin;
6449   /* Handle boundary conditions */
6450   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
6451   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
6452   ierr = DMPlexGetNumBoundary(dm, &numBd);CHKERRQ(ierr);
6453   for (bd = 0; bd < numBd; ++bd) {
6454     PetscBool isEssential;
6455     ierr = DMPlexGetBoundary(dm, bd, &isEssential, NULL, NULL, NULL, NULL, NULL, NULL);CHKERRQ(ierr);
6456     if (isEssential) ++numBC;
6457   }
6458   ierr = PetscMalloc2(numBC,&bcFields,numBC,&bcPoints);CHKERRQ(ierr);
6459   for (bd = 0, bc = 0; bd < numBd; ++bd) {
6460     const char     *bdLabel;
6461     DMLabel         label;
6462     const PetscInt *values;
6463     PetscInt        field, numValues;
6464     PetscBool       isEssential, has;
6465 
6466     ierr = DMPlexGetBoundary(dm, bd, &isEssential, &bdLabel, &field, NULL, &numValues, &values, NULL);CHKERRQ(ierr);
6467     if (numValues != 1) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Bug me and I will fix this");
6468     ierr = DMPlexHasLabel(dm, bdLabel, &has);CHKERRQ(ierr);
6469     if (!has) {
6470       ierr = DMPlexCreateLabel(dm, bdLabel);CHKERRQ(ierr);
6471       ierr = DMPlexGetLabel(dm, bdLabel, &label);CHKERRQ(ierr);
6472       ierr = DMPlexMarkBoundaryFaces(dm, label);CHKERRQ(ierr);
6473     }
6474     ierr = DMPlexGetLabel(dm, bdLabel, &label);CHKERRQ(ierr);
6475     /* Only want to do this for FEM */
6476     ierr = DMPlexLabelComplete(dm, label);CHKERRQ(ierr);
6477     ierr = DMPlexLabelAddCells(dm, label);CHKERRQ(ierr);
6478     /* Filter out cells, if you actually want to constraint cells you need to do things by hand right now */
6479     if (isEssential) {
6480       IS              tmp;
6481       PetscInt       *newidx;
6482       const PetscInt *idx;
6483       PetscInt        cStart, cEnd, n, p, newn = 0;
6484 
6485       bcFields[bc] = field;
6486       ierr = DMPlexGetStratumIS(dm, bdLabel, values[0], &tmp);CHKERRQ(ierr);
6487       ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
6488       ierr = ISGetLocalSize(tmp, &n);CHKERRQ(ierr);
6489       ierr = ISGetIndices(tmp, &idx);CHKERRQ(ierr);
6490       for (p = 0; p < n; ++p) if ((idx[p] < cStart) || (idx[p] >= cEnd)) ++newn;
6491       ierr = PetscMalloc1(newn,&newidx);CHKERRQ(ierr);
6492       newn = 0;
6493       for (p = 0; p < n; ++p) if ((idx[p] < cStart) || (idx[p] >= cEnd)) newidx[newn++] = idx[p];
6494       ierr = ISCreateGeneral(PetscObjectComm((PetscObject) dm), newn, newidx, PETSC_OWN_POINTER, &bcPoints[bc++]);CHKERRQ(ierr);
6495       ierr = ISRestoreIndices(tmp, &idx);CHKERRQ(ierr);
6496       ierr = ISDestroy(&tmp);CHKERRQ(ierr);
6497     }
6498   }
6499   /* Handle discretization */
6500   ierr = DMGetNumFields(dm, &numFields);CHKERRQ(ierr);
6501   ierr = PetscMalloc2(numFields,&numComp,numFields*(dim+1),&numDof);CHKERRQ(ierr);
6502   for (f = 0; f < numFields; ++f) {
6503     PetscFE         fe;
6504     const PetscInt *numFieldDof;
6505     PetscInt        d;
6506 
6507     ierr = DMGetField(dm, f, (PetscObject *) &fe);CHKERRQ(ierr);
6508     ierr = PetscFEGetNumComponents(fe, &numComp[f]);CHKERRQ(ierr);
6509     ierr = PetscFEGetNumDof(fe, &numFieldDof);CHKERRQ(ierr);
6510     for (d = 0; d < dim+1; ++d) numDof[f*(dim+1)+d] = numFieldDof[d];
6511   }
6512   for (f = 0; f < numFields; ++f) {
6513     PetscInt d;
6514     for (d = 1; d < dim; ++d) {
6515       if ((numDof[f*(dim+1)+d] > 0) && (depth < dim)) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Mesh must be interpolated when unknowns are specified on edges or faces.");
6516     }
6517   }
6518   ierr = DMPlexCreateSection(dm, dim, numFields, numComp, numDof, numBC, bcFields, bcPoints, NULL, &section);CHKERRQ(ierr);
6519   for (f = 0; f < numFields; ++f) {
6520     PetscFE     fe;
6521     const char *name;
6522 
6523     ierr = DMGetField(dm, f, (PetscObject *) &fe);CHKERRQ(ierr);
6524     ierr = PetscObjectGetName((PetscObject) fe, &name);CHKERRQ(ierr);
6525     ierr = PetscSectionSetFieldName(section, f, name);CHKERRQ(ierr);
6526   }
6527   ierr = DMSetDefaultSection(dm, section);CHKERRQ(ierr);
6528   ierr = PetscSectionDestroy(&section);CHKERRQ(ierr);
6529   for (bc = 0; bc < numBC; ++bc) {ierr = ISDestroy(&bcPoints[bc]);CHKERRQ(ierr);}
6530   ierr = PetscFree2(bcFields,bcPoints);CHKERRQ(ierr);
6531   ierr = PetscFree2(numComp,numDof);CHKERRQ(ierr);
6532   PetscFunctionReturn(0);
6533 }
6534 
6535 #undef __FUNCT__
6536 #define __FUNCT__ "DMPlexGetCoarseDM"
6537 /*@
6538   DMPlexGetCoarseDM - Get the coarse mesh from which this was obtained by refinement
6539 
6540   Input Parameter:
6541 . dm - The DMPlex object
6542 
6543   Output Parameter:
6544 . cdm - The coarse DM
6545 
6546   Level: intermediate
6547 
6548 .seealso: DMPlexSetCoarseDM()
6549 @*/
6550 PetscErrorCode DMPlexGetCoarseDM(DM dm, DM *cdm)
6551 {
6552   PetscFunctionBegin;
6553   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6554   PetscValidPointer(cdm, 2);
6555   *cdm = ((DM_Plex *) dm->data)->coarseMesh;
6556   PetscFunctionReturn(0);
6557 }
6558 
6559 #undef __FUNCT__
6560 #define __FUNCT__ "DMPlexSetCoarseDM"
6561 /*@
6562   DMPlexSetCoarseDM - Set the coarse mesh from which this was obtained by refinement
6563 
6564   Input Parameters:
6565 + dm - The DMPlex object
6566 - cdm - The coarse DM
6567 
6568   Level: intermediate
6569 
6570 .seealso: DMPlexGetCoarseDM()
6571 @*/
6572 PetscErrorCode DMPlexSetCoarseDM(DM dm, DM cdm)
6573 {
6574   DM_Plex       *mesh;
6575   PetscErrorCode ierr;
6576 
6577   PetscFunctionBegin;
6578   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6579   if (cdm) PetscValidHeaderSpecific(cdm, DM_CLASSID, 2);
6580   mesh = (DM_Plex *) dm->data;
6581   ierr = DMDestroy(&mesh->coarseMesh);CHKERRQ(ierr);
6582   mesh->coarseMesh = cdm;
6583   ierr = PetscObjectReference((PetscObject) mesh->coarseMesh);CHKERRQ(ierr);
6584   PetscFunctionReturn(0);
6585 }
6586