xref: /petsc/src/dm/impls/plex/plex.c (revision 536b137f95b6f40d8df0141fed3efeb26b0ec7af)
1 #include <petsc-private/dmpleximpl.h>   /*I      "petscdmplex.h"   I*/
2 #include <../src/sys/utils/hash.h>
3 #include <petsc-private/isimpl.h>
4 #include <petscsf.h>
5 
6 /* Logging support */
7 PetscLogEvent DMPLEX_Interpolate, DMPLEX_Partition, DMPLEX_Distribute, DMPLEX_DistributeCones, DMPLEX_DistributeLabels, DMPLEX_DistributeSF, DMPLEX_DistributeField, DMPLEX_DistributeData, DMPLEX_Stratify, DMPLEX_Preallocate, DMPLEX_ResidualFEM, DMPLEX_JacobianFEM;
8 
9 PETSC_EXTERN PetscErrorCode VecView_Seq(Vec, PetscViewer);
10 PETSC_EXTERN PetscErrorCode VecView_MPI(Vec, PetscViewer);
11 PETSC_EXTERN PetscErrorCode VecLoad_Default(Vec, PetscViewer);
12 
13 #undef __FUNCT__
14 #define __FUNCT__ "DMPlexGetFieldType_Internal"
15 PetscErrorCode DMPlexGetFieldType_Internal(DM dm, PetscSection section, PetscInt field, PetscInt *sStart, PetscInt *sEnd, PetscViewerVTKFieldType *ft)
16 {
17   PetscInt       dim, pStart, pEnd, vStart, vEnd, cStart, cEnd, vdof = 0, cdof = 0;
18   PetscErrorCode ierr;
19 
20   PetscFunctionBegin;
21   *ft  = PETSC_VTK_POINT_FIELD;
22   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
23   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
24   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
25   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
26   if (field >= 0) {
27     if ((vStart >= pStart) && (vStart < pEnd)) {ierr = PetscSectionGetFieldDof(section, vStart, field, &vdof);CHKERRQ(ierr);}
28     if ((cStart >= pStart) && (cStart < pEnd)) {ierr = PetscSectionGetFieldDof(section, cStart, field, &cdof);CHKERRQ(ierr);}
29   } else {
30     if ((vStart >= pStart) && (vStart < pEnd)) {ierr = PetscSectionGetDof(section, vStart, &vdof);CHKERRQ(ierr);}
31     if ((cStart >= pStart) && (cStart < pEnd)) {ierr = PetscSectionGetDof(section, cStart, &cdof);CHKERRQ(ierr);}
32   }
33   if (vdof) {
34     *sStart = vStart;
35     *sEnd   = vEnd;
36     if (vdof == dim) *ft = PETSC_VTK_POINT_VECTOR_FIELD;
37     else             *ft = PETSC_VTK_POINT_FIELD;
38   } else if (cdof) {
39     *sStart = cStart;
40     *sEnd   = cEnd;
41     if (cdof == dim) *ft = PETSC_VTK_CELL_VECTOR_FIELD;
42     else             *ft = PETSC_VTK_CELL_FIELD;
43   } else SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Could not classify input Vec for VTK");
44   PetscFunctionReturn(0);
45 }
46 
47 #undef __FUNCT__
48 #define __FUNCT__ "VecView_Plex_Local"
49 PetscErrorCode VecView_Plex_Local(Vec v, PetscViewer viewer)
50 {
51   DM             dm;
52   PetscBool      isvtk, ishdf5, isseq;
53   PetscErrorCode ierr;
54 
55   PetscFunctionBegin;
56   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
57   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
58   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,  &isvtk);CHKERRQ(ierr);
59   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
60   ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
61   if (isvtk || ishdf5) {
62     PetscInt  numFields;
63     PetscBool fem = PETSC_FALSE;
64 
65     ierr = DMGetNumFields(dm, &numFields);CHKERRQ(ierr);
66     if (numFields) {
67       PetscObject fe;
68 
69       ierr = DMGetField(dm, 0, &fe);CHKERRQ(ierr);
70       if (fe->classid == PETSCFE_CLASSID) fem = PETSC_TRUE;
71     }
72     if (fem) {ierr = DMPlexInsertBoundaryValuesFEM(dm, v);CHKERRQ(ierr);}
73   }
74   if (isvtk) {
75     PetscSection            section;
76     PetscViewerVTKFieldType ft;
77     PetscInt                pStart, pEnd;
78 
79     ierr = DMGetDefaultSection(dm, &section);CHKERRQ(ierr);
80     ierr = DMPlexGetFieldType_Internal(dm, section, PETSC_DETERMINE, &pStart, &pEnd, &ft);CHKERRQ(ierr);
81     ierr = PetscObjectReference((PetscObject) dm);CHKERRQ(ierr); /* viewer drops reference */
82     ierr = PetscObjectReference((PetscObject) v);CHKERRQ(ierr);  /* viewer drops reference */
83     ierr = PetscViewerVTKAddField(viewer, (PetscObject) dm, DMPlexVTKWriteAll, ft, (PetscObject) v);CHKERRQ(ierr);
84   } else if (ishdf5) {
85 #if defined(PETSC_HAVE_HDF5)
86     ierr = VecView_Plex_Local_HDF5(v, viewer);CHKERRQ(ierr);
87 #else
88     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
89 #endif
90   } else {
91     if (isseq) {ierr = VecView_Seq(v, viewer);CHKERRQ(ierr);}
92     else       {ierr = VecView_MPI(v, viewer);CHKERRQ(ierr);}
93   }
94   PetscFunctionReturn(0);
95 }
96 
97 #undef __FUNCT__
98 #define __FUNCT__ "VecView_Plex"
99 PetscErrorCode VecView_Plex(Vec v, PetscViewer viewer)
100 {
101   DM             dm;
102   PetscBool      isvtk, ishdf5, isseq;
103   PetscErrorCode ierr;
104 
105   PetscFunctionBegin;
106   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
107   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
108   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,  &isvtk);CHKERRQ(ierr);
109   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
110   ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
111   if (isvtk) {
112     Vec         locv;
113     const char *name;
114 
115     ierr = DMGetLocalVector(dm, &locv);CHKERRQ(ierr);
116     ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
117     ierr = PetscObjectSetName((PetscObject) locv, name);CHKERRQ(ierr);
118     ierr = DMGlobalToLocalBegin(dm, v, INSERT_VALUES, locv);CHKERRQ(ierr);
119     ierr = DMGlobalToLocalEnd(dm, v, INSERT_VALUES, locv);CHKERRQ(ierr);
120     ierr = VecView_Plex_Local(locv, viewer);CHKERRQ(ierr);
121     ierr = DMRestoreLocalVector(dm, &locv);CHKERRQ(ierr);
122   } else if (ishdf5) {
123 #if defined(PETSC_HAVE_HDF5)
124     ierr = VecView_Plex_HDF5(v, viewer);CHKERRQ(ierr);
125 #else
126     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
127 #endif
128   } else {
129     if (isseq) {ierr = VecView_Seq(v, viewer);CHKERRQ(ierr);}
130     else       {ierr = VecView_MPI(v, viewer);CHKERRQ(ierr);}
131   }
132   PetscFunctionReturn(0);
133 }
134 
135 #undef __FUNCT__
136 #define __FUNCT__ "VecLoad_Plex_Local"
137 PetscErrorCode VecLoad_Plex_Local(Vec v, PetscViewer viewer)
138 {
139   DM             dm;
140   PetscBool      ishdf5;
141   PetscErrorCode ierr;
142 
143   PetscFunctionBegin;
144   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
145   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
146   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
147   if (ishdf5) {
148     DM          dmBC;
149     Vec         gv;
150     const char *name;
151 
152     ierr = DMGetOutputDM(dm, &dmBC);CHKERRQ(ierr);
153     ierr = DMGetGlobalVector(dmBC, &gv);CHKERRQ(ierr);
154     ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
155     ierr = PetscObjectSetName((PetscObject) gv, name);CHKERRQ(ierr);
156     ierr = VecLoad_Default(gv, viewer);CHKERRQ(ierr);
157     ierr = DMGlobalToLocalBegin(dmBC, gv, INSERT_VALUES, v);CHKERRQ(ierr);
158     ierr = DMGlobalToLocalEnd(dmBC, gv, INSERT_VALUES, v);CHKERRQ(ierr);
159     ierr = DMRestoreGlobalVector(dmBC, &gv);CHKERRQ(ierr);
160   } else {
161     ierr = VecLoad_Default(v, viewer);CHKERRQ(ierr);
162   }
163   PetscFunctionReturn(0);
164 }
165 
166 #undef __FUNCT__
167 #define __FUNCT__ "VecLoad_Plex"
168 PetscErrorCode VecLoad_Plex(Vec v, PetscViewer viewer)
169 {
170   DM             dm;
171   PetscBool      ishdf5;
172   PetscErrorCode ierr;
173 
174   PetscFunctionBegin;
175   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
176   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
177   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
178   if (ishdf5) {
179 #if defined(PETSC_HAVE_HDF5)
180     ierr = VecLoad_Plex_HDF5(v, viewer);CHKERRQ(ierr);
181 #else
182     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
183 #endif
184   } else {
185     ierr = VecLoad_Default(v, viewer);CHKERRQ(ierr);
186   }
187   PetscFunctionReturn(0);
188 }
189 
190 #undef __FUNCT__
191 #define __FUNCT__ "DMPlexView_Ascii"
192 PetscErrorCode DMPlexView_Ascii(DM dm, PetscViewer viewer)
193 {
194   DM_Plex          *mesh = (DM_Plex*) dm->data;
195   DM                cdm;
196   DMLabel           markers;
197   PetscSection      coordSection;
198   Vec               coordinates;
199   PetscViewerFormat format;
200   PetscErrorCode    ierr;
201 
202   PetscFunctionBegin;
203   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
204   ierr = DMGetDefaultSection(cdm, &coordSection);CHKERRQ(ierr);
205   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
206   ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
207   if (format == PETSC_VIEWER_ASCII_INFO_DETAIL) {
208     const char *name;
209     PetscInt    maxConeSize, maxSupportSize;
210     PetscInt    pStart, pEnd, p;
211     PetscMPIInt rank, size;
212 
213     ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRQ(ierr);
214     ierr = MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size);CHKERRQ(ierr);
215     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
216     ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
217     ierr = DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize);CHKERRQ(ierr);
218     ierr = PetscViewerASCIISynchronizedAllow(viewer, PETSC_TRUE);CHKERRQ(ierr);
219     ierr = PetscViewerASCIIPrintf(viewer, "Mesh '%s':\n", name);CHKERRQ(ierr);
220     ierr = PetscViewerASCIISynchronizedPrintf(viewer, "Max sizes cone: %D support: %D\n", maxConeSize, maxSupportSize);CHKERRQ(ierr);
221     ierr = PetscViewerASCIIPrintf(viewer, "orientation is missing\n", name);CHKERRQ(ierr);
222     ierr = PetscViewerASCIIPrintf(viewer, "cap --> base:\n", name);CHKERRQ(ierr);
223     for (p = pStart; p < pEnd; ++p) {
224       PetscInt dof, off, s;
225 
226       ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
227       ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
228       for (s = off; s < off+dof; ++s) {
229         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%D]: %D ----> %D\n", rank, p, mesh->supports[s]);CHKERRQ(ierr);
230       }
231     }
232     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
233     ierr = PetscViewerASCIIPrintf(viewer, "base <-- cap:\n", name);CHKERRQ(ierr);
234     for (p = pStart; p < pEnd; ++p) {
235       PetscInt dof, off, c;
236 
237       ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
238       ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
239       for (c = off; c < off+dof; ++c) {
240         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%D]: %D <---- %D (%D)\n", rank, p, mesh->cones[c], mesh->coneOrientations[c]);CHKERRQ(ierr);
241       }
242     }
243     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
244     ierr = PetscSectionGetChart(coordSection, &pStart, NULL);CHKERRQ(ierr);
245     if (pStart >= 0) {ierr = PetscSectionVecView(coordSection, coordinates, viewer);CHKERRQ(ierr);}
246     ierr = DMPlexGetLabel(dm, "marker", &markers);CHKERRQ(ierr);
247     ierr = DMLabelView(markers,viewer);CHKERRQ(ierr);
248     if (size > 1) {
249       PetscSF sf;
250 
251       ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr);
252       ierr = PetscSFView(sf, viewer);CHKERRQ(ierr);
253     }
254     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
255   } else if (format == PETSC_VIEWER_ASCII_LATEX) {
256     const char  *name;
257     const char  *colors[3] = {"red", "blue", "green"};
258     const int    numColors  = 3;
259     PetscReal    scale      = 2.0;
260     PetscScalar *coords;
261     PetscInt     depth, cStart, cEnd, c, vStart, vEnd, v, eStart = 0, eEnd = 0, e, p;
262     PetscMPIInt  rank, size;
263 
264     ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
265     ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRQ(ierr);
266     ierr = MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size);CHKERRQ(ierr);
267     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
268     ierr = PetscViewerASCIISynchronizedAllow(viewer, PETSC_TRUE);CHKERRQ(ierr);
269     ierr = PetscViewerASCIIPrintf(viewer, "\
270 \\documentclass[crop,multi=false]{standalone}\n\n\
271 \\usepackage{tikz}\n\
272 \\usepackage{pgflibraryshapes}\n\
273 \\usetikzlibrary{backgrounds}\n\
274 \\usetikzlibrary{arrows}\n\
275 \\begin{document}\n\
276 \\section{%s}\n\
277 \\begin{center}\n", name, 8.0/scale);CHKERRQ(ierr);
278     ierr = PetscViewerASCIIPrintf(viewer, "Mesh for process ");CHKERRQ(ierr);
279     for (p = 0; p < size; ++p) {
280       if (p > 0 && p == size-1) {
281         ierr = PetscViewerASCIIPrintf(viewer, ", and ", colors[p%numColors], p);CHKERRQ(ierr);
282       } else if (p > 0) {
283         ierr = PetscViewerASCIIPrintf(viewer, ", ", colors[p%numColors], p);CHKERRQ(ierr);
284       }
285       ierr = PetscViewerASCIIPrintf(viewer, "{\\textcolor{%s}%D}", colors[p%numColors], p);CHKERRQ(ierr);
286     }
287     ierr = PetscViewerASCIIPrintf(viewer, ".\n\n\n\
288 \\begin{tikzpicture}[scale = %g,font=\\fontsize{8}{8}\\selectfont]\n");CHKERRQ(ierr);
289     /* Plot vertices */
290     ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
291     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
292     ierr = PetscViewerASCIIPrintf(viewer, "\\path\n");CHKERRQ(ierr);
293     for (v = vStart; v < vEnd; ++v) {
294       PetscInt off, dof, d;
295 
296       ierr = PetscSectionGetDof(coordSection, v, &dof);CHKERRQ(ierr);
297       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
298       ierr = PetscViewerASCIISynchronizedPrintf(viewer, "(");CHKERRQ(ierr);
299       for (d = 0; d < dof; ++d) {
300         if (d > 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, ",");CHKERRQ(ierr);}
301         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "%g", (double)(scale*PetscRealPart(coords[off+d])));CHKERRQ(ierr);
302       }
303       ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%D) [draw,shape=circle,color=%s] {%D} --\n", v, rank, colors[rank%numColors], v);CHKERRQ(ierr);
304     }
305     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
306     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
307     ierr = PetscViewerASCIIPrintf(viewer, "(0,0);\n");CHKERRQ(ierr);
308     /* Plot edges */
309     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
310     ierr = PetscViewerASCIIPrintf(viewer, "\\path\n");CHKERRQ(ierr);
311     if (depth > 1) {ierr = DMPlexGetDepthStratum(dm, 1, &eStart, &eEnd);CHKERRQ(ierr);}
312     for (e = eStart; e < eEnd; ++e) {
313       const PetscInt *cone;
314       PetscInt        coneSize, offA, offB, dof, d;
315 
316       ierr = DMPlexGetConeSize(dm, e, &coneSize);CHKERRQ(ierr);
317       if (coneSize != 2) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Edge %d cone should have two vertices, not %d", e, coneSize);
318       ierr = DMPlexGetCone(dm, e, &cone);CHKERRQ(ierr);
319       ierr = PetscSectionGetDof(coordSection, cone[0], &dof);CHKERRQ(ierr);
320       ierr = PetscSectionGetOffset(coordSection, cone[0], &offA);CHKERRQ(ierr);
321       ierr = PetscSectionGetOffset(coordSection, cone[1], &offB);CHKERRQ(ierr);
322       ierr = PetscViewerASCIISynchronizedPrintf(viewer, "(");CHKERRQ(ierr);
323       for (d = 0; d < dof; ++d) {
324         if (d > 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, ",");CHKERRQ(ierr);}
325         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "%g", (double)(scale*0.5*PetscRealPart(coords[offA+d]+coords[offB+d])));CHKERRQ(ierr);
326       }
327       ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%D) [draw,shape=circle,color=%s] {%D} --\n", e, rank, colors[rank%numColors], e);CHKERRQ(ierr);
328     }
329     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
330     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
331     ierr = PetscViewerASCIIPrintf(viewer, "(0,0);\n");CHKERRQ(ierr);
332     /* Plot cells */
333     ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
334     for (c = cStart; c < cEnd; ++c) {
335       PetscInt *closure = NULL;
336       PetscInt  closureSize, firstPoint = -1;
337 
338       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
339       ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\draw[color=%s] ", colors[rank%numColors]);CHKERRQ(ierr);
340       for (p = 0; p < closureSize*2; p += 2) {
341         const PetscInt point = closure[p];
342 
343         if ((point < vStart) || (point >= vEnd)) continue;
344         if (firstPoint >= 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- ");CHKERRQ(ierr);}
345         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "(%D_%D)", point, rank);CHKERRQ(ierr);
346         if (firstPoint < 0) firstPoint = point;
347       }
348       /* Why doesn't this work? ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- cycle;\n");CHKERRQ(ierr); */
349       ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- (%D_%D);\n", firstPoint, rank);CHKERRQ(ierr);
350       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
351     }
352     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
353     ierr = PetscViewerASCIIPrintf(viewer, "\\end{tikzpicture}\n\\end{center}\n");CHKERRQ(ierr);
354     ierr = PetscViewerASCIIPrintf(viewer, "\\end{document}\n", name);CHKERRQ(ierr);
355   } else {
356     MPI_Comm    comm;
357     PetscInt   *sizes, *hybsizes;
358     PetscInt    locDepth, depth, dim, d, pMax[4];
359     PetscInt    pStart, pEnd, p;
360     PetscInt    numLabels, l;
361     const char *name;
362     PetscMPIInt size;
363 
364     ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
365     ierr = MPI_Comm_size(comm, &size);CHKERRQ(ierr);
366     ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
367     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
368     if (name) {ierr = PetscViewerASCIIPrintf(viewer, "%s in %D dimensions:\n", name, dim);CHKERRQ(ierr);}
369     else      {ierr = PetscViewerASCIIPrintf(viewer, "Mesh in %D dimensions:\n", dim);CHKERRQ(ierr);}
370     ierr = DMPlexGetDepth(dm, &locDepth);CHKERRQ(ierr);
371     ierr = MPI_Allreduce(&locDepth, &depth, 1, MPIU_INT, MPI_MAX, comm);CHKERRQ(ierr);
372     ierr = DMPlexGetHybridBounds(dm, &pMax[depth], depth > 0 ? &pMax[depth-1] : NULL, &pMax[1], &pMax[0]);CHKERRQ(ierr);
373     ierr = PetscMalloc2(size,&sizes,size,&hybsizes);CHKERRQ(ierr);
374     if (depth == 1) {
375       ierr = DMPlexGetDepthStratum(dm, 0, &pStart, &pEnd);CHKERRQ(ierr);
376       pEnd = pEnd - pStart;
377       ierr = MPI_Gather(&pEnd, 1, MPIU_INT, sizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
378       ierr = PetscViewerASCIIPrintf(viewer, "  %D-cells:", 0);CHKERRQ(ierr);
379       for (p = 0; p < size; ++p) {ierr = PetscViewerASCIIPrintf(viewer, " %D", sizes[p]);CHKERRQ(ierr);}
380       ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
381       ierr = DMPlexGetHeightStratum(dm, 0, &pStart, &pEnd);CHKERRQ(ierr);
382       pEnd = pEnd - pStart;
383       ierr = MPI_Gather(&pEnd, 1, MPIU_INT, sizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
384       ierr = PetscViewerASCIIPrintf(viewer, "  %D-cells:", dim);CHKERRQ(ierr);
385       for (p = 0; p < size; ++p) {ierr = PetscViewerASCIIPrintf(viewer, " %D", sizes[p]);CHKERRQ(ierr);}
386       ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
387     } else {
388       for (d = 0; d <= dim; d++) {
389         ierr = DMPlexGetDepthStratum(dm, d, &pStart, &pEnd);CHKERRQ(ierr);
390         pEnd    -= pStart;
391         pMax[d] -= pStart;
392         ierr = MPI_Gather(&pEnd, 1, MPIU_INT, sizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
393         ierr = MPI_Gather(&pMax[d], 1, MPIU_INT, hybsizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
394         ierr = PetscViewerASCIIPrintf(viewer, "  %D-cells:", d);CHKERRQ(ierr);
395         for (p = 0; p < size; ++p) {
396           if (hybsizes[p] >= 0) {ierr = PetscViewerASCIIPrintf(viewer, " %D (%D)", sizes[p], sizes[p] - hybsizes[p]);CHKERRQ(ierr);}
397           else                  {ierr = PetscViewerASCIIPrintf(viewer, " %D", sizes[p]);CHKERRQ(ierr);}
398         }
399         ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
400       }
401     }
402     ierr = PetscFree2(sizes,hybsizes);CHKERRQ(ierr);
403     ierr = DMPlexGetNumLabels(dm, &numLabels);CHKERRQ(ierr);
404     if (numLabels) {ierr = PetscViewerASCIIPrintf(viewer, "Labels:\n");CHKERRQ(ierr);}
405     for (l = 0; l < numLabels; ++l) {
406       DMLabel         label;
407       const char     *name;
408       IS              valueIS;
409       const PetscInt *values;
410       PetscInt        numValues, v;
411 
412       ierr = DMPlexGetLabelName(dm, l, &name);CHKERRQ(ierr);
413       ierr = DMPlexGetLabel(dm, name, &label);CHKERRQ(ierr);
414       ierr = DMLabelGetNumValues(label, &numValues);CHKERRQ(ierr);
415       ierr = PetscViewerASCIIPrintf(viewer, "  %s: %d strata of sizes (", name, numValues);CHKERRQ(ierr);
416       ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
417       ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
418       for (v = 0; v < numValues; ++v) {
419         PetscInt size;
420 
421         ierr = DMLabelGetStratumSize(label, values[v], &size);CHKERRQ(ierr);
422         if (v > 0) {ierr = PetscViewerASCIIPrintf(viewer, ", ");CHKERRQ(ierr);}
423         ierr = PetscViewerASCIIPrintf(viewer, "%d", size);CHKERRQ(ierr);
424       }
425       ierr = PetscViewerASCIIPrintf(viewer, ")\n");CHKERRQ(ierr);
426       ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
427       ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
428     }
429   }
430   PetscFunctionReturn(0);
431 }
432 
433 #undef __FUNCT__
434 #define __FUNCT__ "DMView_Plex"
435 PetscErrorCode DMView_Plex(DM dm, PetscViewer viewer)
436 {
437   PetscBool      iascii, ishdf5;
438   PetscErrorCode ierr;
439 
440   PetscFunctionBegin;
441   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
442   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
443   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERASCII, &iascii);CHKERRQ(ierr);
444   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,  &ishdf5);CHKERRQ(ierr);
445   if (iascii) {
446     ierr = DMPlexView_Ascii(dm, viewer);CHKERRQ(ierr);
447   } else if (ishdf5) {
448 #if defined(PETSC_HAVE_HDF5)
449     ierr = PetscViewerPushFormat(viewer, PETSC_VIEWER_HDF5_VIZ);CHKERRQ(ierr);
450     ierr = DMPlexView_HDF5(dm, viewer);CHKERRQ(ierr);
451     ierr = PetscViewerPopFormat(viewer);CHKERRQ(ierr);
452 #else
453     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
454 #endif
455   }
456   PetscFunctionReturn(0);
457 }
458 
459 #undef __FUNCT__
460 #define __FUNCT__ "DMLoad_Plex"
461 PetscErrorCode DMLoad_Plex(DM dm, PetscViewer viewer)
462 {
463   PetscBool      isbinary, ishdf5;
464   PetscErrorCode ierr;
465 
466   PetscFunctionBegin;
467   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
468   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
469   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERBINARY, &isbinary);CHKERRQ(ierr);
470   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,   &ishdf5);CHKERRQ(ierr);
471   if (isbinary) {SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Do not yet support binary viewers");}
472   else if (ishdf5) {
473 #if defined(PETSC_HAVE_HDF5)
474     ierr = DMPlexLoad_HDF5(dm, viewer);CHKERRQ(ierr);
475 #else
476     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
477 #endif
478   }
479   PetscFunctionReturn(0);
480 }
481 
482 #undef __FUNCT__
483 #define __FUNCT__ "BoundaryDestroy"
484 static PetscErrorCode BoundaryDestroy(DMBoundary *boundary)
485 {
486   DMBoundary     b, next;
487   PetscErrorCode ierr;
488 
489   PetscFunctionBegin;
490   if (!boundary) PetscFunctionReturn(0);
491   b = *boundary;
492   *boundary = NULL;
493   for (; b; b = next) {
494     next = b->next;
495     ierr = PetscFree(b->ids);CHKERRQ(ierr);
496     ierr = PetscFree(b->name);CHKERRQ(ierr);
497     ierr = PetscFree(b->labelname);CHKERRQ(ierr);
498     ierr = PetscFree(b);CHKERRQ(ierr);
499   }
500   PetscFunctionReturn(0);
501 }
502 
503 #undef __FUNCT__
504 #define __FUNCT__ "DMDestroy_Plex"
505 PetscErrorCode DMDestroy_Plex(DM dm)
506 {
507   DM_Plex       *mesh = (DM_Plex*) dm->data;
508   DMLabel        next  = mesh->labels;
509   PetscErrorCode ierr;
510 
511   PetscFunctionBegin;
512   if (--mesh->refct > 0) PetscFunctionReturn(0);
513   ierr = PetscSectionDestroy(&mesh->coneSection);CHKERRQ(ierr);
514   ierr = PetscFree(mesh->cones);CHKERRQ(ierr);
515   ierr = PetscFree(mesh->coneOrientations);CHKERRQ(ierr);
516   ierr = PetscSectionDestroy(&mesh->supportSection);CHKERRQ(ierr);
517   ierr = PetscFree(mesh->supports);CHKERRQ(ierr);
518   ierr = PetscFree(mesh->facesTmp);CHKERRQ(ierr);
519   while (next) {
520     DMLabel tmp = next->next;
521 
522     ierr = DMLabelDestroy(&next);CHKERRQ(ierr);
523     next = tmp;
524   }
525   ierr = DMDestroy(&mesh->coarseMesh);CHKERRQ(ierr);
526   ierr = DMLabelDestroy(&mesh->subpointMap);CHKERRQ(ierr);
527   ierr = ISDestroy(&mesh->globalVertexNumbers);CHKERRQ(ierr);
528   ierr = ISDestroy(&mesh->globalCellNumbers);CHKERRQ(ierr);
529   ierr = BoundaryDestroy(&mesh->boundary);CHKERRQ(ierr);
530   /* This was originally freed in DMDestroy(), but that prevents reference counting of backend objects */
531   ierr = PetscFree(mesh);CHKERRQ(ierr);
532   PetscFunctionReturn(0);
533 }
534 
535 #undef __FUNCT__
536 #define __FUNCT__ "DMCreateMatrix_Plex"
537 PetscErrorCode DMCreateMatrix_Plex(DM dm, Mat *J)
538 {
539   PetscSection   section, sectionGlobal;
540   PetscInt       bs = -1;
541   PetscInt       localSize;
542   PetscBool      isShell, isBlock, isSeqBlock, isMPIBlock, isSymBlock, isSymSeqBlock, isSymMPIBlock;
543   PetscErrorCode ierr;
544   MatType        mtype;
545 
546   PetscFunctionBegin;
547   ierr = MatInitializePackage();CHKERRQ(ierr);
548   mtype = dm->mattype;
549   ierr = DMGetDefaultSection(dm, &section);CHKERRQ(ierr);
550   ierr = DMGetDefaultGlobalSection(dm, &sectionGlobal);CHKERRQ(ierr);
551   /* ierr = PetscSectionGetStorageSize(sectionGlobal, &localSize);CHKERRQ(ierr); */
552   ierr = PetscSectionGetConstrainedStorageSize(sectionGlobal, &localSize);CHKERRQ(ierr);
553   ierr = MatCreate(PetscObjectComm((PetscObject)dm), J);CHKERRQ(ierr);
554   ierr = MatSetSizes(*J, localSize, localSize, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
555   ierr = MatSetType(*J, mtype);CHKERRQ(ierr);
556   ierr = MatSetFromOptions(*J);CHKERRQ(ierr);
557   ierr = PetscStrcmp(mtype, MATSHELL, &isShell);CHKERRQ(ierr);
558   ierr = PetscStrcmp(mtype, MATBAIJ, &isBlock);CHKERRQ(ierr);
559   ierr = PetscStrcmp(mtype, MATSEQBAIJ, &isSeqBlock);CHKERRQ(ierr);
560   ierr = PetscStrcmp(mtype, MATMPIBAIJ, &isMPIBlock);CHKERRQ(ierr);
561   ierr = PetscStrcmp(mtype, MATSBAIJ, &isSymBlock);CHKERRQ(ierr);
562   ierr = PetscStrcmp(mtype, MATSEQSBAIJ, &isSymSeqBlock);CHKERRQ(ierr);
563   ierr = PetscStrcmp(mtype, MATMPISBAIJ, &isSymMPIBlock);CHKERRQ(ierr);
564   if (!isShell) {
565     PetscBool fillMatrix = (PetscBool) !dm->prealloc_only;
566     PetscInt *dnz, *onz, *dnzu, *onzu, bsLocal, bsMax, bsMin;
567 
568     if (bs < 0) {
569       if (isBlock || isSeqBlock || isMPIBlock || isSymBlock || isSymSeqBlock || isSymMPIBlock) {
570         PetscInt pStart, pEnd, p, dof, cdof;
571 
572         ierr = PetscSectionGetChart(sectionGlobal, &pStart, &pEnd);CHKERRQ(ierr);
573         for (p = pStart; p < pEnd; ++p) {
574           ierr = PetscSectionGetDof(sectionGlobal, p, &dof);CHKERRQ(ierr);
575           ierr = PetscSectionGetConstraintDof(sectionGlobal, p, &cdof);CHKERRQ(ierr);
576           if (dof-cdof) {
577             if (bs < 0) {
578               bs = dof-cdof;
579             } else if (bs != dof-cdof) {
580               /* Layout does not admit a pointwise block size */
581               bs = 1;
582               break;
583             }
584           }
585         }
586         /* Must have same blocksize on all procs (some might have no points) */
587         bsLocal = bs;
588         ierr = MPI_Allreduce(&bsLocal, &bsMax, 1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
589         bsLocal = bs < 0 ? bsMax : bs;
590         ierr = MPI_Allreduce(&bsLocal, &bsMin, 1, MPIU_INT, MPI_MIN, PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
591         if (bsMin != bsMax) {
592           bs = 1;
593         } else {
594           bs = bsMax;
595         }
596       } else {
597         bs = 1;
598       }
599     }
600     ierr = PetscCalloc4(localSize/bs, &dnz, localSize/bs, &onz, localSize/bs, &dnzu, localSize/bs, &onzu);CHKERRQ(ierr);
601     ierr = DMPlexPreallocateOperator(dm, bs, section, sectionGlobal, dnz, onz, dnzu, onzu, *J, fillMatrix);CHKERRQ(ierr);
602     ierr = PetscFree4(dnz, onz, dnzu, onzu);CHKERRQ(ierr);
603   }
604   PetscFunctionReturn(0);
605 }
606 
607 #undef __FUNCT__
608 #define __FUNCT__ "DMPlexGetDimension"
609 /*@
610   DMPlexGetDimension - Return the topological mesh dimension
611 
612   Not collective
613 
614   Input Parameter:
615 . mesh - The DMPlex
616 
617   Output Parameter:
618 . dim - The topological mesh dimension
619 
620   Level: beginner
621 
622 .seealso: DMPlexCreate()
623 @*/
624 PetscErrorCode DMPlexGetDimension(DM dm, PetscInt *dim)
625 {
626   DM_Plex *mesh = (DM_Plex*) dm->data;
627 
628   PetscFunctionBegin;
629   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
630   PetscValidPointer(dim, 2);
631   *dim = mesh->dim;
632   PetscFunctionReturn(0);
633 }
634 
635 #undef __FUNCT__
636 #define __FUNCT__ "DMPlexSetDimension"
637 /*@
638   DMPlexSetDimension - Set the topological mesh dimension
639 
640   Collective on mesh
641 
642   Input Parameters:
643 + mesh - The DMPlex
644 - dim - The topological mesh dimension
645 
646   Level: beginner
647 
648 .seealso: DMPlexCreate()
649 @*/
650 PetscErrorCode DMPlexSetDimension(DM dm, PetscInt dim)
651 {
652   DM_Plex *mesh = (DM_Plex*) dm->data;
653 
654   PetscFunctionBegin;
655   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
656   PetscValidLogicalCollectiveInt(dm, dim, 2);
657   mesh->dim = dim;
658   PetscFunctionReturn(0);
659 }
660 
661 #undef __FUNCT__
662 #define __FUNCT__ "DMPlexGetChart"
663 /*@
664   DMPlexGetChart - Return the interval for all mesh points [pStart, pEnd)
665 
666   Not collective
667 
668   Input Parameter:
669 . mesh - The DMPlex
670 
671   Output Parameters:
672 + pStart - The first mesh point
673 - pEnd   - The upper bound for mesh points
674 
675   Level: beginner
676 
677 .seealso: DMPlexCreate(), DMPlexSetChart()
678 @*/
679 PetscErrorCode DMPlexGetChart(DM dm, PetscInt *pStart, PetscInt *pEnd)
680 {
681   DM_Plex       *mesh = (DM_Plex*) dm->data;
682   PetscErrorCode ierr;
683 
684   PetscFunctionBegin;
685   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
686   ierr = PetscSectionGetChart(mesh->coneSection, pStart, pEnd);CHKERRQ(ierr);
687   PetscFunctionReturn(0);
688 }
689 
690 #undef __FUNCT__
691 #define __FUNCT__ "DMPlexSetChart"
692 /*@
693   DMPlexSetChart - Set the interval for all mesh points [pStart, pEnd)
694 
695   Not collective
696 
697   Input Parameters:
698 + mesh - The DMPlex
699 . pStart - The first mesh point
700 - pEnd   - The upper bound for mesh points
701 
702   Output Parameters:
703 
704   Level: beginner
705 
706 .seealso: DMPlexCreate(), DMPlexGetChart()
707 @*/
708 PetscErrorCode DMPlexSetChart(DM dm, PetscInt pStart, PetscInt pEnd)
709 {
710   DM_Plex       *mesh = (DM_Plex*) dm->data;
711   PetscErrorCode ierr;
712 
713   PetscFunctionBegin;
714   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
715   ierr = PetscSectionSetChart(mesh->coneSection, pStart, pEnd);CHKERRQ(ierr);
716   ierr = PetscSectionSetChart(mesh->supportSection, pStart, pEnd);CHKERRQ(ierr);
717   PetscFunctionReturn(0);
718 }
719 
720 #undef __FUNCT__
721 #define __FUNCT__ "DMPlexGetConeSize"
722 /*@
723   DMPlexGetConeSize - Return the number of in-edges for this point in the Sieve DAG
724 
725   Not collective
726 
727   Input Parameters:
728 + mesh - The DMPlex
729 - p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
730 
731   Output Parameter:
732 . size - The cone size for point p
733 
734   Level: beginner
735 
736 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart()
737 @*/
738 PetscErrorCode DMPlexGetConeSize(DM dm, PetscInt p, PetscInt *size)
739 {
740   DM_Plex       *mesh = (DM_Plex*) dm->data;
741   PetscErrorCode ierr;
742 
743   PetscFunctionBegin;
744   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
745   PetscValidPointer(size, 3);
746   ierr = PetscSectionGetDof(mesh->coneSection, p, size);CHKERRQ(ierr);
747   PetscFunctionReturn(0);
748 }
749 
750 #undef __FUNCT__
751 #define __FUNCT__ "DMPlexSetConeSize"
752 /*@
753   DMPlexSetConeSize - Set the number of in-edges for this point in the Sieve DAG
754 
755   Not collective
756 
757   Input Parameters:
758 + mesh - The DMPlex
759 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
760 - size - The cone size for point p
761 
762   Output Parameter:
763 
764   Note:
765   This should be called after DMPlexSetChart().
766 
767   Level: beginner
768 
769 .seealso: DMPlexCreate(), DMPlexGetConeSize(), DMPlexSetChart()
770 @*/
771 PetscErrorCode DMPlexSetConeSize(DM dm, PetscInt p, PetscInt size)
772 {
773   DM_Plex       *mesh = (DM_Plex*) dm->data;
774   PetscErrorCode ierr;
775 
776   PetscFunctionBegin;
777   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
778   ierr = PetscSectionSetDof(mesh->coneSection, p, size);CHKERRQ(ierr);
779 
780   mesh->maxConeSize = PetscMax(mesh->maxConeSize, size);
781   PetscFunctionReturn(0);
782 }
783 
784 #undef __FUNCT__
785 #define __FUNCT__ "DMPlexAddConeSize"
786 /*@
787   DMPlexAddConeSize - Add the given number of in-edges to this point in the Sieve DAG
788 
789   Not collective
790 
791   Input Parameters:
792 + mesh - The DMPlex
793 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
794 - size - The additional cone size for point p
795 
796   Output Parameter:
797 
798   Note:
799   This should be called after DMPlexSetChart().
800 
801   Level: beginner
802 
803 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexGetConeSize(), DMPlexSetChart()
804 @*/
805 PetscErrorCode DMPlexAddConeSize(DM dm, PetscInt p, PetscInt size)
806 {
807   DM_Plex       *mesh = (DM_Plex*) dm->data;
808   PetscInt       csize;
809   PetscErrorCode ierr;
810 
811   PetscFunctionBegin;
812   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
813   ierr = PetscSectionAddDof(mesh->coneSection, p, size);CHKERRQ(ierr);
814   ierr = PetscSectionGetDof(mesh->coneSection, p, &csize);CHKERRQ(ierr);
815 
816   mesh->maxConeSize = PetscMax(mesh->maxConeSize, csize);
817   PetscFunctionReturn(0);
818 }
819 
820 #undef __FUNCT__
821 #define __FUNCT__ "DMPlexGetCone"
822 /*@C
823   DMPlexGetCone - Return the points on the in-edges for this point in the Sieve DAG
824 
825   Not collective
826 
827   Input Parameters:
828 + mesh - The DMPlex
829 - p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
830 
831   Output Parameter:
832 . cone - An array of points which are on the in-edges for point p
833 
834   Level: beginner
835 
836   Fortran Notes:
837   Since it returns an array, this routine is only available in Fortran 90, and you must
838   include petsc.h90 in your code.
839 
840   You must also call DMPlexRestoreCone() after you finish using the returned array.
841 
842 .seealso: DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart()
843 @*/
844 PetscErrorCode DMPlexGetCone(DM dm, PetscInt p, const PetscInt *cone[])
845 {
846   DM_Plex       *mesh = (DM_Plex*) dm->data;
847   PetscInt       off;
848   PetscErrorCode ierr;
849 
850   PetscFunctionBegin;
851   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
852   PetscValidPointer(cone, 3);
853   ierr  = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
854   *cone = &mesh->cones[off];
855   PetscFunctionReturn(0);
856 }
857 
858 #undef __FUNCT__
859 #define __FUNCT__ "DMPlexSetCone"
860 /*@
861   DMPlexSetCone - Set the points on the in-edges for this point in the Sieve DAG
862 
863   Not collective
864 
865   Input Parameters:
866 + mesh - The DMPlex
867 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
868 - cone - An array of points which are on the in-edges for point p
869 
870   Output Parameter:
871 
872   Note:
873   This should be called after all calls to DMPlexSetConeSize() and DMSetUp().
874 
875   Level: beginner
876 
877 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
878 @*/
879 PetscErrorCode DMPlexSetCone(DM dm, PetscInt p, const PetscInt cone[])
880 {
881   DM_Plex       *mesh = (DM_Plex*) dm->data;
882   PetscInt       pStart, pEnd;
883   PetscInt       dof, off, c;
884   PetscErrorCode ierr;
885 
886   PetscFunctionBegin;
887   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
888   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
889   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
890   if (dof) PetscValidPointer(cone, 3);
891   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
892   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
893   for (c = 0; c < dof; ++c) {
894     if ((cone[c] < pStart) || (cone[c] >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone point %D is not in the valid range [%D, %D)", cone[c], pStart, pEnd);
895     mesh->cones[off+c] = cone[c];
896   }
897   PetscFunctionReturn(0);
898 }
899 
900 #undef __FUNCT__
901 #define __FUNCT__ "DMPlexGetConeOrientation"
902 /*@C
903   DMPlexGetConeOrientation - Return the orientations on the in-edges for this point in the Sieve DAG
904 
905   Not collective
906 
907   Input Parameters:
908 + mesh - The DMPlex
909 - p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
910 
911   Output Parameter:
912 . coneOrientation - An array of orientations which are on the in-edges for point p. An orientation is an
913                     integer giving the prescription for cone traversal. If it is negative, the cone is
914                     traversed in the opposite direction. Its value 'o', or if negative '-(o+1)', gives
915                     the index of the cone point on which to start.
916 
917   Level: beginner
918 
919   Fortran Notes:
920   Since it returns an array, this routine is only available in Fortran 90, and you must
921   include petsc.h90 in your code.
922 
923   You must also call DMPlexRestoreConeOrientation() after you finish using the returned array.
924 
925 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetCone(), DMPlexSetChart()
926 @*/
927 PetscErrorCode DMPlexGetConeOrientation(DM dm, PetscInt p, const PetscInt *coneOrientation[])
928 {
929   DM_Plex       *mesh = (DM_Plex*) dm->data;
930   PetscInt       off;
931   PetscErrorCode ierr;
932 
933   PetscFunctionBegin;
934   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
935 #if defined(PETSC_USE_DEBUG)
936   {
937     PetscInt dof;
938     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
939     if (dof) PetscValidPointer(coneOrientation, 3);
940   }
941 #endif
942   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
943 
944   *coneOrientation = &mesh->coneOrientations[off];
945   PetscFunctionReturn(0);
946 }
947 
948 #undef __FUNCT__
949 #define __FUNCT__ "DMPlexSetConeOrientation"
950 /*@
951   DMPlexSetConeOrientation - Set the orientations on the in-edges for this point in the Sieve DAG
952 
953   Not collective
954 
955   Input Parameters:
956 + mesh - The DMPlex
957 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
958 - coneOrientation - An array of orientations which are on the in-edges for point p. An orientation is an
959                     integer giving the prescription for cone traversal. If it is negative, the cone is
960                     traversed in the opposite direction. Its value 'o', or if negative '-(o+1)', gives
961                     the index of the cone point on which to start.
962 
963   Output Parameter:
964 
965   Note:
966   This should be called after all calls to DMPlexSetConeSize() and DMSetUp().
967 
968   Level: beginner
969 
970 .seealso: DMPlexCreate(), DMPlexGetConeOrientation(), DMPlexSetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
971 @*/
972 PetscErrorCode DMPlexSetConeOrientation(DM dm, PetscInt p, const PetscInt coneOrientation[])
973 {
974   DM_Plex       *mesh = (DM_Plex*) dm->data;
975   PetscInt       pStart, pEnd;
976   PetscInt       dof, off, c;
977   PetscErrorCode ierr;
978 
979   PetscFunctionBegin;
980   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
981   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
982   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
983   if (dof) PetscValidPointer(coneOrientation, 3);
984   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
985   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
986   for (c = 0; c < dof; ++c) {
987     PetscInt cdof, o = coneOrientation[c];
988 
989     ierr = PetscSectionGetDof(mesh->coneSection, mesh->cones[off+c], &cdof);CHKERRQ(ierr);
990     if (o && ((o < -(cdof+1)) || (o >= cdof))) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone orientation %D is not in the valid range [%D. %D)", o, -(cdof+1), cdof);
991     mesh->coneOrientations[off+c] = o;
992   }
993   PetscFunctionReturn(0);
994 }
995 
996 #undef __FUNCT__
997 #define __FUNCT__ "DMPlexInsertCone"
998 PetscErrorCode DMPlexInsertCone(DM dm, PetscInt p, PetscInt conePos, PetscInt conePoint)
999 {
1000   DM_Plex       *mesh = (DM_Plex*) dm->data;
1001   PetscInt       pStart, pEnd;
1002   PetscInt       dof, off;
1003   PetscErrorCode ierr;
1004 
1005   PetscFunctionBegin;
1006   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1007   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
1008   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
1009   if ((conePoint < pStart) || (conePoint >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone point %D is not in the valid range [%D, %D)", conePoint, pStart, pEnd);
1010   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
1011   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1012   if ((conePos < 0) || (conePos >= dof)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone position %D of point %D is not in the valid range [0, %D)", conePos, p, dof);
1013   mesh->cones[off+conePos] = conePoint;
1014   PetscFunctionReturn(0);
1015 }
1016 
1017 #undef __FUNCT__
1018 #define __FUNCT__ "DMPlexInsertConeOrientation"
1019 PetscErrorCode DMPlexInsertConeOrientation(DM dm, PetscInt p, PetscInt conePos, PetscInt coneOrientation)
1020 {
1021   DM_Plex       *mesh = (DM_Plex*) dm->data;
1022   PetscInt       pStart, pEnd;
1023   PetscInt       dof, off;
1024   PetscErrorCode ierr;
1025 
1026   PetscFunctionBegin;
1027   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1028   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
1029   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
1030   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
1031   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1032   if ((conePos < 0) || (conePos >= dof)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone position %D of point %D is not in the valid range [0, %D)", conePos, p, dof);
1033   mesh->coneOrientations[off+conePos] = coneOrientation;
1034   PetscFunctionReturn(0);
1035 }
1036 
1037 #undef __FUNCT__
1038 #define __FUNCT__ "DMPlexGetSupportSize"
1039 /*@
1040   DMPlexGetSupportSize - Return the number of out-edges for this point in the Sieve DAG
1041 
1042   Not collective
1043 
1044   Input Parameters:
1045 + mesh - The DMPlex
1046 - p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1047 
1048   Output Parameter:
1049 . size - The support size for point p
1050 
1051   Level: beginner
1052 
1053 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart(), DMPlexGetConeSize()
1054 @*/
1055 PetscErrorCode DMPlexGetSupportSize(DM dm, PetscInt p, PetscInt *size)
1056 {
1057   DM_Plex       *mesh = (DM_Plex*) dm->data;
1058   PetscErrorCode ierr;
1059 
1060   PetscFunctionBegin;
1061   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1062   PetscValidPointer(size, 3);
1063   ierr = PetscSectionGetDof(mesh->supportSection, p, size);CHKERRQ(ierr);
1064   PetscFunctionReturn(0);
1065 }
1066 
1067 #undef __FUNCT__
1068 #define __FUNCT__ "DMPlexSetSupportSize"
1069 /*@
1070   DMPlexSetSupportSize - Set the number of out-edges for this point in the Sieve DAG
1071 
1072   Not collective
1073 
1074   Input Parameters:
1075 + mesh - The DMPlex
1076 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1077 - size - The support size for point p
1078 
1079   Output Parameter:
1080 
1081   Note:
1082   This should be called after DMPlexSetChart().
1083 
1084   Level: beginner
1085 
1086 .seealso: DMPlexCreate(), DMPlexGetSupportSize(), DMPlexSetChart()
1087 @*/
1088 PetscErrorCode DMPlexSetSupportSize(DM dm, PetscInt p, PetscInt size)
1089 {
1090   DM_Plex       *mesh = (DM_Plex*) dm->data;
1091   PetscErrorCode ierr;
1092 
1093   PetscFunctionBegin;
1094   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1095   ierr = PetscSectionSetDof(mesh->supportSection, p, size);CHKERRQ(ierr);
1096 
1097   mesh->maxSupportSize = PetscMax(mesh->maxSupportSize, size);
1098   PetscFunctionReturn(0);
1099 }
1100 
1101 #undef __FUNCT__
1102 #define __FUNCT__ "DMPlexGetSupport"
1103 /*@C
1104   DMPlexGetSupport - Return the points on the out-edges for this point in the Sieve DAG
1105 
1106   Not collective
1107 
1108   Input Parameters:
1109 + mesh - The DMPlex
1110 - p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1111 
1112   Output Parameter:
1113 . support - An array of points which are on the out-edges for point p
1114 
1115   Level: beginner
1116 
1117   Fortran Notes:
1118   Since it returns an array, this routine is only available in Fortran 90, and you must
1119   include petsc.h90 in your code.
1120 
1121   You must also call DMPlexRestoreSupport() after you finish using the returned array.
1122 
1123 .seealso: DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
1124 @*/
1125 PetscErrorCode DMPlexGetSupport(DM dm, PetscInt p, const PetscInt *support[])
1126 {
1127   DM_Plex       *mesh = (DM_Plex*) dm->data;
1128   PetscInt       off;
1129   PetscErrorCode ierr;
1130 
1131   PetscFunctionBegin;
1132   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1133   PetscValidPointer(support, 3);
1134   ierr     = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
1135   *support = &mesh->supports[off];
1136   PetscFunctionReturn(0);
1137 }
1138 
1139 #undef __FUNCT__
1140 #define __FUNCT__ "DMPlexSetSupport"
1141 /*@
1142   DMPlexSetSupport - Set the points on the out-edges for this point in the Sieve DAG
1143 
1144   Not collective
1145 
1146   Input Parameters:
1147 + mesh - The DMPlex
1148 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1149 - support - An array of points which are on the in-edges for point p
1150 
1151   Output Parameter:
1152 
1153   Note:
1154   This should be called after all calls to DMPlexSetSupportSize() and DMSetUp().
1155 
1156   Level: beginner
1157 
1158 .seealso: DMPlexCreate(), DMPlexGetSupport(), DMPlexSetChart(), DMPlexSetSupportSize(), DMSetUp()
1159 @*/
1160 PetscErrorCode DMPlexSetSupport(DM dm, PetscInt p, const PetscInt support[])
1161 {
1162   DM_Plex       *mesh = (DM_Plex*) dm->data;
1163   PetscInt       pStart, pEnd;
1164   PetscInt       dof, off, c;
1165   PetscErrorCode ierr;
1166 
1167   PetscFunctionBegin;
1168   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1169   ierr = PetscSectionGetChart(mesh->supportSection, &pStart, &pEnd);CHKERRQ(ierr);
1170   ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
1171   if (dof) PetscValidPointer(support, 3);
1172   ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
1173   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
1174   for (c = 0; c < dof; ++c) {
1175     if ((support[c] < pStart) || (support[c] >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support point %D is not in the valid range [%D, %D)", support[c], pStart, pEnd);
1176     mesh->supports[off+c] = support[c];
1177   }
1178   PetscFunctionReturn(0);
1179 }
1180 
1181 #undef __FUNCT__
1182 #define __FUNCT__ "DMPlexInsertSupport"
1183 PetscErrorCode DMPlexInsertSupport(DM dm, PetscInt p, PetscInt supportPos, PetscInt supportPoint)
1184 {
1185   DM_Plex       *mesh = (DM_Plex*) dm->data;
1186   PetscInt       pStart, pEnd;
1187   PetscInt       dof, off;
1188   PetscErrorCode ierr;
1189 
1190   PetscFunctionBegin;
1191   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1192   ierr = PetscSectionGetChart(mesh->supportSection, &pStart, &pEnd);CHKERRQ(ierr);
1193   ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
1194   ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
1195   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
1196   if ((supportPoint < pStart) || (supportPoint >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support point %D is not in the valid range [%D, %D)", supportPoint, pStart, pEnd);
1197   if (supportPos >= dof) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support position %D of point %D is not in the valid range [0, %D)", supportPos, p, dof);
1198   mesh->supports[off+supportPos] = supportPoint;
1199   PetscFunctionReturn(0);
1200 }
1201 
1202 #undef __FUNCT__
1203 #define __FUNCT__ "DMPlexGetTransitiveClosure"
1204 /*@C
1205   DMPlexGetTransitiveClosure - Return the points on the transitive closure of the in-edges or out-edges for this point in the Sieve DAG
1206 
1207   Not collective
1208 
1209   Input Parameters:
1210 + mesh - The DMPlex
1211 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1212 . useCone - PETSC_TRUE for in-edges,  otherwise use out-edges
1213 - points - If points is NULL on input, internal storage will be returned, otherwise the provided array is used
1214 
1215   Output Parameters:
1216 + numPoints - The number of points in the closure, so points[] is of size 2*numPoints
1217 - points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...]
1218 
1219   Note:
1220   If using internal storage (points is NULL on input), each call overwrites the last output.
1221 
1222   Fortran Notes:
1223   Since it returns an array, this routine is only available in Fortran 90, and you must
1224   include petsc.h90 in your code.
1225 
1226   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
1227 
1228   Level: beginner
1229 
1230 .seealso: DMPlexRestoreTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
1231 @*/
1232 PetscErrorCode DMPlexGetTransitiveClosure(DM dm, PetscInt p, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
1233 {
1234   DM_Plex        *mesh = (DM_Plex*) dm->data;
1235   PetscInt       *closure, *fifo;
1236   const PetscInt *tmp = NULL, *tmpO = NULL;
1237   PetscInt        tmpSize, t;
1238   PetscInt        depth       = 0, maxSize;
1239   PetscInt        closureSize = 2, fifoSize = 0, fifoStart = 0;
1240   PetscErrorCode  ierr;
1241 
1242   PetscFunctionBegin;
1243   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1244   ierr    = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
1245   /* This is only 1-level */
1246   if (useCone) {
1247     ierr = DMPlexGetConeSize(dm, p, &tmpSize);CHKERRQ(ierr);
1248     ierr = DMPlexGetCone(dm, p, &tmp);CHKERRQ(ierr);
1249     ierr = DMPlexGetConeOrientation(dm, p, &tmpO);CHKERRQ(ierr);
1250   } else {
1251     ierr = DMPlexGetSupportSize(dm, p, &tmpSize);CHKERRQ(ierr);
1252     ierr = DMPlexGetSupport(dm, p, &tmp);CHKERRQ(ierr);
1253   }
1254   if (depth == 1) {
1255     if (*points) {
1256       closure = *points;
1257     } else {
1258       maxSize = 2*(PetscMax(mesh->maxConeSize, mesh->maxSupportSize)+1);
1259       ierr = DMGetWorkArray(dm, maxSize, PETSC_INT, &closure);CHKERRQ(ierr);
1260     }
1261     closure[0] = p; closure[1] = 0;
1262     for (t = 0; t < tmpSize; ++t, closureSize += 2) {
1263       closure[closureSize]   = tmp[t];
1264       closure[closureSize+1] = tmpO ? tmpO[t] : 0;
1265     }
1266     if (numPoints) *numPoints = closureSize/2;
1267     if (points)    *points    = closure;
1268     PetscFunctionReturn(0);
1269   }
1270   maxSize = 2*PetscMax(PetscMax(PetscPowInt(mesh->maxConeSize,depth+1),PetscPowInt(mesh->maxSupportSize,depth+1)),depth+1);
1271   ierr    = DMGetWorkArray(dm, maxSize, PETSC_INT, &fifo);CHKERRQ(ierr);
1272   if (*points) {
1273     closure = *points;
1274   } else {
1275     ierr = DMGetWorkArray(dm, maxSize, PETSC_INT, &closure);CHKERRQ(ierr);
1276   }
1277   closure[0] = p; closure[1] = 0;
1278   for (t = 0; t < tmpSize; ++t, closureSize += 2, fifoSize += 2) {
1279     const PetscInt cp = tmp[t];
1280     const PetscInt co = tmpO ? tmpO[t] : 0;
1281 
1282     closure[closureSize]   = cp;
1283     closure[closureSize+1] = co;
1284     fifo[fifoSize]         = cp;
1285     fifo[fifoSize+1]       = co;
1286   }
1287   /* Should kick out early when depth is reached, rather than checking all vertices for empty cones */
1288   while (fifoSize - fifoStart) {
1289     const PetscInt q   = fifo[fifoStart];
1290     const PetscInt o   = fifo[fifoStart+1];
1291     const PetscInt rev = o >= 0 ? 0 : 1;
1292     const PetscInt off = rev ? -(o+1) : o;
1293 
1294     if (useCone) {
1295       ierr = DMPlexGetConeSize(dm, q, &tmpSize);CHKERRQ(ierr);
1296       ierr = DMPlexGetCone(dm, q, &tmp);CHKERRQ(ierr);
1297       ierr = DMPlexGetConeOrientation(dm, q, &tmpO);CHKERRQ(ierr);
1298     } else {
1299       ierr = DMPlexGetSupportSize(dm, q, &tmpSize);CHKERRQ(ierr);
1300       ierr = DMPlexGetSupport(dm, q, &tmp);CHKERRQ(ierr);
1301       tmpO = NULL;
1302     }
1303     for (t = 0; t < tmpSize; ++t) {
1304       const PetscInt i  = ((rev ? tmpSize-t : t) + off)%tmpSize;
1305       const PetscInt cp = tmp[i];
1306       /* Must propogate orientation: When we reverse orientation, we both reverse the direction of iteration and start at the other end of the chain. */
1307       /* HACK: It is worse to get the size here, than to change the interpretation of -(*+1)
1308        const PetscInt co = tmpO ? (rev ? -(tmpO[i]+1) : tmpO[i]) : 0; */
1309       PetscInt       co = tmpO ? tmpO[i] : 0;
1310       PetscInt       c;
1311 
1312       if (rev) {
1313         PetscInt childSize, coff;
1314         ierr = DMPlexGetConeSize(dm, cp, &childSize);CHKERRQ(ierr);
1315         coff = tmpO[i] < 0 ? -(tmpO[i]+1) : tmpO[i];
1316         co   = childSize ? -(((coff+childSize-1)%childSize)+1) : 0;
1317       }
1318       /* Check for duplicate */
1319       for (c = 0; c < closureSize; c += 2) {
1320         if (closure[c] == cp) break;
1321       }
1322       if (c == closureSize) {
1323         closure[closureSize]   = cp;
1324         closure[closureSize+1] = co;
1325         fifo[fifoSize]         = cp;
1326         fifo[fifoSize+1]       = co;
1327         closureSize           += 2;
1328         fifoSize              += 2;
1329       }
1330     }
1331     fifoStart += 2;
1332   }
1333   if (numPoints) *numPoints = closureSize/2;
1334   if (points)    *points    = closure;
1335   ierr = DMRestoreWorkArray(dm, maxSize, PETSC_INT, &fifo);CHKERRQ(ierr);
1336   PetscFunctionReturn(0);
1337 }
1338 
1339 #undef __FUNCT__
1340 #define __FUNCT__ "DMPlexGetTransitiveClosure_Internal"
1341 /*@C
1342   DMPlexGetTransitiveClosure_Internal - Return the points on the transitive closure of the in-edges or out-edges for this point in the Sieve DAG with a specified initial orientation
1343 
1344   Not collective
1345 
1346   Input Parameters:
1347 + mesh - The DMPlex
1348 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1349 . orientation - The orientation of the point
1350 . useCone - PETSC_TRUE for in-edges,  otherwise use out-edges
1351 - points - If points is NULL on input, internal storage will be returned, otherwise the provided array is used
1352 
1353   Output Parameters:
1354 + numPoints - The number of points in the closure, so points[] is of size 2*numPoints
1355 - points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...]
1356 
1357   Note:
1358   If using internal storage (points is NULL on input), each call overwrites the last output.
1359 
1360   Fortran Notes:
1361   Since it returns an array, this routine is only available in Fortran 90, and you must
1362   include petsc.h90 in your code.
1363 
1364   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
1365 
1366   Level: beginner
1367 
1368 .seealso: DMPlexRestoreTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
1369 @*/
1370 PetscErrorCode DMPlexGetTransitiveClosure_Internal(DM dm, PetscInt p, PetscInt ornt, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
1371 {
1372   DM_Plex        *mesh = (DM_Plex*) dm->data;
1373   PetscInt       *closure, *fifo;
1374   const PetscInt *tmp = NULL, *tmpO = NULL;
1375   PetscInt        tmpSize, t;
1376   PetscInt        depth       = 0, maxSize;
1377   PetscInt        closureSize = 2, fifoSize = 0, fifoStart = 0;
1378   PetscErrorCode  ierr;
1379 
1380   PetscFunctionBegin;
1381   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1382   ierr    = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
1383   /* This is only 1-level */
1384   if (useCone) {
1385     ierr = DMPlexGetConeSize(dm, p, &tmpSize);CHKERRQ(ierr);
1386     ierr = DMPlexGetCone(dm, p, &tmp);CHKERRQ(ierr);
1387     ierr = DMPlexGetConeOrientation(dm, p, &tmpO);CHKERRQ(ierr);
1388   } else {
1389     ierr = DMPlexGetSupportSize(dm, p, &tmpSize);CHKERRQ(ierr);
1390     ierr = DMPlexGetSupport(dm, p, &tmp);CHKERRQ(ierr);
1391   }
1392   if (depth == 1) {
1393     if (*points) {
1394       closure = *points;
1395     } else {
1396       maxSize = 2*(PetscMax(mesh->maxConeSize, mesh->maxSupportSize)+1);
1397       ierr = DMGetWorkArray(dm, maxSize, PETSC_INT, &closure);CHKERRQ(ierr);
1398     }
1399     closure[0] = p; closure[1] = ornt;
1400     for (t = 0; t < tmpSize; ++t, closureSize += 2) {
1401       const PetscInt i = ornt >= 0 ? (t+ornt)%tmpSize : (-(ornt+1) + tmpSize-t)%tmpSize;
1402       closure[closureSize]   = tmp[i];
1403       closure[closureSize+1] = tmpO ? tmpO[i] : 0;
1404     }
1405     if (numPoints) *numPoints = closureSize/2;
1406     if (points)    *points    = closure;
1407     PetscFunctionReturn(0);
1408   }
1409   maxSize = 2*PetscMax(PetscMax(PetscPowInt(mesh->maxConeSize,depth+1),PetscPowInt(mesh->maxSupportSize,depth+1)),depth+1);
1410   ierr    = DMGetWorkArray(dm, maxSize, PETSC_INT, &fifo);CHKERRQ(ierr);
1411   if (*points) {
1412     closure = *points;
1413   } else {
1414     ierr = DMGetWorkArray(dm, maxSize, PETSC_INT, &closure);CHKERRQ(ierr);
1415   }
1416   closure[0] = p; closure[1] = ornt;
1417   for (t = 0; t < tmpSize; ++t, closureSize += 2, fifoSize += 2) {
1418     const PetscInt i  = ornt >= 0 ? (t+ornt)%tmpSize : (-(ornt+1) + tmpSize-t)%tmpSize;
1419     const PetscInt cp = tmp[i];
1420     PetscInt       co = tmpO ? tmpO[i] : 0;
1421 
1422     if (ornt < 0) {
1423       PetscInt childSize, coff;
1424       ierr = DMPlexGetConeSize(dm, cp, &childSize);CHKERRQ(ierr);
1425       coff = tmpO[i] < 0 ? -(tmpO[i]+1) : tmpO[i];
1426       co   = childSize ? -(((coff+childSize-1)%childSize)+1) : 0;
1427     }
1428     closure[closureSize]   = cp;
1429     closure[closureSize+1] = co;
1430     fifo[fifoSize]         = cp;
1431     fifo[fifoSize+1]       = co;
1432   }
1433   /* Should kick out early when depth is reached, rather than checking all vertices for empty cones */
1434   while (fifoSize - fifoStart) {
1435     const PetscInt q   = fifo[fifoStart];
1436     const PetscInt o   = fifo[fifoStart+1];
1437     const PetscInt rev = o >= 0 ? 0 : 1;
1438     const PetscInt off = rev ? -(o+1) : o;
1439 
1440     if (useCone) {
1441       ierr = DMPlexGetConeSize(dm, q, &tmpSize);CHKERRQ(ierr);
1442       ierr = DMPlexGetCone(dm, q, &tmp);CHKERRQ(ierr);
1443       ierr = DMPlexGetConeOrientation(dm, q, &tmpO);CHKERRQ(ierr);
1444     } else {
1445       ierr = DMPlexGetSupportSize(dm, q, &tmpSize);CHKERRQ(ierr);
1446       ierr = DMPlexGetSupport(dm, q, &tmp);CHKERRQ(ierr);
1447       tmpO = NULL;
1448     }
1449     for (t = 0; t < tmpSize; ++t) {
1450       const PetscInt i  = ((rev ? tmpSize-t : t) + off)%tmpSize;
1451       const PetscInt cp = tmp[i];
1452       /* Must propogate orientation: When we reverse orientation, we both reverse the direction of iteration and start at the other end of the chain. */
1453       /* HACK: It is worse to get the size here, than to change the interpretation of -(*+1)
1454        const PetscInt co = tmpO ? (rev ? -(tmpO[i]+1) : tmpO[i]) : 0; */
1455       PetscInt       co = tmpO ? tmpO[i] : 0;
1456       PetscInt       c;
1457 
1458       if (rev) {
1459         PetscInt childSize, coff;
1460         ierr = DMPlexGetConeSize(dm, cp, &childSize);CHKERRQ(ierr);
1461         coff = tmpO[i] < 0 ? -(tmpO[i]+1) : tmpO[i];
1462         co   = childSize ? -(((coff+childSize-1)%childSize)+1) : 0;
1463       }
1464       /* Check for duplicate */
1465       for (c = 0; c < closureSize; c += 2) {
1466         if (closure[c] == cp) break;
1467       }
1468       if (c == closureSize) {
1469         closure[closureSize]   = cp;
1470         closure[closureSize+1] = co;
1471         fifo[fifoSize]         = cp;
1472         fifo[fifoSize+1]       = co;
1473         closureSize           += 2;
1474         fifoSize              += 2;
1475       }
1476     }
1477     fifoStart += 2;
1478   }
1479   if (numPoints) *numPoints = closureSize/2;
1480   if (points)    *points    = closure;
1481   ierr = DMRestoreWorkArray(dm, maxSize, PETSC_INT, &fifo);CHKERRQ(ierr);
1482   PetscFunctionReturn(0);
1483 }
1484 
1485 #undef __FUNCT__
1486 #define __FUNCT__ "DMPlexRestoreTransitiveClosure"
1487 /*@C
1488   DMPlexRestoreTransitiveClosure - Restore the array of points on the transitive closure of the in-edges or out-edges for this point in the Sieve DAG
1489 
1490   Not collective
1491 
1492   Input Parameters:
1493 + mesh - The DMPlex
1494 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1495 . useCone - PETSC_TRUE for in-edges,  otherwise use out-edges
1496 . numPoints - The number of points in the closure, so points[] is of size 2*numPoints, zeroed on exit
1497 - points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...], zeroed on exit
1498 
1499   Note:
1500   If not using internal storage (points is not NULL on input), this call is unnecessary
1501 
1502   Fortran Notes:
1503   Since it returns an array, this routine is only available in Fortran 90, and you must
1504   include petsc.h90 in your code.
1505 
1506   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
1507 
1508   Level: beginner
1509 
1510 .seealso: DMPlexGetTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
1511 @*/
1512 PetscErrorCode DMPlexRestoreTransitiveClosure(DM dm, PetscInt p, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
1513 {
1514   PetscErrorCode ierr;
1515 
1516   PetscFunctionBegin;
1517   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1518   if (numPoints) PetscValidIntPointer(numPoints,4);
1519   if (points) PetscValidPointer(points,5);
1520   ierr = DMRestoreWorkArray(dm, 0, PETSC_INT, points);CHKERRQ(ierr);
1521   if (numPoints) *numPoints = 0;
1522   PetscFunctionReturn(0);
1523 }
1524 
1525 #undef __FUNCT__
1526 #define __FUNCT__ "DMPlexGetMaxSizes"
1527 /*@
1528   DMPlexGetMaxSizes - Return the maximum number of in-edges (cone) and out-edges (support) for any point in the Sieve DAG
1529 
1530   Not collective
1531 
1532   Input Parameter:
1533 . mesh - The DMPlex
1534 
1535   Output Parameters:
1536 + maxConeSize - The maximum number of in-edges
1537 - maxSupportSize - The maximum number of out-edges
1538 
1539   Level: beginner
1540 
1541 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart()
1542 @*/
1543 PetscErrorCode DMPlexGetMaxSizes(DM dm, PetscInt *maxConeSize, PetscInt *maxSupportSize)
1544 {
1545   DM_Plex *mesh = (DM_Plex*) dm->data;
1546 
1547   PetscFunctionBegin;
1548   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1549   if (maxConeSize)    *maxConeSize    = mesh->maxConeSize;
1550   if (maxSupportSize) *maxSupportSize = mesh->maxSupportSize;
1551   PetscFunctionReturn(0);
1552 }
1553 
1554 #undef __FUNCT__
1555 #define __FUNCT__ "DMSetUp_Plex"
1556 PetscErrorCode DMSetUp_Plex(DM dm)
1557 {
1558   DM_Plex       *mesh = (DM_Plex*) dm->data;
1559   PetscInt       size;
1560   PetscErrorCode ierr;
1561 
1562   PetscFunctionBegin;
1563   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1564   ierr = PetscSectionSetUp(mesh->coneSection);CHKERRQ(ierr);
1565   ierr = PetscSectionGetStorageSize(mesh->coneSection, &size);CHKERRQ(ierr);
1566   ierr = PetscMalloc1(size, &mesh->cones);CHKERRQ(ierr);
1567   ierr = PetscCalloc1(size, &mesh->coneOrientations);CHKERRQ(ierr);
1568   if (mesh->maxSupportSize) {
1569     ierr = PetscSectionSetUp(mesh->supportSection);CHKERRQ(ierr);
1570     ierr = PetscSectionGetStorageSize(mesh->supportSection, &size);CHKERRQ(ierr);
1571     ierr = PetscMalloc1(size, &mesh->supports);CHKERRQ(ierr);
1572   }
1573   PetscFunctionReturn(0);
1574 }
1575 
1576 #undef __FUNCT__
1577 #define __FUNCT__ "DMCreateSubDM_Plex"
1578 PetscErrorCode DMCreateSubDM_Plex(DM dm, PetscInt numFields, PetscInt fields[], IS *is, DM *subdm)
1579 {
1580   PetscErrorCode ierr;
1581 
1582   PetscFunctionBegin;
1583   if (subdm) {ierr = DMClone(dm, subdm);CHKERRQ(ierr);}
1584   ierr = DMCreateSubDM_Section_Private(dm, numFields, fields, is, subdm);CHKERRQ(ierr);
1585   PetscFunctionReturn(0);
1586 }
1587 
1588 #undef __FUNCT__
1589 #define __FUNCT__ "DMPlexSymmetrize"
1590 /*@
1591   DMPlexSymmetrize - Creates support (out-edge) information from cone (in-edge) inoformation
1592 
1593   Not collective
1594 
1595   Input Parameter:
1596 . mesh - The DMPlex
1597 
1598   Output Parameter:
1599 
1600   Note:
1601   This should be called after all calls to DMPlexSetCone()
1602 
1603   Level: beginner
1604 
1605 .seealso: DMPlexCreate(), DMPlexSetChart(), DMPlexSetConeSize(), DMPlexSetCone()
1606 @*/
1607 PetscErrorCode DMPlexSymmetrize(DM dm)
1608 {
1609   DM_Plex       *mesh = (DM_Plex*) dm->data;
1610   PetscInt      *offsets;
1611   PetscInt       supportSize;
1612   PetscInt       pStart, pEnd, p;
1613   PetscErrorCode ierr;
1614 
1615   PetscFunctionBegin;
1616   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1617   if (mesh->supports) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONGSTATE, "Supports were already setup in this DMPlex");
1618   /* Calculate support sizes */
1619   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
1620   for (p = pStart; p < pEnd; ++p) {
1621     PetscInt dof, off, c;
1622 
1623     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
1624     ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1625     for (c = off; c < off+dof; ++c) {
1626       ierr = PetscSectionAddDof(mesh->supportSection, mesh->cones[c], 1);CHKERRQ(ierr);
1627     }
1628   }
1629   for (p = pStart; p < pEnd; ++p) {
1630     PetscInt dof;
1631 
1632     ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
1633 
1634     mesh->maxSupportSize = PetscMax(mesh->maxSupportSize, dof);
1635   }
1636   ierr = PetscSectionSetUp(mesh->supportSection);CHKERRQ(ierr);
1637   /* Calculate supports */
1638   ierr = PetscSectionGetStorageSize(mesh->supportSection, &supportSize);CHKERRQ(ierr);
1639   ierr = PetscMalloc1(supportSize, &mesh->supports);CHKERRQ(ierr);
1640   ierr = PetscCalloc1(pEnd - pStart, &offsets);CHKERRQ(ierr);
1641   for (p = pStart; p < pEnd; ++p) {
1642     PetscInt dof, off, c;
1643 
1644     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
1645     ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1646     for (c = off; c < off+dof; ++c) {
1647       const PetscInt q = mesh->cones[c];
1648       PetscInt       offS;
1649 
1650       ierr = PetscSectionGetOffset(mesh->supportSection, q, &offS);CHKERRQ(ierr);
1651 
1652       mesh->supports[offS+offsets[q]] = p;
1653       ++offsets[q];
1654     }
1655   }
1656   ierr = PetscFree(offsets);CHKERRQ(ierr);
1657   PetscFunctionReturn(0);
1658 }
1659 
1660 #undef __FUNCT__
1661 #define __FUNCT__ "DMPlexStratify"
1662 /*@
1663   DMPlexStratify - The Sieve DAG for most topologies is a graded poset (http://en.wikipedia.org/wiki/Graded_poset), and
1664   can be illustrated by Hasse Diagram (a http://en.wikipedia.org/wiki/Hasse_diagram). The strata group all points of the
1665   same grade, and this function calculates the strata. This grade can be seen as the height (or depth) of the point in
1666   the DAG.
1667 
1668   Not collective
1669 
1670   Input Parameter:
1671 . mesh - The DMPlex
1672 
1673   Output Parameter:
1674 
1675   Notes:
1676   The normal association for the point grade is element dimension (or co-dimension). For instance, all vertices would
1677   have depth 0, and all edges depth 1. Likewise, all cells heights would have height 0, and all faces height 1.
1678 
1679   This should be called after all calls to DMPlexSymmetrize()
1680 
1681   Level: beginner
1682 
1683 .seealso: DMPlexCreate(), DMPlexSymmetrize()
1684 @*/
1685 PetscErrorCode DMPlexStratify(DM dm)
1686 {
1687   DMLabel        label;
1688   PetscInt       pStart, pEnd, p;
1689   PetscInt       numRoots = 0, numLeaves = 0;
1690   PetscErrorCode ierr;
1691 
1692   PetscFunctionBegin;
1693   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1694   ierr = PetscLogEventBegin(DMPLEX_Stratify,dm,0,0,0);CHKERRQ(ierr);
1695   /* Calculate depth */
1696   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
1697   ierr = DMPlexCreateLabel(dm, "depth");CHKERRQ(ierr);
1698   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
1699   /* Initialize roots and count leaves */
1700   for (p = pStart; p < pEnd; ++p) {
1701     PetscInt coneSize, supportSize;
1702 
1703     ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
1704     ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
1705     if (!coneSize && supportSize) {
1706       ++numRoots;
1707       ierr = DMLabelSetValue(label, p, 0);CHKERRQ(ierr);
1708     } else if (!supportSize && coneSize) {
1709       ++numLeaves;
1710     } else if (!supportSize && !coneSize) {
1711       /* Isolated points */
1712       ierr = DMLabelSetValue(label, p, 0);CHKERRQ(ierr);
1713     }
1714   }
1715   if (numRoots + numLeaves == (pEnd - pStart)) {
1716     for (p = pStart; p < pEnd; ++p) {
1717       PetscInt coneSize, supportSize;
1718 
1719       ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
1720       ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
1721       if (!supportSize && coneSize) {
1722         ierr = DMLabelSetValue(label, p, 1);CHKERRQ(ierr);
1723       }
1724     }
1725   } else {
1726     IS       pointIS;
1727     PetscInt numPoints = 0, level = 0;
1728 
1729     ierr = DMLabelGetStratumIS(label, level, &pointIS);CHKERRQ(ierr);
1730     if (pointIS) {ierr = ISGetLocalSize(pointIS, &numPoints);CHKERRQ(ierr);}
1731     while (numPoints) {
1732       const PetscInt *points;
1733       const PetscInt  newLevel = level+1;
1734 
1735       ierr = ISGetIndices(pointIS, &points);CHKERRQ(ierr);
1736       for (p = 0; p < numPoints; ++p) {
1737         const PetscInt  point = points[p];
1738         const PetscInt *support;
1739         PetscInt        supportSize, s;
1740 
1741         ierr = DMPlexGetSupportSize(dm, point, &supportSize);CHKERRQ(ierr);
1742         ierr = DMPlexGetSupport(dm, point, &support);CHKERRQ(ierr);
1743         for (s = 0; s < supportSize; ++s) {
1744           ierr = DMLabelSetValue(label, support[s], newLevel);CHKERRQ(ierr);
1745         }
1746       }
1747       ierr = ISRestoreIndices(pointIS, &points);CHKERRQ(ierr);
1748       ++level;
1749       ierr = ISDestroy(&pointIS);CHKERRQ(ierr);
1750       ierr = DMLabelGetStratumIS(label, level, &pointIS);CHKERRQ(ierr);
1751       if (pointIS) {ierr = ISGetLocalSize(pointIS, &numPoints);CHKERRQ(ierr);}
1752       else         {numPoints = 0;}
1753     }
1754     ierr = ISDestroy(&pointIS);CHKERRQ(ierr);
1755   }
1756   ierr = PetscLogEventEnd(DMPLEX_Stratify,dm,0,0,0);CHKERRQ(ierr);
1757   PetscFunctionReturn(0);
1758 }
1759 
1760 #undef __FUNCT__
1761 #define __FUNCT__ "DMPlexGetJoin"
1762 /*@C
1763   DMPlexGetJoin - Get an array for the join of the set of points
1764 
1765   Not Collective
1766 
1767   Input Parameters:
1768 + dm - The DMPlex object
1769 . numPoints - The number of input points for the join
1770 - points - The input points
1771 
1772   Output Parameters:
1773 + numCoveredPoints - The number of points in the join
1774 - coveredPoints - The points in the join
1775 
1776   Level: intermediate
1777 
1778   Note: Currently, this is restricted to a single level join
1779 
1780   Fortran Notes:
1781   Since it returns an array, this routine is only available in Fortran 90, and you must
1782   include petsc.h90 in your code.
1783 
1784   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
1785 
1786 .keywords: mesh
1787 .seealso: DMPlexRestoreJoin(), DMPlexGetMeet()
1788 @*/
1789 PetscErrorCode DMPlexGetJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
1790 {
1791   DM_Plex       *mesh = (DM_Plex*) dm->data;
1792   PetscInt      *join[2];
1793   PetscInt       joinSize, i = 0;
1794   PetscInt       dof, off, p, c, m;
1795   PetscErrorCode ierr;
1796 
1797   PetscFunctionBegin;
1798   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1799   PetscValidPointer(points, 2);
1800   PetscValidPointer(numCoveredPoints, 3);
1801   PetscValidPointer(coveredPoints, 4);
1802   ierr = DMGetWorkArray(dm, mesh->maxSupportSize, PETSC_INT, &join[0]);CHKERRQ(ierr);
1803   ierr = DMGetWorkArray(dm, mesh->maxSupportSize, PETSC_INT, &join[1]);CHKERRQ(ierr);
1804   /* Copy in support of first point */
1805   ierr = PetscSectionGetDof(mesh->supportSection, points[0], &dof);CHKERRQ(ierr);
1806   ierr = PetscSectionGetOffset(mesh->supportSection, points[0], &off);CHKERRQ(ierr);
1807   for (joinSize = 0; joinSize < dof; ++joinSize) {
1808     join[i][joinSize] = mesh->supports[off+joinSize];
1809   }
1810   /* Check each successive support */
1811   for (p = 1; p < numPoints; ++p) {
1812     PetscInt newJoinSize = 0;
1813 
1814     ierr = PetscSectionGetDof(mesh->supportSection, points[p], &dof);CHKERRQ(ierr);
1815     ierr = PetscSectionGetOffset(mesh->supportSection, points[p], &off);CHKERRQ(ierr);
1816     for (c = 0; c < dof; ++c) {
1817       const PetscInt point = mesh->supports[off+c];
1818 
1819       for (m = 0; m < joinSize; ++m) {
1820         if (point == join[i][m]) {
1821           join[1-i][newJoinSize++] = point;
1822           break;
1823         }
1824       }
1825     }
1826     joinSize = newJoinSize;
1827     i        = 1-i;
1828   }
1829   *numCoveredPoints = joinSize;
1830   *coveredPoints    = join[i];
1831   ierr              = DMRestoreWorkArray(dm, mesh->maxSupportSize, PETSC_INT, &join[1-i]);CHKERRQ(ierr);
1832   PetscFunctionReturn(0);
1833 }
1834 
1835 #undef __FUNCT__
1836 #define __FUNCT__ "DMPlexRestoreJoin"
1837 /*@C
1838   DMPlexRestoreJoin - Restore an array for the join of the set of points
1839 
1840   Not Collective
1841 
1842   Input Parameters:
1843 + dm - The DMPlex object
1844 . numPoints - The number of input points for the join
1845 - points - The input points
1846 
1847   Output Parameters:
1848 + numCoveredPoints - The number of points in the join
1849 - coveredPoints - The points in the join
1850 
1851   Fortran Notes:
1852   Since it returns an array, this routine is only available in Fortran 90, and you must
1853   include petsc.h90 in your code.
1854 
1855   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
1856 
1857   Level: intermediate
1858 
1859 .keywords: mesh
1860 .seealso: DMPlexGetJoin(), DMPlexGetFullJoin(), DMPlexGetMeet()
1861 @*/
1862 PetscErrorCode DMPlexRestoreJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
1863 {
1864   PetscErrorCode ierr;
1865 
1866   PetscFunctionBegin;
1867   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1868   if (points) PetscValidIntPointer(points,3);
1869   if (numCoveredPoints) PetscValidIntPointer(numCoveredPoints,4);
1870   PetscValidPointer(coveredPoints, 5);
1871   ierr = DMRestoreWorkArray(dm, 0, PETSC_INT, (void*) coveredPoints);CHKERRQ(ierr);
1872   if (numCoveredPoints) *numCoveredPoints = 0;
1873   PetscFunctionReturn(0);
1874 }
1875 
1876 #undef __FUNCT__
1877 #define __FUNCT__ "DMPlexGetFullJoin"
1878 /*@C
1879   DMPlexGetFullJoin - Get an array for the join of the set of points
1880 
1881   Not Collective
1882 
1883   Input Parameters:
1884 + dm - The DMPlex object
1885 . numPoints - The number of input points for the join
1886 - points - The input points
1887 
1888   Output Parameters:
1889 + numCoveredPoints - The number of points in the join
1890 - coveredPoints - The points in the join
1891 
1892   Fortran Notes:
1893   Since it returns an array, this routine is only available in Fortran 90, and you must
1894   include petsc.h90 in your code.
1895 
1896   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
1897 
1898   Level: intermediate
1899 
1900 .keywords: mesh
1901 .seealso: DMPlexGetJoin(), DMPlexRestoreJoin(), DMPlexGetMeet()
1902 @*/
1903 PetscErrorCode DMPlexGetFullJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
1904 {
1905   DM_Plex       *mesh = (DM_Plex*) dm->data;
1906   PetscInt      *offsets, **closures;
1907   PetscInt      *join[2];
1908   PetscInt       depth = 0, maxSize, joinSize = 0, i = 0;
1909   PetscInt       p, d, c, m;
1910   PetscErrorCode ierr;
1911 
1912   PetscFunctionBegin;
1913   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1914   PetscValidPointer(points, 2);
1915   PetscValidPointer(numCoveredPoints, 3);
1916   PetscValidPointer(coveredPoints, 4);
1917 
1918   ierr    = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
1919   ierr    = PetscCalloc1(numPoints, &closures);CHKERRQ(ierr);
1920   ierr    = DMGetWorkArray(dm, numPoints*(depth+2), PETSC_INT, &offsets);CHKERRQ(ierr);
1921   maxSize = PetscPowInt(mesh->maxSupportSize,depth+1);
1922   ierr    = DMGetWorkArray(dm, maxSize, PETSC_INT, &join[0]);CHKERRQ(ierr);
1923   ierr    = DMGetWorkArray(dm, maxSize, PETSC_INT, &join[1]);CHKERRQ(ierr);
1924 
1925   for (p = 0; p < numPoints; ++p) {
1926     PetscInt closureSize;
1927 
1928     ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_FALSE, &closureSize, &closures[p]);CHKERRQ(ierr);
1929 
1930     offsets[p*(depth+2)+0] = 0;
1931     for (d = 0; d < depth+1; ++d) {
1932       PetscInt pStart, pEnd, i;
1933 
1934       ierr = DMPlexGetDepthStratum(dm, d, &pStart, &pEnd);CHKERRQ(ierr);
1935       for (i = offsets[p*(depth+2)+d]; i < closureSize; ++i) {
1936         if ((pStart > closures[p][i*2]) || (pEnd <= closures[p][i*2])) {
1937           offsets[p*(depth+2)+d+1] = i;
1938           break;
1939         }
1940       }
1941       if (i == closureSize) offsets[p*(depth+2)+d+1] = i;
1942     }
1943     if (offsets[p*(depth+2)+depth+1] != closureSize) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Total size of closure %D should be %D", offsets[p*(depth+2)+depth+1], closureSize);
1944   }
1945   for (d = 0; d < depth+1; ++d) {
1946     PetscInt dof;
1947 
1948     /* Copy in support of first point */
1949     dof = offsets[d+1] - offsets[d];
1950     for (joinSize = 0; joinSize < dof; ++joinSize) {
1951       join[i][joinSize] = closures[0][(offsets[d]+joinSize)*2];
1952     }
1953     /* Check each successive cone */
1954     for (p = 1; p < numPoints && joinSize; ++p) {
1955       PetscInt newJoinSize = 0;
1956 
1957       dof = offsets[p*(depth+2)+d+1] - offsets[p*(depth+2)+d];
1958       for (c = 0; c < dof; ++c) {
1959         const PetscInt point = closures[p][(offsets[p*(depth+2)+d]+c)*2];
1960 
1961         for (m = 0; m < joinSize; ++m) {
1962           if (point == join[i][m]) {
1963             join[1-i][newJoinSize++] = point;
1964             break;
1965           }
1966         }
1967       }
1968       joinSize = newJoinSize;
1969       i        = 1-i;
1970     }
1971     if (joinSize) break;
1972   }
1973   *numCoveredPoints = joinSize;
1974   *coveredPoints    = join[i];
1975   for (p = 0; p < numPoints; ++p) {
1976     ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_FALSE, NULL, &closures[p]);CHKERRQ(ierr);
1977   }
1978   ierr = PetscFree(closures);CHKERRQ(ierr);
1979   ierr = DMRestoreWorkArray(dm, numPoints*(depth+2), PETSC_INT, &offsets);CHKERRQ(ierr);
1980   ierr = DMRestoreWorkArray(dm, mesh->maxSupportSize, PETSC_INT, &join[1-i]);CHKERRQ(ierr);
1981   PetscFunctionReturn(0);
1982 }
1983 
1984 #undef __FUNCT__
1985 #define __FUNCT__ "DMPlexGetMeet"
1986 /*@C
1987   DMPlexGetMeet - Get an array for the meet of the set of points
1988 
1989   Not Collective
1990 
1991   Input Parameters:
1992 + dm - The DMPlex object
1993 . numPoints - The number of input points for the meet
1994 - points - The input points
1995 
1996   Output Parameters:
1997 + numCoveredPoints - The number of points in the meet
1998 - coveredPoints - The points in the meet
1999 
2000   Level: intermediate
2001 
2002   Note: Currently, this is restricted to a single level meet
2003 
2004   Fortran Notes:
2005   Since it returns an array, this routine is only available in Fortran 90, and you must
2006   include petsc.h90 in your code.
2007 
2008   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2009 
2010 .keywords: mesh
2011 .seealso: DMPlexRestoreMeet(), DMPlexGetJoin()
2012 @*/
2013 PetscErrorCode DMPlexGetMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveringPoints, const PetscInt **coveringPoints)
2014 {
2015   DM_Plex       *mesh = (DM_Plex*) dm->data;
2016   PetscInt      *meet[2];
2017   PetscInt       meetSize, i = 0;
2018   PetscInt       dof, off, p, c, m;
2019   PetscErrorCode ierr;
2020 
2021   PetscFunctionBegin;
2022   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2023   PetscValidPointer(points, 2);
2024   PetscValidPointer(numCoveringPoints, 3);
2025   PetscValidPointer(coveringPoints, 4);
2026   ierr = DMGetWorkArray(dm, mesh->maxConeSize, PETSC_INT, &meet[0]);CHKERRQ(ierr);
2027   ierr = DMGetWorkArray(dm, mesh->maxConeSize, PETSC_INT, &meet[1]);CHKERRQ(ierr);
2028   /* Copy in cone of first point */
2029   ierr = PetscSectionGetDof(mesh->coneSection, points[0], &dof);CHKERRQ(ierr);
2030   ierr = PetscSectionGetOffset(mesh->coneSection, points[0], &off);CHKERRQ(ierr);
2031   for (meetSize = 0; meetSize < dof; ++meetSize) {
2032     meet[i][meetSize] = mesh->cones[off+meetSize];
2033   }
2034   /* Check each successive cone */
2035   for (p = 1; p < numPoints; ++p) {
2036     PetscInt newMeetSize = 0;
2037 
2038     ierr = PetscSectionGetDof(mesh->coneSection, points[p], &dof);CHKERRQ(ierr);
2039     ierr = PetscSectionGetOffset(mesh->coneSection, points[p], &off);CHKERRQ(ierr);
2040     for (c = 0; c < dof; ++c) {
2041       const PetscInt point = mesh->cones[off+c];
2042 
2043       for (m = 0; m < meetSize; ++m) {
2044         if (point == meet[i][m]) {
2045           meet[1-i][newMeetSize++] = point;
2046           break;
2047         }
2048       }
2049     }
2050     meetSize = newMeetSize;
2051     i        = 1-i;
2052   }
2053   *numCoveringPoints = meetSize;
2054   *coveringPoints    = meet[i];
2055   ierr               = DMRestoreWorkArray(dm, mesh->maxConeSize, PETSC_INT, &meet[1-i]);CHKERRQ(ierr);
2056   PetscFunctionReturn(0);
2057 }
2058 
2059 #undef __FUNCT__
2060 #define __FUNCT__ "DMPlexRestoreMeet"
2061 /*@C
2062   DMPlexRestoreMeet - Restore an array for the meet of the set of points
2063 
2064   Not Collective
2065 
2066   Input Parameters:
2067 + dm - The DMPlex object
2068 . numPoints - The number of input points for the meet
2069 - points - The input points
2070 
2071   Output Parameters:
2072 + numCoveredPoints - The number of points in the meet
2073 - coveredPoints - The points in the meet
2074 
2075   Level: intermediate
2076 
2077   Fortran Notes:
2078   Since it returns an array, this routine is only available in Fortran 90, and you must
2079   include petsc.h90 in your code.
2080 
2081   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2082 
2083 .keywords: mesh
2084 .seealso: DMPlexGetMeet(), DMPlexGetFullMeet(), DMPlexGetJoin()
2085 @*/
2086 PetscErrorCode DMPlexRestoreMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
2087 {
2088   PetscErrorCode ierr;
2089 
2090   PetscFunctionBegin;
2091   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2092   if (points) PetscValidIntPointer(points,3);
2093   if (numCoveredPoints) PetscValidIntPointer(numCoveredPoints,4);
2094   PetscValidPointer(coveredPoints,5);
2095   ierr = DMRestoreWorkArray(dm, 0, PETSC_INT, (void*) coveredPoints);CHKERRQ(ierr);
2096   if (numCoveredPoints) *numCoveredPoints = 0;
2097   PetscFunctionReturn(0);
2098 }
2099 
2100 #undef __FUNCT__
2101 #define __FUNCT__ "DMPlexGetFullMeet"
2102 /*@C
2103   DMPlexGetFullMeet - Get an array for the meet of the set of points
2104 
2105   Not Collective
2106 
2107   Input Parameters:
2108 + dm - The DMPlex object
2109 . numPoints - The number of input points for the meet
2110 - points - The input points
2111 
2112   Output Parameters:
2113 + numCoveredPoints - The number of points in the meet
2114 - coveredPoints - The points in the meet
2115 
2116   Level: intermediate
2117 
2118   Fortran Notes:
2119   Since it returns an array, this routine is only available in Fortran 90, and you must
2120   include petsc.h90 in your code.
2121 
2122   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2123 
2124 .keywords: mesh
2125 .seealso: DMPlexGetMeet(), DMPlexRestoreMeet(), DMPlexGetJoin()
2126 @*/
2127 PetscErrorCode DMPlexGetFullMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
2128 {
2129   DM_Plex       *mesh = (DM_Plex*) dm->data;
2130   PetscInt      *offsets, **closures;
2131   PetscInt      *meet[2];
2132   PetscInt       height = 0, maxSize, meetSize = 0, i = 0;
2133   PetscInt       p, h, c, m;
2134   PetscErrorCode ierr;
2135 
2136   PetscFunctionBegin;
2137   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2138   PetscValidPointer(points, 2);
2139   PetscValidPointer(numCoveredPoints, 3);
2140   PetscValidPointer(coveredPoints, 4);
2141 
2142   ierr    = DMPlexGetDepth(dm, &height);CHKERRQ(ierr);
2143   ierr    = PetscMalloc1(numPoints, &closures);CHKERRQ(ierr);
2144   ierr    = DMGetWorkArray(dm, numPoints*(height+2), PETSC_INT, &offsets);CHKERRQ(ierr);
2145   maxSize = PetscPowInt(mesh->maxConeSize,height+1);
2146   ierr    = DMGetWorkArray(dm, maxSize, PETSC_INT, &meet[0]);CHKERRQ(ierr);
2147   ierr    = DMGetWorkArray(dm, maxSize, PETSC_INT, &meet[1]);CHKERRQ(ierr);
2148 
2149   for (p = 0; p < numPoints; ++p) {
2150     PetscInt closureSize;
2151 
2152     ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_TRUE, &closureSize, &closures[p]);CHKERRQ(ierr);
2153 
2154     offsets[p*(height+2)+0] = 0;
2155     for (h = 0; h < height+1; ++h) {
2156       PetscInt pStart, pEnd, i;
2157 
2158       ierr = DMPlexGetHeightStratum(dm, h, &pStart, &pEnd);CHKERRQ(ierr);
2159       for (i = offsets[p*(height+2)+h]; i < closureSize; ++i) {
2160         if ((pStart > closures[p][i*2]) || (pEnd <= closures[p][i*2])) {
2161           offsets[p*(height+2)+h+1] = i;
2162           break;
2163         }
2164       }
2165       if (i == closureSize) offsets[p*(height+2)+h+1] = i;
2166     }
2167     if (offsets[p*(height+2)+height+1] != closureSize) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Total size of closure %D should be %D", offsets[p*(height+2)+height+1], closureSize);
2168   }
2169   for (h = 0; h < height+1; ++h) {
2170     PetscInt dof;
2171 
2172     /* Copy in cone of first point */
2173     dof = offsets[h+1] - offsets[h];
2174     for (meetSize = 0; meetSize < dof; ++meetSize) {
2175       meet[i][meetSize] = closures[0][(offsets[h]+meetSize)*2];
2176     }
2177     /* Check each successive cone */
2178     for (p = 1; p < numPoints && meetSize; ++p) {
2179       PetscInt newMeetSize = 0;
2180 
2181       dof = offsets[p*(height+2)+h+1] - offsets[p*(height+2)+h];
2182       for (c = 0; c < dof; ++c) {
2183         const PetscInt point = closures[p][(offsets[p*(height+2)+h]+c)*2];
2184 
2185         for (m = 0; m < meetSize; ++m) {
2186           if (point == meet[i][m]) {
2187             meet[1-i][newMeetSize++] = point;
2188             break;
2189           }
2190         }
2191       }
2192       meetSize = newMeetSize;
2193       i        = 1-i;
2194     }
2195     if (meetSize) break;
2196   }
2197   *numCoveredPoints = meetSize;
2198   *coveredPoints    = meet[i];
2199   for (p = 0; p < numPoints; ++p) {
2200     ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_TRUE, NULL, &closures[p]);CHKERRQ(ierr);
2201   }
2202   ierr = PetscFree(closures);CHKERRQ(ierr);
2203   ierr = DMRestoreWorkArray(dm, numPoints*(height+2), PETSC_INT, &offsets);CHKERRQ(ierr);
2204   ierr = DMRestoreWorkArray(dm, mesh->maxConeSize, PETSC_INT, &meet[1-i]);CHKERRQ(ierr);
2205   PetscFunctionReturn(0);
2206 }
2207 
2208 #undef __FUNCT__
2209 #define __FUNCT__ "DMPlexEqual"
2210 /*@C
2211   DMPlexEqual - Determine if two DMs have the same topology
2212 
2213   Not Collective
2214 
2215   Input Parameters:
2216 + dmA - A DMPlex object
2217 - dmB - A DMPlex object
2218 
2219   Output Parameters:
2220 . equal - PETSC_TRUE if the topologies are identical
2221 
2222   Level: intermediate
2223 
2224   Notes:
2225   We are not solving graph isomorphism, so we do not permutation.
2226 
2227 .keywords: mesh
2228 .seealso: DMPlexGetCone()
2229 @*/
2230 PetscErrorCode DMPlexEqual(DM dmA, DM dmB, PetscBool *equal)
2231 {
2232   PetscInt       depth, depthB, pStart, pEnd, pStartB, pEndB, p;
2233   PetscErrorCode ierr;
2234 
2235   PetscFunctionBegin;
2236   PetscValidHeaderSpecific(dmA, DM_CLASSID, 1);
2237   PetscValidHeaderSpecific(dmB, DM_CLASSID, 2);
2238   PetscValidPointer(equal, 3);
2239 
2240   *equal = PETSC_FALSE;
2241   ierr = DMPlexGetDepth(dmA, &depth);CHKERRQ(ierr);
2242   ierr = DMPlexGetDepth(dmB, &depthB);CHKERRQ(ierr);
2243   if (depth != depthB) PetscFunctionReturn(0);
2244   ierr = DMPlexGetChart(dmA, &pStart,  &pEnd);CHKERRQ(ierr);
2245   ierr = DMPlexGetChart(dmB, &pStartB, &pEndB);CHKERRQ(ierr);
2246   if ((pStart != pStartB) || (pEnd != pEndB)) PetscFunctionReturn(0);
2247   for (p = pStart; p < pEnd; ++p) {
2248     const PetscInt *cone, *coneB, *ornt, *orntB, *support, *supportB;
2249     PetscInt        coneSize, coneSizeB, c, supportSize, supportSizeB, s;
2250 
2251     ierr = DMPlexGetConeSize(dmA, p, &coneSize);CHKERRQ(ierr);
2252     ierr = DMPlexGetCone(dmA, p, &cone);CHKERRQ(ierr);
2253     ierr = DMPlexGetConeOrientation(dmA, p, &ornt);CHKERRQ(ierr);
2254     ierr = DMPlexGetConeSize(dmB, p, &coneSizeB);CHKERRQ(ierr);
2255     ierr = DMPlexGetCone(dmB, p, &coneB);CHKERRQ(ierr);
2256     ierr = DMPlexGetConeOrientation(dmB, p, &orntB);CHKERRQ(ierr);
2257     if (coneSize != coneSizeB) PetscFunctionReturn(0);
2258     for (c = 0; c < coneSize; ++c) {
2259       if (cone[c] != coneB[c]) PetscFunctionReturn(0);
2260       if (ornt[c] != orntB[c]) PetscFunctionReturn(0);
2261     }
2262     ierr = DMPlexGetSupportSize(dmA, p, &supportSize);CHKERRQ(ierr);
2263     ierr = DMPlexGetSupport(dmA, p, &support);CHKERRQ(ierr);
2264     ierr = DMPlexGetSupportSize(dmB, p, &supportSizeB);CHKERRQ(ierr);
2265     ierr = DMPlexGetSupport(dmB, p, &supportB);CHKERRQ(ierr);
2266     if (supportSize != supportSizeB) PetscFunctionReturn(0);
2267     for (s = 0; s < supportSize; ++s) {
2268       if (support[s] != supportB[s]) PetscFunctionReturn(0);
2269     }
2270   }
2271   *equal = PETSC_TRUE;
2272   PetscFunctionReturn(0);
2273 }
2274 
2275 #undef __FUNCT__
2276 #define __FUNCT__ "DMPlexGetNumFaceVertices"
2277 PetscErrorCode DMPlexGetNumFaceVertices(DM dm, PetscInt cellDim, PetscInt numCorners, PetscInt *numFaceVertices)
2278 {
2279   MPI_Comm       comm;
2280   PetscErrorCode ierr;
2281 
2282   PetscFunctionBegin;
2283   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
2284   PetscValidPointer(numFaceVertices,3);
2285   switch (cellDim) {
2286   case 0:
2287     *numFaceVertices = 0;
2288     break;
2289   case 1:
2290     *numFaceVertices = 1;
2291     break;
2292   case 2:
2293     switch (numCorners) {
2294     case 3: /* triangle */
2295       *numFaceVertices = 2; /* Edge has 2 vertices */
2296       break;
2297     case 4: /* quadrilateral */
2298       *numFaceVertices = 2; /* Edge has 2 vertices */
2299       break;
2300     case 6: /* quadratic triangle, tri and quad cohesive Lagrange cells */
2301       *numFaceVertices = 3; /* Edge has 3 vertices */
2302       break;
2303     case 9: /* quadratic quadrilateral, quadratic quad cohesive Lagrange cells */
2304       *numFaceVertices = 3; /* Edge has 3 vertices */
2305       break;
2306     default:
2307       SETERRQ2(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid number of face corners %d for dimension %d", numCorners, cellDim);
2308     }
2309     break;
2310   case 3:
2311     switch (numCorners) {
2312     case 4: /* tetradehdron */
2313       *numFaceVertices = 3; /* Face has 3 vertices */
2314       break;
2315     case 6: /* tet cohesive cells */
2316       *numFaceVertices = 4; /* Face has 4 vertices */
2317       break;
2318     case 8: /* hexahedron */
2319       *numFaceVertices = 4; /* Face has 4 vertices */
2320       break;
2321     case 9: /* tet cohesive Lagrange cells */
2322       *numFaceVertices = 6; /* Face has 6 vertices */
2323       break;
2324     case 10: /* quadratic tetrahedron */
2325       *numFaceVertices = 6; /* Face has 6 vertices */
2326       break;
2327     case 12: /* hex cohesive Lagrange cells */
2328       *numFaceVertices = 6; /* Face has 6 vertices */
2329       break;
2330     case 18: /* quadratic tet cohesive Lagrange cells */
2331       *numFaceVertices = 6; /* Face has 6 vertices */
2332       break;
2333     case 27: /* quadratic hexahedron, quadratic hex cohesive Lagrange cells */
2334       *numFaceVertices = 9; /* Face has 9 vertices */
2335       break;
2336     default:
2337       SETERRQ2(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid number of face corners %d for dimension %d", numCorners, cellDim);
2338     }
2339     break;
2340   default:
2341     SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid cell dimension %d", cellDim);
2342   }
2343   PetscFunctionReturn(0);
2344 }
2345 
2346 #undef __FUNCT__
2347 #define __FUNCT__ "DMPlexOrient"
2348 /* Trys to give the mesh a consistent orientation */
2349 PetscErrorCode DMPlexOrient(DM dm)
2350 {
2351   PetscBT        seenCells, flippedCells, seenFaces;
2352   PetscInt      *faceFIFO, fTop, fBottom;
2353   PetscInt       dim, h, cStart, cEnd, c, fStart, fEnd, face, maxConeSize, *revcone, *revconeO;
2354   PetscErrorCode ierr;
2355 
2356   PetscFunctionBegin;
2357   /* Truth Table
2358      mismatch    flips   do action   mismatch   flipA ^ flipB   action
2359          F       0 flips     no         F             F           F
2360          F       1 flip      yes        F             T           T
2361          F       2 flips     no         T             F           T
2362          T       0 flips     yes        T             T           F
2363          T       1 flip      no
2364          T       2 flips     yes
2365   */
2366   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
2367   ierr = DMPlexGetVTKCellHeight(dm, &h);CHKERRQ(ierr);
2368   ierr = DMPlexGetHeightStratum(dm, h,   &cStart, &cEnd);CHKERRQ(ierr);
2369   ierr = DMPlexGetHeightStratum(dm, h+1, &fStart, &fEnd);CHKERRQ(ierr);
2370   ierr = PetscBTCreate(cEnd - cStart, &seenCells);CHKERRQ(ierr);
2371   ierr = PetscBTMemzero(cEnd - cStart, seenCells);CHKERRQ(ierr);
2372   ierr = PetscBTCreate(cEnd - cStart, &flippedCells);CHKERRQ(ierr);
2373   ierr = PetscBTMemzero(cEnd - cStart, flippedCells);CHKERRQ(ierr);
2374   ierr = PetscBTCreate(fEnd - fStart, &seenFaces);CHKERRQ(ierr);
2375   ierr = PetscBTMemzero(fEnd - fStart, seenFaces);CHKERRQ(ierr);
2376   ierr = PetscMalloc1((fEnd - fStart), &faceFIFO);CHKERRQ(ierr);
2377   fTop = fBottom = 0;
2378   /* Initialize FIFO with first cell */
2379   if (cEnd > cStart) {
2380     const PetscInt *cone;
2381     PetscInt        coneSize;
2382 
2383     ierr = DMPlexGetConeSize(dm, cStart, &coneSize);CHKERRQ(ierr);
2384     ierr = DMPlexGetCone(dm, cStart, &cone);CHKERRQ(ierr);
2385     for (c = 0; c < coneSize; ++c) {
2386       faceFIFO[fBottom++] = cone[c];
2387       ierr = PetscBTSet(seenFaces, cone[c]-fStart);CHKERRQ(ierr);
2388     }
2389   }
2390   /* Consider each face in FIFO */
2391   while (fTop < fBottom) {
2392     const PetscInt *support, *coneA, *coneB, *coneOA, *coneOB;
2393     PetscInt        supportSize, coneSizeA, coneSizeB, posA = -1, posB = -1;
2394     PetscInt        seenA, flippedA, seenB, flippedB, mismatch;
2395 
2396     face = faceFIFO[fTop++];
2397     ierr = DMPlexGetSupportSize(dm, face, &supportSize);CHKERRQ(ierr);
2398     ierr = DMPlexGetSupport(dm, face, &support);CHKERRQ(ierr);
2399     if (supportSize < 2) continue;
2400     if (supportSize != 2) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Faces should separate only two cells, not %d", supportSize);
2401     seenA    = PetscBTLookup(seenCells,    support[0]-cStart);
2402     flippedA = PetscBTLookup(flippedCells, support[0]-cStart) ? 1 : 0;
2403     seenB    = PetscBTLookup(seenCells,    support[1]-cStart);
2404     flippedB = PetscBTLookup(flippedCells, support[1]-cStart) ? 1 : 0;
2405 
2406     ierr = DMPlexGetConeSize(dm, support[0], &coneSizeA);CHKERRQ(ierr);
2407     ierr = DMPlexGetConeSize(dm, support[1], &coneSizeB);CHKERRQ(ierr);
2408     ierr = DMPlexGetCone(dm, support[0], &coneA);CHKERRQ(ierr);
2409     ierr = DMPlexGetCone(dm, support[1], &coneB);CHKERRQ(ierr);
2410     ierr = DMPlexGetConeOrientation(dm, support[0], &coneOA);CHKERRQ(ierr);
2411     ierr = DMPlexGetConeOrientation(dm, support[1], &coneOB);CHKERRQ(ierr);
2412     for (c = 0; c < coneSizeA; ++c) {
2413       if (!PetscBTLookup(seenFaces, coneA[c]-fStart)) {
2414         faceFIFO[fBottom++] = coneA[c];
2415         ierr = PetscBTSet(seenFaces, coneA[c]-fStart);CHKERRQ(ierr);
2416       }
2417       if (coneA[c] == face) posA = c;
2418       if (fBottom > fEnd-fStart) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Face %d was pushed exceeding capacity %d > %d", coneA[c], fBottom, fEnd-fStart);
2419     }
2420     if (posA < 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %d could not be located in cell %d", face, support[0]);
2421     for (c = 0; c < coneSizeB; ++c) {
2422       if (!PetscBTLookup(seenFaces, coneB[c]-fStart)) {
2423         faceFIFO[fBottom++] = coneB[c];
2424         ierr = PetscBTSet(seenFaces, coneB[c]-fStart);CHKERRQ(ierr);
2425       }
2426       if (coneB[c] == face) posB = c;
2427       if (fBottom > fEnd-fStart) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Face %d was pushed exceeding capacity %d > %d", coneA[c], fBottom, fEnd-fStart);
2428     }
2429     if (posB < 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %d could not be located in cell %d", face, support[1]);
2430 
2431     if (dim == 1) {
2432       mismatch = posA == posB;
2433     } else {
2434       mismatch = coneOA[posA] == coneOB[posB];
2435     }
2436 
2437     if (mismatch ^ (flippedA ^ flippedB)) {
2438       if (seenA && seenB) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Previously seen cells %d and %d do not match: Fault mesh is non-orientable", support[0], support[1]);
2439       if (!seenA && !flippedA) {
2440         ierr = PetscBTSet(flippedCells, support[0]-cStart);CHKERRQ(ierr);
2441       } else if (!seenB && !flippedB) {
2442         ierr = PetscBTSet(flippedCells, support[1]-cStart);CHKERRQ(ierr);
2443       } else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Inconsistent mesh orientation: Fault mesh is non-orientable");
2444     } else if (mismatch && flippedA && flippedB) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Attempt to flip already flipped cell: Fault mesh is non-orientable");
2445     ierr = PetscBTSet(seenCells, support[0]-cStart);CHKERRQ(ierr);
2446     ierr = PetscBTSet(seenCells, support[1]-cStart);CHKERRQ(ierr);
2447   }
2448   /* Now all subdomains are oriented, but we need a consistent parallel orientation */
2449   {
2450     /* Find a representative face (edge) separating pairs of procs */
2451     PetscSF            sf;
2452     const PetscInt    *lpoints;
2453     const PetscSFNode *rpoints;
2454     PetscInt          *neighbors, *nranks;
2455     PetscInt           numLeaves, numRoots, numNeighbors = 0, l, n;
2456 
2457     ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr);
2458     ierr = PetscSFGetGraph(sf, &numRoots, &numLeaves, &lpoints, &rpoints);CHKERRQ(ierr);
2459     if (numLeaves >= 0) {
2460       const PetscInt *cone, *ornt, *support;
2461       PetscInt        coneSize, supportSize;
2462       int            *rornt, *lornt; /* PetscSF cannot handle smaller than int */
2463       PetscBool      *match, flipped = PETSC_FALSE;
2464 
2465       ierr = PetscMalloc1(numLeaves,&neighbors);CHKERRQ(ierr);
2466       /* I know this is p^2 time in general, but for bounded degree its alright */
2467       for (l = 0; l < numLeaves; ++l) {
2468         const PetscInt face = lpoints[l];
2469         if ((face >= fStart) && (face < fEnd)) {
2470           const PetscInt rank = rpoints[l].rank;
2471           for (n = 0; n < numNeighbors; ++n) if (rank == rpoints[neighbors[n]].rank) break;
2472           if (n >= numNeighbors) {
2473             PetscInt supportSize;
2474             ierr = DMPlexGetSupportSize(dm, face, &supportSize);CHKERRQ(ierr);
2475             if (supportSize != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Boundary faces should see one cell, not %d", supportSize);
2476             neighbors[numNeighbors++] = l;
2477           }
2478         }
2479       }
2480       ierr = PetscCalloc4(numNeighbors,&match,numNeighbors,&nranks,numRoots,&rornt,numRoots,&lornt);CHKERRQ(ierr);
2481       for (face = fStart; face < fEnd; ++face) {
2482         ierr = DMPlexGetSupportSize(dm, face, &supportSize);CHKERRQ(ierr);
2483         if (supportSize != 1) continue;
2484         ierr = DMPlexGetSupport(dm, face, &support);CHKERRQ(ierr);
2485 
2486         ierr = DMPlexGetCone(dm, support[0], &cone);CHKERRQ(ierr);
2487         ierr = DMPlexGetConeSize(dm, support[0], &coneSize);CHKERRQ(ierr);
2488         ierr = DMPlexGetConeOrientation(dm, support[0], &ornt);CHKERRQ(ierr);
2489         for (c = 0; c < coneSize; ++c) if (cone[c] == face) break;
2490         if (dim == 1) {
2491           /* Use cone position instead, shifted to -1 or 1 */
2492           rornt[face] = c*2-1;
2493         } else {
2494           if (PetscBTLookup(flippedCells, support[0]-cStart)) rornt[face] = ornt[c] < 0 ? -1 :  1;
2495           else                                                rornt[face] = ornt[c] < 0 ?  1 : -1;
2496         }
2497       }
2498       /* Mark each edge with match or nomatch */
2499       ierr = PetscSFBcastBegin(sf, MPI_INT, rornt, lornt);CHKERRQ(ierr);
2500       ierr = PetscSFBcastEnd(sf, MPI_INT, rornt, lornt);CHKERRQ(ierr);
2501       for (n = 0; n < numNeighbors; ++n) {
2502         const PetscInt face = lpoints[neighbors[n]];
2503 
2504         if (rornt[face]*lornt[face] < 0) match[n] = PETSC_TRUE;
2505         else                             match[n] = PETSC_FALSE;
2506         nranks[n] = rpoints[neighbors[n]].rank;
2507       }
2508       /* Collect the graph on 0 */
2509       {
2510         MPI_Comm     comm = PetscObjectComm((PetscObject) sf);
2511         Mat          G;
2512         PetscBT      seenProcs, flippedProcs;
2513         PetscInt    *procFIFO, pTop, pBottom;
2514         PetscInt    *adj = NULL;
2515         PetscBool   *val = NULL;
2516         PetscMPIInt *recvcounts = NULL, *displs = NULL, p;
2517         PetscMPIInt  N = numNeighbors, numProcs = 0, rank;
2518         PetscInt     debug = 0;
2519 
2520         ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
2521         if (!rank) {ierr = MPI_Comm_size(comm, &numProcs);CHKERRQ(ierr);}
2522         ierr = PetscCalloc2(numProcs,&recvcounts,numProcs+1,&displs);CHKERRQ(ierr);
2523         ierr = MPI_Gather(&N, 1, MPI_INT, recvcounts, 1, MPI_INT, 0, comm);CHKERRQ(ierr);
2524         for (p = 0; p < numProcs; ++p) {
2525           displs[p+1] = displs[p] + recvcounts[p];
2526         }
2527         if (!rank) {ierr = PetscMalloc2(displs[numProcs],&adj,displs[numProcs],&val);CHKERRQ(ierr);}
2528         ierr = MPI_Gatherv(nranks, numNeighbors, MPIU_INT, adj, recvcounts, displs, MPIU_INT, 0, comm);CHKERRQ(ierr);
2529         ierr = MPI_Gatherv(match, numNeighbors, MPIU_BOOL, val, recvcounts, displs, MPIU_BOOL, 0, comm);CHKERRQ(ierr);
2530         if (debug) {
2531           for (p = 0; p < numProcs; ++p) {
2532             ierr = PetscPrintf(comm, "Proc %d:\n", p);
2533             for (n = 0; n < recvcounts[p]; ++n) {
2534               ierr = PetscPrintf(comm, "  edge %d (%d):\n", adj[displs[p]+n], val[displs[p]+n]);
2535             }
2536           }
2537         }
2538         /* Symmetrize the graph */
2539         ierr = MatCreate(PETSC_COMM_SELF, &G);CHKERRQ(ierr);
2540         ierr = MatSetSizes(G, numProcs, numProcs, numProcs, numProcs);CHKERRQ(ierr);
2541         ierr = MatSetUp(G);CHKERRQ(ierr);
2542         for (p = 0; p < numProcs; ++p) {
2543           for (n = 0; n < recvcounts[p]; ++n) {
2544             const PetscInt    r = p;
2545             const PetscInt    q = adj[displs[p]+n];
2546             const PetscScalar o = val[displs[p]+n] ? 1.0 : 0.0;
2547 
2548             ierr = MatSetValues(G, 1, &r, 1, &q, &o, INSERT_VALUES);CHKERRQ(ierr);
2549             ierr = MatSetValues(G, 1, &q, 1, &r, &o, INSERT_VALUES);CHKERRQ(ierr);
2550           }
2551         }
2552         ierr = MatAssemblyBegin(G, MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
2553         ierr = MatAssemblyEnd(G, MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
2554 
2555         ierr = PetscBTCreate(numProcs, &seenProcs);CHKERRQ(ierr);
2556         ierr = PetscBTMemzero(numProcs, seenProcs);CHKERRQ(ierr);
2557         ierr = PetscBTCreate(numProcs, &flippedProcs);CHKERRQ(ierr);
2558         ierr = PetscBTMemzero(numProcs, flippedProcs);CHKERRQ(ierr);
2559         ierr = PetscMalloc1(numProcs,&procFIFO);CHKERRQ(ierr);
2560         pTop = pBottom = 0;
2561         for (p = 0; p < numProcs; ++p) {
2562           if (PetscBTLookup(seenProcs, p)) continue;
2563           /* Initialize FIFO with next proc */
2564           procFIFO[pBottom++] = p;
2565           ierr = PetscBTSet(seenProcs, p);CHKERRQ(ierr);
2566           /* Consider each proc in FIFO */
2567           while (pTop < pBottom) {
2568             const PetscScalar *ornt;
2569             const PetscInt    *neighbors;
2570             PetscInt           proc, nproc, seen, flippedA, flippedB, mismatch, numNeighbors;
2571 
2572             proc     = procFIFO[pTop++];
2573             flippedA = PetscBTLookup(flippedProcs, proc) ? 1 : 0;
2574             ierr = MatGetRow(G, proc, &numNeighbors, &neighbors, &ornt);CHKERRQ(ierr);
2575             /* Loop over neighboring procs */
2576             for (n = 0; n < numNeighbors; ++n) {
2577               nproc    = neighbors[n];
2578               mismatch = PetscRealPart(ornt[n]) > 0.5 ? 0 : 1;
2579               seen     = PetscBTLookup(seenProcs, nproc);
2580               flippedB = PetscBTLookup(flippedProcs, nproc) ? 1 : 0;
2581 
2582               if (mismatch ^ (flippedA ^ flippedB)) {
2583                 if (seen) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Previously seen procs %d and %d do not match: Fault mesh is non-orientable", proc, nproc);
2584                 if (!flippedB) {
2585                   ierr = PetscBTSet(flippedProcs, nproc);CHKERRQ(ierr);
2586               } else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Inconsistent mesh orientation: Fault mesh is non-orientable");
2587               } else if (mismatch && flippedA && flippedB) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Attempt to flip already flipped cell: Fault mesh is non-orientable");
2588               if (!seen) {
2589                 procFIFO[pBottom++] = nproc;
2590                 ierr = PetscBTSet(seenProcs, nproc);CHKERRQ(ierr);
2591               }
2592             }
2593           }
2594         }
2595         ierr = PetscFree(procFIFO);CHKERRQ(ierr);
2596         ierr = MatDestroy(&G);CHKERRQ(ierr);
2597 
2598         ierr = PetscFree2(recvcounts,displs);CHKERRQ(ierr);
2599         ierr = PetscFree2(adj,val);CHKERRQ(ierr);
2600         {
2601           PetscBool *flips;
2602 
2603           ierr = PetscMalloc1(numProcs,&flips);CHKERRQ(ierr);
2604           for (p = 0; p < numProcs; ++p) {
2605             flips[p] = PetscBTLookup(flippedProcs, p) ? PETSC_TRUE : PETSC_FALSE;
2606             if (debug && flips[p]) {ierr = PetscPrintf(comm, "Flipping Proc %d:\n", p);}
2607           }
2608           ierr = MPI_Scatter(flips, 1, MPIU_BOOL, &flipped, 1, MPIU_BOOL, 0, comm);CHKERRQ(ierr);
2609           ierr = PetscFree(flips);CHKERRQ(ierr);
2610         }
2611         ierr = PetscBTDestroy(&seenProcs);CHKERRQ(ierr);
2612         ierr = PetscBTDestroy(&flippedProcs);CHKERRQ(ierr);
2613       }
2614       ierr = PetscFree4(match,nranks,rornt,lornt);CHKERRQ(ierr);
2615       ierr = PetscFree(neighbors);CHKERRQ(ierr);
2616       if (flipped) {for (c = cStart; c < cEnd; ++c) {ierr = PetscBTNegate(flippedCells, c-cStart);CHKERRQ(ierr);}}
2617     }
2618   }
2619   /* Reverse flipped cells in the mesh */
2620   ierr = DMPlexGetMaxSizes(dm, &maxConeSize, NULL);CHKERRQ(ierr);
2621   ierr = DMGetWorkArray(dm, maxConeSize, PETSC_INT, &revcone);CHKERRQ(ierr);
2622   ierr = DMGetWorkArray(dm, maxConeSize, PETSC_INT, &revconeO);CHKERRQ(ierr);
2623   for (c = cStart; c < cEnd; ++c) {
2624     const PetscInt *cone, *coneO, *support;
2625     PetscInt        coneSize, supportSize, faceSize, cp, sp;
2626 
2627     if (!PetscBTLookup(flippedCells, c-cStart)) continue;
2628     ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
2629     ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
2630     ierr = DMPlexGetConeOrientation(dm, c, &coneO);CHKERRQ(ierr);
2631     for (cp = 0; cp < coneSize; ++cp) {
2632       const PetscInt rcp = coneSize-cp-1;
2633 
2634       ierr = DMPlexGetConeSize(dm, cone[rcp], &faceSize);CHKERRQ(ierr);
2635       revcone[cp]  = cone[rcp];
2636       revconeO[cp] = coneO[rcp] >= 0 ? -(faceSize-coneO[rcp]) : faceSize+coneO[rcp];
2637     }
2638     ierr = DMPlexSetCone(dm, c, revcone);CHKERRQ(ierr);
2639     ierr = DMPlexSetConeOrientation(dm, c, revconeO);CHKERRQ(ierr);
2640     /* Reverse orientations of support */
2641     faceSize = coneSize;
2642     ierr = DMPlexGetSupportSize(dm, c, &supportSize);CHKERRQ(ierr);
2643     ierr = DMPlexGetSupport(dm, c, &support);CHKERRQ(ierr);
2644     for (sp = 0; sp < supportSize; ++sp) {
2645       ierr = DMPlexGetConeSize(dm, support[sp], &coneSize);CHKERRQ(ierr);
2646       ierr = DMPlexGetCone(dm, support[sp], &cone);CHKERRQ(ierr);
2647       ierr = DMPlexGetConeOrientation(dm, support[sp], &coneO);CHKERRQ(ierr);
2648       for (cp = 0; cp < coneSize; ++cp) {
2649         if (cone[cp] != c) continue;
2650         ierr = DMPlexInsertConeOrientation(dm, support[sp], cp, coneO[cp] >= 0 ? -(faceSize-coneO[cp]) : faceSize+coneO[cp]);CHKERRQ(ierr);
2651       }
2652     }
2653   }
2654   ierr = DMRestoreWorkArray(dm, maxConeSize, PETSC_INT, &revcone);CHKERRQ(ierr);
2655   ierr = DMRestoreWorkArray(dm, maxConeSize, PETSC_INT, &revconeO);CHKERRQ(ierr);
2656   ierr = PetscBTDestroy(&seenCells);CHKERRQ(ierr);
2657   ierr = PetscBTDestroy(&flippedCells);CHKERRQ(ierr);
2658   ierr = PetscBTDestroy(&seenFaces);CHKERRQ(ierr);
2659   ierr = PetscFree(faceFIFO);CHKERRQ(ierr);
2660   PetscFunctionReturn(0);
2661 }
2662 
2663 #undef __FUNCT__
2664 #define __FUNCT__ "DMPlexInvertCell_Internal"
2665 PetscErrorCode DMPlexInvertCell_Internal(PetscInt dim, PetscInt numCorners, PetscInt cone[])
2666 {
2667   int tmpc;
2668 
2669   PetscFunctionBegin;
2670   if (dim != 3) PetscFunctionReturn(0);
2671   switch (numCorners) {
2672   case 4:
2673     tmpc    = cone[0];
2674     cone[0] = cone[1];
2675     cone[1] = tmpc;
2676     break;
2677   case 8:
2678     tmpc    = cone[1];
2679     cone[1] = cone[3];
2680     cone[3] = tmpc;
2681     break;
2682   default: break;
2683   }
2684   PetscFunctionReturn(0);
2685 }
2686 
2687 #undef __FUNCT__
2688 #define __FUNCT__ "DMPlexInvertCell"
2689 /*@C
2690   DMPlexInvertCell - This flips tetrahedron and hexahedron orientation since Plex stores them internally with outward normals. Other cells are left untouched.
2691 
2692   Input Parameters:
2693 + numCorners - The number of vertices in a cell
2694 - cone - The incoming cone
2695 
2696   Output Parameter:
2697 . cone - The inverted cone (in-place)
2698 
2699   Level: developer
2700 
2701 .seealso: DMPlexGenerate()
2702 @*/
2703 PetscErrorCode DMPlexInvertCell(PetscInt dim, PetscInt numCorners, int cone[])
2704 {
2705   int tmpc;
2706 
2707   PetscFunctionBegin;
2708   if (dim != 3) PetscFunctionReturn(0);
2709   switch (numCorners) {
2710   case 4:
2711     tmpc    = cone[0];
2712     cone[0] = cone[1];
2713     cone[1] = tmpc;
2714     break;
2715   case 8:
2716     tmpc    = cone[1];
2717     cone[1] = cone[3];
2718     cone[3] = tmpc;
2719     break;
2720   default: break;
2721   }
2722   PetscFunctionReturn(0);
2723 }
2724 
2725 #undef __FUNCT__
2726 #define __FUNCT__ "DMPlexInvertCells_Internal"
2727 /* This is to fix the tetrahedron orientation from TetGen */
2728 PETSC_UNUSED static PetscErrorCode DMPlexInvertCells_Internal(PetscInt dim, PetscInt numCells, PetscInt numCorners, int cells[])
2729 {
2730   PetscInt       bound = numCells*numCorners, coff;
2731   PetscErrorCode ierr;
2732 
2733   PetscFunctionBegin;
2734   for (coff = 0; coff < bound; coff += numCorners) {
2735     ierr = DMPlexInvertCell(dim, numCorners, &cells[coff]);CHKERRQ(ierr);
2736   }
2737   PetscFunctionReturn(0);
2738 }
2739 
2740 #if defined(PETSC_HAVE_TRIANGLE)
2741 #include <triangle.h>
2742 
2743 #undef __FUNCT__
2744 #define __FUNCT__ "InitInput_Triangle"
2745 PetscErrorCode InitInput_Triangle(struct triangulateio *inputCtx)
2746 {
2747   PetscFunctionBegin;
2748   inputCtx->numberofpoints             = 0;
2749   inputCtx->numberofpointattributes    = 0;
2750   inputCtx->pointlist                  = NULL;
2751   inputCtx->pointattributelist         = NULL;
2752   inputCtx->pointmarkerlist            = NULL;
2753   inputCtx->numberofsegments           = 0;
2754   inputCtx->segmentlist                = NULL;
2755   inputCtx->segmentmarkerlist          = NULL;
2756   inputCtx->numberoftriangleattributes = 0;
2757   inputCtx->trianglelist               = NULL;
2758   inputCtx->numberofholes              = 0;
2759   inputCtx->holelist                   = NULL;
2760   inputCtx->numberofregions            = 0;
2761   inputCtx->regionlist                 = NULL;
2762   PetscFunctionReturn(0);
2763 }
2764 
2765 #undef __FUNCT__
2766 #define __FUNCT__ "InitOutput_Triangle"
2767 PetscErrorCode InitOutput_Triangle(struct triangulateio *outputCtx)
2768 {
2769   PetscFunctionBegin;
2770   outputCtx->numberofpoints        = 0;
2771   outputCtx->pointlist             = NULL;
2772   outputCtx->pointattributelist    = NULL;
2773   outputCtx->pointmarkerlist       = NULL;
2774   outputCtx->numberoftriangles     = 0;
2775   outputCtx->trianglelist          = NULL;
2776   outputCtx->triangleattributelist = NULL;
2777   outputCtx->neighborlist          = NULL;
2778   outputCtx->segmentlist           = NULL;
2779   outputCtx->segmentmarkerlist     = NULL;
2780   outputCtx->numberofedges         = 0;
2781   outputCtx->edgelist              = NULL;
2782   outputCtx->edgemarkerlist        = NULL;
2783   PetscFunctionReturn(0);
2784 }
2785 
2786 #undef __FUNCT__
2787 #define __FUNCT__ "FiniOutput_Triangle"
2788 PetscErrorCode FiniOutput_Triangle(struct triangulateio *outputCtx)
2789 {
2790   PetscFunctionBegin;
2791   free(outputCtx->pointlist);
2792   free(outputCtx->pointmarkerlist);
2793   free(outputCtx->segmentlist);
2794   free(outputCtx->segmentmarkerlist);
2795   free(outputCtx->edgelist);
2796   free(outputCtx->edgemarkerlist);
2797   free(outputCtx->trianglelist);
2798   free(outputCtx->neighborlist);
2799   PetscFunctionReturn(0);
2800 }
2801 
2802 #undef __FUNCT__
2803 #define __FUNCT__ "DMPlexGenerate_Triangle"
2804 PetscErrorCode DMPlexGenerate_Triangle(DM boundary, PetscBool interpolate, DM *dm)
2805 {
2806   MPI_Comm             comm;
2807   PetscInt             dim              = 2;
2808   const PetscBool      createConvexHull = PETSC_FALSE;
2809   const PetscBool      constrained      = PETSC_FALSE;
2810   struct triangulateio in;
2811   struct triangulateio out;
2812   PetscInt             vStart, vEnd, v, eStart, eEnd, e;
2813   PetscMPIInt          rank;
2814   PetscErrorCode       ierr;
2815 
2816   PetscFunctionBegin;
2817   ierr = PetscObjectGetComm((PetscObject)boundary,&comm);CHKERRQ(ierr);
2818   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
2819   ierr = InitInput_Triangle(&in);CHKERRQ(ierr);
2820   ierr = InitOutput_Triangle(&out);CHKERRQ(ierr);
2821   ierr = DMPlexGetDepthStratum(boundary, 0, &vStart, &vEnd);CHKERRQ(ierr);
2822 
2823   in.numberofpoints = vEnd - vStart;
2824   if (in.numberofpoints > 0) {
2825     PetscSection coordSection;
2826     Vec          coordinates;
2827     PetscScalar *array;
2828 
2829     ierr = PetscMalloc1(in.numberofpoints*dim, &in.pointlist);CHKERRQ(ierr);
2830     ierr = PetscMalloc1(in.numberofpoints, &in.pointmarkerlist);CHKERRQ(ierr);
2831     ierr = DMGetCoordinatesLocal(boundary, &coordinates);CHKERRQ(ierr);
2832     ierr = DMGetCoordinateSection(boundary, &coordSection);CHKERRQ(ierr);
2833     ierr = VecGetArray(coordinates, &array);CHKERRQ(ierr);
2834     for (v = vStart; v < vEnd; ++v) {
2835       const PetscInt idx = v - vStart;
2836       PetscInt       off, d;
2837 
2838       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
2839       for (d = 0; d < dim; ++d) {
2840         in.pointlist[idx*dim + d] = PetscRealPart(array[off+d]);
2841       }
2842       ierr = DMPlexGetLabelValue(boundary, "marker", v, &in.pointmarkerlist[idx]);CHKERRQ(ierr);
2843     }
2844     ierr = VecRestoreArray(coordinates, &array);CHKERRQ(ierr);
2845   }
2846   ierr  = DMPlexGetHeightStratum(boundary, 0, &eStart, &eEnd);CHKERRQ(ierr);
2847   in.numberofsegments = eEnd - eStart;
2848   if (in.numberofsegments > 0) {
2849     ierr = PetscMalloc1(in.numberofsegments*2, &in.segmentlist);CHKERRQ(ierr);
2850     ierr = PetscMalloc1(in.numberofsegments, &in.segmentmarkerlist);CHKERRQ(ierr);
2851     for (e = eStart; e < eEnd; ++e) {
2852       const PetscInt  idx = e - eStart;
2853       const PetscInt *cone;
2854 
2855       ierr = DMPlexGetCone(boundary, e, &cone);CHKERRQ(ierr);
2856 
2857       in.segmentlist[idx*2+0] = cone[0] - vStart;
2858       in.segmentlist[idx*2+1] = cone[1] - vStart;
2859 
2860       ierr = DMPlexGetLabelValue(boundary, "marker", e, &in.segmentmarkerlist[idx]);CHKERRQ(ierr);
2861     }
2862   }
2863 #if 0 /* Do not currently support holes */
2864   PetscReal *holeCoords;
2865   PetscInt   h, d;
2866 
2867   ierr = DMPlexGetHoles(boundary, &in.numberofholes, &holeCords);CHKERRQ(ierr);
2868   if (in.numberofholes > 0) {
2869     ierr = PetscMalloc1(in.numberofholes*dim, &in.holelist);CHKERRQ(ierr);
2870     for (h = 0; h < in.numberofholes; ++h) {
2871       for (d = 0; d < dim; ++d) {
2872         in.holelist[h*dim+d] = holeCoords[h*dim+d];
2873       }
2874     }
2875   }
2876 #endif
2877   if (!rank) {
2878     char args[32];
2879 
2880     /* Take away 'Q' for verbose output */
2881     ierr = PetscStrcpy(args, "pqezQ");CHKERRQ(ierr);
2882     if (createConvexHull) {
2883       ierr = PetscStrcat(args, "c");CHKERRQ(ierr);
2884     }
2885     if (constrained) {
2886       ierr = PetscStrcpy(args, "zepDQ");CHKERRQ(ierr);
2887     }
2888     triangulate(args, &in, &out, NULL);
2889   }
2890   ierr = PetscFree(in.pointlist);CHKERRQ(ierr);
2891   ierr = PetscFree(in.pointmarkerlist);CHKERRQ(ierr);
2892   ierr = PetscFree(in.segmentlist);CHKERRQ(ierr);
2893   ierr = PetscFree(in.segmentmarkerlist);CHKERRQ(ierr);
2894   ierr = PetscFree(in.holelist);CHKERRQ(ierr);
2895 
2896   {
2897     const PetscInt numCorners  = 3;
2898     const PetscInt numCells    = out.numberoftriangles;
2899     const PetscInt numVertices = out.numberofpoints;
2900     const int     *cells      = out.trianglelist;
2901     const double  *meshCoords = out.pointlist;
2902 
2903     ierr = DMPlexCreateFromCellList(comm, dim, numCells, numVertices, numCorners, interpolate, cells, dim, meshCoords, dm);CHKERRQ(ierr);
2904     /* Set labels */
2905     for (v = 0; v < numVertices; ++v) {
2906       if (out.pointmarkerlist[v]) {
2907         ierr = DMPlexSetLabelValue(*dm, "marker", v+numCells, out.pointmarkerlist[v]);CHKERRQ(ierr);
2908       }
2909     }
2910     if (interpolate) {
2911       for (e = 0; e < out.numberofedges; e++) {
2912         if (out.edgemarkerlist[e]) {
2913           const PetscInt  vertices[2] = {out.edgelist[e*2+0]+numCells, out.edgelist[e*2+1]+numCells};
2914           const PetscInt *edges;
2915           PetscInt        numEdges;
2916 
2917           ierr = DMPlexGetJoin(*dm, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
2918           if (numEdges != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Two vertices must cover only one edge, not %D", numEdges);
2919           ierr = DMPlexSetLabelValue(*dm, "marker", edges[0], out.edgemarkerlist[e]);CHKERRQ(ierr);
2920           ierr = DMPlexRestoreJoin(*dm, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
2921         }
2922       }
2923     }
2924     ierr = DMPlexSetRefinementUniform(*dm, PETSC_FALSE);CHKERRQ(ierr);
2925   }
2926 #if 0 /* Do not currently support holes */
2927   ierr = DMPlexCopyHoles(*dm, boundary);CHKERRQ(ierr);
2928 #endif
2929   ierr = FiniOutput_Triangle(&out);CHKERRQ(ierr);
2930   PetscFunctionReturn(0);
2931 }
2932 
2933 #undef __FUNCT__
2934 #define __FUNCT__ "DMPlexRefine_Triangle"
2935 PetscErrorCode DMPlexRefine_Triangle(DM dm, double *maxVolumes, DM *dmRefined)
2936 {
2937   MPI_Comm             comm;
2938   PetscInt             dim  = 2;
2939   struct triangulateio in;
2940   struct triangulateio out;
2941   PetscInt             vStart, vEnd, v, cStart, cEnd, c, depth, depthGlobal;
2942   PetscMPIInt          rank;
2943   PetscErrorCode       ierr;
2944 
2945   PetscFunctionBegin;
2946   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
2947   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
2948   ierr = InitInput_Triangle(&in);CHKERRQ(ierr);
2949   ierr = InitOutput_Triangle(&out);CHKERRQ(ierr);
2950   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
2951   ierr = MPI_Allreduce(&depth, &depthGlobal, 1, MPIU_INT, MPI_MAX, comm);CHKERRQ(ierr);
2952   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
2953 
2954   in.numberofpoints = vEnd - vStart;
2955   if (in.numberofpoints > 0) {
2956     PetscSection coordSection;
2957     Vec          coordinates;
2958     PetscScalar *array;
2959 
2960     ierr = PetscMalloc1(in.numberofpoints*dim, &in.pointlist);CHKERRQ(ierr);
2961     ierr = PetscMalloc1(in.numberofpoints, &in.pointmarkerlist);CHKERRQ(ierr);
2962     ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
2963     ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
2964     ierr = VecGetArray(coordinates, &array);CHKERRQ(ierr);
2965     for (v = vStart; v < vEnd; ++v) {
2966       const PetscInt idx = v - vStart;
2967       PetscInt       off, d;
2968 
2969       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
2970       for (d = 0; d < dim; ++d) {
2971         in.pointlist[idx*dim + d] = PetscRealPart(array[off+d]);
2972       }
2973       ierr = DMPlexGetLabelValue(dm, "marker", v, &in.pointmarkerlist[idx]);CHKERRQ(ierr);
2974     }
2975     ierr = VecRestoreArray(coordinates, &array);CHKERRQ(ierr);
2976   }
2977   ierr  = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
2978 
2979   in.numberofcorners   = 3;
2980   in.numberoftriangles = cEnd - cStart;
2981 
2982   in.trianglearealist  = (double*) maxVolumes;
2983   if (in.numberoftriangles > 0) {
2984     ierr = PetscMalloc1(in.numberoftriangles*in.numberofcorners, &in.trianglelist);CHKERRQ(ierr);
2985     for (c = cStart; c < cEnd; ++c) {
2986       const PetscInt idx      = c - cStart;
2987       PetscInt      *closure = NULL;
2988       PetscInt       closureSize;
2989 
2990       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
2991       if ((closureSize != 4) && (closureSize != 7)) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Mesh has cell which is not a triangle, %D vertices in closure", closureSize);
2992       for (v = 0; v < 3; ++v) {
2993         in.trianglelist[idx*in.numberofcorners + v] = closure[(v+closureSize-3)*2] - vStart;
2994       }
2995       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
2996     }
2997   }
2998   /* TODO: Segment markers are missing on input */
2999 #if 0 /* Do not currently support holes */
3000   PetscReal *holeCoords;
3001   PetscInt   h, d;
3002 
3003   ierr = DMPlexGetHoles(boundary, &in.numberofholes, &holeCords);CHKERRQ(ierr);
3004   if (in.numberofholes > 0) {
3005     ierr = PetscMalloc1(in.numberofholes*dim, &in.holelist);CHKERRQ(ierr);
3006     for (h = 0; h < in.numberofholes; ++h) {
3007       for (d = 0; d < dim; ++d) {
3008         in.holelist[h*dim+d] = holeCoords[h*dim+d];
3009       }
3010     }
3011   }
3012 #endif
3013   if (!rank) {
3014     char args[32];
3015 
3016     /* Take away 'Q' for verbose output */
3017     ierr = PetscStrcpy(args, "pqezQra");CHKERRQ(ierr);
3018     triangulate(args, &in, &out, NULL);
3019   }
3020   ierr = PetscFree(in.pointlist);CHKERRQ(ierr);
3021   ierr = PetscFree(in.pointmarkerlist);CHKERRQ(ierr);
3022   ierr = PetscFree(in.segmentlist);CHKERRQ(ierr);
3023   ierr = PetscFree(in.segmentmarkerlist);CHKERRQ(ierr);
3024   ierr = PetscFree(in.trianglelist);CHKERRQ(ierr);
3025 
3026   {
3027     const PetscInt numCorners  = 3;
3028     const PetscInt numCells    = out.numberoftriangles;
3029     const PetscInt numVertices = out.numberofpoints;
3030     const int     *cells      = out.trianglelist;
3031     const double  *meshCoords = out.pointlist;
3032     PetscBool      interpolate = depthGlobal > 1 ? PETSC_TRUE : PETSC_FALSE;
3033 
3034     ierr = DMPlexCreateFromCellList(comm, dim, numCells, numVertices, numCorners, interpolate, cells, dim, meshCoords, dmRefined);CHKERRQ(ierr);
3035     /* Set labels */
3036     for (v = 0; v < numVertices; ++v) {
3037       if (out.pointmarkerlist[v]) {
3038         ierr = DMPlexSetLabelValue(*dmRefined, "marker", v+numCells, out.pointmarkerlist[v]);CHKERRQ(ierr);
3039       }
3040     }
3041     if (interpolate) {
3042       PetscInt e;
3043 
3044       for (e = 0; e < out.numberofedges; e++) {
3045         if (out.edgemarkerlist[e]) {
3046           const PetscInt  vertices[2] = {out.edgelist[e*2+0]+numCells, out.edgelist[e*2+1]+numCells};
3047           const PetscInt *edges;
3048           PetscInt        numEdges;
3049 
3050           ierr = DMPlexGetJoin(*dmRefined, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3051           if (numEdges != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Two vertices must cover only one edge, not %D", numEdges);
3052           ierr = DMPlexSetLabelValue(*dmRefined, "marker", edges[0], out.edgemarkerlist[e]);CHKERRQ(ierr);
3053           ierr = DMPlexRestoreJoin(*dmRefined, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3054         }
3055       }
3056     }
3057     ierr = DMPlexSetRefinementUniform(*dmRefined, PETSC_FALSE);CHKERRQ(ierr);
3058   }
3059 #if 0 /* Do not currently support holes */
3060   ierr = DMPlexCopyHoles(*dm, boundary);CHKERRQ(ierr);
3061 #endif
3062   ierr = FiniOutput_Triangle(&out);CHKERRQ(ierr);
3063   PetscFunctionReturn(0);
3064 }
3065 #endif
3066 
3067 #if defined(PETSC_HAVE_TETGEN)
3068 #include <tetgen.h>
3069 #undef __FUNCT__
3070 #define __FUNCT__ "DMPlexGenerate_Tetgen"
3071 PetscErrorCode DMPlexGenerate_Tetgen(DM boundary, PetscBool interpolate, DM *dm)
3072 {
3073   MPI_Comm       comm;
3074   const PetscInt dim  = 3;
3075   ::tetgenio     in;
3076   ::tetgenio     out;
3077   PetscInt       vStart, vEnd, v, fStart, fEnd, f;
3078   PetscMPIInt    rank;
3079   PetscErrorCode ierr;
3080 
3081   PetscFunctionBegin;
3082   ierr              = PetscObjectGetComm((PetscObject)boundary,&comm);CHKERRQ(ierr);
3083   ierr              = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
3084   ierr              = DMPlexGetDepthStratum(boundary, 0, &vStart, &vEnd);CHKERRQ(ierr);
3085   in.numberofpoints = vEnd - vStart;
3086   if (in.numberofpoints > 0) {
3087     PetscSection coordSection;
3088     Vec          coordinates;
3089     PetscScalar *array;
3090 
3091     in.pointlist       = new double[in.numberofpoints*dim];
3092     in.pointmarkerlist = new int[in.numberofpoints];
3093 
3094     ierr = DMGetCoordinatesLocal(boundary, &coordinates);CHKERRQ(ierr);
3095     ierr = DMGetCoordinateSection(boundary, &coordSection);CHKERRQ(ierr);
3096     ierr = VecGetArray(coordinates, &array);CHKERRQ(ierr);
3097     for (v = vStart; v < vEnd; ++v) {
3098       const PetscInt idx = v - vStart;
3099       PetscInt       off, d;
3100 
3101       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
3102       for (d = 0; d < dim; ++d) in.pointlist[idx*dim + d] = array[off+d];
3103       ierr = DMPlexGetLabelValue(boundary, "marker", v, &in.pointmarkerlist[idx]);CHKERRQ(ierr);
3104     }
3105     ierr = VecRestoreArray(coordinates, &array);CHKERRQ(ierr);
3106   }
3107   ierr  = DMPlexGetHeightStratum(boundary, 0, &fStart, &fEnd);CHKERRQ(ierr);
3108 
3109   in.numberoffacets = fEnd - fStart;
3110   if (in.numberoffacets > 0) {
3111     in.facetlist       = new tetgenio::facet[in.numberoffacets];
3112     in.facetmarkerlist = new int[in.numberoffacets];
3113     for (f = fStart; f < fEnd; ++f) {
3114       const PetscInt idx     = f - fStart;
3115       PetscInt      *points = NULL, numPoints, p, numVertices = 0, v;
3116 
3117       in.facetlist[idx].numberofpolygons = 1;
3118       in.facetlist[idx].polygonlist      = new tetgenio::polygon[in.facetlist[idx].numberofpolygons];
3119       in.facetlist[idx].numberofholes    = 0;
3120       in.facetlist[idx].holelist         = NULL;
3121 
3122       ierr = DMPlexGetTransitiveClosure(boundary, f, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
3123       for (p = 0; p < numPoints*2; p += 2) {
3124         const PetscInt point = points[p];
3125         if ((point >= vStart) && (point < vEnd)) points[numVertices++] = point;
3126       }
3127 
3128       tetgenio::polygon *poly = in.facetlist[idx].polygonlist;
3129       poly->numberofvertices = numVertices;
3130       poly->vertexlist       = new int[poly->numberofvertices];
3131       for (v = 0; v < numVertices; ++v) {
3132         const PetscInt vIdx = points[v] - vStart;
3133         poly->vertexlist[v] = vIdx;
3134       }
3135       ierr = DMPlexGetLabelValue(boundary, "marker", f, &in.facetmarkerlist[idx]);CHKERRQ(ierr);
3136       ierr = DMPlexRestoreTransitiveClosure(boundary, f, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
3137     }
3138   }
3139   if (!rank) {
3140     char args[32];
3141 
3142     /* Take away 'Q' for verbose output */
3143     ierr = PetscStrcpy(args, "pqezQ");CHKERRQ(ierr);
3144     ::tetrahedralize(args, &in, &out);
3145   }
3146   {
3147     const PetscInt numCorners  = 4;
3148     const PetscInt numCells    = out.numberoftetrahedra;
3149     const PetscInt numVertices = out.numberofpoints;
3150     const double   *meshCoords = out.pointlist;
3151     int            *cells      = out.tetrahedronlist;
3152 
3153     ierr = DMPlexInvertCells_Internal(dim, numCells, numCorners, cells);CHKERRQ(ierr);
3154     ierr = DMPlexCreateFromCellList(comm, dim, numCells, numVertices, numCorners, interpolate, cells, dim, meshCoords, dm);CHKERRQ(ierr);
3155     /* Set labels */
3156     for (v = 0; v < numVertices; ++v) {
3157       if (out.pointmarkerlist[v]) {
3158         ierr = DMPlexSetLabelValue(*dm, "marker", v+numCells, out.pointmarkerlist[v]);CHKERRQ(ierr);
3159       }
3160     }
3161     if (interpolate) {
3162       PetscInt e;
3163 
3164       for (e = 0; e < out.numberofedges; e++) {
3165         if (out.edgemarkerlist[e]) {
3166           const PetscInt  vertices[2] = {out.edgelist[e*2+0]+numCells, out.edgelist[e*2+1]+numCells};
3167           const PetscInt *edges;
3168           PetscInt        numEdges;
3169 
3170           ierr = DMPlexGetJoin(*dm, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3171           if (numEdges != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Two vertices must cover only one edge, not %D", numEdges);
3172           ierr = DMPlexSetLabelValue(*dm, "marker", edges[0], out.edgemarkerlist[e]);CHKERRQ(ierr);
3173           ierr = DMPlexRestoreJoin(*dm, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3174         }
3175       }
3176       for (f = 0; f < out.numberoftrifaces; f++) {
3177         if (out.trifacemarkerlist[f]) {
3178           const PetscInt  vertices[3] = {out.trifacelist[f*3+0]+numCells, out.trifacelist[f*3+1]+numCells, out.trifacelist[f*3+2]+numCells};
3179           const PetscInt *faces;
3180           PetscInt        numFaces;
3181 
3182           ierr = DMPlexGetJoin(*dm, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
3183           if (numFaces != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Three vertices must cover only one face, not %D", numFaces);
3184           ierr = DMPlexSetLabelValue(*dm, "marker", faces[0], out.trifacemarkerlist[f]);CHKERRQ(ierr);
3185           ierr = DMPlexRestoreJoin(*dm, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
3186         }
3187       }
3188     }
3189     ierr = DMPlexSetRefinementUniform(*dm, PETSC_FALSE);CHKERRQ(ierr);
3190   }
3191   PetscFunctionReturn(0);
3192 }
3193 
3194 #undef __FUNCT__
3195 #define __FUNCT__ "DMPlexRefine_Tetgen"
3196 PetscErrorCode DMPlexRefine_Tetgen(DM dm, double *maxVolumes, DM *dmRefined)
3197 {
3198   MPI_Comm       comm;
3199   const PetscInt dim  = 3;
3200   ::tetgenio     in;
3201   ::tetgenio     out;
3202   PetscInt       vStart, vEnd, v, cStart, cEnd, c, depth, depthGlobal;
3203   PetscMPIInt    rank;
3204   PetscErrorCode ierr;
3205 
3206   PetscFunctionBegin;
3207   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
3208   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
3209   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3210   ierr = MPI_Allreduce(&depth, &depthGlobal, 1, MPIU_INT, MPI_MAX, comm);CHKERRQ(ierr);
3211   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
3212 
3213   in.numberofpoints = vEnd - vStart;
3214   if (in.numberofpoints > 0) {
3215     PetscSection coordSection;
3216     Vec          coordinates;
3217     PetscScalar *array;
3218 
3219     in.pointlist       = new double[in.numberofpoints*dim];
3220     in.pointmarkerlist = new int[in.numberofpoints];
3221 
3222     ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
3223     ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
3224     ierr = VecGetArray(coordinates, &array);CHKERRQ(ierr);
3225     for (v = vStart; v < vEnd; ++v) {
3226       const PetscInt idx = v - vStart;
3227       PetscInt       off, d;
3228 
3229       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
3230       for (d = 0; d < dim; ++d) in.pointlist[idx*dim + d] = array[off+d];
3231       ierr = DMPlexGetLabelValue(dm, "marker", v, &in.pointmarkerlist[idx]);CHKERRQ(ierr);
3232     }
3233     ierr = VecRestoreArray(coordinates, &array);CHKERRQ(ierr);
3234   }
3235   ierr  = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
3236 
3237   in.numberofcorners       = 4;
3238   in.numberoftetrahedra    = cEnd - cStart;
3239   in.tetrahedronvolumelist = (double*) maxVolumes;
3240   if (in.numberoftetrahedra > 0) {
3241     in.tetrahedronlist = new int[in.numberoftetrahedra*in.numberofcorners];
3242     for (c = cStart; c < cEnd; ++c) {
3243       const PetscInt idx      = c - cStart;
3244       PetscInt      *closure = NULL;
3245       PetscInt       closureSize;
3246 
3247       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
3248       if ((closureSize != 5) && (closureSize != 15)) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Mesh has cell which is not a tetrahedron, %D vertices in closure", closureSize);
3249       for (v = 0; v < 4; ++v) {
3250         in.tetrahedronlist[idx*in.numberofcorners + v] = closure[(v+closureSize-4)*2] - vStart;
3251       }
3252       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
3253     }
3254   }
3255   /* TODO: Put in boundary faces with markers */
3256   if (!rank) {
3257     char args[32];
3258 
3259     /* Take away 'Q' for verbose output */
3260     /*ierr = PetscStrcpy(args, "qezQra");CHKERRQ(ierr); */
3261     ierr = PetscStrcpy(args, "qezraVVVV");CHKERRQ(ierr);
3262     ::tetrahedralize(args, &in, &out);
3263   }
3264   in.tetrahedronvolumelist = NULL;
3265 
3266   {
3267     const PetscInt numCorners  = 4;
3268     const PetscInt numCells    = out.numberoftetrahedra;
3269     const PetscInt numVertices = out.numberofpoints;
3270     const double   *meshCoords = out.pointlist;
3271     int            *cells      = out.tetrahedronlist;
3272 
3273     PetscBool      interpolate = depthGlobal > 1 ? PETSC_TRUE : PETSC_FALSE;
3274 
3275     ierr = DMPlexInvertCells_Internal(dim, numCells, numCorners, cells);CHKERRQ(ierr);
3276     ierr = DMPlexCreateFromCellList(comm, dim, numCells, numVertices, numCorners, interpolate, cells, dim, meshCoords, dmRefined);CHKERRQ(ierr);
3277     /* Set labels */
3278     for (v = 0; v < numVertices; ++v) {
3279       if (out.pointmarkerlist[v]) {
3280         ierr = DMPlexSetLabelValue(*dmRefined, "marker", v+numCells, out.pointmarkerlist[v]);CHKERRQ(ierr);
3281       }
3282     }
3283     if (interpolate) {
3284       PetscInt e, f;
3285 
3286       for (e = 0; e < out.numberofedges; e++) {
3287         if (out.edgemarkerlist[e]) {
3288           const PetscInt  vertices[2] = {out.edgelist[e*2+0]+numCells, out.edgelist[e*2+1]+numCells};
3289           const PetscInt *edges;
3290           PetscInt        numEdges;
3291 
3292           ierr = DMPlexGetJoin(*dmRefined, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3293           if (numEdges != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Two vertices must cover only one edge, not %D", numEdges);
3294           ierr = DMPlexSetLabelValue(*dmRefined, "marker", edges[0], out.edgemarkerlist[e]);CHKERRQ(ierr);
3295           ierr = DMPlexRestoreJoin(*dmRefined, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3296         }
3297       }
3298       for (f = 0; f < out.numberoftrifaces; f++) {
3299         if (out.trifacemarkerlist[f]) {
3300           const PetscInt  vertices[3] = {out.trifacelist[f*3+0]+numCells, out.trifacelist[f*3+1]+numCells, out.trifacelist[f*3+2]+numCells};
3301           const PetscInt *faces;
3302           PetscInt        numFaces;
3303 
3304           ierr = DMPlexGetJoin(*dmRefined, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
3305           if (numFaces != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Three vertices must cover only one face, not %D", numFaces);
3306           ierr = DMPlexSetLabelValue(*dmRefined, "marker", faces[0], out.trifacemarkerlist[f]);CHKERRQ(ierr);
3307           ierr = DMPlexRestoreJoin(*dmRefined, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
3308         }
3309       }
3310     }
3311     ierr = DMPlexSetRefinementUniform(*dmRefined, PETSC_FALSE);CHKERRQ(ierr);
3312   }
3313   PetscFunctionReturn(0);
3314 }
3315 #endif
3316 
3317 #if defined(PETSC_HAVE_CTETGEN)
3318 #include <ctetgen.h>
3319 
3320 #undef __FUNCT__
3321 #define __FUNCT__ "DMPlexGenerate_CTetgen"
3322 PetscErrorCode DMPlexGenerate_CTetgen(DM boundary, PetscBool interpolate, DM *dm)
3323 {
3324   MPI_Comm       comm;
3325   const PetscInt dim  = 3;
3326   PLC           *in, *out;
3327   PetscInt       verbose = 0, vStart, vEnd, v, fStart, fEnd, f;
3328   PetscMPIInt    rank;
3329   PetscErrorCode ierr;
3330 
3331   PetscFunctionBegin;
3332   ierr = PetscObjectGetComm((PetscObject)boundary,&comm);CHKERRQ(ierr);
3333   ierr = PetscOptionsGetInt(((PetscObject) boundary)->prefix, "-ctetgen_verbose", &verbose, NULL);CHKERRQ(ierr);
3334   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
3335   ierr = DMPlexGetDepthStratum(boundary, 0, &vStart, &vEnd);CHKERRQ(ierr);
3336   ierr = PLCCreate(&in);CHKERRQ(ierr);
3337   ierr = PLCCreate(&out);CHKERRQ(ierr);
3338 
3339   in->numberofpoints = vEnd - vStart;
3340   if (in->numberofpoints > 0) {
3341     PetscSection coordSection;
3342     Vec          coordinates;
3343     PetscScalar *array;
3344 
3345     ierr = PetscMalloc1(in->numberofpoints*dim, &in->pointlist);CHKERRQ(ierr);
3346     ierr = PetscMalloc1(in->numberofpoints,       &in->pointmarkerlist);CHKERRQ(ierr);
3347     ierr = DMGetCoordinatesLocal(boundary, &coordinates);CHKERRQ(ierr);
3348     ierr = DMGetCoordinateSection(boundary, &coordSection);CHKERRQ(ierr);
3349     ierr = VecGetArray(coordinates, &array);CHKERRQ(ierr);
3350     for (v = vStart; v < vEnd; ++v) {
3351       const PetscInt idx = v - vStart;
3352       PetscInt       off, d, m;
3353 
3354       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
3355       for (d = 0; d < dim; ++d) {
3356         in->pointlist[idx*dim + d] = PetscRealPart(array[off+d]);
3357       }
3358       ierr = DMPlexGetLabelValue(boundary, "marker", v, &m);CHKERRQ(ierr);
3359 
3360       in->pointmarkerlist[idx] = (int) m;
3361     }
3362     ierr = VecRestoreArray(coordinates, &array);CHKERRQ(ierr);
3363   }
3364   ierr  = DMPlexGetHeightStratum(boundary, 0, &fStart, &fEnd);CHKERRQ(ierr);
3365 
3366   in->numberoffacets = fEnd - fStart;
3367   if (in->numberoffacets > 0) {
3368     ierr = PetscMalloc1(in->numberoffacets, &in->facetlist);CHKERRQ(ierr);
3369     ierr = PetscMalloc1(in->numberoffacets,   &in->facetmarkerlist);CHKERRQ(ierr);
3370     for (f = fStart; f < fEnd; ++f) {
3371       const PetscInt idx     = f - fStart;
3372       PetscInt      *points = NULL, numPoints, p, numVertices = 0, v, m;
3373       polygon       *poly;
3374 
3375       in->facetlist[idx].numberofpolygons = 1;
3376 
3377       ierr = PetscMalloc1(in->facetlist[idx].numberofpolygons, &in->facetlist[idx].polygonlist);CHKERRQ(ierr);
3378 
3379       in->facetlist[idx].numberofholes    = 0;
3380       in->facetlist[idx].holelist         = NULL;
3381 
3382       ierr = DMPlexGetTransitiveClosure(boundary, f, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
3383       for (p = 0; p < numPoints*2; p += 2) {
3384         const PetscInt point = points[p];
3385         if ((point >= vStart) && (point < vEnd)) points[numVertices++] = point;
3386       }
3387 
3388       poly                   = in->facetlist[idx].polygonlist;
3389       poly->numberofvertices = numVertices;
3390       ierr                   = PetscMalloc1(poly->numberofvertices, &poly->vertexlist);CHKERRQ(ierr);
3391       for (v = 0; v < numVertices; ++v) {
3392         const PetscInt vIdx = points[v] - vStart;
3393         poly->vertexlist[v] = vIdx;
3394       }
3395       ierr                     = DMPlexGetLabelValue(boundary, "marker", f, &m);CHKERRQ(ierr);
3396       in->facetmarkerlist[idx] = (int) m;
3397       ierr                     = DMPlexRestoreTransitiveClosure(boundary, f, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
3398     }
3399   }
3400   if (!rank) {
3401     TetGenOpts t;
3402 
3403     ierr        = TetGenOptsInitialize(&t);CHKERRQ(ierr);
3404     t.in        = boundary; /* Should go away */
3405     t.plc       = 1;
3406     t.quality   = 1;
3407     t.edgesout  = 1;
3408     t.zeroindex = 1;
3409     t.quiet     = 1;
3410     t.verbose   = verbose;
3411     ierr        = TetGenCheckOpts(&t);CHKERRQ(ierr);
3412     ierr        = TetGenTetrahedralize(&t, in, out);CHKERRQ(ierr);
3413   }
3414   {
3415     const PetscInt numCorners  = 4;
3416     const PetscInt numCells    = out->numberoftetrahedra;
3417     const PetscInt numVertices = out->numberofpoints;
3418     const double   *meshCoords = out->pointlist;
3419     int            *cells      = out->tetrahedronlist;
3420 
3421     ierr = DMPlexInvertCells_Internal(dim, numCells, numCorners, cells);CHKERRQ(ierr);
3422     ierr = DMPlexCreateFromCellList(comm, dim, numCells, numVertices, numCorners, interpolate, cells, dim, meshCoords, dm);CHKERRQ(ierr);
3423     /* Set labels */
3424     for (v = 0; v < numVertices; ++v) {
3425       if (out->pointmarkerlist[v]) {
3426         ierr = DMPlexSetLabelValue(*dm, "marker", v+numCells, out->pointmarkerlist[v]);CHKERRQ(ierr);
3427       }
3428     }
3429     if (interpolate) {
3430       PetscInt e;
3431 
3432       for (e = 0; e < out->numberofedges; e++) {
3433         if (out->edgemarkerlist[e]) {
3434           const PetscInt  vertices[2] = {out->edgelist[e*2+0]+numCells, out->edgelist[e*2+1]+numCells};
3435           const PetscInt *edges;
3436           PetscInt        numEdges;
3437 
3438           ierr = DMPlexGetJoin(*dm, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3439           if (numEdges != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Two vertices must cover only one edge, not %D", numEdges);
3440           ierr = DMPlexSetLabelValue(*dm, "marker", edges[0], out->edgemarkerlist[e]);CHKERRQ(ierr);
3441           ierr = DMPlexRestoreJoin(*dm, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3442         }
3443       }
3444       for (f = 0; f < out->numberoftrifaces; f++) {
3445         if (out->trifacemarkerlist[f]) {
3446           const PetscInt  vertices[3] = {out->trifacelist[f*3+0]+numCells, out->trifacelist[f*3+1]+numCells, out->trifacelist[f*3+2]+numCells};
3447           const PetscInt *faces;
3448           PetscInt        numFaces;
3449 
3450           ierr = DMPlexGetFullJoin(*dm, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
3451           if (numFaces != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Three vertices must cover only one face, not %D", numFaces);
3452           ierr = DMPlexSetLabelValue(*dm, "marker", faces[0], out->trifacemarkerlist[f]);CHKERRQ(ierr);
3453           ierr = DMPlexRestoreJoin(*dm, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
3454         }
3455       }
3456     }
3457     ierr = DMPlexSetRefinementUniform(*dm, PETSC_FALSE);CHKERRQ(ierr);
3458   }
3459 
3460   ierr = PLCDestroy(&in);CHKERRQ(ierr);
3461   ierr = PLCDestroy(&out);CHKERRQ(ierr);
3462   PetscFunctionReturn(0);
3463 }
3464 
3465 #undef __FUNCT__
3466 #define __FUNCT__ "DMPlexRefine_CTetgen"
3467 PetscErrorCode DMPlexRefine_CTetgen(DM dm, PetscReal *maxVolumes, DM *dmRefined)
3468 {
3469   MPI_Comm       comm;
3470   const PetscInt dim  = 3;
3471   PLC           *in, *out;
3472   PetscInt       verbose = 0, vStart, vEnd, v, cStart, cEnd, c, depth, depthGlobal;
3473   PetscMPIInt    rank;
3474   PetscErrorCode ierr;
3475 
3476   PetscFunctionBegin;
3477   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
3478   ierr = PetscOptionsGetInt(((PetscObject) dm)->prefix, "-ctetgen_verbose", &verbose, NULL);CHKERRQ(ierr);
3479   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
3480   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3481   ierr = MPI_Allreduce(&depth, &depthGlobal, 1, MPIU_INT, MPI_MAX, comm);CHKERRQ(ierr);
3482   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
3483   ierr = PLCCreate(&in);CHKERRQ(ierr);
3484   ierr = PLCCreate(&out);CHKERRQ(ierr);
3485 
3486   in->numberofpoints = vEnd - vStart;
3487   if (in->numberofpoints > 0) {
3488     PetscSection coordSection;
3489     Vec          coordinates;
3490     PetscScalar *array;
3491 
3492     ierr = PetscMalloc1(in->numberofpoints*dim, &in->pointlist);CHKERRQ(ierr);
3493     ierr = PetscMalloc1(in->numberofpoints,       &in->pointmarkerlist);CHKERRQ(ierr);
3494     ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
3495     ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
3496     ierr = VecGetArray(coordinates, &array);CHKERRQ(ierr);
3497     for (v = vStart; v < vEnd; ++v) {
3498       const PetscInt idx = v - vStart;
3499       PetscInt       off, d, m;
3500 
3501       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
3502       for (d = 0; d < dim; ++d) {
3503         in->pointlist[idx*dim + d] = PetscRealPart(array[off+d]);
3504       }
3505       ierr = DMPlexGetLabelValue(dm, "marker", v, &m);CHKERRQ(ierr);
3506 
3507       in->pointmarkerlist[idx] = (int) m;
3508     }
3509     ierr = VecRestoreArray(coordinates, &array);CHKERRQ(ierr);
3510   }
3511   ierr  = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
3512 
3513   in->numberofcorners       = 4;
3514   in->numberoftetrahedra    = cEnd - cStart;
3515   in->tetrahedronvolumelist = maxVolumes;
3516   if (in->numberoftetrahedra > 0) {
3517     ierr = PetscMalloc1(in->numberoftetrahedra*in->numberofcorners, &in->tetrahedronlist);CHKERRQ(ierr);
3518     for (c = cStart; c < cEnd; ++c) {
3519       const PetscInt idx      = c - cStart;
3520       PetscInt      *closure = NULL;
3521       PetscInt       closureSize;
3522 
3523       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
3524       if ((closureSize != 5) && (closureSize != 15)) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Mesh has cell which is not a tetrahedron, %D vertices in closure", closureSize);
3525       for (v = 0; v < 4; ++v) {
3526         in->tetrahedronlist[idx*in->numberofcorners + v] = closure[(v+closureSize-4)*2] - vStart;
3527       }
3528       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
3529     }
3530   }
3531   if (!rank) {
3532     TetGenOpts t;
3533 
3534     ierr = TetGenOptsInitialize(&t);CHKERRQ(ierr);
3535 
3536     t.in        = dm; /* Should go away */
3537     t.refine    = 1;
3538     t.varvolume = 1;
3539     t.quality   = 1;
3540     t.edgesout  = 1;
3541     t.zeroindex = 1;
3542     t.quiet     = 1;
3543     t.verbose   = verbose; /* Change this */
3544 
3545     ierr = TetGenCheckOpts(&t);CHKERRQ(ierr);
3546     ierr = TetGenTetrahedralize(&t, in, out);CHKERRQ(ierr);
3547   }
3548   {
3549     const PetscInt numCorners  = 4;
3550     const PetscInt numCells    = out->numberoftetrahedra;
3551     const PetscInt numVertices = out->numberofpoints;
3552     const double   *meshCoords = out->pointlist;
3553     int            *cells      = out->tetrahedronlist;
3554     PetscBool      interpolate = depthGlobal > 1 ? PETSC_TRUE : PETSC_FALSE;
3555 
3556     ierr = DMPlexInvertCells_Internal(dim, numCells, numCorners, cells);CHKERRQ(ierr);
3557     ierr = DMPlexCreateFromCellList(comm, dim, numCells, numVertices, numCorners, interpolate, cells, dim, meshCoords, dmRefined);CHKERRQ(ierr);
3558     /* Set labels */
3559     for (v = 0; v < numVertices; ++v) {
3560       if (out->pointmarkerlist[v]) {
3561         ierr = DMPlexSetLabelValue(*dmRefined, "marker", v+numCells, out->pointmarkerlist[v]);CHKERRQ(ierr);
3562       }
3563     }
3564     if (interpolate) {
3565       PetscInt e, f;
3566 
3567       for (e = 0; e < out->numberofedges; e++) {
3568         if (out->edgemarkerlist[e]) {
3569           const PetscInt  vertices[2] = {out->edgelist[e*2+0]+numCells, out->edgelist[e*2+1]+numCells};
3570           const PetscInt *edges;
3571           PetscInt        numEdges;
3572 
3573           ierr = DMPlexGetJoin(*dmRefined, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3574           if (numEdges != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Two vertices must cover only one edge, not %D", numEdges);
3575           ierr = DMPlexSetLabelValue(*dmRefined, "marker", edges[0], out->edgemarkerlist[e]);CHKERRQ(ierr);
3576           ierr = DMPlexRestoreJoin(*dmRefined, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3577         }
3578       }
3579       for (f = 0; f < out->numberoftrifaces; f++) {
3580         if (out->trifacemarkerlist[f]) {
3581           const PetscInt  vertices[3] = {out->trifacelist[f*3+0]+numCells, out->trifacelist[f*3+1]+numCells, out->trifacelist[f*3+2]+numCells};
3582           const PetscInt *faces;
3583           PetscInt        numFaces;
3584 
3585           ierr = DMPlexGetFullJoin(*dmRefined, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
3586           if (numFaces != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Three vertices must cover only one face, not %D", numFaces);
3587           ierr = DMPlexSetLabelValue(*dmRefined, "marker", faces[0], out->trifacemarkerlist[f]);CHKERRQ(ierr);
3588           ierr = DMPlexRestoreJoin(*dmRefined, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
3589         }
3590       }
3591     }
3592     ierr = DMPlexSetRefinementUniform(*dmRefined, PETSC_FALSE);CHKERRQ(ierr);
3593   }
3594   ierr = PLCDestroy(&in);CHKERRQ(ierr);
3595   ierr = PLCDestroy(&out);CHKERRQ(ierr);
3596   PetscFunctionReturn(0);
3597 }
3598 #endif
3599 
3600 #undef __FUNCT__
3601 #define __FUNCT__ "DMPlexGenerate"
3602 /*@C
3603   DMPlexGenerate - Generates a mesh.
3604 
3605   Not Collective
3606 
3607   Input Parameters:
3608 + boundary - The DMPlex boundary object
3609 . name - The mesh generation package name
3610 - interpolate - Flag to create intermediate mesh elements
3611 
3612   Output Parameter:
3613 . mesh - The DMPlex object
3614 
3615   Level: intermediate
3616 
3617 .keywords: mesh, elements
3618 .seealso: DMPlexCreate(), DMRefine()
3619 @*/
3620 PetscErrorCode DMPlexGenerate(DM boundary, const char name[], PetscBool interpolate, DM *mesh)
3621 {
3622   PetscInt       dim;
3623   char           genname[1024];
3624   PetscBool      isTriangle = PETSC_FALSE, isTetgen = PETSC_FALSE, isCTetgen = PETSC_FALSE, flg;
3625   PetscErrorCode ierr;
3626 
3627   PetscFunctionBegin;
3628   PetscValidHeaderSpecific(boundary, DM_CLASSID, 1);
3629   PetscValidLogicalCollectiveBool(boundary, interpolate, 2);
3630   ierr = DMPlexGetDimension(boundary, &dim);CHKERRQ(ierr);
3631   ierr = PetscOptionsGetString(((PetscObject) boundary)->prefix, "-dm_plex_generator", genname, 1024, &flg);CHKERRQ(ierr);
3632   if (flg) name = genname;
3633   if (name) {
3634     ierr = PetscStrcmp(name, "triangle", &isTriangle);CHKERRQ(ierr);
3635     ierr = PetscStrcmp(name, "tetgen",   &isTetgen);CHKERRQ(ierr);
3636     ierr = PetscStrcmp(name, "ctetgen",  &isCTetgen);CHKERRQ(ierr);
3637   }
3638   switch (dim) {
3639   case 1:
3640     if (!name || isTriangle) {
3641 #if defined(PETSC_HAVE_TRIANGLE)
3642       ierr = DMPlexGenerate_Triangle(boundary, interpolate, mesh);CHKERRQ(ierr);
3643 #else
3644       SETERRQ(PetscObjectComm((PetscObject)boundary), PETSC_ERR_SUP, "Mesh generation needs external package support.\nPlease reconfigure with --download-triangle.");
3645 #endif
3646     } else SETERRQ1(PetscObjectComm((PetscObject)boundary), PETSC_ERR_SUP, "Unknown 2D mesh generation package %s", name);
3647     break;
3648   case 2:
3649     if (!name || isCTetgen) {
3650 #if defined(PETSC_HAVE_CTETGEN)
3651       ierr = DMPlexGenerate_CTetgen(boundary, interpolate, mesh);CHKERRQ(ierr);
3652 #else
3653       SETERRQ(PetscObjectComm((PetscObject)boundary), PETSC_ERR_SUP, "CTetgen needs external package support.\nPlease reconfigure with --download-ctetgen.");
3654 #endif
3655     } else if (isTetgen) {
3656 #if defined(PETSC_HAVE_TETGEN)
3657       ierr = DMPlexGenerate_Tetgen(boundary, interpolate, mesh);CHKERRQ(ierr);
3658 #else
3659       SETERRQ(PetscObjectComm((PetscObject)boundary), PETSC_ERR_SUP, "Tetgen needs external package support.\nPlease reconfigure with --with-c-language=cxx --download-tetgen.");
3660 #endif
3661     } else SETERRQ1(PetscObjectComm((PetscObject)boundary), PETSC_ERR_SUP, "Unknown 3D mesh generation package %s", name);
3662     break;
3663   default:
3664     SETERRQ1(PetscObjectComm((PetscObject)boundary), PETSC_ERR_SUP, "Mesh generation for a dimension %d boundary is not supported.", dim);
3665   }
3666   PetscFunctionReturn(0);
3667 }
3668 
3669 #undef __FUNCT__
3670 #define __FUNCT__ "DMRefine_Plex"
3671 PetscErrorCode DMRefine_Plex(DM dm, MPI_Comm comm, DM *dmRefined)
3672 {
3673   PetscReal      refinementLimit;
3674   PetscInt       dim, cStart, cEnd;
3675   char           genname[1024], *name = NULL;
3676   PetscBool      isUniform, isTriangle = PETSC_FALSE, isTetgen = PETSC_FALSE, isCTetgen = PETSC_FALSE, flg;
3677   PetscErrorCode ierr;
3678 
3679   PetscFunctionBegin;
3680   ierr = DMPlexGetRefinementUniform(dm, &isUniform);CHKERRQ(ierr);
3681   if (isUniform) {
3682     CellRefiner cellRefiner;
3683 
3684     ierr = DMPlexGetCellRefiner_Internal(dm, &cellRefiner);CHKERRQ(ierr);
3685     ierr = DMPlexRefineUniform_Internal(dm, cellRefiner, dmRefined);CHKERRQ(ierr);
3686     ierr = DMPlexCopyBoundary(dm, *dmRefined);CHKERRQ(ierr);
3687     PetscFunctionReturn(0);
3688   }
3689   ierr = DMPlexGetRefinementLimit(dm, &refinementLimit);CHKERRQ(ierr);
3690   if (refinementLimit == 0.0) PetscFunctionReturn(0);
3691   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
3692   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
3693   ierr = PetscOptionsGetString(((PetscObject) dm)->prefix, "-dm_plex_generator", genname, 1024, &flg);CHKERRQ(ierr);
3694   if (flg) name = genname;
3695   if (name) {
3696     ierr = PetscStrcmp(name, "triangle", &isTriangle);CHKERRQ(ierr);
3697     ierr = PetscStrcmp(name, "tetgen",   &isTetgen);CHKERRQ(ierr);
3698     ierr = PetscStrcmp(name, "ctetgen",  &isCTetgen);CHKERRQ(ierr);
3699   }
3700   switch (dim) {
3701   case 2:
3702     if (!name || isTriangle) {
3703 #if defined(PETSC_HAVE_TRIANGLE)
3704       double  *maxVolumes;
3705       PetscInt c;
3706 
3707       ierr = PetscMalloc1((cEnd - cStart), &maxVolumes);CHKERRQ(ierr);
3708       for (c = 0; c < cEnd-cStart; ++c) maxVolumes[c] = refinementLimit;
3709       ierr = DMPlexRefine_Triangle(dm, maxVolumes, dmRefined);CHKERRQ(ierr);
3710       ierr = PetscFree(maxVolumes);CHKERRQ(ierr);
3711 #else
3712       SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Mesh refinement needs external package support.\nPlease reconfigure with --download-triangle.");
3713 #endif
3714     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Unknown 2D mesh generation package %s", name);
3715     break;
3716   case 3:
3717     if (!name || isCTetgen) {
3718 #if defined(PETSC_HAVE_CTETGEN)
3719       PetscReal *maxVolumes;
3720       PetscInt   c;
3721 
3722       ierr = PetscMalloc1((cEnd - cStart), &maxVolumes);CHKERRQ(ierr);
3723       for (c = 0; c < cEnd-cStart; ++c) maxVolumes[c] = refinementLimit;
3724       ierr = DMPlexRefine_CTetgen(dm, maxVolumes, dmRefined);CHKERRQ(ierr);
3725 #else
3726       SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "CTetgen needs external package support.\nPlease reconfigure with --download-ctetgen.");
3727 #endif
3728     } else if (isTetgen) {
3729 #if defined(PETSC_HAVE_TETGEN)
3730       double  *maxVolumes;
3731       PetscInt c;
3732 
3733       ierr = PetscMalloc1((cEnd - cStart), &maxVolumes);CHKERRQ(ierr);
3734       for (c = 0; c < cEnd-cStart; ++c) maxVolumes[c] = refinementLimit;
3735       ierr = DMPlexRefine_Tetgen(dm, maxVolumes, dmRefined);CHKERRQ(ierr);
3736 #else
3737       SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Tetgen needs external package support.\nPlease reconfigure with --with-c-language=cxx --download-tetgen.");
3738 #endif
3739     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Unknown 3D mesh generation package %s", name);
3740     break;
3741   default:
3742     SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Mesh refinement in dimension %d is not supported.", dim);
3743   }
3744   ierr = DMPlexCopyBoundary(dm, *dmRefined);CHKERRQ(ierr);
3745   PetscFunctionReturn(0);
3746 }
3747 
3748 #undef __FUNCT__
3749 #define __FUNCT__ "DMRefineHierarchy_Plex"
3750 PetscErrorCode DMRefineHierarchy_Plex(DM dm, PetscInt nlevels, DM dmRefined[])
3751 {
3752   DM             cdm = dm;
3753   PetscInt       r;
3754   PetscBool      isUniform;
3755   PetscErrorCode ierr;
3756 
3757   PetscFunctionBegin;
3758   ierr = DMPlexGetRefinementUniform(dm, &isUniform);CHKERRQ(ierr);
3759   if (!isUniform) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Non-uniform refinement is incompatible with the hierarchy");
3760   for (r = 0; r < nlevels; ++r) {
3761     CellRefiner cellRefiner;
3762 
3763     ierr = DMPlexGetCellRefiner_Internal(cdm, &cellRefiner);CHKERRQ(ierr);
3764     ierr = DMPlexRefineUniform_Internal(cdm, cellRefiner, &dmRefined[r]);CHKERRQ(ierr);
3765     ierr = DMPlexSetCoarseDM(dmRefined[r], cdm);CHKERRQ(ierr);
3766     cdm  = dmRefined[r];
3767   }
3768   PetscFunctionReturn(0);
3769 }
3770 
3771 #undef __FUNCT__
3772 #define __FUNCT__ "DMCoarsen_Plex"
3773 PetscErrorCode DMCoarsen_Plex(DM dm, MPI_Comm comm, DM *dmCoarsened)
3774 {
3775   DM_Plex       *mesh = (DM_Plex*) dm->data;
3776   PetscErrorCode ierr;
3777 
3778   PetscFunctionBegin;
3779   ierr = PetscObjectReference((PetscObject) mesh->coarseMesh);CHKERRQ(ierr);
3780   *dmCoarsened = mesh->coarseMesh;
3781   PetscFunctionReturn(0);
3782 }
3783 
3784 #undef __FUNCT__
3785 #define __FUNCT__ "DMPlexLocalizeCoordinate_Internal"
3786 PetscErrorCode DMPlexLocalizeCoordinate_Internal(DM dm, PetscInt dim, const PetscScalar anchor[], const PetscScalar in[], PetscScalar out[])
3787 {
3788   PetscInt d;
3789 
3790   PetscFunctionBegin;
3791   if (!dm->maxCell) {
3792     for (d = 0; d < dim; ++d) out[d] = in[d];
3793   } else {
3794     for (d = 0; d < dim; ++d) {
3795       if (PetscAbsScalar(anchor[d] - in[d]) > dm->maxCell[d]) {
3796         out[d] = PetscRealPart(anchor[d]) > PetscRealPart(in[d]) ? dm->L[d] + in[d] : in[d] - dm->L[d];
3797       } else {
3798         out[d] = in[d];
3799       }
3800     }
3801   }
3802   PetscFunctionReturn(0);
3803 }
3804 
3805 #undef __FUNCT__
3806 #define __FUNCT__ "DMPlexLocalizeAddCoordinate_Internal"
3807 PetscErrorCode DMPlexLocalizeAddCoordinate_Internal(DM dm, PetscInt dim, const PetscScalar anchor[], const PetscScalar in[], PetscScalar out[])
3808 {
3809   PetscInt d;
3810 
3811   PetscFunctionBegin;
3812   if (!dm->maxCell) {
3813     for (d = 0; d < dim; ++d) out[d] += in[d];
3814   } else {
3815     for (d = 0; d < dim; ++d) {
3816       if (PetscAbsScalar(anchor[d] - in[d]) > dm->maxCell[d]) {
3817         out[d] += PetscRealPart(anchor[d]) > PetscRealPart(in[d]) ? dm->L[d] + in[d] : in[d] - dm->L[d];
3818       } else {
3819         out[d] += in[d];
3820       }
3821     }
3822   }
3823   PetscFunctionReturn(0);
3824 }
3825 
3826 #undef __FUNCT__
3827 #define __FUNCT__ "DMPlexLocalizeCoordinates"
3828 PetscErrorCode DMPlexLocalizeCoordinates(DM dm)
3829 {
3830   PetscSection   coordSection, cSection;
3831   Vec            coordinates,  cVec;
3832   PetscScalar   *coords, *coords2, *anchor;
3833   PetscInt       cStart, cEnd, c, vStart, vEnd, v, dof, d, off, off2, bs, coordSize;
3834   PetscErrorCode ierr;
3835 
3836   PetscFunctionBegin;
3837   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3838   if (!dm->maxCell) PetscFunctionReturn(0);
3839   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
3840   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
3841   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
3842   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
3843   ierr = PetscSectionCreate(PetscObjectComm((PetscObject) dm), &cSection);CHKERRQ(ierr);
3844   ierr = PetscSectionSetChart(cSection, cStart, vEnd);CHKERRQ(ierr);
3845   for (v = vStart; v < vEnd; ++v) {
3846     ierr = PetscSectionGetDof(coordSection, v, &dof);CHKERRQ(ierr);
3847     ierr = PetscSectionSetDof(cSection,     v,  dof);CHKERRQ(ierr);
3848   }
3849   for (c = cStart; c < cEnd; ++c) {
3850     ierr = DMPlexVecGetClosure(dm, coordSection, coordinates, c, &dof, NULL);CHKERRQ(ierr);
3851     ierr = PetscSectionSetDof(cSection, c, dof);CHKERRQ(ierr);
3852   }
3853   ierr = PetscSectionSetUp(cSection);CHKERRQ(ierr);
3854   ierr = PetscSectionGetStorageSize(cSection, &coordSize);CHKERRQ(ierr);
3855   ierr = VecCreate(PetscObjectComm((PetscObject) dm), &cVec);CHKERRQ(ierr);
3856   ierr = VecGetBlockSize(coordinates, &bs);CHKERRQ(ierr);
3857   ierr = VecSetBlockSize(cVec,         bs);CHKERRQ(ierr);
3858   ierr = VecSetSizes(cVec, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
3859   ierr = VecSetType(cVec,VECSTANDARD);CHKERRQ(ierr);
3860   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
3861   ierr = VecGetArray(cVec,        &coords2);CHKERRQ(ierr);
3862   for (v = vStart; v < vEnd; ++v) {
3863     ierr = PetscSectionGetDof(coordSection, v, &dof);CHKERRQ(ierr);
3864     ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
3865     ierr = PetscSectionGetOffset(cSection,     v, &off2);CHKERRQ(ierr);
3866     for (d = 0; d < dof; ++d) coords2[off2+d] = coords[off+d];
3867   }
3868   ierr = DMGetWorkArray(dm, 3, PETSC_SCALAR, &anchor);CHKERRQ(ierr);
3869   for (c = cStart; c < cEnd; ++c) {
3870     PetscScalar *cellCoords = NULL;
3871     PetscInt     b;
3872 
3873     ierr = DMPlexVecGetClosure(dm, coordSection, coordinates, c, &dof, &cellCoords);CHKERRQ(ierr);
3874     ierr = PetscSectionGetOffset(cSection, c, &off2);CHKERRQ(ierr);
3875     for (b = 0; b < bs; ++b) anchor[b] = cellCoords[b];
3876     for (d = 0; d < dof/bs; ++d) {ierr = DMPlexLocalizeCoordinate_Internal(dm, bs, anchor, &cellCoords[d*bs], &coords2[off2+d*bs]);CHKERRQ(ierr);}
3877     ierr = DMPlexVecRestoreClosure(dm, coordSection, coordinates, c, &dof, &cellCoords);CHKERRQ(ierr);
3878   }
3879   ierr = DMRestoreWorkArray(dm, 3, PETSC_SCALAR, &anchor);CHKERRQ(ierr);
3880   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
3881   ierr = VecRestoreArray(cVec,        &coords2);CHKERRQ(ierr);
3882   ierr = DMSetCoordinateSection(dm, cSection);CHKERRQ(ierr);
3883   ierr = DMSetCoordinatesLocal(dm, cVec);CHKERRQ(ierr);
3884   ierr = VecDestroy(&cVec);CHKERRQ(ierr);
3885   ierr = PetscSectionDestroy(&cSection);CHKERRQ(ierr);
3886   PetscFunctionReturn(0);
3887 }
3888 
3889 #undef __FUNCT__
3890 #define __FUNCT__ "DMPlexGetDepthLabel"
3891 /*@
3892   DMPlexGetDepthLabel - Get the DMLabel recording the depth of each point
3893 
3894   Not Collective
3895 
3896   Input Parameter:
3897 . dm    - The DMPlex object
3898 
3899   Output Parameter:
3900 . depthLabel - The DMLabel recording point depth
3901 
3902   Level: developer
3903 
3904 .keywords: mesh, points
3905 .seealso: DMPlexGetDepth(), DMPlexGetHeightStratum(), DMPlexGetDepthStratum()
3906 @*/
3907 PetscErrorCode DMPlexGetDepthLabel(DM dm, DMLabel *depthLabel)
3908 {
3909   DM_Plex       *mesh = (DM_Plex*) dm->data;
3910   PetscErrorCode ierr;
3911 
3912   PetscFunctionBegin;
3913   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3914   PetscValidPointer(depthLabel, 2);
3915   if (!mesh->depthLabel) {ierr = DMPlexGetLabel(dm, "depth", &mesh->depthLabel);CHKERRQ(ierr);}
3916   *depthLabel = mesh->depthLabel;
3917   PetscFunctionReturn(0);
3918 }
3919 
3920 #undef __FUNCT__
3921 #define __FUNCT__ "DMPlexGetDepth"
3922 /*@
3923   DMPlexGetDepth - Get the depth of the DAG representing this mesh
3924 
3925   Not Collective
3926 
3927   Input Parameter:
3928 . dm    - The DMPlex object
3929 
3930   Output Parameter:
3931 . depth - The number of strata (breadth first levels) in the DAG
3932 
3933   Level: developer
3934 
3935 .keywords: mesh, points
3936 .seealso: DMPlexGetDepthLabel(), DMPlexGetHeightStratum(), DMPlexGetDepthStratum()
3937 @*/
3938 PetscErrorCode DMPlexGetDepth(DM dm, PetscInt *depth)
3939 {
3940   DMLabel        label;
3941   PetscInt       d = 0;
3942   PetscErrorCode ierr;
3943 
3944   PetscFunctionBegin;
3945   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3946   PetscValidPointer(depth, 2);
3947   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
3948   if (label) {ierr = DMLabelGetNumValues(label, &d);CHKERRQ(ierr);}
3949   *depth = d-1;
3950   PetscFunctionReturn(0);
3951 }
3952 
3953 #undef __FUNCT__
3954 #define __FUNCT__ "DMPlexGetDepthStratum"
3955 /*@
3956   DMPlexGetDepthStratum - Get the bounds [start, end) for all points at a certain depth.
3957 
3958   Not Collective
3959 
3960   Input Parameters:
3961 + dm           - The DMPlex object
3962 - stratumValue - The requested depth
3963 
3964   Output Parameters:
3965 + start - The first point at this depth
3966 - end   - One beyond the last point at this depth
3967 
3968   Level: developer
3969 
3970 .keywords: mesh, points
3971 .seealso: DMPlexGetHeightStratum(), DMPlexGetDepth()
3972 @*/
3973 PetscErrorCode DMPlexGetDepthStratum(DM dm, PetscInt stratumValue, PetscInt *start, PetscInt *end)
3974 {
3975   DMLabel        label;
3976   PetscInt       pStart, pEnd;
3977   PetscErrorCode ierr;
3978 
3979   PetscFunctionBegin;
3980   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3981   if (start) {PetscValidPointer(start, 3); *start = 0;}
3982   if (end)   {PetscValidPointer(end,   4); *end   = 0;}
3983   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
3984   if (pStart == pEnd) PetscFunctionReturn(0);
3985   if (stratumValue < 0) {
3986     if (start) *start = pStart;
3987     if (end)   *end   = pEnd;
3988     PetscFunctionReturn(0);
3989   }
3990   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
3991   if (!label) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "No label named depth was found");
3992   ierr = DMLabelGetStratumBounds(label, stratumValue, start, end);CHKERRQ(ierr);
3993   PetscFunctionReturn(0);
3994 }
3995 
3996 #undef __FUNCT__
3997 #define __FUNCT__ "DMPlexGetHeightStratum"
3998 /*@
3999   DMPlexGetHeightStratum - Get the bounds [start, end) for all points at a certain height.
4000 
4001   Not Collective
4002 
4003   Input Parameters:
4004 + dm           - The DMPlex object
4005 - stratumValue - The requested height
4006 
4007   Output Parameters:
4008 + start - The first point at this height
4009 - end   - One beyond the last point at this height
4010 
4011   Level: developer
4012 
4013 .keywords: mesh, points
4014 .seealso: DMPlexGetDepthStratum(), DMPlexGetDepth()
4015 @*/
4016 PetscErrorCode DMPlexGetHeightStratum(DM dm, PetscInt stratumValue, PetscInt *start, PetscInt *end)
4017 {
4018   DMLabel        label;
4019   PetscInt       depth, pStart, pEnd;
4020   PetscErrorCode ierr;
4021 
4022   PetscFunctionBegin;
4023   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4024   if (start) {PetscValidPointer(start, 3); *start = 0;}
4025   if (end)   {PetscValidPointer(end,   4); *end   = 0;}
4026   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
4027   if (pStart == pEnd) PetscFunctionReturn(0);
4028   if (stratumValue < 0) {
4029     if (start) *start = pStart;
4030     if (end)   *end   = pEnd;
4031     PetscFunctionReturn(0);
4032   }
4033   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
4034   if (!label) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "No label named depth was found");CHKERRQ(ierr);
4035   ierr = DMLabelGetNumValues(label, &depth);CHKERRQ(ierr);
4036   ierr = DMLabelGetStratumBounds(label, depth-1-stratumValue, start, end);CHKERRQ(ierr);
4037   PetscFunctionReturn(0);
4038 }
4039 
4040 #undef __FUNCT__
4041 #define __FUNCT__ "DMPlexCreateSectionInitial"
4042 /* Set the number of dof on each point and separate by fields */
4043 PetscErrorCode DMPlexCreateSectionInitial(DM dm, PetscInt dim, PetscInt numFields,const PetscInt numComp[],const PetscInt numDof[], PetscSection *section)
4044 {
4045   PetscInt      *numDofTot;
4046   PetscInt       depth, pStart = 0, pEnd = 0;
4047   PetscInt       p, d, dep, f;
4048   PetscErrorCode ierr;
4049 
4050   PetscFunctionBegin;
4051   ierr = PetscMalloc1((dim+1), &numDofTot);CHKERRQ(ierr);
4052   for (d = 0; d <= dim; ++d) {
4053     numDofTot[d] = 0;
4054     for (f = 0; f < numFields; ++f) numDofTot[d] += numDof[f*(dim+1)+d];
4055   }
4056   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), section);CHKERRQ(ierr);
4057   if (numFields > 0) {
4058     ierr = PetscSectionSetNumFields(*section, numFields);CHKERRQ(ierr);
4059     if (numComp) {
4060       for (f = 0; f < numFields; ++f) {
4061         ierr = PetscSectionSetFieldComponents(*section, f, numComp[f]);CHKERRQ(ierr);
4062       }
4063     }
4064   }
4065   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
4066   ierr = PetscSectionSetChart(*section, pStart, pEnd);CHKERRQ(ierr);
4067   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
4068   for (dep = 0; dep <= depth; ++dep) {
4069     d    = dim == depth ? dep : (!dep ? 0 : dim);
4070     ierr = DMPlexGetDepthStratum(dm, dep, &pStart, &pEnd);CHKERRQ(ierr);
4071     for (p = pStart; p < pEnd; ++p) {
4072       for (f = 0; f < numFields; ++f) {
4073         ierr = PetscSectionSetFieldDof(*section, p, f, numDof[f*(dim+1)+d]);CHKERRQ(ierr);
4074       }
4075       ierr = PetscSectionSetDof(*section, p, numDofTot[d]);CHKERRQ(ierr);
4076     }
4077   }
4078   ierr = PetscFree(numDofTot);CHKERRQ(ierr);
4079   PetscFunctionReturn(0);
4080 }
4081 
4082 #undef __FUNCT__
4083 #define __FUNCT__ "DMPlexCreateSectionBCDof"
4084 /* Set the number of dof on each point and separate by fields
4085    If constDof is PETSC_DETERMINE, constrain every dof on the point
4086 */
4087 PetscErrorCode DMPlexCreateSectionBCDof(DM dm, PetscInt numBC,const PetscInt bcField[],const IS bcPoints[], PetscInt constDof, PetscSection section)
4088 {
4089   PetscInt       numFields;
4090   PetscInt       bc;
4091   PetscErrorCode ierr;
4092 
4093   PetscFunctionBegin;
4094   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
4095   for (bc = 0; bc < numBC; ++bc) {
4096     PetscInt        field = 0;
4097     const PetscInt *idx;
4098     PetscInt        n, i;
4099 
4100     if (numFields) field = bcField[bc];
4101     ierr = ISGetLocalSize(bcPoints[bc], &n);CHKERRQ(ierr);
4102     ierr = ISGetIndices(bcPoints[bc], &idx);CHKERRQ(ierr);
4103     for (i = 0; i < n; ++i) {
4104       const PetscInt p        = idx[i];
4105       PetscInt       numConst = constDof;
4106 
4107       /* Constrain every dof on the point */
4108       if (numConst < 0) {
4109         if (numFields) {
4110           ierr = PetscSectionGetFieldDof(section, p, field, &numConst);CHKERRQ(ierr);
4111         } else {
4112           ierr = PetscSectionGetDof(section, p, &numConst);CHKERRQ(ierr);
4113         }
4114       }
4115       if (numFields) {
4116         ierr = PetscSectionAddFieldConstraintDof(section, p, field, numConst);CHKERRQ(ierr);
4117       }
4118       ierr = PetscSectionAddConstraintDof(section, p, numConst);CHKERRQ(ierr);
4119     }
4120     ierr = ISRestoreIndices(bcPoints[bc], &idx);CHKERRQ(ierr);
4121   }
4122   PetscFunctionReturn(0);
4123 }
4124 
4125 #undef __FUNCT__
4126 #define __FUNCT__ "DMPlexCreateSectionBCIndicesAll"
4127 /* Set the constrained indices on each point and separate by fields */
4128 PetscErrorCode DMPlexCreateSectionBCIndicesAll(DM dm, PetscSection section)
4129 {
4130   PetscInt      *maxConstraints;
4131   PetscInt       numFields, f, pStart = 0, pEnd = 0, p;
4132   PetscErrorCode ierr;
4133 
4134   PetscFunctionBegin;
4135   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
4136   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
4137   ierr = PetscMalloc1((numFields+1), &maxConstraints);CHKERRQ(ierr);
4138   for (f = 0; f <= numFields; ++f) maxConstraints[f] = 0;
4139   for (p = pStart; p < pEnd; ++p) {
4140     PetscInt cdof;
4141 
4142     if (numFields) {
4143       for (f = 0; f < numFields; ++f) {
4144         ierr              = PetscSectionGetFieldConstraintDof(section, p, f, &cdof);CHKERRQ(ierr);
4145         maxConstraints[f] = PetscMax(maxConstraints[f], cdof);
4146       }
4147     } else {
4148       ierr              = PetscSectionGetConstraintDof(section, p, &cdof);CHKERRQ(ierr);
4149       maxConstraints[0] = PetscMax(maxConstraints[0], cdof);
4150     }
4151   }
4152   for (f = 0; f < numFields; ++f) {
4153     maxConstraints[numFields] += maxConstraints[f];
4154   }
4155   if (maxConstraints[numFields]) {
4156     PetscInt *indices;
4157 
4158     ierr = PetscMalloc1(maxConstraints[numFields], &indices);CHKERRQ(ierr);
4159     for (p = pStart; p < pEnd; ++p) {
4160       PetscInt cdof, d;
4161 
4162       ierr = PetscSectionGetConstraintDof(section, p, &cdof);CHKERRQ(ierr);
4163       if (cdof) {
4164         if (cdof > maxConstraints[numFields]) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_LIB, "Likely memory corruption, point %D cDof %D > maxConstraints %D", p, cdof, maxConstraints[numFields]);
4165         if (numFields) {
4166           PetscInt numConst = 0, foff = 0;
4167 
4168           for (f = 0; f < numFields; ++f) {
4169             PetscInt cfdof, fdof;
4170 
4171             ierr = PetscSectionGetFieldDof(section, p, f, &fdof);CHKERRQ(ierr);
4172             ierr = PetscSectionGetFieldConstraintDof(section, p, f, &cfdof);CHKERRQ(ierr);
4173             /* Change constraint numbering from absolute local dof number to field relative local dof number */
4174             for (d = 0; d < cfdof; ++d) indices[numConst+d] = d;
4175             ierr = PetscSectionSetFieldConstraintIndices(section, p, f, &indices[numConst]);CHKERRQ(ierr);
4176             for (d = 0; d < cfdof; ++d) indices[numConst+d] += foff;
4177             numConst += cfdof;
4178             foff     += fdof;
4179           }
4180           if (cdof != numConst) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_LIB, "Total number of field constraints %D should be %D", numConst, cdof);
4181         } else {
4182           for (d = 0; d < cdof; ++d) indices[d] = d;
4183         }
4184         ierr = PetscSectionSetConstraintIndices(section, p, indices);CHKERRQ(ierr);
4185       }
4186     }
4187     ierr = PetscFree(indices);CHKERRQ(ierr);
4188   }
4189   ierr = PetscFree(maxConstraints);CHKERRQ(ierr);
4190   PetscFunctionReturn(0);
4191 }
4192 
4193 #undef __FUNCT__
4194 #define __FUNCT__ "DMPlexCreateSectionBCIndicesField"
4195 /* Set the constrained field indices on each point */
4196 PetscErrorCode DMPlexCreateSectionBCIndicesField(DM dm, PetscInt field, IS bcPoints, IS constraintIndices, PetscSection section)
4197 {
4198   const PetscInt *points, *indices;
4199   PetscInt        numFields, maxDof, numPoints, p, numConstraints;
4200   PetscErrorCode  ierr;
4201 
4202   PetscFunctionBegin;
4203   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
4204   if ((field < 0) || (field >= numFields)) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Section field %d should be in [%d, %d)", field, 0, numFields);
4205 
4206   ierr = ISGetLocalSize(bcPoints, &numPoints);CHKERRQ(ierr);
4207   ierr = ISGetIndices(bcPoints, &points);CHKERRQ(ierr);
4208   if (!constraintIndices) {
4209     PetscInt *idx, i;
4210 
4211     ierr = PetscSectionGetMaxDof(section, &maxDof);CHKERRQ(ierr);
4212     ierr = PetscMalloc1(maxDof, &idx);CHKERRQ(ierr);
4213     for (i = 0; i < maxDof; ++i) idx[i] = i;
4214     for (p = 0; p < numPoints; ++p) {
4215       ierr = PetscSectionSetFieldConstraintIndices(section, points[p], field, idx);CHKERRQ(ierr);
4216     }
4217     ierr = PetscFree(idx);CHKERRQ(ierr);
4218   } else {
4219     ierr = ISGetLocalSize(constraintIndices, &numConstraints);CHKERRQ(ierr);
4220     ierr = ISGetIndices(constraintIndices, &indices);CHKERRQ(ierr);
4221     for (p = 0; p < numPoints; ++p) {
4222       PetscInt fcdof;
4223 
4224       ierr = PetscSectionGetFieldConstraintDof(section, points[p], field, &fcdof);CHKERRQ(ierr);
4225       if (fcdof != numConstraints) SETERRQ4(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Section point %d field %d has %d constraints, but yo ugave %d indices", p, field, fcdof, numConstraints);
4226       ierr = PetscSectionSetFieldConstraintIndices(section, points[p], field, indices);CHKERRQ(ierr);
4227     }
4228     ierr = ISRestoreIndices(constraintIndices, &indices);CHKERRQ(ierr);
4229   }
4230   ierr = ISRestoreIndices(bcPoints, &points);CHKERRQ(ierr);
4231   PetscFunctionReturn(0);
4232 }
4233 
4234 #undef __FUNCT__
4235 #define __FUNCT__ "DMPlexCreateSectionBCIndices"
4236 /* Set the constrained indices on each point and separate by fields */
4237 PetscErrorCode DMPlexCreateSectionBCIndices(DM dm, PetscSection section)
4238 {
4239   PetscInt      *indices;
4240   PetscInt       numFields, maxDof, f, pStart = 0, pEnd = 0, p;
4241   PetscErrorCode ierr;
4242 
4243   PetscFunctionBegin;
4244   ierr = PetscSectionGetMaxDof(section, &maxDof);CHKERRQ(ierr);
4245   ierr = PetscMalloc1(maxDof, &indices);CHKERRQ(ierr);
4246   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
4247   if (!numFields) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "This function only works after users have set field constraint indices.");
4248   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
4249   for (p = pStart; p < pEnd; ++p) {
4250     PetscInt cdof, d;
4251 
4252     ierr = PetscSectionGetConstraintDof(section, p, &cdof);CHKERRQ(ierr);
4253     if (cdof) {
4254       PetscInt numConst = 0, foff = 0;
4255 
4256       for (f = 0; f < numFields; ++f) {
4257         const PetscInt *fcind;
4258         PetscInt        fdof, fcdof;
4259 
4260         ierr = PetscSectionGetFieldDof(section, p, f, &fdof);CHKERRQ(ierr);
4261         ierr = PetscSectionGetFieldConstraintDof(section, p, f, &fcdof);CHKERRQ(ierr);
4262         if (fcdof) {ierr = PetscSectionGetFieldConstraintIndices(section, p, f, &fcind);CHKERRQ(ierr);}
4263         /* Change constraint numbering from field relative local dof number to absolute local dof number */
4264         for (d = 0; d < fcdof; ++d) indices[numConst+d] = fcind[d]+foff;
4265         foff     += fdof;
4266         numConst += fcdof;
4267       }
4268       if (cdof != numConst) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_LIB, "Total number of field constraints %D should be %D", numConst, cdof);
4269       ierr = PetscSectionSetConstraintIndices(section, p, indices);CHKERRQ(ierr);
4270     }
4271   }
4272   ierr = PetscFree(indices);CHKERRQ(ierr);
4273   PetscFunctionReturn(0);
4274 }
4275 
4276 #undef __FUNCT__
4277 #define __FUNCT__ "DMPlexCreateSection"
4278 /*@C
4279   DMPlexCreateSection - Create a PetscSection based upon the dof layout specification provided.
4280 
4281   Not Collective
4282 
4283   Input Parameters:
4284 + dm        - The DMPlex object
4285 . dim       - The spatial dimension of the problem
4286 . numFields - The number of fields in the problem
4287 . numComp   - An array of size numFields that holds the number of components for each field
4288 . numDof    - An array of size numFields*(dim+1) which holds the number of dof for each field on a mesh piece of dimension d
4289 . numBC     - The number of boundary conditions
4290 . bcField   - An array of size numBC giving the field number for each boundry condition
4291 . bcPoints  - An array of size numBC giving an IS holding the sieve points to which each boundary condition applies
4292 - perm      - Optional permutation of the chart, or NULL
4293 
4294   Output Parameter:
4295 . section - The PetscSection object
4296 
4297   Notes: numDof[f*(dim+1)+d] gives the number of dof for field f on sieve points of dimension d. For instance, numDof[1] is the
4298   number of dof for field 0 on each edge.
4299 
4300   The chart permutation is the same one set using PetscSectionSetPermutation()
4301 
4302   Level: developer
4303 
4304   Fortran Notes:
4305   A Fortran 90 version is available as DMPlexCreateSectionF90()
4306 
4307 .keywords: mesh, elements
4308 .seealso: DMPlexCreate(), PetscSectionCreate(), PetscSectionSetPermutation()
4309 @*/
4310 PetscErrorCode DMPlexCreateSection(DM dm, PetscInt dim, PetscInt numFields,const PetscInt numComp[],const PetscInt numDof[], PetscInt numBC,const PetscInt bcField[],const IS bcPoints[], IS perm, PetscSection *section)
4311 {
4312   PetscErrorCode ierr;
4313 
4314   PetscFunctionBegin;
4315   ierr = DMPlexCreateSectionInitial(dm, dim, numFields, numComp, numDof, section);CHKERRQ(ierr);
4316   ierr = DMPlexCreateSectionBCDof(dm, numBC, bcField, bcPoints, PETSC_DETERMINE, *section);CHKERRQ(ierr);
4317   if (perm) {ierr = PetscSectionSetPermutation(*section, perm);CHKERRQ(ierr);}
4318   ierr = PetscSectionSetUp(*section);CHKERRQ(ierr);
4319   if (numBC) {ierr = DMPlexCreateSectionBCIndicesAll(dm, *section);CHKERRQ(ierr);}
4320   ierr = PetscSectionViewFromOptions(*section,NULL,"-section_view");CHKERRQ(ierr);
4321   PetscFunctionReturn(0);
4322 }
4323 
4324 #undef __FUNCT__
4325 #define __FUNCT__ "DMCreateCoordinateDM_Plex"
4326 PetscErrorCode DMCreateCoordinateDM_Plex(DM dm, DM *cdm)
4327 {
4328   PetscSection   section;
4329   PetscErrorCode ierr;
4330 
4331   PetscFunctionBegin;
4332   ierr = DMClone(dm, cdm);CHKERRQ(ierr);
4333   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &section);CHKERRQ(ierr);
4334   ierr = DMSetDefaultSection(*cdm, section);CHKERRQ(ierr);
4335   ierr = PetscSectionDestroy(&section);CHKERRQ(ierr);
4336   PetscFunctionReturn(0);
4337 }
4338 
4339 #undef __FUNCT__
4340 #define __FUNCT__ "DMPlexGetConeSection"
4341 PetscErrorCode DMPlexGetConeSection(DM dm, PetscSection *section)
4342 {
4343   DM_Plex *mesh = (DM_Plex*) dm->data;
4344 
4345   PetscFunctionBegin;
4346   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4347   if (section) *section = mesh->coneSection;
4348   PetscFunctionReturn(0);
4349 }
4350 
4351 #undef __FUNCT__
4352 #define __FUNCT__ "DMPlexGetSupportSection"
4353 PetscErrorCode DMPlexGetSupportSection(DM dm, PetscSection *section)
4354 {
4355   DM_Plex *mesh = (DM_Plex*) dm->data;
4356 
4357   PetscFunctionBegin;
4358   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4359   if (section) *section = mesh->supportSection;
4360   PetscFunctionReturn(0);
4361 }
4362 
4363 #undef __FUNCT__
4364 #define __FUNCT__ "DMPlexGetCones"
4365 PetscErrorCode DMPlexGetCones(DM dm, PetscInt *cones[])
4366 {
4367   DM_Plex *mesh = (DM_Plex*) dm->data;
4368 
4369   PetscFunctionBegin;
4370   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4371   if (cones) *cones = mesh->cones;
4372   PetscFunctionReturn(0);
4373 }
4374 
4375 #undef __FUNCT__
4376 #define __FUNCT__ "DMPlexGetConeOrientations"
4377 PetscErrorCode DMPlexGetConeOrientations(DM dm, PetscInt *coneOrientations[])
4378 {
4379   DM_Plex *mesh = (DM_Plex*) dm->data;
4380 
4381   PetscFunctionBegin;
4382   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4383   if (coneOrientations) *coneOrientations = mesh->coneOrientations;
4384   PetscFunctionReturn(0);
4385 }
4386 
4387 /******************************** FEM Support **********************************/
4388 
4389 #undef __FUNCT__
4390 #define __FUNCT__ "DMPlexVecGetClosure_Depth1_Static"
4391 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecGetClosure_Depth1_Static(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
4392 {
4393   PetscScalar    *array, *vArray;
4394   const PetscInt *cone, *coneO;
4395   PetscInt        pStart, pEnd, p, numPoints, size = 0, offset = 0;
4396   PetscErrorCode  ierr;
4397 
4398   PetscFunctionBeginHot;
4399   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
4400   ierr = DMPlexGetConeSize(dm, point, &numPoints);CHKERRQ(ierr);
4401   ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
4402   ierr = DMPlexGetConeOrientation(dm, point, &coneO);CHKERRQ(ierr);
4403   if (!values || !*values) {
4404     if ((point >= pStart) && (point < pEnd)) {
4405       PetscInt dof;
4406 
4407       ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4408       size += dof;
4409     }
4410     for (p = 0; p < numPoints; ++p) {
4411       const PetscInt cp = cone[p];
4412       PetscInt       dof;
4413 
4414       if ((cp < pStart) || (cp >= pEnd)) continue;
4415       ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
4416       size += dof;
4417     }
4418     if (!values) {
4419       if (csize) *csize = size;
4420       PetscFunctionReturn(0);
4421     }
4422     ierr = DMGetWorkArray(dm, size, PETSC_SCALAR, &array);CHKERRQ(ierr);
4423   } else {
4424     array = *values;
4425   }
4426   size = 0;
4427   ierr = VecGetArray(v, &vArray);CHKERRQ(ierr);
4428   if ((point >= pStart) && (point < pEnd)) {
4429     PetscInt     dof, off, d;
4430     PetscScalar *varr;
4431 
4432     ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4433     ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
4434     varr = &vArray[off];
4435     for (d = 0; d < dof; ++d, ++offset) {
4436       array[offset] = varr[d];
4437     }
4438     size += dof;
4439   }
4440   for (p = 0; p < numPoints; ++p) {
4441     const PetscInt cp = cone[p];
4442     PetscInt       o  = coneO[p];
4443     PetscInt       dof, off, d;
4444     PetscScalar   *varr;
4445 
4446     if ((cp < pStart) || (cp >= pEnd)) continue;
4447     ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
4448     ierr = PetscSectionGetOffset(section, cp, &off);CHKERRQ(ierr);
4449     varr = &vArray[off];
4450     if (o >= 0) {
4451       for (d = 0; d < dof; ++d, ++offset) {
4452         array[offset] = varr[d];
4453       }
4454     } else {
4455       for (d = dof-1; d >= 0; --d, ++offset) {
4456         array[offset] = varr[d];
4457       }
4458     }
4459     size += dof;
4460   }
4461   ierr = VecRestoreArray(v, &vArray);CHKERRQ(ierr);
4462   if (!*values) {
4463     if (csize) *csize = size;
4464     *values = array;
4465   } else {
4466     if (size > *csize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Size of input array %d < actual size %d", *csize, size);
4467     *csize = size;
4468   }
4469   PetscFunctionReturn(0);
4470 }
4471 
4472 #undef __FUNCT__
4473 #define __FUNCT__ "DMPlexVecGetClosure_Static"
4474 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecGetClosure_Static(PetscSection section, PetscInt numPoints, const PetscInt points[], const PetscScalar vArray[], PetscInt *size, PetscScalar array[])
4475 {
4476   PetscInt       offset = 0, p;
4477   PetscErrorCode ierr;
4478 
4479   PetscFunctionBeginHot;
4480   *size = 0;
4481   for (p = 0; p < numPoints*2; p += 2) {
4482     const PetscInt point = points[p];
4483     const PetscInt o     = points[p+1];
4484     PetscInt       dof, off, d;
4485     const PetscScalar *varr;
4486 
4487     ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4488     ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
4489     varr = &vArray[off];
4490     if (o >= 0) {
4491       for (d = 0; d < dof; ++d, ++offset)    array[offset] = varr[d];
4492     } else {
4493       for (d = dof-1; d >= 0; --d, ++offset) array[offset] = varr[d];
4494     }
4495   }
4496   *size = offset;
4497   PetscFunctionReturn(0);
4498 }
4499 
4500 #undef __FUNCT__
4501 #define __FUNCT__ "DMPlexVecGetClosure_Fields_Static"
4502 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecGetClosure_Fields_Static(PetscSection section, PetscInt numPoints, const PetscInt points[], PetscInt numFields, const PetscScalar vArray[], PetscInt *size, PetscScalar array[])
4503 {
4504   PetscInt       offset = 0, f;
4505   PetscErrorCode ierr;
4506 
4507   PetscFunctionBeginHot;
4508   *size = 0;
4509   for (f = 0; f < numFields; ++f) {
4510     PetscInt fcomp, p;
4511 
4512     ierr = PetscSectionGetFieldComponents(section, f, &fcomp);CHKERRQ(ierr);
4513     for (p = 0; p < numPoints*2; p += 2) {
4514       const PetscInt point = points[p];
4515       const PetscInt o     = points[p+1];
4516       PetscInt       fdof, foff, d, c;
4517       const PetscScalar *varr;
4518 
4519       ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
4520       ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
4521       varr = &vArray[foff];
4522       if (o >= 0) {
4523         for (d = 0; d < fdof; ++d, ++offset) array[offset] = varr[d];
4524       } else {
4525         for (d = fdof/fcomp-1; d >= 0; --d) {
4526           for (c = 0; c < fcomp; ++c, ++offset) {
4527             array[offset] = varr[d*fcomp+c];
4528           }
4529         }
4530       }
4531     }
4532   }
4533   *size = offset;
4534   PetscFunctionReturn(0);
4535 }
4536 
4537 #undef __FUNCT__
4538 #define __FUNCT__ "DMPlexVecGetClosure"
4539 /*@C
4540   DMPlexVecGetClosure - Get an array of the values on the closure of 'point'
4541 
4542   Not collective
4543 
4544   Input Parameters:
4545 + dm - The DM
4546 . section - The section describing the layout in v, or NULL to use the default section
4547 . v - The local vector
4548 - point - The sieve point in the DM
4549 
4550   Output Parameters:
4551 + csize - The number of values in the closure, or NULL
4552 - values - The array of values, which is a borrowed array and should not be freed
4553 
4554   Fortran Notes:
4555   Since it returns an array, this routine is only available in Fortran 90, and you must
4556   include petsc.h90 in your code.
4557 
4558   The csize argument is not present in the Fortran 90 binding since it is internal to the array.
4559 
4560   Level: intermediate
4561 
4562 .seealso DMPlexVecRestoreClosure(), DMPlexVecSetClosure(), DMPlexMatSetClosure()
4563 @*/
4564 PetscErrorCode DMPlexVecGetClosure(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
4565 {
4566   PetscSection    clSection;
4567   IS              clPoints;
4568   PetscScalar    *array, *vArray;
4569   PetscInt       *points = NULL;
4570   const PetscInt *clp;
4571   PetscInt        depth, numFields, numPoints, size;
4572   PetscErrorCode  ierr;
4573 
4574   PetscFunctionBeginHot;
4575   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4576   if (!section) {ierr = DMGetDefaultSection(dm, &section);CHKERRQ(ierr);}
4577   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
4578   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
4579   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
4580   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
4581   if (depth == 1 && numFields < 2) {
4582     ierr = DMPlexVecGetClosure_Depth1_Static(dm, section, v, point, csize, values);CHKERRQ(ierr);
4583     PetscFunctionReturn(0);
4584   }
4585   /* Get points */
4586   ierr = PetscSectionGetClosureIndex(section, (PetscObject) dm, &clSection, &clPoints);CHKERRQ(ierr);
4587   if (!clPoints) {
4588     PetscInt pStart, pEnd, p, q;
4589 
4590     ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
4591     ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
4592     /* Compress out points not in the section */
4593     for (p = 0, q = 0; p < numPoints*2; p += 2) {
4594       if ((points[p] >= pStart) && (points[p] < pEnd)) {
4595         points[q*2]   = points[p];
4596         points[q*2+1] = points[p+1];
4597         ++q;
4598       }
4599     }
4600     numPoints = q;
4601   } else {
4602     PetscInt dof, off;
4603 
4604     ierr = PetscSectionGetDof(clSection, point, &dof);CHKERRQ(ierr);
4605     ierr = PetscSectionGetOffset(clSection, point, &off);CHKERRQ(ierr);
4606     ierr = ISGetIndices(clPoints, &clp);CHKERRQ(ierr);
4607     numPoints = dof/2;
4608     points    = (PetscInt *) &clp[off];
4609   }
4610   /* Get array */
4611   if (!values || !*values) {
4612     PetscInt asize = 0, dof, p;
4613 
4614     for (p = 0; p < numPoints*2; p += 2) {
4615       ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
4616       asize += dof;
4617     }
4618     if (!values) {
4619       if (!clPoints) {ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);}
4620       else           {ierr = ISRestoreIndices(clPoints, &clp);CHKERRQ(ierr);}
4621       if (csize) *csize = asize;
4622       PetscFunctionReturn(0);
4623     }
4624     ierr = DMGetWorkArray(dm, asize, PETSC_SCALAR, &array);CHKERRQ(ierr);
4625   } else {
4626     array = *values;
4627   }
4628   ierr = VecGetArray(v, &vArray);CHKERRQ(ierr);
4629   /* Get values */
4630   if (numFields > 0) {ierr = DMPlexVecGetClosure_Fields_Static(section, numPoints, points, numFields, vArray, &size, array);CHKERRQ(ierr);}
4631   else               {ierr = DMPlexVecGetClosure_Static(section, numPoints, points, vArray, &size, array);CHKERRQ(ierr);}
4632   /* Cleanup points */
4633   if (!clPoints) {ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);}
4634   else           {ierr = ISRestoreIndices(clPoints, &clp);CHKERRQ(ierr);}
4635   /* Cleanup array */
4636   ierr = VecRestoreArray(v, &vArray);CHKERRQ(ierr);
4637   if (!*values) {
4638     if (csize) *csize = size;
4639     *values = array;
4640   } else {
4641     if (size > *csize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Size of input array %d < actual size %d", *csize, size);
4642     *csize = size;
4643   }
4644   PetscFunctionReturn(0);
4645 }
4646 
4647 #undef __FUNCT__
4648 #define __FUNCT__ "DMPlexVecRestoreClosure"
4649 /*@C
4650   DMPlexVecRestoreClosure - Restore the array of the values on the closure of 'point'
4651 
4652   Not collective
4653 
4654   Input Parameters:
4655 + dm - The DM
4656 . section - The section describing the layout in v, or NULL to use the default section
4657 . v - The local vector
4658 . point - The sieve point in the DM
4659 . csize - The number of values in the closure, or NULL
4660 - values - The array of values, which is a borrowed array and should not be freed
4661 
4662   Fortran Notes:
4663   Since it returns an array, this routine is only available in Fortran 90, and you must
4664   include petsc.h90 in your code.
4665 
4666   The csize argument is not present in the Fortran 90 binding since it is internal to the array.
4667 
4668   Level: intermediate
4669 
4670 .seealso DMPlexVecGetClosure(), DMPlexVecSetClosure(), DMPlexMatSetClosure()
4671 @*/
4672 PetscErrorCode DMPlexVecRestoreClosure(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
4673 {
4674   PetscInt       size = 0;
4675   PetscErrorCode ierr;
4676 
4677   PetscFunctionBegin;
4678   /* Should work without recalculating size */
4679   ierr = DMRestoreWorkArray(dm, size, PETSC_SCALAR, (void*) values);CHKERRQ(ierr);
4680   PetscFunctionReturn(0);
4681 }
4682 
4683 PETSC_STATIC_INLINE void add   (PetscScalar *x, PetscScalar y) {*x += y;}
4684 PETSC_STATIC_INLINE void insert(PetscScalar *x, PetscScalar y) {*x  = y;}
4685 
4686 #undef __FUNCT__
4687 #define __FUNCT__ "updatePoint_private"
4688 PETSC_STATIC_INLINE PetscErrorCode updatePoint_private(PetscSection section, PetscInt point, PetscInt dof, void (*fuse)(PetscScalar*, PetscScalar), PetscBool setBC, PetscInt orientation, const PetscScalar values[], PetscScalar array[])
4689 {
4690   PetscInt        cdof;   /* The number of constraints on this point */
4691   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
4692   PetscScalar    *a;
4693   PetscInt        off, cind = 0, k;
4694   PetscErrorCode  ierr;
4695 
4696   PetscFunctionBegin;
4697   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
4698   ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
4699   a    = &array[off];
4700   if (!cdof || setBC) {
4701     if (orientation >= 0) {
4702       for (k = 0; k < dof; ++k) {
4703         fuse(&a[k], values[k]);
4704       }
4705     } else {
4706       for (k = 0; k < dof; ++k) {
4707         fuse(&a[k], values[dof-k-1]);
4708       }
4709     }
4710   } else {
4711     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
4712     if (orientation >= 0) {
4713       for (k = 0; k < dof; ++k) {
4714         if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
4715         fuse(&a[k], values[k]);
4716       }
4717     } else {
4718       for (k = 0; k < dof; ++k) {
4719         if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
4720         fuse(&a[k], values[dof-k-1]);
4721       }
4722     }
4723   }
4724   PetscFunctionReturn(0);
4725 }
4726 
4727 #undef __FUNCT__
4728 #define __FUNCT__ "updatePointBC_private"
4729 PETSC_STATIC_INLINE PetscErrorCode updatePointBC_private(PetscSection section, PetscInt point, PetscInt dof, void (*fuse)(PetscScalar*, PetscScalar), PetscInt orientation, const PetscScalar values[], PetscScalar array[])
4730 {
4731   PetscInt        cdof;   /* The number of constraints on this point */
4732   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
4733   PetscScalar    *a;
4734   PetscInt        off, cind = 0, k;
4735   PetscErrorCode  ierr;
4736 
4737   PetscFunctionBegin;
4738   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
4739   ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
4740   a    = &array[off];
4741   if (cdof) {
4742     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
4743     if (orientation >= 0) {
4744       for (k = 0; k < dof; ++k) {
4745         if ((cind < cdof) && (k == cdofs[cind])) {
4746           fuse(&a[k], values[k]);
4747           ++cind;
4748         }
4749       }
4750     } else {
4751       for (k = 0; k < dof; ++k) {
4752         if ((cind < cdof) && (k == cdofs[cind])) {
4753           fuse(&a[k], values[dof-k-1]);
4754           ++cind;
4755         }
4756       }
4757     }
4758   }
4759   PetscFunctionReturn(0);
4760 }
4761 
4762 #undef __FUNCT__
4763 #define __FUNCT__ "updatePointFields_private"
4764 PETSC_STATIC_INLINE PetscErrorCode updatePointFields_private(PetscSection section, PetscInt point, PetscInt o, PetscInt f, PetscInt fcomp, void (*fuse)(PetscScalar*, PetscScalar), PetscBool setBC, const PetscScalar values[], PetscInt *offset, PetscScalar array[])
4765 {
4766   PetscScalar    *a;
4767   PetscInt        fdof, foff, fcdof, foffset = *offset;
4768   const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
4769   PetscInt        cind = 0, k, c;
4770   PetscErrorCode  ierr;
4771 
4772   PetscFunctionBegin;
4773   ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
4774   ierr = PetscSectionGetFieldConstraintDof(section, point, f, &fcdof);CHKERRQ(ierr);
4775   ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
4776   a    = &array[foff];
4777   if (!fcdof || setBC) {
4778     if (o >= 0) {
4779       for (k = 0; k < fdof; ++k) fuse(&a[k], values[foffset+k]);
4780     } else {
4781       for (k = fdof/fcomp-1; k >= 0; --k) {
4782         for (c = 0; c < fcomp; ++c) {
4783           fuse(&a[(fdof/fcomp-1-k)*fcomp+c], values[foffset+k*fcomp+c]);
4784         }
4785       }
4786     }
4787   } else {
4788     ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
4789     if (o >= 0) {
4790       for (k = 0; k < fdof; ++k) {
4791         if ((cind < fcdof) && (k == fcdofs[cind])) {++cind; continue;}
4792         fuse(&a[k], values[foffset+k]);
4793       }
4794     } else {
4795       for (k = fdof/fcomp-1; k >= 0; --k) {
4796         for (c = 0; c < fcomp; ++c) {
4797           if ((cind < fcdof) && (k*fcomp+c == fcdofs[cind])) {++cind; continue;}
4798           fuse(&a[(fdof/fcomp-1-k)*fcomp+c], values[foffset+k*fcomp+c]);
4799         }
4800       }
4801     }
4802   }
4803   *offset += fdof;
4804   PetscFunctionReturn(0);
4805 }
4806 
4807 #undef __FUNCT__
4808 #define __FUNCT__ "updatePointFieldsBC_private"
4809 PETSC_STATIC_INLINE PetscErrorCode updatePointFieldsBC_private(PetscSection section, PetscInt point, PetscInt o, PetscInt f, PetscInt fcomp, void (*fuse)(PetscScalar*, PetscScalar), const PetscScalar values[], PetscInt *offset, PetscScalar array[])
4810 {
4811   PetscScalar    *a;
4812   PetscInt        fdof, foff, fcdof, foffset = *offset;
4813   const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
4814   PetscInt        cind = 0, k, c;
4815   PetscErrorCode  ierr;
4816 
4817   PetscFunctionBegin;
4818   ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
4819   ierr = PetscSectionGetFieldConstraintDof(section, point, f, &fcdof);CHKERRQ(ierr);
4820   ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
4821   a    = &array[foff];
4822   if (fcdof) {
4823     ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
4824     if (o >= 0) {
4825       for (k = 0; k < fdof; ++k) {
4826         if ((cind < fcdof) && (k == fcdofs[cind])) {
4827           fuse(&a[k], values[foffset+k]);
4828           ++cind;
4829         }
4830       }
4831     } else {
4832       for (k = fdof/fcomp-1; k >= 0; --k) {
4833         for (c = 0; c < fcomp; ++c) {
4834           if ((cind < fcdof) && (k*fcomp+c == fcdofs[cind])) {
4835             fuse(&a[(fdof/fcomp-1-k)*fcomp+c], values[foffset+k*fcomp+c]);
4836             ++cind;
4837           }
4838         }
4839       }
4840     }
4841   }
4842   *offset += fdof;
4843   PetscFunctionReturn(0);
4844 }
4845 
4846 #undef __FUNCT__
4847 #define __FUNCT__ "DMPlexVecSetClosure_Static"
4848 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecSetClosure_Static(DM dm, PetscSection section, Vec v, PetscInt point, const PetscScalar values[], InsertMode mode)
4849 {
4850   PetscScalar    *array;
4851   const PetscInt *cone, *coneO;
4852   PetscInt        pStart, pEnd, p, numPoints, off, dof;
4853   PetscErrorCode  ierr;
4854 
4855   PetscFunctionBeginHot;
4856   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
4857   ierr = DMPlexGetConeSize(dm, point, &numPoints);CHKERRQ(ierr);
4858   ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
4859   ierr = DMPlexGetConeOrientation(dm, point, &coneO);CHKERRQ(ierr);
4860   ierr = VecGetArray(v, &array);CHKERRQ(ierr);
4861   for (p = 0, off = 0; p <= numPoints; ++p, off += dof) {
4862     const PetscInt cp = !p ? point : cone[p-1];
4863     const PetscInt o  = !p ? 0     : coneO[p-1];
4864 
4865     if ((cp < pStart) || (cp >= pEnd)) {dof = 0; continue;}
4866     ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
4867     /* ADD_VALUES */
4868     {
4869       const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
4870       PetscScalar    *a;
4871       PetscInt        cdof, coff, cind = 0, k;
4872 
4873       ierr = PetscSectionGetConstraintDof(section, cp, &cdof);CHKERRQ(ierr);
4874       ierr = PetscSectionGetOffset(section, cp, &coff);CHKERRQ(ierr);
4875       a    = &array[coff];
4876       if (!cdof) {
4877         if (o >= 0) {
4878           for (k = 0; k < dof; ++k) {
4879             a[k] += values[off+k];
4880           }
4881         } else {
4882           for (k = 0; k < dof; ++k) {
4883             a[k] += values[off+dof-k-1];
4884           }
4885         }
4886       } else {
4887         ierr = PetscSectionGetConstraintIndices(section, cp, &cdofs);CHKERRQ(ierr);
4888         if (o >= 0) {
4889           for (k = 0; k < dof; ++k) {
4890             if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
4891             a[k] += values[off+k];
4892           }
4893         } else {
4894           for (k = 0; k < dof; ++k) {
4895             if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
4896             a[k] += values[off+dof-k-1];
4897           }
4898         }
4899       }
4900     }
4901   }
4902   ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
4903   PetscFunctionReturn(0);
4904 }
4905 
4906 #undef __FUNCT__
4907 #define __FUNCT__ "DMPlexVecSetClosure"
4908 /*@C
4909   DMPlexVecSetClosure - Set an array of the values on the closure of 'point'
4910 
4911   Not collective
4912 
4913   Input Parameters:
4914 + dm - The DM
4915 . section - The section describing the layout in v, or NULL to use the default section
4916 . v - The local vector
4917 . point - The sieve point in the DM
4918 . values - The array of values
4919 - mode - The insert mode, where INSERT_ALL_VALUES and ADD_ALL_VALUES also overwrite boundary conditions
4920 
4921   Fortran Notes:
4922   This routine is only available in Fortran 90, and you must include petsc.h90 in your code.
4923 
4924   Level: intermediate
4925 
4926 .seealso DMPlexVecGetClosure(), DMPlexMatSetClosure()
4927 @*/
4928 PetscErrorCode DMPlexVecSetClosure(DM dm, PetscSection section, Vec v, PetscInt point, const PetscScalar values[], InsertMode mode)
4929 {
4930   PetscSection    clSection;
4931   IS              clPoints;
4932   PetscScalar    *array;
4933   PetscInt       *points = NULL;
4934   const PetscInt *clp;
4935   PetscInt        depth, numFields, numPoints, p;
4936   PetscErrorCode  ierr;
4937 
4938   PetscFunctionBeginHot;
4939   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4940   if (!section) {ierr = DMGetDefaultSection(dm, &section);CHKERRQ(ierr);}
4941   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
4942   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
4943   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
4944   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
4945   if (depth == 1 && numFields < 2 && mode == ADD_VALUES) {
4946     ierr = DMPlexVecSetClosure_Static(dm, section, v, point, values, mode);CHKERRQ(ierr);
4947     PetscFunctionReturn(0);
4948   }
4949   /* Get points */
4950   ierr = PetscSectionGetClosureIndex(section, (PetscObject) dm, &clSection, &clPoints);CHKERRQ(ierr);
4951   if (!clPoints) {
4952     PetscInt pStart, pEnd, q;
4953 
4954     ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
4955     ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
4956     /* Compress out points not in the section */
4957     for (p = 0, q = 0; p < numPoints*2; p += 2) {
4958       if ((points[p] >= pStart) && (points[p] < pEnd)) {
4959         points[q*2]   = points[p];
4960         points[q*2+1] = points[p+1];
4961         ++q;
4962       }
4963     }
4964     numPoints = q;
4965   } else {
4966     PetscInt dof, off;
4967 
4968     ierr = PetscSectionGetDof(clSection, point, &dof);CHKERRQ(ierr);
4969     ierr = PetscSectionGetOffset(clSection, point, &off);CHKERRQ(ierr);
4970     ierr = ISGetIndices(clPoints, &clp);CHKERRQ(ierr);
4971     numPoints = dof/2;
4972     points    = (PetscInt *) &clp[off];
4973   }
4974   /* Get array */
4975   ierr = VecGetArray(v, &array);CHKERRQ(ierr);
4976   /* Get values */
4977   if (numFields > 0) {
4978     PetscInt offset = 0, fcomp, f;
4979     for (f = 0; f < numFields; ++f) {
4980       ierr = PetscSectionGetFieldComponents(section, f, &fcomp);CHKERRQ(ierr);
4981       switch (mode) {
4982       case INSERT_VALUES:
4983         for (p = 0; p < numPoints*2; p += 2) {
4984           const PetscInt point = points[p];
4985           const PetscInt o     = points[p+1];
4986           updatePointFields_private(section, point, o, f, fcomp, insert, PETSC_FALSE, values, &offset, array);
4987         } break;
4988       case INSERT_ALL_VALUES:
4989         for (p = 0; p < numPoints*2; p += 2) {
4990           const PetscInt point = points[p];
4991           const PetscInt o     = points[p+1];
4992           updatePointFields_private(section, point, o, f, fcomp, insert, PETSC_TRUE, values, &offset, array);
4993         } break;
4994       case INSERT_BC_VALUES:
4995         for (p = 0; p < numPoints*2; p += 2) {
4996           const PetscInt point = points[p];
4997           const PetscInt o     = points[p+1];
4998           updatePointFieldsBC_private(section, point, o, f, fcomp, insert, values, &offset, array);
4999         } break;
5000       case ADD_VALUES:
5001         for (p = 0; p < numPoints*2; p += 2) {
5002           const PetscInt point = points[p];
5003           const PetscInt o     = points[p+1];
5004           updatePointFields_private(section, point, o, f, fcomp, add, PETSC_FALSE, values, &offset, array);
5005         } break;
5006       case ADD_ALL_VALUES:
5007         for (p = 0; p < numPoints*2; p += 2) {
5008           const PetscInt point = points[p];
5009           const PetscInt o     = points[p+1];
5010           updatePointFields_private(section, point, o, f, fcomp, add, PETSC_TRUE, values, &offset, array);
5011         } break;
5012       default:
5013         SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %D", mode);
5014       }
5015     }
5016   } else {
5017     PetscInt dof, off;
5018 
5019     switch (mode) {
5020     case INSERT_VALUES:
5021       for (p = 0, off = 0; p < numPoints*2; p += 2, off += dof) {
5022         PetscInt o = points[p+1];
5023         ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5024         updatePoint_private(section, points[p], dof, insert, PETSC_FALSE, o, &values[off], array);
5025       } break;
5026     case INSERT_ALL_VALUES:
5027       for (p = 0, off = 0; p < numPoints*2; p += 2, off += dof) {
5028         PetscInt o = points[p+1];
5029         ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5030         updatePoint_private(section, points[p], dof, insert, PETSC_TRUE,  o, &values[off], array);
5031       } break;
5032     case INSERT_BC_VALUES:
5033       for (p = 0, off = 0; p < numPoints*2; p += 2, off += dof) {
5034         PetscInt o = points[p+1];
5035         ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5036         updatePointBC_private(section, points[p], dof, insert,  o, &values[off], array);
5037       } break;
5038     case ADD_VALUES:
5039       for (p = 0, off = 0; p < numPoints*2; p += 2, off += dof) {
5040         PetscInt o = points[p+1];
5041         ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5042         updatePoint_private(section, points[p], dof, add,    PETSC_FALSE, o, &values[off], array);
5043       } break;
5044     case ADD_ALL_VALUES:
5045       for (p = 0, off = 0; p < numPoints*2; p += 2, off += dof) {
5046         PetscInt o = points[p+1];
5047         ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5048         updatePoint_private(section, points[p], dof, add,    PETSC_TRUE,  o, &values[off], array);
5049       } break;
5050     default:
5051       SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %D", mode);
5052     }
5053   }
5054   /* Cleanup points */
5055   if (!clPoints) {ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);}
5056   else           {ierr = ISRestoreIndices(clPoints, &clp);CHKERRQ(ierr);}
5057   /* Cleanup array */
5058   ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
5059   PetscFunctionReturn(0);
5060 }
5061 
5062 #undef __FUNCT__
5063 #define __FUNCT__ "DMPlexVecSetFieldClosure_Internal"
5064 PetscErrorCode DMPlexVecSetFieldClosure_Internal(DM dm, PetscSection section, Vec v, PetscBool fieldActive[], PetscInt point, const PetscScalar values[], InsertMode mode)
5065 {
5066   PetscSection    clSection;
5067   IS              clPoints;
5068   PetscScalar    *array;
5069   PetscInt       *points = NULL;
5070   const PetscInt *clp;
5071   PetscInt        numFields, numPoints, p;
5072   PetscInt        offset = 0, fcomp, f;
5073   PetscErrorCode  ierr;
5074 
5075   PetscFunctionBeginHot;
5076   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5077   if (!section) {ierr = DMGetDefaultSection(dm, &section);CHKERRQ(ierr);}
5078   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
5079   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
5080   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5081   /* Get points */
5082   ierr = PetscSectionGetClosureIndex(section, (PetscObject) dm, &clSection, &clPoints);CHKERRQ(ierr);
5083   if (!clPoints) {
5084     PetscInt pStart, pEnd, q;
5085 
5086     ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
5087     ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
5088     /* Compress out points not in the section */
5089     for (p = 0, q = 0; p < numPoints*2; p += 2) {
5090       if ((points[p] >= pStart) && (points[p] < pEnd)) {
5091         points[q*2]   = points[p];
5092         points[q*2+1] = points[p+1];
5093         ++q;
5094       }
5095     }
5096     numPoints = q;
5097   } else {
5098     PetscInt dof, off;
5099 
5100     ierr = PetscSectionGetDof(clSection, point, &dof);CHKERRQ(ierr);
5101     ierr = PetscSectionGetOffset(clSection, point, &off);CHKERRQ(ierr);
5102     ierr = ISGetIndices(clPoints, &clp);CHKERRQ(ierr);
5103     numPoints = dof/2;
5104     points    = (PetscInt *) &clp[off];
5105   }
5106   /* Get array */
5107   ierr = VecGetArray(v, &array);CHKERRQ(ierr);
5108   /* Get values */
5109   for (f = 0; f < numFields; ++f) {
5110     ierr = PetscSectionGetFieldComponents(section, f, &fcomp);CHKERRQ(ierr);
5111     if (!fieldActive[f]) {
5112       for (p = 0; p < numPoints*2; p += 2) {
5113         PetscInt fdof;
5114         ierr = PetscSectionGetFieldDof(section, points[p], f, &fdof);CHKERRQ(ierr);
5115         offset += fdof;
5116       }
5117       continue;
5118     }
5119     switch (mode) {
5120     case INSERT_VALUES:
5121       for (p = 0; p < numPoints*2; p += 2) {
5122         const PetscInt point = points[p];
5123         const PetscInt o     = points[p+1];
5124         updatePointFields_private(section, point, o, f, fcomp, insert, PETSC_FALSE, values, &offset, array);
5125       } break;
5126     case INSERT_ALL_VALUES:
5127       for (p = 0; p < numPoints*2; p += 2) {
5128         const PetscInt point = points[p];
5129         const PetscInt o     = points[p+1];
5130         updatePointFields_private(section, point, o, f, fcomp, insert, PETSC_TRUE, values, &offset, array);
5131         } break;
5132     case INSERT_BC_VALUES:
5133       for (p = 0; p < numPoints*2; p += 2) {
5134         const PetscInt point = points[p];
5135         const PetscInt o     = points[p+1];
5136         updatePointFieldsBC_private(section, point, o, f, fcomp, insert, values, &offset, array);
5137       } break;
5138     case ADD_VALUES:
5139       for (p = 0; p < numPoints*2; p += 2) {
5140         const PetscInt point = points[p];
5141         const PetscInt o     = points[p+1];
5142         updatePointFields_private(section, point, o, f, fcomp, add, PETSC_FALSE, values, &offset, array);
5143       } break;
5144     case ADD_ALL_VALUES:
5145       for (p = 0; p < numPoints*2; p += 2) {
5146         const PetscInt point = points[p];
5147         const PetscInt o     = points[p+1];
5148         updatePointFields_private(section, point, o, f, fcomp, add, PETSC_TRUE, values, &offset, array);
5149       } break;
5150     default:
5151       SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %D", mode);
5152     }
5153   }
5154   /* Cleanup points */
5155   if (!clPoints) {ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);}
5156   else           {ierr = ISRestoreIndices(clPoints, &clp);CHKERRQ(ierr);}
5157   /* Cleanup array */
5158   ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
5159   PetscFunctionReturn(0);
5160 }
5161 
5162 #undef __FUNCT__
5163 #define __FUNCT__ "DMPlexPrintMatSetValues"
5164 PetscErrorCode DMPlexPrintMatSetValues(PetscViewer viewer, Mat A, PetscInt point, PetscInt numRIndices, const PetscInt rindices[], PetscInt numCIndices, const PetscInt cindices[], const PetscScalar values[])
5165 {
5166   PetscMPIInt    rank;
5167   PetscInt       i, j;
5168   PetscErrorCode ierr;
5169 
5170   PetscFunctionBegin;
5171   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRQ(ierr);
5172   ierr = PetscViewerASCIIPrintf(viewer, "[%D]mat for sieve point %D\n", rank, point);CHKERRQ(ierr);
5173   for (i = 0; i < numRIndices; i++) {ierr = PetscViewerASCIIPrintf(viewer, "[%D]mat row indices[%D] = %D\n", rank, i, rindices[i]);CHKERRQ(ierr);}
5174   for (i = 0; i < numCIndices; i++) {ierr = PetscViewerASCIIPrintf(viewer, "[%D]mat col indices[%D] = %D\n", rank, i, cindices[i]);CHKERRQ(ierr);}
5175   numCIndices = numCIndices ? numCIndices : numRIndices;
5176   for (i = 0; i < numRIndices; i++) {
5177     ierr = PetscViewerASCIIPrintf(viewer, "[%D]", rank);CHKERRQ(ierr);
5178     for (j = 0; j < numCIndices; j++) {
5179 #if defined(PETSC_USE_COMPLEX)
5180       ierr = PetscViewerASCIIPrintf(viewer, " (%g,%g)", (double)PetscRealPart(values[i*numCIndices+j]), (double)PetscImaginaryPart(values[i*numCIndices+j]));CHKERRQ(ierr);
5181 #else
5182       ierr = PetscViewerASCIIPrintf(viewer, " %g", (double)values[i*numCIndices+j]);CHKERRQ(ierr);
5183 #endif
5184     }
5185     ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
5186   }
5187   PetscFunctionReturn(0);
5188 }
5189 
5190 #undef __FUNCT__
5191 #define __FUNCT__ "indicesPoint_private"
5192 /* . off - The global offset of this point */
5193 PetscErrorCode indicesPoint_private(PetscSection section, PetscInt point, PetscInt off, PetscInt *loff, PetscBool setBC, PetscInt orientation, PetscInt indices[])
5194 {
5195   PetscInt        dof;    /* The number of unknowns on this point */
5196   PetscInt        cdof;   /* The number of constraints on this point */
5197   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
5198   PetscInt        cind = 0, k;
5199   PetscErrorCode  ierr;
5200 
5201   PetscFunctionBegin;
5202   ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
5203   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
5204   if (!cdof || setBC) {
5205     if (orientation >= 0) {
5206       for (k = 0; k < dof; ++k) indices[*loff+k] = off+k;
5207     } else {
5208       for (k = 0; k < dof; ++k) indices[*loff+dof-k-1] = off+k;
5209     }
5210   } else {
5211     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
5212     if (orientation >= 0) {
5213       for (k = 0; k < dof; ++k) {
5214         if ((cind < cdof) && (k == cdofs[cind])) {
5215           /* Insert check for returning constrained indices */
5216           indices[*loff+k] = -(off+k+1);
5217           ++cind;
5218         } else {
5219           indices[*loff+k] = off+k-cind;
5220         }
5221       }
5222     } else {
5223       for (k = 0; k < dof; ++k) {
5224         if ((cind < cdof) && (k == cdofs[cind])) {
5225           /* Insert check for returning constrained indices */
5226           indices[*loff+dof-k-1] = -(off+k+1);
5227           ++cind;
5228         } else {
5229           indices[*loff+dof-k-1] = off+k-cind;
5230         }
5231       }
5232     }
5233   }
5234   *loff += dof;
5235   PetscFunctionReturn(0);
5236 }
5237 
5238 #undef __FUNCT__
5239 #define __FUNCT__ "indicesPointFields_private"
5240 /* . off - The global offset of this point */
5241 PetscErrorCode indicesPointFields_private(PetscSection section, PetscInt point, PetscInt off, PetscInt foffs[], PetscBool setBC, PetscInt orientation, PetscInt indices[])
5242 {
5243   PetscInt       numFields, foff, f;
5244   PetscErrorCode ierr;
5245 
5246   PetscFunctionBegin;
5247   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5248   for (f = 0, foff = 0; f < numFields; ++f) {
5249     PetscInt        fdof, fcomp, cfdof;
5250     const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
5251     PetscInt        cind = 0, k, c;
5252 
5253     ierr = PetscSectionGetFieldComponents(section, f, &fcomp);CHKERRQ(ierr);
5254     ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
5255     ierr = PetscSectionGetFieldConstraintDof(section, point, f, &cfdof);CHKERRQ(ierr);
5256     if (!cfdof || setBC) {
5257       if (orientation >= 0) {
5258         for (k = 0; k < fdof; ++k) indices[foffs[f]+k] = off+foff+k;
5259       } else {
5260         for (k = fdof/fcomp-1; k >= 0; --k) {
5261           for (c = 0; c < fcomp; ++c) {
5262             indices[foffs[f]+k*fcomp+c] = off+foff+(fdof/fcomp-1-k)*fcomp+c;
5263           }
5264         }
5265       }
5266     } else {
5267       ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
5268       if (orientation >= 0) {
5269         for (k = 0; k < fdof; ++k) {
5270           if ((cind < cfdof) && (k == fcdofs[cind])) {
5271             indices[foffs[f]+k] = -(off+foff+k+1);
5272             ++cind;
5273           } else {
5274             indices[foffs[f]+k] = off+foff+k-cind;
5275           }
5276         }
5277       } else {
5278         for (k = fdof/fcomp-1; k >= 0; --k) {
5279           for (c = 0; c < fcomp; ++c) {
5280             if ((cind < cfdof) && ((fdof/fcomp-1-k)*fcomp+c == fcdofs[cind])) {
5281               indices[foffs[f]+k*fcomp+c] = -(off+foff+(fdof/fcomp-1-k)*fcomp+c+1);
5282               ++cind;
5283             } else {
5284               indices[foffs[f]+k*fcomp+c] = off+foff+(fdof/fcomp-1-k)*fcomp+c-cind;
5285             }
5286           }
5287         }
5288       }
5289     }
5290     foff     += fdof - cfdof;
5291     foffs[f] += fdof;
5292   }
5293   PetscFunctionReturn(0);
5294 }
5295 
5296 #undef __FUNCT__
5297 #define __FUNCT__ "DMPlexMatSetClosure"
5298 /*@C
5299   DMPlexMatSetClosure - Set an array of the values on the closure of 'point'
5300 
5301   Not collective
5302 
5303   Input Parameters:
5304 + dm - The DM
5305 . section - The section describing the layout in v, or NULL to use the default section
5306 . globalSection - The section describing the layout in v, or NULL to use the default global section
5307 . A - The matrix
5308 . point - The sieve point in the DM
5309 . values - The array of values
5310 - mode - The insert mode, where INSERT_ALL_VALUES and ADD_ALL_VALUES also overwrite boundary conditions
5311 
5312   Fortran Notes:
5313   This routine is only available in Fortran 90, and you must include petsc.h90 in your code.
5314 
5315   Level: intermediate
5316 
5317 .seealso DMPlexVecGetClosure(), DMPlexVecSetClosure()
5318 @*/
5319 PetscErrorCode DMPlexMatSetClosure(DM dm, PetscSection section, PetscSection globalSection, Mat A, PetscInt point, const PetscScalar values[], InsertMode mode)
5320 {
5321   DM_Plex        *mesh   = (DM_Plex*) dm->data;
5322   PetscSection    clSection;
5323   IS              clPoints;
5324   PetscInt       *points = NULL;
5325   const PetscInt *clp;
5326   PetscInt       *indices;
5327   PetscInt        offsets[32];
5328   PetscInt        numFields, numPoints, numIndices, dof, off, globalOff, pStart, pEnd, p, q, f;
5329   PetscErrorCode  ierr;
5330 
5331   PetscFunctionBegin;
5332   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5333   if (!section) {ierr = DMGetDefaultSection(dm, &section);CHKERRQ(ierr);}
5334   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
5335   if (!globalSection) {ierr = DMGetDefaultGlobalSection(dm, &globalSection);CHKERRQ(ierr);}
5336   PetscValidHeaderSpecific(globalSection, PETSC_SECTION_CLASSID, 3);
5337   PetscValidHeaderSpecific(A, MAT_CLASSID, 4);
5338   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5339   if (numFields > 31) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", numFields);
5340   ierr = PetscMemzero(offsets, 32 * sizeof(PetscInt));CHKERRQ(ierr);
5341   ierr = PetscSectionGetClosureIndex(section, (PetscObject) dm, &clSection, &clPoints);CHKERRQ(ierr);
5342   if (!clPoints) {
5343     ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
5344     /* Compress out points not in the section */
5345     ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
5346     for (p = 0, q = 0; p < numPoints*2; p += 2) {
5347       if ((points[p] >= pStart) && (points[p] < pEnd)) {
5348         points[q*2]   = points[p];
5349         points[q*2+1] = points[p+1];
5350         ++q;
5351       }
5352     }
5353     numPoints = q;
5354   } else {
5355     PetscInt dof, off;
5356 
5357     ierr = PetscSectionGetDof(clSection, point, &dof);CHKERRQ(ierr);
5358     numPoints = dof/2;
5359     ierr = PetscSectionGetOffset(clSection, point, &off);CHKERRQ(ierr);
5360     ierr = ISGetIndices(clPoints, &clp);CHKERRQ(ierr);
5361     points = (PetscInt *) &clp[off];
5362   }
5363   for (p = 0, numIndices = 0; p < numPoints*2; p += 2) {
5364     PetscInt fdof;
5365 
5366     ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5367     for (f = 0; f < numFields; ++f) {
5368       ierr          = PetscSectionGetFieldDof(section, points[p], f, &fdof);CHKERRQ(ierr);
5369       offsets[f+1] += fdof;
5370     }
5371     numIndices += dof;
5372   }
5373   for (f = 1; f < numFields; ++f) offsets[f+1] += offsets[f];
5374 
5375   if (numFields && offsets[numFields] != numIndices) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid size for closure %d should be %d", offsets[numFields], numIndices);
5376   ierr = DMGetWorkArray(dm, numIndices, PETSC_INT, &indices);CHKERRQ(ierr);
5377   if (numFields) {
5378     for (p = 0; p < numPoints*2; p += 2) {
5379       PetscInt o = points[p+1];
5380       ierr = PetscSectionGetOffset(globalSection, points[p], &globalOff);CHKERRQ(ierr);
5381       indicesPointFields_private(section, points[p], globalOff < 0 ? -(globalOff+1) : globalOff, offsets, PETSC_FALSE, o, indices);
5382     }
5383   } else {
5384     for (p = 0, off = 0; p < numPoints*2; p += 2) {
5385       PetscInt o = points[p+1];
5386       ierr = PetscSectionGetOffset(globalSection, points[p], &globalOff);CHKERRQ(ierr);
5387       indicesPoint_private(section, points[p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, o, indices);
5388     }
5389   }
5390   if (mesh->printSetValues) {ierr = DMPlexPrintMatSetValues(PETSC_VIEWER_STDOUT_SELF, A, point, numIndices, indices, 0, NULL, values);CHKERRQ(ierr);}
5391   ierr = MatSetValues(A, numIndices, indices, numIndices, indices, values, mode);
5392   if (ierr) {
5393     PetscMPIInt    rank;
5394     PetscErrorCode ierr2;
5395 
5396     ierr2 = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRQ(ierr2);
5397     ierr2 = (*PetscErrorPrintf)("[%D]ERROR in DMPlexMatSetClosure\n", rank);CHKERRQ(ierr2);
5398     ierr2 = DMPlexPrintMatSetValues(PETSC_VIEWER_STDERR_SELF, A, point, numIndices, indices, 0, NULL, values);CHKERRQ(ierr2);
5399     ierr2 = DMRestoreWorkArray(dm, numIndices, PETSC_INT, &indices);CHKERRQ(ierr2);
5400     CHKERRQ(ierr);
5401   }
5402   if (!clPoints) {
5403     ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
5404   } else {
5405     ierr = ISRestoreIndices(clPoints, &clp);CHKERRQ(ierr);
5406   }
5407   ierr = DMRestoreWorkArray(dm, numIndices, PETSC_INT, &indices);CHKERRQ(ierr);
5408   PetscFunctionReturn(0);
5409 }
5410 
5411 #undef __FUNCT__
5412 #define __FUNCT__ "DMPlexMatSetClosureRefined"
5413 PetscErrorCode DMPlexMatSetClosureRefined(DM dmf, PetscSection fsection, PetscSection globalFSection, DM dmc, PetscSection csection, PetscSection globalCSection, Mat A, PetscInt point, const PetscScalar values[], InsertMode mode)
5414 {
5415   DM_Plex        *mesh   = (DM_Plex*) dmf->data;
5416   PetscInt       *fpoints = NULL, *ftotpoints = NULL;
5417   PetscInt       *cpoints = NULL;
5418   PetscInt       *findices, *cindices;
5419   PetscInt        foffsets[32], coffsets[32];
5420   CellRefiner     cellRefiner;
5421   PetscInt        numFields, numSubcells, maxFPoints, numFPoints, numCPoints, numFIndices, numCIndices, dof, off, globalOff, pStart, pEnd, p, q, r, s, f;
5422   PetscErrorCode  ierr;
5423 
5424   PetscFunctionBegin;
5425   PetscValidHeaderSpecific(dmf, DM_CLASSID, 1);
5426   PetscValidHeaderSpecific(dmc, DM_CLASSID, 4);
5427   if (!fsection) {ierr = DMGetDefaultSection(dmf, &fsection);CHKERRQ(ierr);}
5428   PetscValidHeaderSpecific(fsection, PETSC_SECTION_CLASSID, 2);
5429   if (!csection) {ierr = DMGetDefaultSection(dmc, &csection);CHKERRQ(ierr);}
5430   PetscValidHeaderSpecific(csection, PETSC_SECTION_CLASSID, 5);
5431   if (!globalFSection) {ierr = DMGetDefaultGlobalSection(dmf, &globalFSection);CHKERRQ(ierr);}
5432   PetscValidHeaderSpecific(globalFSection, PETSC_SECTION_CLASSID, 3);
5433   if (!globalCSection) {ierr = DMGetDefaultGlobalSection(dmc, &globalCSection);CHKERRQ(ierr);}
5434   PetscValidHeaderSpecific(globalCSection, PETSC_SECTION_CLASSID, 6);
5435   PetscValidHeaderSpecific(A, MAT_CLASSID, 7);
5436   ierr = PetscSectionGetNumFields(fsection, &numFields);CHKERRQ(ierr);
5437   if (numFields > 31) SETERRQ1(PetscObjectComm((PetscObject)dmf), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", numFields);
5438   ierr = PetscMemzero(foffsets, 32 * sizeof(PetscInt));CHKERRQ(ierr);
5439   ierr = PetscMemzero(coffsets, 32 * sizeof(PetscInt));CHKERRQ(ierr);
5440   /* Column indices */
5441   ierr = DMPlexGetTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
5442   maxFPoints = numCPoints;
5443   /* Compress out points not in the section */
5444   /*   TODO: Squeeze out points with 0 dof as well */
5445   ierr = PetscSectionGetChart(csection, &pStart, &pEnd);CHKERRQ(ierr);
5446   for (p = 0, q = 0; p < numCPoints*2; p += 2) {
5447     if ((cpoints[p] >= pStart) && (cpoints[p] < pEnd)) {
5448       cpoints[q*2]   = cpoints[p];
5449       cpoints[q*2+1] = cpoints[p+1];
5450       ++q;
5451     }
5452   }
5453   numCPoints = q;
5454   for (p = 0, numCIndices = 0; p < numCPoints*2; p += 2) {
5455     PetscInt fdof;
5456 
5457     ierr = PetscSectionGetDof(csection, cpoints[p], &dof);CHKERRQ(ierr);
5458     if (!dof) continue;
5459     for (f = 0; f < numFields; ++f) {
5460       ierr           = PetscSectionGetFieldDof(csection, cpoints[p], f, &fdof);CHKERRQ(ierr);
5461       coffsets[f+1] += fdof;
5462     }
5463     numCIndices += dof;
5464   }
5465   for (f = 1; f < numFields; ++f) coffsets[f+1] += coffsets[f];
5466   /* Row indices */
5467   ierr = DMPlexGetCellRefiner_Internal(dmc, &cellRefiner);CHKERRQ(ierr);
5468   ierr = CellRefinerGetAffineTransforms_Internal(cellRefiner, &numSubcells, NULL, NULL, NULL);CHKERRQ(ierr);
5469   ierr = DMGetWorkArray(dmf, maxFPoints*2*numSubcells, PETSC_INT, &ftotpoints);CHKERRQ(ierr);
5470   for (r = 0, q = 0; r < numSubcells; ++r) {
5471     /* TODO Map from coarse to fine cells */
5472     ierr = DMPlexGetTransitiveClosure(dmf, point*numSubcells + r, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
5473     /* Compress out points not in the section */
5474     ierr = PetscSectionGetChart(fsection, &pStart, &pEnd);CHKERRQ(ierr);
5475     for (p = 0; p < numFPoints*2; p += 2) {
5476       if ((fpoints[p] >= pStart) && (fpoints[p] < pEnd)) {
5477         ierr = PetscSectionGetDof(fsection, fpoints[p], &dof);CHKERRQ(ierr);
5478         if (!dof) continue;
5479         for (s = 0; s < q; ++s) if (fpoints[p] == ftotpoints[s*2]) break;
5480         if (s < q) continue;
5481         ftotpoints[q*2]   = fpoints[p];
5482         ftotpoints[q*2+1] = fpoints[p+1];
5483         ++q;
5484       }
5485     }
5486     ierr = DMPlexRestoreTransitiveClosure(dmf, point, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
5487   }
5488   numFPoints = q;
5489   for (p = 0, numFIndices = 0; p < numFPoints*2; p += 2) {
5490     PetscInt fdof;
5491 
5492     ierr = PetscSectionGetDof(fsection, ftotpoints[p], &dof);CHKERRQ(ierr);
5493     if (!dof) continue;
5494     for (f = 0; f < numFields; ++f) {
5495       ierr           = PetscSectionGetFieldDof(fsection, ftotpoints[p], f, &fdof);CHKERRQ(ierr);
5496       foffsets[f+1] += fdof;
5497     }
5498     numFIndices += dof;
5499   }
5500   for (f = 1; f < numFields; ++f) foffsets[f+1] += foffsets[f];
5501 
5502   if (numFields && foffsets[numFields] != numFIndices) SETERRQ2(PetscObjectComm((PetscObject)dmf), PETSC_ERR_PLIB, "Invalid size for closure %d should be %d", foffsets[numFields], numFIndices);
5503   if (numFields && coffsets[numFields] != numCIndices) SETERRQ2(PetscObjectComm((PetscObject)dmc), PETSC_ERR_PLIB, "Invalid size for closure %d should be %d", coffsets[numFields], numCIndices);
5504   ierr = DMGetWorkArray(dmf, numFIndices, PETSC_INT, &findices);CHKERRQ(ierr);
5505   ierr = DMGetWorkArray(dmc, numCIndices, PETSC_INT, &cindices);CHKERRQ(ierr);
5506   if (numFields) {
5507     for (p = 0; p < numFPoints*2; p += 2) {
5508       PetscInt o = ftotpoints[p+1];
5509       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[p], &globalOff);CHKERRQ(ierr);
5510       indicesPointFields_private(fsection, ftotpoints[p], globalOff < 0 ? -(globalOff+1) : globalOff, foffsets, PETSC_FALSE, o, findices);
5511     }
5512     for (p = 0; p < numCPoints*2; p += 2) {
5513       PetscInt o = cpoints[p+1];
5514       ierr = PetscSectionGetOffset(globalCSection, cpoints[p], &globalOff);CHKERRQ(ierr);
5515       indicesPointFields_private(csection, cpoints[p], globalOff < 0 ? -(globalOff+1) : globalOff, coffsets, PETSC_FALSE, o, cindices);
5516     }
5517   } else {
5518     for (p = 0, off = 0; p < numFPoints*2; p += 2) {
5519       PetscInt o = ftotpoints[p+1];
5520       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[p], &globalOff);CHKERRQ(ierr);
5521       indicesPoint_private(fsection, ftotpoints[p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, o, findices);
5522     }
5523     for (p = 0, off = 0; p < numCPoints*2; p += 2) {
5524       PetscInt o = cpoints[p+1];
5525       ierr = PetscSectionGetOffset(globalCSection, cpoints[p], &globalOff);CHKERRQ(ierr);
5526       indicesPoint_private(csection, cpoints[p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, o, cindices);
5527     }
5528   }
5529   if (mesh->printSetValues) {ierr = DMPlexPrintMatSetValues(PETSC_VIEWER_STDOUT_SELF, A, point, numFIndices, findices, numCIndices, cindices, values);CHKERRQ(ierr);}
5530   ierr = MatSetValues(A, numFIndices, findices, numCIndices, cindices, values, mode);
5531   if (ierr) {
5532     PetscMPIInt    rank;
5533     PetscErrorCode ierr2;
5534 
5535     ierr2 = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRQ(ierr2);
5536     ierr2 = (*PetscErrorPrintf)("[%D]ERROR in DMPlexMatSetClosure\n", rank);CHKERRQ(ierr2);
5537     ierr2 = DMPlexPrintMatSetValues(PETSC_VIEWER_STDERR_SELF, A, point, numFIndices, findices, numCIndices, cindices, values);CHKERRQ(ierr2);
5538     ierr2 = DMRestoreWorkArray(dmf, numFIndices, PETSC_INT, &findices);CHKERRQ(ierr2);
5539     ierr2 = DMRestoreWorkArray(dmc, numCIndices, PETSC_INT, &cindices);CHKERRQ(ierr2);
5540     CHKERRQ(ierr);
5541   }
5542   ierr = DMRestoreWorkArray(dmf, numCPoints*2*4, PETSC_INT, &ftotpoints);CHKERRQ(ierr);
5543   ierr = DMPlexRestoreTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
5544   ierr = DMRestoreWorkArray(dmf, numFIndices, PETSC_INT, &findices);CHKERRQ(ierr);
5545   ierr = DMRestoreWorkArray(dmc, numCIndices, PETSC_INT, &cindices);CHKERRQ(ierr);
5546   PetscFunctionReturn(0);
5547 }
5548 
5549 #undef __FUNCT__
5550 #define __FUNCT__ "DMPlexGetHybridBounds"
5551 /*@
5552   DMPlexGetHybridBounds - Get the first mesh point of each dimension which is a hybrid
5553 
5554   Input Parameter:
5555 . dm - The DMPlex object
5556 
5557   Output Parameters:
5558 + cMax - The first hybrid cell
5559 . fMax - The first hybrid face
5560 . eMax - The first hybrid edge
5561 - vMax - The first hybrid vertex
5562 
5563   Level: developer
5564 
5565 .seealso DMPlexCreateHybridMesh(), DMPlexSetHybridBounds()
5566 @*/
5567 PetscErrorCode DMPlexGetHybridBounds(DM dm, PetscInt *cMax, PetscInt *fMax, PetscInt *eMax, PetscInt *vMax)
5568 {
5569   DM_Plex       *mesh = (DM_Plex*) dm->data;
5570   PetscInt       dim;
5571   PetscErrorCode ierr;
5572 
5573   PetscFunctionBegin;
5574   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5575   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
5576   if (cMax) *cMax = mesh->hybridPointMax[dim];
5577   if (fMax) *fMax = mesh->hybridPointMax[dim-1];
5578   if (eMax) *eMax = mesh->hybridPointMax[1];
5579   if (vMax) *vMax = mesh->hybridPointMax[0];
5580   PetscFunctionReturn(0);
5581 }
5582 
5583 #undef __FUNCT__
5584 #define __FUNCT__ "DMPlexSetHybridBounds"
5585 /*@
5586   DMPlexSetHybridBounds - Set the first mesh point of each dimension which is a hybrid
5587 
5588   Input Parameters:
5589 . dm   - The DMPlex object
5590 . cMax - The first hybrid cell
5591 . fMax - The first hybrid face
5592 . eMax - The first hybrid edge
5593 - vMax - The first hybrid vertex
5594 
5595   Level: developer
5596 
5597 .seealso DMPlexCreateHybridMesh(), DMPlexGetHybridBounds()
5598 @*/
5599 PetscErrorCode DMPlexSetHybridBounds(DM dm, PetscInt cMax, PetscInt fMax, PetscInt eMax, PetscInt vMax)
5600 {
5601   DM_Plex       *mesh = (DM_Plex*) dm->data;
5602   PetscInt       dim;
5603   PetscErrorCode ierr;
5604 
5605   PetscFunctionBegin;
5606   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5607   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
5608   if (cMax >= 0) mesh->hybridPointMax[dim]   = cMax;
5609   if (fMax >= 0) mesh->hybridPointMax[dim-1] = fMax;
5610   if (eMax >= 0) mesh->hybridPointMax[1]     = eMax;
5611   if (vMax >= 0) mesh->hybridPointMax[0]     = vMax;
5612   PetscFunctionReturn(0);
5613 }
5614 
5615 #undef __FUNCT__
5616 #define __FUNCT__ "DMPlexGetVTKCellHeight"
5617 PetscErrorCode DMPlexGetVTKCellHeight(DM dm, PetscInt *cellHeight)
5618 {
5619   DM_Plex *mesh = (DM_Plex*) dm->data;
5620 
5621   PetscFunctionBegin;
5622   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5623   PetscValidPointer(cellHeight, 2);
5624   *cellHeight = mesh->vtkCellHeight;
5625   PetscFunctionReturn(0);
5626 }
5627 
5628 #undef __FUNCT__
5629 #define __FUNCT__ "DMPlexSetVTKCellHeight"
5630 PetscErrorCode DMPlexSetVTKCellHeight(DM dm, PetscInt cellHeight)
5631 {
5632   DM_Plex *mesh = (DM_Plex*) dm->data;
5633 
5634   PetscFunctionBegin;
5635   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5636   mesh->vtkCellHeight = cellHeight;
5637   PetscFunctionReturn(0);
5638 }
5639 
5640 #undef __FUNCT__
5641 #define __FUNCT__ "DMPlexCreateNumbering_Private"
5642 /* We can easily have a form that takes an IS instead */
5643 static PetscErrorCode DMPlexCreateNumbering_Private(DM dm, PetscInt pStart, PetscInt pEnd, PetscInt shift, PetscInt *globalSize, PetscSF sf, IS *numbering)
5644 {
5645   PetscSection   section, globalSection;
5646   PetscInt      *numbers, p;
5647   PetscErrorCode ierr;
5648 
5649   PetscFunctionBegin;
5650   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &section);CHKERRQ(ierr);
5651   ierr = PetscSectionSetChart(section, pStart, pEnd);CHKERRQ(ierr);
5652   for (p = pStart; p < pEnd; ++p) {
5653     ierr = PetscSectionSetDof(section, p, 1);CHKERRQ(ierr);
5654   }
5655   ierr = PetscSectionSetUp(section);CHKERRQ(ierr);
5656   ierr = PetscSectionCreateGlobalSection(section, sf, PETSC_FALSE, &globalSection);CHKERRQ(ierr);
5657   ierr = PetscMalloc1((pEnd - pStart), &numbers);CHKERRQ(ierr);
5658   for (p = pStart; p < pEnd; ++p) {
5659     ierr = PetscSectionGetOffset(globalSection, p, &numbers[p-pStart]);CHKERRQ(ierr);
5660     if (numbers[p-pStart] < 0) numbers[p-pStart] -= shift;
5661     else                       numbers[p-pStart] += shift;
5662   }
5663   ierr = ISCreateGeneral(PetscObjectComm((PetscObject) dm), pEnd - pStart, numbers, PETSC_OWN_POINTER, numbering);CHKERRQ(ierr);
5664   if (globalSize) {
5665     PetscLayout layout;
5666     ierr = PetscSectionGetPointLayout(PetscObjectComm((PetscObject) dm), globalSection, &layout);CHKERRQ(ierr);
5667     ierr = PetscLayoutGetSize(layout, globalSize);CHKERRQ(ierr);
5668     ierr = PetscLayoutDestroy(&layout);CHKERRQ(ierr);
5669   }
5670   ierr = PetscSectionDestroy(&section);CHKERRQ(ierr);
5671   ierr = PetscSectionDestroy(&globalSection);CHKERRQ(ierr);
5672   PetscFunctionReturn(0);
5673 }
5674 
5675 #undef __FUNCT__
5676 #define __FUNCT__ "DMPlexGetCellNumbering"
5677 PetscErrorCode DMPlexGetCellNumbering(DM dm, IS *globalCellNumbers)
5678 {
5679   DM_Plex       *mesh = (DM_Plex*) dm->data;
5680   PetscInt       cellHeight, cStart, cEnd, cMax;
5681   PetscErrorCode ierr;
5682 
5683   PetscFunctionBegin;
5684   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5685   if (!mesh->globalCellNumbers) {
5686     ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
5687     ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
5688     ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
5689     if (cMax >= 0) cEnd = PetscMin(cEnd, cMax);
5690     ierr = DMPlexCreateNumbering_Private(dm, cStart, cEnd, 0, NULL, dm->sf, &mesh->globalCellNumbers);CHKERRQ(ierr);
5691   }
5692   *globalCellNumbers = mesh->globalCellNumbers;
5693   PetscFunctionReturn(0);
5694 }
5695 
5696 #undef __FUNCT__
5697 #define __FUNCT__ "DMPlexGetVertexNumbering"
5698 PetscErrorCode DMPlexGetVertexNumbering(DM dm, IS *globalVertexNumbers)
5699 {
5700   DM_Plex       *mesh = (DM_Plex*) dm->data;
5701   PetscInt       vStart, vEnd, vMax;
5702   PetscErrorCode ierr;
5703 
5704   PetscFunctionBegin;
5705   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5706   if (!mesh->globalVertexNumbers) {
5707     ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
5708     ierr = DMPlexGetHybridBounds(dm, NULL, NULL, NULL, &vMax);CHKERRQ(ierr);
5709     if (vMax >= 0) vEnd = PetscMin(vEnd, vMax);
5710     ierr = DMPlexCreateNumbering_Private(dm, vStart, vEnd, 0, NULL, dm->sf, &mesh->globalVertexNumbers);CHKERRQ(ierr);
5711   }
5712   *globalVertexNumbers = mesh->globalVertexNumbers;
5713   PetscFunctionReturn(0);
5714 }
5715 
5716 #undef __FUNCT__
5717 #define __FUNCT__ "DMPlexCreatePointNumbering"
5718 PetscErrorCode DMPlexCreatePointNumbering(DM dm, IS *globalPointNumbers)
5719 {
5720   IS             nums[4];
5721   PetscInt       depths[4];
5722   PetscInt       depth, d, shift = 0;
5723   PetscErrorCode ierr;
5724 
5725   PetscFunctionBegin;
5726   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5727   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
5728   depths[0] = depth; depths[1] = 0;
5729   for (d = 2; d <= depth; ++d) depths[d] = depth-d+1;
5730   for (d = 0; d <= depth; ++d) {
5731     PetscInt pStart, pEnd, gsize;
5732 
5733     ierr = DMPlexGetDepthStratum(dm, depths[d], &pStart, &pEnd);CHKERRQ(ierr);
5734     ierr = DMPlexCreateNumbering_Private(dm, pStart, pEnd, shift, &gsize, dm->sf, &nums[d]);CHKERRQ(ierr);
5735     shift += gsize;
5736   }
5737   ierr = ISConcatenate(PetscObjectComm((PetscObject) dm), depth+1, nums, globalPointNumbers);
5738   for (d = 0; d <= depth; ++d) {ierr = ISDestroy(&nums[d]);CHKERRQ(ierr);}
5739   PetscFunctionReturn(0);
5740 }
5741 
5742 
5743 #undef __FUNCT__
5744 #define __FUNCT__ "PetscSectionCreateGlobalSectionLabel"
5745 /*@C
5746   PetscSectionCreateGlobalSectionLabel - Create a section describing the global field layout using
5747   the local section and an SF describing the section point overlap.
5748 
5749   Input Parameters:
5750   + s - The PetscSection for the local field layout
5751   . sf - The SF describing parallel layout of the section points
5752   . includeConstraints - By default this is PETSC_FALSE, meaning that the global field vector will not possess constrained dofs
5753   . label - The label specifying the points
5754   - labelValue - The label stratum specifying the points
5755 
5756   Output Parameter:
5757   . gsection - The PetscSection for the global field layout
5758 
5759   Note: This gives negative sizes and offsets to points not owned by this process
5760 
5761   Level: developer
5762 
5763 .seealso: PetscSectionCreate()
5764 @*/
5765 PetscErrorCode PetscSectionCreateGlobalSectionLabel(PetscSection s, PetscSF sf, PetscBool includeConstraints, DMLabel label, PetscInt labelValue, PetscSection *gsection)
5766 {
5767   PetscInt      *neg = NULL, *tmpOff = NULL;
5768   PetscInt       pStart, pEnd, p, dof, cdof, off, globalOff = 0, nroots;
5769   PetscErrorCode ierr;
5770 
5771   PetscFunctionBegin;
5772   ierr = PetscSectionCreate(PetscObjectComm((PetscObject) s), gsection);CHKERRQ(ierr);
5773   ierr = PetscSectionGetChart(s, &pStart, &pEnd);CHKERRQ(ierr);
5774   ierr = PetscSectionSetChart(*gsection, pStart, pEnd);CHKERRQ(ierr);
5775   ierr = PetscSFGetGraph(sf, &nroots, NULL, NULL, NULL);CHKERRQ(ierr);
5776   if (nroots >= 0) {
5777     if (nroots < pEnd-pStart) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "PetscSF nroots %d < %d section size", nroots, pEnd-pStart);
5778     ierr = PetscCalloc1(nroots, &neg);CHKERRQ(ierr);
5779     if (nroots > pEnd-pStart) {
5780       ierr = PetscCalloc1(nroots, &tmpOff);CHKERRQ(ierr);
5781     } else {
5782       tmpOff = &(*gsection)->atlasDof[-pStart];
5783     }
5784   }
5785   /* Mark ghost points with negative dof */
5786   for (p = pStart; p < pEnd; ++p) {
5787     PetscInt value;
5788 
5789     ierr = DMLabelGetValue(label, p, &value);CHKERRQ(ierr);
5790     if (value != labelValue) continue;
5791     ierr = PetscSectionGetDof(s, p, &dof);CHKERRQ(ierr);
5792     ierr = PetscSectionSetDof(*gsection, p, dof);CHKERRQ(ierr);
5793     ierr = PetscSectionGetConstraintDof(s, p, &cdof);CHKERRQ(ierr);
5794     if (!includeConstraints && cdof > 0) {ierr = PetscSectionSetConstraintDof(*gsection, p, cdof);CHKERRQ(ierr);}
5795     if (neg) neg[p] = -(dof+1);
5796   }
5797   ierr = PetscSectionSetUpBC(*gsection);CHKERRQ(ierr);
5798   if (nroots >= 0) {
5799     ierr = PetscSFBcastBegin(sf, MPIU_INT, neg, tmpOff);CHKERRQ(ierr);
5800     ierr = PetscSFBcastEnd(sf, MPIU_INT, neg, tmpOff);CHKERRQ(ierr);
5801     if (nroots > pEnd-pStart) {
5802       for (p = pStart; p < pEnd; ++p) {if (tmpOff[p] < 0) (*gsection)->atlasDof[p-pStart] = tmpOff[p];}
5803     }
5804   }
5805   /* Calculate new sizes, get proccess offset, and calculate point offsets */
5806   for (p = 0, off = 0; p < pEnd-pStart; ++p) {
5807     cdof = (!includeConstraints && s->bc) ? s->bc->atlasDof[p] : 0;
5808     (*gsection)->atlasOff[p] = off;
5809     off += (*gsection)->atlasDof[p] > 0 ? (*gsection)->atlasDof[p]-cdof : 0;
5810   }
5811   ierr       = MPI_Scan(&off, &globalOff, 1, MPIU_INT, MPI_SUM, PetscObjectComm((PetscObject) s));CHKERRQ(ierr);
5812   globalOff -= off;
5813   for (p = 0, off = 0; p < pEnd-pStart; ++p) {
5814     (*gsection)->atlasOff[p] += globalOff;
5815     if (neg) neg[p] = -((*gsection)->atlasOff[p]+1);
5816   }
5817   /* Put in negative offsets for ghost points */
5818   if (nroots >= 0) {
5819     ierr = PetscSFBcastBegin(sf, MPIU_INT, neg, tmpOff);CHKERRQ(ierr);
5820     ierr = PetscSFBcastEnd(sf, MPIU_INT, neg, tmpOff);CHKERRQ(ierr);
5821     if (nroots > pEnd-pStart) {
5822       for (p = pStart; p < pEnd; ++p) {if (tmpOff[p] < 0) (*gsection)->atlasOff[p-pStart] = tmpOff[p];}
5823     }
5824   }
5825   if (nroots >= 0 && nroots > pEnd-pStart) {ierr = PetscFree(tmpOff);CHKERRQ(ierr);}
5826   ierr = PetscFree(neg);CHKERRQ(ierr);
5827   PetscFunctionReturn(0);
5828 }
5829 
5830 #undef __FUNCT__
5831 #define __FUNCT__ "DMPlexCheckSymmetry"
5832 /*@
5833   DMPlexCheckSymmetry - Check that the adjacency information in the mesh is symmetric.
5834 
5835   Input Parameters:
5836   + dm - The DMPlex object
5837 
5838   Note: This is a useful diagnostic when creating meshes programmatically.
5839 
5840   Level: developer
5841 
5842 .seealso: DMCreate(), DMCheckSkeleton(), DMCheckFaces()
5843 @*/
5844 PetscErrorCode DMPlexCheckSymmetry(DM dm)
5845 {
5846   PetscSection    coneSection, supportSection;
5847   const PetscInt *cone, *support;
5848   PetscInt        coneSize, c, supportSize, s;
5849   PetscInt        pStart, pEnd, p, csize, ssize;
5850   PetscErrorCode  ierr;
5851 
5852   PetscFunctionBegin;
5853   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5854   ierr = DMPlexGetConeSection(dm, &coneSection);CHKERRQ(ierr);
5855   ierr = DMPlexGetSupportSection(dm, &supportSection);CHKERRQ(ierr);
5856   /* Check that point p is found in the support of its cone points, and vice versa */
5857   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
5858   for (p = pStart; p < pEnd; ++p) {
5859     ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
5860     ierr = DMPlexGetCone(dm, p, &cone);CHKERRQ(ierr);
5861     for (c = 0; c < coneSize; ++c) {
5862       PetscBool dup = PETSC_FALSE;
5863       PetscInt  d;
5864       for (d = c-1; d >= 0; --d) {
5865         if (cone[c] == cone[d]) {dup = PETSC_TRUE; break;}
5866       }
5867       ierr = DMPlexGetSupportSize(dm, cone[c], &supportSize);CHKERRQ(ierr);
5868       ierr = DMPlexGetSupport(dm, cone[c], &support);CHKERRQ(ierr);
5869       for (s = 0; s < supportSize; ++s) {
5870         if (support[s] == p) break;
5871       }
5872       if ((s >= supportSize) || (dup && (support[s+1] != p))) {
5873         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %d cone: ", p);
5874         for (s = 0; s < coneSize; ++s) {
5875           ierr = PetscPrintf(PETSC_COMM_SELF, "%d, ", cone[s]);
5876         }
5877         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");
5878         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %d support: ", cone[c]);
5879         for (s = 0; s < supportSize; ++s) {
5880           ierr = PetscPrintf(PETSC_COMM_SELF, "%d, ", support[s]);
5881         }
5882         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");
5883         if (dup) {
5884           SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %d not repeatedly found in support of repeated cone point %d", p, cone[c]);
5885         } else {
5886           SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %d not found in support of cone point %d", p, cone[c]);
5887         }
5888       }
5889     }
5890     ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
5891     ierr = DMPlexGetSupport(dm, p, &support);CHKERRQ(ierr);
5892     for (s = 0; s < supportSize; ++s) {
5893       ierr = DMPlexGetConeSize(dm, support[s], &coneSize);CHKERRQ(ierr);
5894       ierr = DMPlexGetCone(dm, support[s], &cone);CHKERRQ(ierr);
5895       for (c = 0; c < coneSize; ++c) {
5896         if (cone[c] == p) break;
5897       }
5898       if (c >= coneSize) {
5899         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %d support: ", p);
5900         for (c = 0; c < supportSize; ++c) {
5901           ierr = PetscPrintf(PETSC_COMM_SELF, "%d, ", support[c]);
5902         }
5903         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");
5904         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %d cone: ", support[s]);
5905         for (c = 0; c < coneSize; ++c) {
5906           ierr = PetscPrintf(PETSC_COMM_SELF, "%d, ", cone[c]);
5907         }
5908         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");
5909         SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %d not found in cone of support point %d", p, support[s]);
5910       }
5911     }
5912   }
5913   ierr = PetscSectionGetStorageSize(coneSection, &csize);CHKERRQ(ierr);
5914   ierr = PetscSectionGetStorageSize(supportSection, &ssize);CHKERRQ(ierr);
5915   if (csize != ssize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "Total cone size %d != Total support size %d", csize, ssize);
5916   PetscFunctionReturn(0);
5917 }
5918 
5919 #undef __FUNCT__
5920 #define __FUNCT__ "DMPlexCheckSkeleton"
5921 /*@
5922   DMPlexCheckSkeleton - Check that each cell has the correct number of vertices
5923 
5924   Input Parameters:
5925 + dm - The DMPlex object
5926 . isSimplex - Are the cells simplices or tensor products
5927 - cellHeight - Normally 0
5928 
5929   Note: This is a useful diagnostic when creating meshes programmatically.
5930 
5931   Level: developer
5932 
5933 .seealso: DMCreate(), DMCheckSymmetry(), DMCheckFaces()
5934 @*/
5935 PetscErrorCode DMPlexCheckSkeleton(DM dm, PetscBool isSimplex, PetscInt cellHeight)
5936 {
5937   PetscInt       dim, numCorners, numHybridCorners, vStart, vEnd, cStart, cEnd, cMax, c;
5938   PetscErrorCode ierr;
5939 
5940   PetscFunctionBegin;
5941   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5942   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
5943   switch (dim) {
5944   case 1: numCorners = isSimplex ? 2 : 2; numHybridCorners = isSimplex ? 2 : 2; break;
5945   case 2: numCorners = isSimplex ? 3 : 4; numHybridCorners = isSimplex ? 4 : 4; break;
5946   case 3: numCorners = isSimplex ? 4 : 8; numHybridCorners = isSimplex ? 6 : 8; break;
5947   default:
5948     SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_OUTOFRANGE, "Cannot handle meshes of dimension %d", dim);
5949   }
5950   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
5951   ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
5952   ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
5953   cMax = cMax >= 0 ? cMax : cEnd;
5954   for (c = cStart; c < cMax; ++c) {
5955     PetscInt *closure = NULL, closureSize, cl, coneSize = 0;
5956 
5957     ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
5958     for (cl = 0; cl < closureSize*2; cl += 2) {
5959       const PetscInt p = closure[cl];
5960       if ((p >= vStart) && (p < vEnd)) ++coneSize;
5961     }
5962     ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
5963     if (coneSize != numCorners) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %d has  %d vertices != %d", c, coneSize, numCorners);
5964   }
5965   for (c = cMax; c < cEnd; ++c) {
5966     PetscInt *closure = NULL, closureSize, cl, coneSize = 0;
5967 
5968     ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
5969     for (cl = 0; cl < closureSize*2; cl += 2) {
5970       const PetscInt p = closure[cl];
5971       if ((p >= vStart) && (p < vEnd)) ++coneSize;
5972     }
5973     ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
5974     if (coneSize > numHybridCorners) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Hybrid cell %d has  %d vertices > %d", c, coneSize, numHybridCorners);
5975   }
5976   PetscFunctionReturn(0);
5977 }
5978 
5979 #undef __FUNCT__
5980 #define __FUNCT__ "DMPlexCheckFaces"
5981 /*@
5982   DMPlexCheckFaces - Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type
5983 
5984   Input Parameters:
5985 + dm - The DMPlex object
5986 . isSimplex - Are the cells simplices or tensor products
5987 - cellHeight - Normally 0
5988 
5989   Note: This is a useful diagnostic when creating meshes programmatically.
5990 
5991   Level: developer
5992 
5993 .seealso: DMCreate(), DMCheckSymmetry(), DMCheckSkeleton()
5994 @*/
5995 PetscErrorCode DMPlexCheckFaces(DM dm, PetscBool isSimplex, PetscInt cellHeight)
5996 {
5997   PetscInt       pMax[4];
5998   PetscInt       dim, vStart, vEnd, cStart, cEnd, c, h;
5999   PetscErrorCode ierr;
6000 
6001   PetscFunctionBegin;
6002   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6003   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
6004   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
6005   ierr = DMPlexGetHybridBounds(dm, &pMax[dim], &pMax[dim-1], &pMax[1], &pMax[0]);CHKERRQ(ierr);
6006   for (h = cellHeight; h < dim; ++h) {
6007     ierr = DMPlexGetHeightStratum(dm, h, &cStart, &cEnd);CHKERRQ(ierr);
6008     for (c = cStart; c < cEnd; ++c) {
6009       const PetscInt *cone, *ornt, *faces;
6010       PetscInt        numFaces, faceSize, coneSize,f;
6011       PetscInt       *closure = NULL, closureSize, cl, numCorners = 0;
6012 
6013       if (pMax[dim-h] >= 0 && c >= pMax[dim-h]) continue;
6014       ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
6015       ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
6016       ierr = DMPlexGetConeOrientation(dm, c, &ornt);CHKERRQ(ierr);
6017       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6018       for (cl = 0; cl < closureSize*2; cl += 2) {
6019         const PetscInt p = closure[cl];
6020         if ((p >= vStart) && (p < vEnd)) closure[numCorners++] = p;
6021       }
6022       ierr = DMPlexGetRawFaces_Internal(dm, dim-h, numCorners, closure, &numFaces, &faceSize, &faces);CHKERRQ(ierr);
6023       if (coneSize != numFaces) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %d has %d faces but should have %d", c, coneSize, numFaces);
6024       for (f = 0; f < numFaces; ++f) {
6025         PetscInt *fclosure = NULL, fclosureSize, cl, fnumCorners = 0, v;
6026 
6027         ierr = DMPlexGetTransitiveClosure_Internal(dm, cone[f], ornt[f], PETSC_TRUE, &fclosureSize, &fclosure);CHKERRQ(ierr);
6028         for (cl = 0; cl < fclosureSize*2; cl += 2) {
6029           const PetscInt p = fclosure[cl];
6030           if ((p >= vStart) && (p < vEnd)) fclosure[fnumCorners++] = p;
6031         }
6032         if (fnumCorners != faceSize) SETERRQ5(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %d (%d) of cell %d has %d vertices but should have %d", cone[f], f, c, fnumCorners, faceSize);
6033         for (v = 0; v < fnumCorners; ++v) {
6034           if (fclosure[v] != faces[f*faceSize+v]) SETERRQ6(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %d (%d) of cell %d vertex %d, %d != %d", cone[f], f, c, v, fclosure[v], faces[f*faceSize+v]);
6035         }
6036         ierr = DMPlexRestoreTransitiveClosure(dm, cone[f], PETSC_TRUE, &fclosureSize, &fclosure);CHKERRQ(ierr);
6037       }
6038       ierr = DMPlexRestoreFaces_Internal(dm, dim, c, &numFaces, &faceSize, &faces);CHKERRQ(ierr);
6039       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6040     }
6041   }
6042   PetscFunctionReturn(0);
6043 }
6044 
6045 #undef __FUNCT__
6046 #define __FUNCT__ "DMCreateInterpolation_Plex"
6047 /* Pointwise interpolation
6048      Just code FEM for now
6049      u^f = I u^c
6050      sum_k u^f_k phi^f_k = I sum_j u^c_j phi^c_j
6051      u^f_i = sum_j psi^f_i I phi^c_j u^c_j
6052      I_{ij} = psi^f_i phi^c_j
6053 */
6054 PetscErrorCode DMCreateInterpolation_Plex(DM dmCoarse, DM dmFine, Mat *interpolation, Vec *scaling)
6055 {
6056   PetscSection   gsc, gsf;
6057   PetscInt       m, n;
6058   void          *ctx;
6059   PetscErrorCode ierr;
6060 
6061   PetscFunctionBegin;
6062   /*
6063   Loop over coarse cells
6064     Loop over coarse basis functions
6065       Loop over fine cells in coarse cell
6066         Loop over fine dual basis functions
6067           Evaluate coarse basis on fine dual basis quad points
6068           Sum
6069           Update local element matrix
6070     Accumulate to interpolation matrix
6071 
6072    Can extend PetscFEIntegrateJacobian_Basic() to do a specialized cell loop
6073   */
6074   ierr = DMGetDefaultGlobalSection(dmFine, &gsf);CHKERRQ(ierr);
6075   ierr = PetscSectionGetConstrainedStorageSize(gsf, &m);CHKERRQ(ierr);
6076   ierr = DMGetDefaultGlobalSection(dmCoarse, &gsc);CHKERRQ(ierr);
6077   ierr = PetscSectionGetConstrainedStorageSize(gsc, &n);CHKERRQ(ierr);
6078   /* We need to preallocate properly */
6079   ierr = MatCreate(PetscObjectComm((PetscObject) dmCoarse), interpolation);CHKERRQ(ierr);
6080   ierr = MatSetSizes(*interpolation, m, n, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
6081   ierr = MatSetType(*interpolation, dmCoarse->mattype);CHKERRQ(ierr);
6082   ierr = MatSetUp(*interpolation);CHKERRQ(ierr);
6083   ierr = MatSetFromOptions(*interpolation);CHKERRQ(ierr);
6084   ierr = MatSetOption(*interpolation, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_FALSE);CHKERRQ(ierr);
6085   ierr = DMGetApplicationContext(dmFine, &ctx);CHKERRQ(ierr);
6086   ierr = DMPlexComputeInterpolatorFEM(dmCoarse, dmFine, *interpolation, ctx);CHKERRQ(ierr);
6087   /* Use naive scaling */
6088   ierr = DMCreateInterpolationScale(dmCoarse, dmFine, *interpolation, scaling);CHKERRQ(ierr);
6089   PetscFunctionReturn(0);
6090 }
6091 
6092 #undef __FUNCT__
6093 #define __FUNCT__ "DMCreateInjection_Plex"
6094 PetscErrorCode DMCreateInjection_Plex(DM dmCoarse, DM dmFine, VecScatter *ctx)
6095 {
6096   Vec             cv,  fv;
6097   IS              cis, fis, fpointIS;
6098   PetscSection    sc, gsc, gsf;
6099   const PetscInt *fpoints;
6100   PetscInt       *cindices, *findices;
6101   PetscInt        cpStart, cpEnd, m, off, cp;
6102   PetscErrorCode  ierr;
6103 
6104   PetscFunctionBegin;
6105   ierr = DMGetDefaultGlobalSection(dmFine, &gsf);CHKERRQ(ierr);
6106   ierr = DMGetGlobalVector(dmFine, &fv);CHKERRQ(ierr);
6107   ierr = DMGetDefaultSection(dmCoarse, &sc);CHKERRQ(ierr);
6108   ierr = DMGetDefaultGlobalSection(dmCoarse, &gsc);CHKERRQ(ierr);
6109   ierr = DMGetGlobalVector(dmCoarse, &cv);CHKERRQ(ierr);
6110   ierr = DMPlexCreateCoarsePointIS(dmCoarse, &fpointIS);CHKERRQ(ierr);
6111   ierr = PetscSectionGetConstrainedStorageSize(gsc, &m);CHKERRQ(ierr);
6112   ierr = PetscMalloc2(m,&cindices,m,&findices);CHKERRQ(ierr);
6113   ierr = PetscSectionGetChart(gsc, &cpStart, &cpEnd);CHKERRQ(ierr);
6114   ierr = ISGetIndices(fpointIS, &fpoints);CHKERRQ(ierr);
6115   for (cp = cpStart, off = 0; cp < cpEnd; ++cp) {
6116     const PetscInt *cdofsC = NULL;
6117     PetscInt        fp     = fpoints[cp-cpStart], dofC, cdofC, dofF, offC, offF, d, e;
6118 
6119     ierr = PetscSectionGetDof(gsc, cp, &dofC);CHKERRQ(ierr);
6120     if (dofC <= 0) continue;
6121     ierr = PetscSectionGetConstraintDof(sc, cp, &cdofC);CHKERRQ(ierr);
6122     ierr = PetscSectionGetDof(gsf, fp, &dofF);CHKERRQ(ierr);
6123     ierr = PetscSectionGetOffset(gsc, cp, &offC);CHKERRQ(ierr);
6124     ierr = PetscSectionGetOffset(gsf, fp, &offF);CHKERRQ(ierr);
6125     if (cdofC) {ierr = PetscSectionGetConstraintIndices(sc, cp, &cdofsC);CHKERRQ(ierr);}
6126     if (dofC != dofF) SETERRQ4(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Point %d (%d) has %d coarse dofs != %d fine dofs", cp, fp, dofC, dofF);
6127     if (offC < 0 || offF < 0) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Coarse point %d has invalid offset %d (%d)", cp, offC, offF);
6128     for (d = 0, e = 0; d < dofC; ++d) {
6129       if (cdofsC && cdofsC[e] == d) {++e; continue;}
6130       cindices[off+d-e] = offC+d; findices[off+d-e] = offF+d;
6131     }
6132     if (e != cdofC) SETERRQ4(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Point %d (%d) has invalid number of constraints %d != %d", cp, fp, e, cdofC);
6133     off += dofC-cdofC;
6134   }
6135   ierr = ISRestoreIndices(fpointIS, &fpoints);CHKERRQ(ierr);
6136   if (off != m) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of coarse dofs %d != %d", off, m);
6137   ierr = ISCreateGeneral(PETSC_COMM_SELF, m, cindices, PETSC_OWN_POINTER, &cis);CHKERRQ(ierr);
6138   ierr = ISCreateGeneral(PETSC_COMM_SELF, m, findices, PETSC_OWN_POINTER, &fis);CHKERRQ(ierr);
6139   ierr = VecScatterCreate(cv, cis, fv, fis, ctx);CHKERRQ(ierr);
6140   ierr = ISDestroy(&cis);CHKERRQ(ierr);
6141   ierr = ISDestroy(&fis);CHKERRQ(ierr);
6142   ierr = DMRestoreGlobalVector(dmFine, &fv);CHKERRQ(ierr);
6143   ierr = DMRestoreGlobalVector(dmCoarse, &cv);CHKERRQ(ierr);
6144   ierr = ISDestroy(&fpointIS);CHKERRQ(ierr);
6145   PetscFunctionReturn(0);
6146 }
6147 
6148 #undef __FUNCT__
6149 #define __FUNCT__ "DMCreateDefaultSection_Plex"
6150 /* Pointwise interpolation
6151      Just code FEM for now
6152      u^f = I u^c
6153      sum_k u^f_k phi^f_k = I sum_l u^c_l phi^c_l
6154      u^f_i = sum_l int psi^f_i I phi^c_l u^c_l
6155      I_{ij} = int psi^f_i phi^c_j
6156 */
6157 PetscErrorCode DMCreateDefaultSection_Plex(DM dm)
6158 {
6159   PetscSection   section;
6160   IS            *bcPoints;
6161   PetscInt      *bcFields, *numComp, *numDof;
6162   PetscInt       depth, dim, numBd, numBC = 0, numFields, bd, bc, f;
6163   PetscErrorCode ierr;
6164 
6165   PetscFunctionBegin;
6166   /* Handle boundary conditions */
6167   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
6168   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
6169   ierr = DMPlexGetNumBoundary(dm, &numBd);CHKERRQ(ierr);
6170   for (bd = 0; bd < numBd; ++bd) {
6171     PetscBool isEssential;
6172     ierr = DMPlexGetBoundary(dm, bd, &isEssential, NULL, NULL, NULL, NULL, NULL, NULL, NULL);CHKERRQ(ierr);
6173     if (isEssential) ++numBC;
6174   }
6175   ierr = PetscMalloc2(numBC,&bcFields,numBC,&bcPoints);CHKERRQ(ierr);
6176   for (bd = 0, bc = 0; bd < numBd; ++bd) {
6177     const char     *bdLabel;
6178     DMLabel         label;
6179     const PetscInt *values;
6180     PetscInt        bd2, field, numValues;
6181     PetscBool       isEssential, duplicate = PETSC_FALSE;
6182 
6183     ierr = DMPlexGetBoundary(dm, bd, &isEssential, NULL, &bdLabel, &field, NULL, &numValues, &values, NULL);CHKERRQ(ierr);
6184     if (numValues != 1) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Bug me and I will fix this");
6185     ierr = DMPlexGetLabel(dm, bdLabel, &label);CHKERRQ(ierr);
6186     /* Only want to do this for FEM, and only once */
6187     for (bd2 = 0; bd2 < bd; ++bd2) {
6188       const char *bdname;
6189       ierr = DMPlexGetBoundary(dm, bd2, NULL, NULL, &bdname, NULL, NULL, NULL, NULL, NULL);CHKERRQ(ierr);
6190       ierr = PetscStrcmp(bdname, bdLabel, &duplicate);CHKERRQ(ierr);
6191       if (duplicate) break;
6192     }
6193     if (!duplicate) {
6194       ierr = DMPlexLabelComplete(dm, label);CHKERRQ(ierr);
6195       ierr = DMPlexLabelAddCells(dm, label);CHKERRQ(ierr);
6196     }
6197     /* Filter out cells, if you actually want to constraint cells you need to do things by hand right now */
6198     if (isEssential) {
6199       IS              tmp;
6200       PetscInt       *newidx;
6201       const PetscInt *idx;
6202       PetscInt        cStart, cEnd, n, p, newn = 0;
6203 
6204       bcFields[bc] = field;
6205       ierr = DMPlexGetStratumIS(dm, bdLabel, values[0], &tmp);CHKERRQ(ierr);
6206       ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
6207       ierr = ISGetLocalSize(tmp, &n);CHKERRQ(ierr);
6208       ierr = ISGetIndices(tmp, &idx);CHKERRQ(ierr);
6209       for (p = 0; p < n; ++p) if ((idx[p] < cStart) || (idx[p] >= cEnd)) ++newn;
6210       ierr = PetscMalloc1(newn,&newidx);CHKERRQ(ierr);
6211       newn = 0;
6212       for (p = 0; p < n; ++p) if ((idx[p] < cStart) || (idx[p] >= cEnd)) newidx[newn++] = idx[p];
6213       ierr = ISCreateGeneral(PetscObjectComm((PetscObject) dm), newn, newidx, PETSC_OWN_POINTER, &bcPoints[bc++]);CHKERRQ(ierr);
6214       ierr = ISRestoreIndices(tmp, &idx);CHKERRQ(ierr);
6215       ierr = ISDestroy(&tmp);CHKERRQ(ierr);
6216     }
6217   }
6218   /* Handle discretization */
6219   ierr = DMGetNumFields(dm, &numFields);CHKERRQ(ierr);
6220   ierr = PetscMalloc2(numFields,&numComp,numFields*(dim+1),&numDof);CHKERRQ(ierr);
6221   for (f = 0; f < numFields; ++f) {
6222     PetscFE         fe;
6223     const PetscInt *numFieldDof;
6224     PetscInt        d;
6225 
6226     ierr = DMGetField(dm, f, (PetscObject *) &fe);CHKERRQ(ierr);
6227     ierr = PetscFEGetNumComponents(fe, &numComp[f]);CHKERRQ(ierr);
6228     ierr = PetscFEGetNumDof(fe, &numFieldDof);CHKERRQ(ierr);
6229     for (d = 0; d < dim+1; ++d) numDof[f*(dim+1)+d] = numFieldDof[d];
6230   }
6231   for (f = 0; f < numFields; ++f) {
6232     PetscInt d;
6233     for (d = 1; d < dim; ++d) {
6234       if ((numDof[f*(dim+1)+d] > 0) && (depth < dim)) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Mesh must be interpolated when unknowns are specified on edges or faces.");
6235     }
6236   }
6237   ierr = DMPlexCreateSection(dm, dim, numFields, numComp, numDof, numBC, bcFields, bcPoints, NULL, &section);CHKERRQ(ierr);
6238   for (f = 0; f < numFields; ++f) {
6239     PetscFE     fe;
6240     const char *name;
6241 
6242     ierr = DMGetField(dm, f, (PetscObject *) &fe);CHKERRQ(ierr);
6243     ierr = PetscObjectGetName((PetscObject) fe, &name);CHKERRQ(ierr);
6244     ierr = PetscSectionSetFieldName(section, f, name);CHKERRQ(ierr);
6245   }
6246   ierr = DMSetDefaultSection(dm, section);CHKERRQ(ierr);
6247   ierr = PetscSectionDestroy(&section);CHKERRQ(ierr);
6248   for (bc = 0; bc < numBC; ++bc) {ierr = ISDestroy(&bcPoints[bc]);CHKERRQ(ierr);}
6249   ierr = PetscFree2(bcFields,bcPoints);CHKERRQ(ierr);
6250   ierr = PetscFree2(numComp,numDof);CHKERRQ(ierr);
6251   PetscFunctionReturn(0);
6252 }
6253 
6254 #undef __FUNCT__
6255 #define __FUNCT__ "DMPlexGetCoarseDM"
6256 /*@
6257   DMPlexGetCoarseDM - Get the coarse mesh from which this was obtained by refinement
6258 
6259   Input Parameter:
6260 . dm - The DMPlex object
6261 
6262   Output Parameter:
6263 . cdm - The coarse DM
6264 
6265   Level: intermediate
6266 
6267 .seealso: DMPlexSetCoarseDM()
6268 @*/
6269 PetscErrorCode DMPlexGetCoarseDM(DM dm, DM *cdm)
6270 {
6271   PetscFunctionBegin;
6272   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6273   PetscValidPointer(cdm, 2);
6274   *cdm = ((DM_Plex *) dm->data)->coarseMesh;
6275   PetscFunctionReturn(0);
6276 }
6277 
6278 #undef __FUNCT__
6279 #define __FUNCT__ "DMPlexSetCoarseDM"
6280 /*@
6281   DMPlexSetCoarseDM - Set the coarse mesh from which this was obtained by refinement
6282 
6283   Input Parameters:
6284 + dm - The DMPlex object
6285 - cdm - The coarse DM
6286 
6287   Level: intermediate
6288 
6289 .seealso: DMPlexGetCoarseDM()
6290 @*/
6291 PetscErrorCode DMPlexSetCoarseDM(DM dm, DM cdm)
6292 {
6293   DM_Plex       *mesh;
6294   PetscErrorCode ierr;
6295 
6296   PetscFunctionBegin;
6297   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6298   if (cdm) PetscValidHeaderSpecific(cdm, DM_CLASSID, 2);
6299   mesh = (DM_Plex *) dm->data;
6300   ierr = DMDestroy(&mesh->coarseMesh);CHKERRQ(ierr);
6301   mesh->coarseMesh = cdm;
6302   ierr = PetscObjectReference((PetscObject) mesh->coarseMesh);CHKERRQ(ierr);
6303   PetscFunctionReturn(0);
6304 }
6305