xref: /petsc/src/dm/impls/plex/plexcreate.c (revision b41ce5d507ea9a58bfa83cf403107a702e77a67d)
1 #define PETSCDM_DLL
2 #include <petsc/private/dmpleximpl.h>    /*I   "petscdmplex.h"   I*/
3 #include <petscdmda.h>
4 #include <petscsf.h>
5 
6 /*@
7   DMPlexCreateDoublet - Creates a mesh of two cells of the specified type, optionally with later refinement.
8 
9   Collective on MPI_Comm
10 
11   Input Parameters:
12 + comm - The communicator for the DM object
13 . dim - The spatial dimension
14 . simplex - Flag for simplicial cells, otherwise they are tensor product cells
15 . interpolate - Flag to create intermediate mesh pieces (edges, faces)
16 . refinementUniform - Flag for uniform parallel refinement
17 - refinementLimit - A nonzero number indicates the largest admissible volume for a refined cell
18 
19   Output Parameter:
20 . dm  - The DM object
21 
22   Level: beginner
23 
24 .keywords: DM, create
25 .seealso: DMSetType(), DMCreate()
26 @*/
27 PetscErrorCode DMPlexCreateDoublet(MPI_Comm comm, PetscInt dim, PetscBool simplex, PetscBool interpolate, PetscBool refinementUniform, PetscReal refinementLimit, DM *newdm)
28 {
29   DM             dm;
30   PetscInt       p;
31   PetscMPIInt    rank;
32   PetscErrorCode ierr;
33 
34   PetscFunctionBegin;
35   ierr = DMCreate(comm, &dm);CHKERRQ(ierr);
36   ierr = DMSetType(dm, DMPLEX);CHKERRQ(ierr);
37   ierr = DMSetDimension(dm, dim);CHKERRQ(ierr);
38   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
39   switch (dim) {
40   case 2:
41     if (simplex) {ierr = PetscObjectSetName((PetscObject) dm, "triangular");CHKERRQ(ierr);}
42     else         {ierr = PetscObjectSetName((PetscObject) dm, "quadrilateral");CHKERRQ(ierr);}
43     break;
44   case 3:
45     if (simplex) {ierr = PetscObjectSetName((PetscObject) dm, "tetrahedral");CHKERRQ(ierr);}
46     else         {ierr = PetscObjectSetName((PetscObject) dm, "hexahedral");CHKERRQ(ierr);}
47     break;
48   default:
49     SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make meshes for dimension %d", dim);
50   }
51   if (rank) {
52     PetscInt numPoints[2] = {0, 0};
53     ierr = DMPlexCreateFromDAG(dm, 1, numPoints, NULL, NULL, NULL, NULL);CHKERRQ(ierr);
54   } else {
55     switch (dim) {
56     case 2:
57       if (simplex) {
58         PetscInt    numPoints[2]        = {4, 2};
59         PetscInt    coneSize[6]         = {3, 3, 0, 0, 0, 0};
60         PetscInt    cones[6]            = {2, 3, 4,  5, 4, 3};
61         PetscInt    coneOrientations[6] = {0, 0, 0,  0, 0, 0};
62         PetscScalar vertexCoords[8]     = {-0.5, 0.5, 0.0, 0.0, 0.0, 1.0, 0.5, 0.5};
63         PetscInt    markerPoints[8]     = {2, 1, 3, 1, 4, 1, 5, 1};
64 
65         ierr = DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
66         for (p = 0; p < 4; ++p) {ierr = DMSetLabelValue(dm, "marker", markerPoints[p*2], markerPoints[p*2+1]);CHKERRQ(ierr);}
67       } else {
68         PetscInt    numPoints[2]        = {6, 2};
69         PetscInt    coneSize[8]         = {4, 4, 0, 0, 0, 0, 0, 0};
70         PetscInt    cones[8]            = {2, 3, 4, 5,  3, 6, 7, 4};
71         PetscInt    coneOrientations[8] = {0, 0, 0, 0,  0, 0, 0, 0};
72         PetscScalar vertexCoords[12]    = {-1.0, -0.5,  0.0, -0.5,  0.0, 0.5,  -1.0, 0.5,  1.0, -0.5,  1.0, 0.5};
73 
74         ierr = DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
75       }
76       break;
77     case 3:
78       if (simplex) {
79         PetscInt    numPoints[2]        = {5, 2};
80         PetscInt    coneSize[7]         = {4, 4, 0, 0, 0, 0, 0};
81         PetscInt    cones[8]            = {4, 3, 5, 2,  5, 3, 4, 6};
82         PetscInt    coneOrientations[8] = {0, 0, 0, 0,  0, 0, 0, 0};
83         PetscScalar vertexCoords[15]    = {-1.0, 0.0, 0.0,  0.0, -1.0, 0.0,  0.0, 0.0, 1.0,  0.0, 1.0, 0.0,  1.0, 0.0, 0.0};
84         PetscInt    markerPoints[10]    = {2, 1, 3, 1, 4, 1, 5, 1, 6, 1};
85 
86         ierr = DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
87         for (p = 0; p < 5; ++p) {ierr = DMSetLabelValue(dm, "marker", markerPoints[p*2], markerPoints[p*2+1]);CHKERRQ(ierr);}
88       } else {
89         PetscInt    numPoints[2]         = {12, 2};
90         PetscInt    coneSize[14]         = {8, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
91         PetscInt    cones[16]            = {2, 3, 4, 5, 6, 7, 8, 9,  5, 4, 10, 11, 7, 12, 13, 8};
92         PetscInt    coneOrientations[16] = {0, 0, 0, 0, 0, 0, 0, 0,  0, 0,  0,  0, 0,  0,  0, 0};
93         PetscScalar vertexCoords[36]     = {-1.0, -0.5, -0.5,  -1.0,  0.5, -0.5,  0.0,  0.5, -0.5,   0.0, -0.5, -0.5,
94                                             -1.0, -0.5,  0.5,   0.0, -0.5,  0.5,  0.0,  0.5,  0.5,  -1.0,  0.5,  0.5,
95                                              1.0,  0.5, -0.5,   1.0, -0.5, -0.5,  1.0, -0.5,  0.5,   1.0,  0.5,  0.5};
96 
97         ierr = DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
98       }
99       break;
100     default:
101       SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make meshes for dimension %d", dim);
102     }
103   }
104   *newdm = dm;
105   if (refinementLimit > 0.0) {
106     DM rdm;
107     const char *name;
108 
109     ierr = DMPlexSetRefinementUniform(*newdm, PETSC_FALSE);CHKERRQ(ierr);
110     ierr = DMPlexSetRefinementLimit(*newdm, refinementLimit);CHKERRQ(ierr);
111     ierr = DMRefine(*newdm, comm, &rdm);CHKERRQ(ierr);
112     ierr = PetscObjectGetName((PetscObject) *newdm, &name);CHKERRQ(ierr);
113     ierr = PetscObjectSetName((PetscObject)    rdm,  name);CHKERRQ(ierr);
114     ierr = DMDestroy(newdm);CHKERRQ(ierr);
115     *newdm = rdm;
116   }
117   if (interpolate) {
118     DM idm = NULL;
119     const char *name;
120 
121     ierr = DMPlexInterpolate(*newdm, &idm);CHKERRQ(ierr);
122     ierr = PetscObjectGetName((PetscObject) *newdm, &name);CHKERRQ(ierr);
123     ierr = PetscObjectSetName((PetscObject)    idm,  name);CHKERRQ(ierr);
124     ierr = DMPlexCopyCoordinates(*newdm, idm);CHKERRQ(ierr);
125     ierr = DMCopyLabels(*newdm, idm);CHKERRQ(ierr);
126     ierr = DMDestroy(newdm);CHKERRQ(ierr);
127     *newdm = idm;
128   }
129   {
130     DM refinedMesh     = NULL;
131     DM distributedMesh = NULL;
132 
133     /* Distribute mesh over processes */
134     ierr = DMPlexDistribute(*newdm, 0, NULL, &distributedMesh);CHKERRQ(ierr);
135     if (distributedMesh) {
136       ierr = DMDestroy(newdm);CHKERRQ(ierr);
137       *newdm = distributedMesh;
138     }
139     if (refinementUniform) {
140       ierr = DMPlexSetRefinementUniform(*newdm, refinementUniform);CHKERRQ(ierr);
141       ierr = DMRefine(*newdm, comm, &refinedMesh);CHKERRQ(ierr);
142       if (refinedMesh) {
143         ierr = DMDestroy(newdm);CHKERRQ(ierr);
144         *newdm = refinedMesh;
145       }
146     }
147   }
148   PetscFunctionReturn(0);
149 }
150 
151 /*@
152   DMPlexCreateSquareBoundary - Creates a 1D mesh the is the boundary of a square lattice.
153 
154   Collective on MPI_Comm
155 
156   Input Parameters:
157 + comm  - The communicator for the DM object
158 . lower - The lower left corner coordinates
159 . upper - The upper right corner coordinates
160 - edges - The number of cells in each direction
161 
162   Output Parameter:
163 . dm  - The DM object
164 
165   Note: Here is the numbering returned for 2 cells in each direction:
166 $ 18--5-17--4--16
167 $  |     |     |
168 $  6    10     3
169 $  |     |     |
170 $ 19-11-20--9--15
171 $  |     |     |
172 $  7     8     2
173 $  |     |     |
174 $ 12--0-13--1--14
175 
176   Level: beginner
177 
178 .keywords: DM, create
179 .seealso: DMPlexCreateBoxMesh(), DMPlexCreateCubeBoundary(), DMSetType(), DMCreate()
180 @*/
181 PetscErrorCode DMPlexCreateSquareBoundary(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[])
182 {
183   const PetscInt numVertices    = (edges[0]+1)*(edges[1]+1);
184   const PetscInt numEdges       = edges[0]*(edges[1]+1) + (edges[0]+1)*edges[1];
185   PetscInt       markerTop      = 1;
186   PetscInt       markerBottom   = 1;
187   PetscInt       markerRight    = 1;
188   PetscInt       markerLeft     = 1;
189   PetscBool      markerSeparate = PETSC_FALSE;
190   Vec            coordinates;
191   PetscSection   coordSection;
192   PetscScalar    *coords;
193   PetscInt       coordSize;
194   PetscMPIInt    rank;
195   PetscInt       v, vx, vy;
196   PetscErrorCode ierr;
197 
198   PetscFunctionBegin;
199   ierr = PetscOptionsGetBool(((PetscObject) dm)->options,((PetscObject) dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL);CHKERRQ(ierr);
200   if (markerSeparate) {
201     markerTop    = 3;
202     markerBottom = 1;
203     markerRight  = 2;
204     markerLeft   = 4;
205   }
206   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRQ(ierr);
207   if (!rank) {
208     PetscInt e, ex, ey;
209 
210     ierr = DMPlexSetChart(dm, 0, numEdges+numVertices);CHKERRQ(ierr);
211     for (e = 0; e < numEdges; ++e) {
212       ierr = DMPlexSetConeSize(dm, e, 2);CHKERRQ(ierr);
213     }
214     ierr = DMSetUp(dm);CHKERRQ(ierr); /* Allocate space for cones */
215     for (vx = 0; vx <= edges[0]; vx++) {
216       for (ey = 0; ey < edges[1]; ey++) {
217         PetscInt edge   = vx*edges[1] + ey + edges[0]*(edges[1]+1);
218         PetscInt vertex = ey*(edges[0]+1) + vx + numEdges;
219         PetscInt cone[2];
220 
221         cone[0] = vertex; cone[1] = vertex+edges[0]+1;
222         ierr    = DMPlexSetCone(dm, edge, cone);CHKERRQ(ierr);
223         if (vx == edges[0]) {
224           ierr = DMSetLabelValue(dm, "marker", edge,    markerRight);CHKERRQ(ierr);
225           ierr = DMSetLabelValue(dm, "marker", cone[0], markerRight);CHKERRQ(ierr);
226           if (ey == edges[1]-1) {
227             ierr = DMSetLabelValue(dm, "marker", cone[1], markerRight);CHKERRQ(ierr);
228             ierr = DMSetLabelValue(dm, "Face Sets", cone[1], markerRight);CHKERRQ(ierr);
229           }
230         } else if (vx == 0) {
231           ierr = DMSetLabelValue(dm, "marker", edge,    markerLeft);CHKERRQ(ierr);
232           ierr = DMSetLabelValue(dm, "marker", cone[0], markerLeft);CHKERRQ(ierr);
233           if (ey == edges[1]-1) {
234             ierr = DMSetLabelValue(dm, "marker", cone[1], markerLeft);CHKERRQ(ierr);
235             ierr = DMSetLabelValue(dm, "Face Sets", cone[1], markerLeft);CHKERRQ(ierr);
236           }
237         }
238       }
239     }
240     for (vy = 0; vy <= edges[1]; vy++) {
241       for (ex = 0; ex < edges[0]; ex++) {
242         PetscInt edge   = vy*edges[0]     + ex;
243         PetscInt vertex = vy*(edges[0]+1) + ex + numEdges;
244         PetscInt cone[2];
245 
246         cone[0] = vertex; cone[1] = vertex+1;
247         ierr    = DMPlexSetCone(dm, edge, cone);CHKERRQ(ierr);
248         if (vy == edges[1]) {
249           ierr = DMSetLabelValue(dm, "marker", edge,    markerTop);CHKERRQ(ierr);
250           ierr = DMSetLabelValue(dm, "marker", cone[0], markerTop);CHKERRQ(ierr);
251           if (ex == edges[0]-1) {
252             ierr = DMSetLabelValue(dm, "marker", cone[1], markerTop);CHKERRQ(ierr);
253             ierr = DMSetLabelValue(dm, "Face Sets", cone[1], markerTop);CHKERRQ(ierr);
254           }
255         } else if (vy == 0) {
256           ierr = DMSetLabelValue(dm, "marker", edge,    markerBottom);CHKERRQ(ierr);
257           ierr = DMSetLabelValue(dm, "marker", cone[0], markerBottom);CHKERRQ(ierr);
258           if (ex == edges[0]-1) {
259             ierr = DMSetLabelValue(dm, "marker", cone[1], markerBottom);CHKERRQ(ierr);
260             ierr = DMSetLabelValue(dm, "Face Sets", cone[1], markerBottom);CHKERRQ(ierr);
261           }
262         }
263       }
264     }
265   }
266   ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
267   ierr = DMPlexStratify(dm);CHKERRQ(ierr);
268   /* Build coordinates */
269   ierr = DMSetCoordinateDim(dm, 2);CHKERRQ(ierr);
270   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
271   ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
272   ierr = PetscSectionSetChart(coordSection, numEdges, numEdges + numVertices);CHKERRQ(ierr);
273   ierr = PetscSectionSetFieldComponents(coordSection, 0, 2);CHKERRQ(ierr);
274   for (v = numEdges; v < numEdges+numVertices; ++v) {
275     ierr = PetscSectionSetDof(coordSection, v, 2);CHKERRQ(ierr);
276     ierr = PetscSectionSetFieldDof(coordSection, v, 0, 2);CHKERRQ(ierr);
277   }
278   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
279   ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
280   ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
281   ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
282   ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
283   ierr = VecSetBlockSize(coordinates, 2);CHKERRQ(ierr);
284   ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
285   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
286   for (vy = 0; vy <= edges[1]; ++vy) {
287     for (vx = 0; vx <= edges[0]; ++vx) {
288       coords[(vy*(edges[0]+1)+vx)*2+0] = lower[0] + ((upper[0] - lower[0])/edges[0])*vx;
289       coords[(vy*(edges[0]+1)+vx)*2+1] = lower[1] + ((upper[1] - lower[1])/edges[1])*vy;
290     }
291   }
292   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
293   ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
294   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
295   PetscFunctionReturn(0);
296 }
297 
298 /*@
299   DMPlexCreateCubeBoundary - Creates a 2D mesh the is the boundary of a cubic lattice.
300 
301   Collective on MPI_Comm
302 
303   Input Parameters:
304 + comm  - The communicator for the DM object
305 . lower - The lower left front corner coordinates
306 . upper - The upper right back corner coordinates
307 - edges - The number of cells in each direction
308 
309   Output Parameter:
310 . dm  - The DM object
311 
312   Level: beginner
313 
314 .keywords: DM, create
315 .seealso: DMPlexCreateBoxMesh(), DMPlexCreateSquareBoundary(), DMSetType(), DMCreate()
316 @*/
317 PetscErrorCode DMPlexCreateCubeBoundary(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt faces[])
318 {
319   PetscInt       vertices[3], numVertices;
320   PetscInt       numFaces    = 2*faces[0]*faces[1] + 2*faces[1]*faces[2] + 2*faces[0]*faces[2];
321   Vec            coordinates;
322   PetscSection   coordSection;
323   PetscScalar    *coords;
324   PetscInt       coordSize;
325   PetscMPIInt    rank;
326   PetscInt       v, vx, vy, vz;
327   PetscInt       voffset, iface=0, cone[4];
328   PetscErrorCode ierr;
329 
330   PetscFunctionBegin;
331   if ((faces[0] < 1) || (faces[1] < 1) || (faces[2] < 1)) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Must have at least 1 face per side");
332   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRQ(ierr);
333   vertices[0] = faces[0]+1; vertices[1] = faces[1]+1; vertices[2] = faces[2]+1;
334   numVertices = vertices[0]*vertices[1]*vertices[2];
335   if (!rank) {
336     PetscInt f;
337 
338     ierr = DMPlexSetChart(dm, 0, numFaces+numVertices);CHKERRQ(ierr);
339     for (f = 0; f < numFaces; ++f) {
340       ierr = DMPlexSetConeSize(dm, f, 4);CHKERRQ(ierr);
341     }
342     ierr = DMSetUp(dm);CHKERRQ(ierr); /* Allocate space for cones */
343     for (v = 0; v < numFaces+numVertices; ++v) {
344       ierr = DMSetLabelValue(dm, "marker", v, 1);CHKERRQ(ierr);
345     }
346 
347     /* Side 0 (Top) */
348     for (vy = 0; vy < faces[1]; vy++) {
349       for (vx = 0; vx < faces[0]; vx++) {
350         voffset = numFaces + vertices[0]*vertices[1]*(vertices[2]-1) + vy*vertices[0] + vx;
351         cone[0] = voffset; cone[1] = voffset+1; cone[2] = voffset+vertices[0]+1; cone[3] = voffset+vertices[0];
352         ierr    = DMPlexSetCone(dm, iface, cone);CHKERRQ(ierr);
353         iface++;
354       }
355     }
356 
357     /* Side 1 (Bottom) */
358     for (vy = 0; vy < faces[1]; vy++) {
359       for (vx = 0; vx < faces[0]; vx++) {
360         voffset = numFaces + vy*(faces[0]+1) + vx;
361         cone[0] = voffset+1; cone[1] = voffset; cone[2] = voffset+vertices[0]; cone[3] = voffset+vertices[0]+1;
362         ierr    = DMPlexSetCone(dm, iface, cone);CHKERRQ(ierr);
363         iface++;
364       }
365     }
366 
367     /* Side 2 (Front) */
368     for (vz = 0; vz < faces[2]; vz++) {
369       for (vx = 0; vx < faces[0]; vx++) {
370         voffset = numFaces + vz*vertices[0]*vertices[1] + vx;
371         cone[0] = voffset; cone[1] = voffset+1; cone[2] = voffset+vertices[0]*vertices[1]+1; cone[3] = voffset+vertices[0]*vertices[1];
372         ierr    = DMPlexSetCone(dm, iface, cone);CHKERRQ(ierr);
373         iface++;
374       }
375     }
376 
377     /* Side 3 (Back) */
378     for (vz = 0; vz < faces[2]; vz++) {
379       for (vx = 0; vx < faces[0]; vx++) {
380         voffset = numFaces + vz*vertices[0]*vertices[1] + vertices[0]*(vertices[1]-1) + vx;
381         cone[0] = voffset+vertices[0]*vertices[1]; cone[1] = voffset+vertices[0]*vertices[1]+1;
382         cone[2] = voffset+1; cone[3] = voffset;
383         ierr    = DMPlexSetCone(dm, iface, cone);CHKERRQ(ierr);
384         iface++;
385       }
386     }
387 
388     /* Side 4 (Left) */
389     for (vz = 0; vz < faces[2]; vz++) {
390       for (vy = 0; vy < faces[1]; vy++) {
391         voffset = numFaces + vz*vertices[0]*vertices[1] + vy*vertices[0];
392         cone[0] = voffset; cone[1] = voffset+vertices[0]*vertices[1];
393         cone[2] = voffset+vertices[0]*vertices[1]+vertices[0]; cone[3] = voffset+vertices[0];
394         ierr    = DMPlexSetCone(dm, iface, cone);CHKERRQ(ierr);
395         iface++;
396       }
397     }
398 
399     /* Side 5 (Right) */
400     for (vz = 0; vz < faces[2]; vz++) {
401       for (vy = 0; vy < faces[1]; vy++) {
402         voffset = numFaces + vz*vertices[0]*vertices[1] + vy*vertices[0] + faces[0];
403         cone[0] = voffset+vertices[0]*vertices[1]; cone[1] = voffset;
404         cone[2] = voffset+vertices[0]; cone[3] = voffset+vertices[0]*vertices[1]+vertices[0];
405         ierr    = DMPlexSetCone(dm, iface, cone);CHKERRQ(ierr);
406         iface++;
407       }
408     }
409   }
410   ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
411   ierr = DMPlexStratify(dm);CHKERRQ(ierr);
412   /* Build coordinates */
413   ierr = DMSetCoordinateDim(dm, 3);CHKERRQ(ierr);
414   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
415   ierr = PetscSectionSetChart(coordSection, numFaces, numFaces + numVertices);CHKERRQ(ierr);
416   for (v = numFaces; v < numFaces+numVertices; ++v) {
417     ierr = PetscSectionSetDof(coordSection, v, 3);CHKERRQ(ierr);
418   }
419   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
420   ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
421   ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
422   ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
423   ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
424   ierr = VecSetBlockSize(coordinates, 3);CHKERRQ(ierr);
425   ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
426   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
427   for (vz = 0; vz <= faces[2]; ++vz) {
428     for (vy = 0; vy <= faces[1]; ++vy) {
429       for (vx = 0; vx <= faces[0]; ++vx) {
430         coords[((vz*(faces[1]+1)+vy)*(faces[0]+1)+vx)*3+0] = lower[0] + ((upper[0] - lower[0])/faces[0])*vx;
431         coords[((vz*(faces[1]+1)+vy)*(faces[0]+1)+vx)*3+1] = lower[1] + ((upper[1] - lower[1])/faces[1])*vy;
432         coords[((vz*(faces[1]+1)+vy)*(faces[0]+1)+vx)*3+2] = lower[2] + ((upper[2] - lower[2])/faces[2])*vz;
433       }
434     }
435   }
436   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
437   ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
438   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
439   PetscFunctionReturn(0);
440 }
441 
442 /*@
443   DMPlexCreateBoxMesh - Creates a mesh on the tensor product of unit intervals (box) using simplices.
444 
445   Collective on MPI_Comm
446 
447   Input Parameters:
448 + comm - The communicator for the DM object
449 . dim - The spatial dimension
450 . numFaces - Number of faces per dimension
451 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
452 
453   Output Parameter:
454 . dm  - The DM object
455 
456   Level: beginner
457 
458 .keywords: DM, create
459 .seealso: DMPlexCreateHexBoxMesh(), DMSetType(), DMCreate()
460 @*/
461 PetscErrorCode DMPlexCreateBoxMesh(MPI_Comm comm, PetscInt dim, PetscInt numFaces, PetscBool interpolate, DM *dm)
462 {
463   DM             boundary;
464   PetscErrorCode ierr;
465 
466   PetscFunctionBegin;
467   PetscValidPointer(dm, 4);
468   ierr = DMCreate(comm, &boundary);CHKERRQ(ierr);
469   PetscValidLogicalCollectiveInt(boundary,dim,2);
470   ierr = DMSetType(boundary, DMPLEX);CHKERRQ(ierr);
471   ierr = DMSetDimension(boundary, dim-1);CHKERRQ(ierr);
472   ierr = DMSetCoordinateDim(boundary, dim);CHKERRQ(ierr);
473   switch (dim) {
474   case 2:
475   {
476     PetscReal lower[2] = {0.0, 0.0};
477     PetscReal upper[2] = {1.0, 1.0};
478     PetscInt  edges[2];
479 
480     edges[0] = numFaces; edges[1] = numFaces;
481     ierr = DMPlexCreateSquareBoundary(boundary, lower, upper, edges);CHKERRQ(ierr);
482     break;
483   }
484   case 3:
485   {
486     PetscReal lower[3] = {0.0, 0.0, 0.0};
487     PetscReal upper[3] = {1.0, 1.0, 1.0};
488     PetscInt  faces[3];
489 
490     faces[0] = numFaces; faces[1] = numFaces; faces[2] = numFaces;
491     ierr = DMPlexCreateCubeBoundary(boundary, lower, upper, faces);CHKERRQ(ierr);
492     break;
493   }
494   default:
495     SETERRQ1(comm, PETSC_ERR_SUP, "Dimension not supported: %d", dim);
496   }
497   ierr = DMPlexGenerate(boundary, NULL, interpolate, dm);CHKERRQ(ierr);
498   ierr = DMDestroy(&boundary);CHKERRQ(ierr);
499   PetscFunctionReturn(0);
500 }
501 
502 static PetscErrorCode DMPlexCreateCubeMesh_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[], DMBoundaryType bdX, DMBoundaryType bdY, DMBoundaryType bdZ)
503 {
504   DMLabel        cutLabel = NULL;
505   PetscInt       markerTop      = 1, faceMarkerTop      = 1;
506   PetscInt       markerBottom   = 1, faceMarkerBottom   = 1;
507   PetscInt       markerFront    = 1, faceMarkerFront    = 1;
508   PetscInt       markerBack     = 1, faceMarkerBack     = 1;
509   PetscInt       markerRight    = 1, faceMarkerRight    = 1;
510   PetscInt       markerLeft     = 1, faceMarkerLeft     = 1;
511   PetscInt       dim;
512   PetscBool      markerSeparate = PETSC_FALSE, cutMarker = PETSC_FALSE;
513   PetscMPIInt    rank;
514   PetscErrorCode ierr;
515 
516   PetscFunctionBegin;
517   ierr = DMGetDimension(dm,&dim);CHKERRQ(ierr);
518   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRQ(ierr);
519   ierr = DMCreateLabel(dm,"marker");CHKERRQ(ierr);
520   ierr = DMCreateLabel(dm,"Face Sets");CHKERRQ(ierr);
521   if (bdX == DM_BOUNDARY_PERIODIC || bdX == DM_BOUNDARY_TWIST ||
522       bdY == DM_BOUNDARY_PERIODIC || bdY == DM_BOUNDARY_TWIST ||
523       bdZ == DM_BOUNDARY_PERIODIC || bdZ == DM_BOUNDARY_TWIST) {
524     ierr = PetscOptionsGetBool(((PetscObject) dm)->options,((PetscObject) dm)->prefix, "-dm_plex_periodic_cut", &cutMarker, NULL);CHKERRQ(ierr);
525     if (cutMarker) {ierr = DMCreateLabel(dm, "periodic_cut");CHKERRQ(ierr); ierr = DMGetLabel(dm, "periodic_cut", &cutLabel);CHKERRQ(ierr);}
526   }
527   switch (dim) {
528   case 2:
529     faceMarkerTop    = 3;
530     faceMarkerBottom = 1;
531     faceMarkerRight  = 2;
532     faceMarkerLeft   = 4;
533     break;
534   case 3:
535     faceMarkerBottom = 1;
536     faceMarkerTop    = 2;
537     faceMarkerFront  = 3;
538     faceMarkerBack   = 4;
539     faceMarkerRight  = 5;
540     faceMarkerLeft   = 6;
541     break;
542   default:
543     SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SUP,"Dimension %d not supported",dim);
544     break;
545   }
546   ierr = PetscOptionsGetBool(((PetscObject) dm)->options,((PetscObject) dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL);CHKERRQ(ierr);
547   if (markerSeparate) {
548     markerBottom = faceMarkerBottom;
549     markerTop    = faceMarkerTop;
550     markerFront  = faceMarkerFront;
551     markerBack   = faceMarkerBack;
552     markerRight  = faceMarkerRight;
553     markerLeft   = faceMarkerLeft;
554   }
555   {
556     const PetscInt numXEdges    = !rank ? edges[0] : 0;
557     const PetscInt numYEdges    = !rank ? edges[1] : 0;
558     const PetscInt numZEdges    = !rank ? edges[2] : 0;
559     const PetscInt numXVertices = !rank ? (bdX == DM_BOUNDARY_PERIODIC || bdX == DM_BOUNDARY_TWIST ? edges[0] : edges[0]+1) : 0;
560     const PetscInt numYVertices = !rank ? (bdY == DM_BOUNDARY_PERIODIC || bdY == DM_BOUNDARY_TWIST ? edges[1] : edges[1]+1) : 0;
561     const PetscInt numZVertices = !rank ? (bdZ == DM_BOUNDARY_PERIODIC || bdZ == DM_BOUNDARY_TWIST ? edges[2] : edges[2]+1) : 0;
562     const PetscInt numCells     = numXEdges*numYEdges*numZEdges;
563     const PetscInt numXFaces    = numYEdges*numZEdges;
564     const PetscInt numYFaces    = numXEdges*numZEdges;
565     const PetscInt numZFaces    = numXEdges*numYEdges;
566     const PetscInt numTotXFaces = numXVertices*numXFaces;
567     const PetscInt numTotYFaces = numYVertices*numYFaces;
568     const PetscInt numTotZFaces = numZVertices*numZFaces;
569     const PetscInt numFaces     = numTotXFaces + numTotYFaces + numTotZFaces;
570     const PetscInt numTotXEdges = numXEdges*numYVertices*numZVertices;
571     const PetscInt numTotYEdges = numYEdges*numXVertices*numZVertices;
572     const PetscInt numTotZEdges = numZEdges*numXVertices*numYVertices;
573     const PetscInt numVertices  = numXVertices*numYVertices*numZVertices;
574     const PetscInt numEdges     = numTotXEdges + numTotYEdges + numTotZEdges;
575     const PetscInt firstVertex  = (dim == 2) ? numFaces : numCells;
576     const PetscInt firstXFace   = (dim == 2) ? 0 : numCells + numVertices;
577     const PetscInt firstYFace   = firstXFace + numTotXFaces;
578     const PetscInt firstZFace   = firstYFace + numTotYFaces;
579     const PetscInt firstXEdge   = numCells + numFaces + numVertices;
580     const PetscInt firstYEdge   = firstXEdge + numTotXEdges;
581     const PetscInt firstZEdge   = firstYEdge + numTotYEdges;
582     Vec            coordinates;
583     PetscSection   coordSection;
584     PetscScalar   *coords;
585     PetscInt       coordSize;
586     PetscInt       v, vx, vy, vz;
587     PetscInt       c, f, fx, fy, fz, e, ex, ey, ez;
588 
589     ierr = DMPlexSetChart(dm, 0, numCells+numFaces+numEdges+numVertices);CHKERRQ(ierr);
590     for (c = 0; c < numCells; c++) {
591       ierr = DMPlexSetConeSize(dm, c, 6);CHKERRQ(ierr);
592     }
593     for (f = firstXFace; f < firstXFace+numFaces; ++f) {
594       ierr = DMPlexSetConeSize(dm, f, 4);CHKERRQ(ierr);
595     }
596     for (e = firstXEdge; e < firstXEdge+numEdges; ++e) {
597       ierr = DMPlexSetConeSize(dm, e, 2);CHKERRQ(ierr);
598     }
599     ierr = DMSetUp(dm);CHKERRQ(ierr); /* Allocate space for cones */
600     /* Build cells */
601     for (fz = 0; fz < numZEdges; ++fz) {
602       for (fy = 0; fy < numYEdges; ++fy) {
603         for (fx = 0; fx < numXEdges; ++fx) {
604           PetscInt cell    = (fz*numYEdges + fy)*numXEdges + fx;
605           PetscInt faceB   = firstZFace + (fy*numXEdges+fx)*numZVertices +   fz;
606           PetscInt faceT   = firstZFace + (fy*numXEdges+fx)*numZVertices + ((fz+1)%numZVertices);
607           PetscInt faceF   = firstYFace + (fz*numXEdges+fx)*numYVertices +   fy;
608           PetscInt faceK   = firstYFace + (fz*numXEdges+fx)*numYVertices + ((fy+1)%numYVertices);
609           PetscInt faceL   = firstXFace + (fz*numYEdges+fy)*numXVertices +   fx;
610           PetscInt faceR   = firstXFace + (fz*numYEdges+fy)*numXVertices + ((fx+1)%numXVertices);
611                             /* B,  T,  F,  K,  R,  L */
612           PetscInt ornt[6] = {-4,  0,  0, -1,  0, -4}; /* ??? */
613           PetscInt cone[6];
614 
615           /* no boundary twisting in 3D */
616           cone[0] = faceB; cone[1] = faceT; cone[2] = faceF; cone[3] = faceK; cone[4] = faceR; cone[5] = faceL;
617           ierr    = DMPlexSetCone(dm, cell, cone);CHKERRQ(ierr);
618           ierr    = DMPlexSetConeOrientation(dm, cell, ornt);CHKERRQ(ierr);
619           if (bdX != DM_BOUNDARY_NONE && fx == numXEdges-1 && cutLabel) {ierr = DMLabelSetValue(cutLabel, cell, 2);CHKERRQ(ierr);}
620           if (bdY != DM_BOUNDARY_NONE && fy == numYEdges-1 && cutLabel) {ierr = DMLabelSetValue(cutLabel, cell, 2);CHKERRQ(ierr);}
621           if (bdZ != DM_BOUNDARY_NONE && fz == numZEdges-1 && cutLabel) {ierr = DMLabelSetValue(cutLabel, cell, 2);CHKERRQ(ierr);}
622         }
623       }
624     }
625     /* Build x faces */
626     for (fz = 0; fz < numZEdges; ++fz) {
627       for (fy = 0; fy < numYEdges; ++fy) {
628         for (fx = 0; fx < numXVertices; ++fx) {
629           PetscInt face    = firstXFace + (fz*numYEdges+fy)*numXVertices + fx;
630           PetscInt edgeL   = firstZEdge + (  fy*                 numXVertices+fx)*numZEdges + fz;
631           PetscInt edgeR   = firstZEdge + (((fy+1)%numYVertices)*numXVertices+fx)*numZEdges + fz;
632           PetscInt edgeB   = firstYEdge + (  fz*                 numXVertices+fx)*numYEdges + fy;
633           PetscInt edgeT   = firstYEdge + (((fz+1)%numZVertices)*numXVertices+fx)*numYEdges + fy;
634           PetscInt ornt[4] = {0, 0, -2, -2};
635           PetscInt cone[4];
636 
637           if (dim == 3) {
638             /* markers */
639             if (bdX != DM_BOUNDARY_PERIODIC) {
640               if (fx == numXVertices-1) {
641                 ierr = DMSetLabelValue(dm, "Face Sets", face, faceMarkerRight);CHKERRQ(ierr);
642                 ierr = DMSetLabelValue(dm, "marker", face, markerRight);CHKERRQ(ierr);
643               }
644               else if (fx == 0) {
645                 ierr = DMSetLabelValue(dm, "Face Sets", face, faceMarkerLeft);CHKERRQ(ierr);
646                 ierr = DMSetLabelValue(dm, "marker", face, markerLeft);CHKERRQ(ierr);
647               }
648             }
649           }
650           cone[0] = edgeB; cone[1] = edgeR; cone[2] = edgeT; cone[3] = edgeL;
651           ierr    = DMPlexSetCone(dm, face, cone);CHKERRQ(ierr);
652           ierr    = DMPlexSetConeOrientation(dm, face, ornt);CHKERRQ(ierr);
653         }
654       }
655     }
656     /* Build y faces */
657     for (fz = 0; fz < numZEdges; ++fz) {
658       for (fx = 0; fx < numXEdges; ++fx) {
659         for (fy = 0; fy < numYVertices; ++fy) {
660           PetscInt face    = firstYFace + (fz*numXEdges+fx)*numYVertices + fy;
661           PetscInt edgeL   = firstZEdge + (fy*numXVertices+  fx                 )*numZEdges + fz;
662           PetscInt edgeR   = firstZEdge + (fy*numXVertices+((fx+1)%numXVertices))*numZEdges + fz;
663           PetscInt edgeB   = firstXEdge + (  fz                 *numYVertices+fy)*numXEdges + fx;
664           PetscInt edgeT   = firstXEdge + (((fz+1)%numZVertices)*numYVertices+fy)*numXEdges + fx;
665           PetscInt ornt[4] = {0, 0, -2, -2};
666           PetscInt cone[4];
667 
668           if (dim == 3) {
669             /* markers */
670             if (bdY != DM_BOUNDARY_PERIODIC) {
671               if (fy == numYVertices-1) {
672                 ierr = DMSetLabelValue(dm, "Face Sets", face, faceMarkerBack);CHKERRQ(ierr);
673                 ierr = DMSetLabelValue(dm, "marker", face, markerBack);CHKERRQ(ierr);
674               }
675               else if (fy == 0) {
676                 ierr = DMSetLabelValue(dm, "Face Sets", face, faceMarkerFront);CHKERRQ(ierr);
677                 ierr = DMSetLabelValue(dm, "marker", face, markerFront);CHKERRQ(ierr);
678               }
679             }
680           }
681           cone[0] = edgeB; cone[1] = edgeR; cone[2] = edgeT; cone[3] = edgeL;
682           ierr    = DMPlexSetCone(dm, face, cone);CHKERRQ(ierr);
683           ierr    = DMPlexSetConeOrientation(dm, face, ornt);CHKERRQ(ierr);
684         }
685       }
686     }
687     /* Build z faces */
688     for (fy = 0; fy < numYEdges; ++fy) {
689       for (fx = 0; fx < numXEdges; ++fx) {
690         for (fz = 0; fz < numZVertices; fz++) {
691           PetscInt face    = firstZFace + (fy*numXEdges+fx)*numZVertices + fz;
692           PetscInt edgeL   = firstYEdge + (fz*numXVertices+  fx                 )*numYEdges + fy;
693           PetscInt edgeR   = firstYEdge + (fz*numXVertices+((fx+1)%numXVertices))*numYEdges + fy;
694           PetscInt edgeB   = firstXEdge + (fz*numYVertices+  fy                 )*numXEdges + fx;
695           PetscInt edgeT   = firstXEdge + (fz*numYVertices+((fy+1)%numYVertices))*numXEdges + fx;
696           PetscInt ornt[4] = {0, 0, -2, -2};
697           PetscInt cone[4];
698 
699           if (dim == 2) {
700             if (bdX == DM_BOUNDARY_TWIST && fx == numXEdges-1) {edgeR += numYEdges-1-2*fy; ornt[1] = -2;}
701             if (bdY == DM_BOUNDARY_TWIST && fy == numYEdges-1) {edgeT += numXEdges-1-2*fx; ornt[2] =  0;}
702             if (bdX != DM_BOUNDARY_NONE && fx == numXEdges-1 && cutLabel) {ierr = DMLabelSetValue(cutLabel, face, 2);CHKERRQ(ierr);}
703             if (bdY != DM_BOUNDARY_NONE && fy == numYEdges-1 && cutLabel) {ierr = DMLabelSetValue(cutLabel, face, 2);CHKERRQ(ierr);}
704           } else {
705             /* markers */
706             if (bdZ != DM_BOUNDARY_PERIODIC) {
707               if (fz == numZVertices-1) {
708                 ierr = DMSetLabelValue(dm, "Face Sets", face, faceMarkerTop);CHKERRQ(ierr);
709                 ierr = DMSetLabelValue(dm, "marker", face, markerTop);CHKERRQ(ierr);
710               }
711               else if (fz == 0) {
712                 ierr = DMSetLabelValue(dm, "Face Sets", face, faceMarkerBottom);CHKERRQ(ierr);
713                 ierr = DMSetLabelValue(dm, "marker", face, markerBottom);CHKERRQ(ierr);
714               }
715             }
716           }
717           cone[0] = edgeB; cone[1] = edgeR; cone[2] = edgeT; cone[3] = edgeL;
718           ierr    = DMPlexSetCone(dm, face, cone);CHKERRQ(ierr);
719           ierr    = DMPlexSetConeOrientation(dm, face, ornt);CHKERRQ(ierr);
720         }
721       }
722     }
723     /* Build Z edges*/
724     for (vy = 0; vy < numYVertices; vy++) {
725       for (vx = 0; vx < numXVertices; vx++) {
726         for (ez = 0; ez < numZEdges; ez++) {
727           const PetscInt edge    = firstZEdge  + (vy*numXVertices+vx)*numZEdges + ez;
728           const PetscInt vertexB = firstVertex + (  ez                 *numYVertices+vy)*numXVertices + vx;
729           const PetscInt vertexT = firstVertex + (((ez+1)%numZVertices)*numYVertices+vy)*numXVertices + vx;
730           PetscInt       cone[2];
731 
732           if (dim == 3) {
733             if (bdX != DM_BOUNDARY_PERIODIC) {
734               if (vx == numXVertices-1) {
735                 ierr = DMSetLabelValue(dm, "marker", edge, markerRight);CHKERRQ(ierr);
736               }
737               else if (vx == 0) {
738                 ierr = DMSetLabelValue(dm, "marker", edge, markerLeft);CHKERRQ(ierr);
739               }
740             }
741             if (bdY != DM_BOUNDARY_PERIODIC) {
742               if (vy == numYVertices-1) {
743                 ierr = DMSetLabelValue(dm, "marker", edge, markerBack);CHKERRQ(ierr);
744               }
745               else if (vy == 0) {
746                 ierr = DMSetLabelValue(dm, "marker", edge, markerFront);CHKERRQ(ierr);
747               }
748             }
749           }
750           cone[0] = vertexB; cone[1] = vertexT;
751           ierr = DMPlexSetCone(dm, edge, cone);CHKERRQ(ierr);
752         }
753       }
754     }
755     /* Build Y edges*/
756     for (vz = 0; vz < numZVertices; vz++) {
757       for (vx = 0; vx < numXVertices; vx++) {
758         for (ey = 0; ey < numYEdges; ey++) {
759           const PetscInt nextv   = (dim == 2 && bdY == DM_BOUNDARY_TWIST && ey == numYEdges-1) ? (numXVertices-vx-1) : (vz*numYVertices+((ey+1)%numYVertices))*numXVertices + vx;
760           const PetscInt edge    = firstYEdge  + (vz*numXVertices+vx)*numYEdges + ey;
761           const PetscInt vertexF = firstVertex + (vz*numYVertices+ey)*numXVertices + vx;
762           const PetscInt vertexK = firstVertex + nextv;
763           PetscInt       cone[2];
764 
765           cone[0] = vertexF; cone[1] = vertexK;
766           ierr = DMPlexSetCone(dm, edge, cone);CHKERRQ(ierr);
767           if (dim == 2) {
768             if ((bdX != DM_BOUNDARY_PERIODIC) && (bdX != DM_BOUNDARY_TWIST)) {
769               if (vx == numXVertices-1) {
770                 ierr = DMSetLabelValue(dm, "Face Sets", edge, faceMarkerRight);CHKERRQ(ierr);
771                 ierr = DMSetLabelValue(dm, "marker", edge,    markerRight);CHKERRQ(ierr);
772                 ierr = DMSetLabelValue(dm, "marker", cone[0], markerRight);CHKERRQ(ierr);
773                 if (ey == numYEdges-1) {
774                   ierr = DMSetLabelValue(dm, "marker", cone[1], markerRight);CHKERRQ(ierr);
775                 }
776               } else if (vx == 0) {
777                 ierr = DMSetLabelValue(dm, "Face Sets", edge, faceMarkerLeft);CHKERRQ(ierr);
778                 ierr = DMSetLabelValue(dm, "marker", edge,    markerLeft);CHKERRQ(ierr);
779                 ierr = DMSetLabelValue(dm, "marker", cone[0], markerLeft);CHKERRQ(ierr);
780                 if (ey == numYEdges-1) {
781                   ierr = DMSetLabelValue(dm, "marker", cone[1], markerLeft);CHKERRQ(ierr);
782                 }
783               }
784             } else {
785               if (vx == 0 && cutMarker) {
786                 ierr = DMLabelSetValue(cutLabel, edge,    1);CHKERRQ(ierr);
787                 ierr = DMLabelSetValue(cutLabel, cone[0], 1);CHKERRQ(ierr);
788                 if (ey == numYEdges-1) {
789                   ierr = DMLabelSetValue(cutLabel, cone[1], 1);CHKERRQ(ierr);
790                 }
791               }
792             }
793           } else {
794             if (bdX != DM_BOUNDARY_PERIODIC) {
795               if (vx == numXVertices-1) {
796                 ierr = DMSetLabelValue(dm, "marker", edge, markerRight);CHKERRQ(ierr);
797               } else if (vx == 0) {
798                 ierr = DMSetLabelValue(dm, "marker", edge, markerLeft);CHKERRQ(ierr);
799               }
800             }
801             if (bdZ != DM_BOUNDARY_PERIODIC) {
802               if (vz == numZVertices-1) {
803                 ierr = DMSetLabelValue(dm, "marker", edge, markerTop);CHKERRQ(ierr);
804               } else if (vz == 0) {
805                 ierr = DMSetLabelValue(dm, "marker", edge, markerBottom);CHKERRQ(ierr);
806               }
807             }
808           }
809         }
810       }
811     }
812     /* Build X edges*/
813     for (vz = 0; vz < numZVertices; vz++) {
814       for (vy = 0; vy < numYVertices; vy++) {
815         for (ex = 0; ex < numXEdges; ex++) {
816           const PetscInt nextv   = (dim == 2 && bdX == DM_BOUNDARY_TWIST && ex == numXEdges-1) ? (numYVertices-vy-1)*numXVertices : (vz*numYVertices+vy)*numXVertices + (ex+1)%numXVertices;
817           const PetscInt edge    = firstXEdge  + (vz*numYVertices+vy)*numXEdges + ex;
818           const PetscInt vertexL = firstVertex + (vz*numYVertices+vy)*numXVertices + ex;
819           const PetscInt vertexR = firstVertex + nextv;
820           PetscInt       cone[2];
821 
822           cone[0] = vertexL; cone[1] = vertexR;
823           ierr = DMPlexSetCone(dm, edge, cone);CHKERRQ(ierr);
824           if (dim == 2) {
825             if ((bdY != DM_BOUNDARY_PERIODIC) && (bdY != DM_BOUNDARY_TWIST)) {
826               if (vy == numYVertices-1) {
827                 ierr = DMSetLabelValue(dm, "Face Sets", edge, faceMarkerTop);CHKERRQ(ierr);
828                 ierr = DMSetLabelValue(dm, "marker", edge,    markerTop);CHKERRQ(ierr);
829                 ierr = DMSetLabelValue(dm, "marker", cone[0], markerTop);CHKERRQ(ierr);
830                 if (ex == numXEdges-1) {
831                   ierr = DMSetLabelValue(dm, "marker", cone[1], markerTop);CHKERRQ(ierr);
832                 }
833               } else if (vy == 0) {
834                 ierr = DMSetLabelValue(dm, "Face Sets", edge, faceMarkerBottom);CHKERRQ(ierr);
835                 ierr = DMSetLabelValue(dm, "marker", edge,    markerBottom);CHKERRQ(ierr);
836                 ierr = DMSetLabelValue(dm, "marker", cone[0], markerBottom);CHKERRQ(ierr);
837                 if (ex == numXEdges-1) {
838                   ierr = DMSetLabelValue(dm, "marker", cone[1], markerBottom);CHKERRQ(ierr);
839                 }
840               }
841             } else {
842               if (vy == 0 && cutMarker) {
843                 ierr = DMLabelSetValue(cutLabel, edge,    1);CHKERRQ(ierr);
844                 ierr = DMLabelSetValue(cutLabel, cone[0], 1);CHKERRQ(ierr);
845                 if (ex == numXEdges-1) {
846                   ierr = DMLabelSetValue(cutLabel, cone[1], 1);CHKERRQ(ierr);
847                 }
848               }
849             }
850           } else {
851             if (bdY != DM_BOUNDARY_PERIODIC) {
852               if (vy == numYVertices-1) {
853                 ierr = DMSetLabelValue(dm, "marker", edge, markerBack);CHKERRQ(ierr);
854               }
855               else if (vy == 0) {
856                 ierr = DMSetLabelValue(dm, "marker", edge, markerFront);CHKERRQ(ierr);
857               }
858             }
859             if (bdZ != DM_BOUNDARY_PERIODIC) {
860               if (vz == numZVertices-1) {
861                 ierr = DMSetLabelValue(dm, "marker", edge, markerTop);CHKERRQ(ierr);
862               }
863               else if (vz == 0) {
864                 ierr = DMSetLabelValue(dm, "marker", edge, markerBottom);CHKERRQ(ierr);
865               }
866             }
867           }
868         }
869       }
870     }
871     ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
872     ierr = DMPlexStratify(dm);CHKERRQ(ierr);
873     /* Build coordinates */
874     ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
875     ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
876     ierr = PetscSectionSetFieldComponents(coordSection, 0, dim);CHKERRQ(ierr);
877     ierr = PetscSectionSetChart(coordSection, firstVertex, firstVertex+numVertices);CHKERRQ(ierr);
878     for (v = firstVertex; v < firstVertex+numVertices; ++v) {
879       ierr = PetscSectionSetDof(coordSection, v, dim);CHKERRQ(ierr);
880       ierr = PetscSectionSetFieldDof(coordSection, v, 0, dim);CHKERRQ(ierr);
881     }
882     ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
883     ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
884     ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
885     ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
886     ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
887     ierr = VecSetBlockSize(coordinates, dim);CHKERRQ(ierr);
888     ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
889     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
890     for (vz = 0; vz < numZVertices; ++vz) {
891       for (vy = 0; vy < numYVertices; ++vy) {
892         for (vx = 0; vx < numXVertices; ++vx) {
893           coords[((vz*numYVertices+vy)*numXVertices+vx)*dim+0] = lower[0] + ((upper[0] - lower[0])/numXEdges)*vx;
894           coords[((vz*numYVertices+vy)*numXVertices+vx)*dim+1] = lower[1] + ((upper[1] - lower[1])/numYEdges)*vy;
895           if (dim == 3) {
896             coords[((vz*numYVertices+vy)*numXVertices+vx)*dim+2] = lower[2] + ((upper[2] - lower[2])/numZEdges)*vz;
897           }
898         }
899       }
900     }
901     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
902     ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
903     ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
904   }
905   PetscFunctionReturn(0);
906 }
907 
908 /*@
909   DMPlexCreateHexBoxMesh - Creates a mesh on the tensor product of unit intervals (box) using hexahedra.
910 
911   Collective on MPI_Comm
912 
913   Input Parameters:
914 + comm  - The communicator for the DM object
915 . dim   - The spatial dimension
916 . periodicX - The boundary type for the X direction
917 . periodicY - The boundary type for the Y direction
918 . periodicZ - The boundary type for the Z direction
919 - cells - The number of cells in each direction
920 
921   Output Parameter:
922 . dm  - The DM object
923 
924   Note: Here is the numbering returned for 2 cells in each direction:
925 $ 22--8-23--9--24
926 $  |     |     |
927 $ 13  2 14  3  15
928 $  |     |     |
929 $ 19--6-20--7--21
930 $  |     |     |
931 $ 10  0 11  1 12
932 $  |     |     |
933 $ 16--4-17--5--18
934 
935   Level: beginner
936 
937 .keywords: DM, create
938 .seealso: DMPlexCreateBoxMesh(), DMSetType(), DMCreate()
939 @*/
940 PetscErrorCode DMPlexCreateHexBoxMesh(MPI_Comm comm, PetscInt dim, const PetscInt cells[], DMBoundaryType periodicX, DMBoundaryType periodicY, DMBoundaryType periodicZ, DM *dm)
941 {
942   PetscInt       i;
943   PetscErrorCode ierr;
944 
945   PetscFunctionBegin;
946   PetscValidPointer(dm, 7);
947   ierr = DMCreate(comm, dm);CHKERRQ(ierr);
948   PetscValidLogicalCollectiveInt(*dm,dim,2);
949   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
950   ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
951   switch (dim) {
952   case 2:
953   {
954     PetscReal lower[3] = {0.0, 0.0, 0.0};
955     PetscReal upper[3] = {1.0, 1.0, 0.0};
956     PetscInt  edges[3];
957 
958     edges[0] = cells[0];
959     edges[1] = cells[1];
960     edges[2] = 0;
961 
962     ierr = DMPlexCreateCubeMesh_Internal(*dm, lower, upper, edges, periodicX, periodicY, DM_BOUNDARY_NONE);CHKERRQ(ierr);
963     if (periodicX == DM_BOUNDARY_PERIODIC || periodicX == DM_BOUNDARY_TWIST ||
964         periodicY == DM_BOUNDARY_PERIODIC || periodicY == DM_BOUNDARY_TWIST) {
965       PetscReal      L[2];
966       PetscReal      maxCell[2];
967       DMBoundaryType bdType[2];
968 
969       bdType[0] = periodicX;
970       bdType[1] = periodicY;
971       for (i = 0; i < dim; i++) {
972         L[i]       = upper[i] - lower[i];
973         maxCell[i] = 1.1 * (L[i] / PetscMax(1,cells[i]));
974       }
975 
976       ierr = DMSetPeriodicity(*dm,PETSC_TRUE,maxCell,L,bdType);CHKERRQ(ierr);
977     }
978     break;
979   }
980   case 3:
981   {
982     PetscReal lower[3] = {0.0, 0.0, 0.0};
983     PetscReal upper[3] = {1.0, 1.0, 1.0};
984 
985     ierr = DMPlexCreateCubeMesh_Internal(*dm, lower, upper, cells, periodicX, periodicY, periodicZ);CHKERRQ(ierr);
986     if (periodicX == DM_BOUNDARY_PERIODIC || periodicX == DM_BOUNDARY_TWIST ||
987         periodicY == DM_BOUNDARY_PERIODIC || periodicY == DM_BOUNDARY_TWIST ||
988         periodicZ == DM_BOUNDARY_PERIODIC || periodicZ == DM_BOUNDARY_TWIST) {
989       PetscReal      L[3];
990       PetscReal      maxCell[3];
991       DMBoundaryType bdType[3];
992 
993       bdType[0] = periodicX;
994       bdType[1] = periodicY;
995       bdType[2] = periodicZ;
996       for (i = 0; i < dim; i++) {
997         L[i]       = upper[i] - lower[i];
998         maxCell[i] = 1.1 * (L[i] / cells[i]);
999       }
1000 
1001       ierr = DMSetPeriodicity(*dm,PETSC_TRUE,maxCell,L,bdType);CHKERRQ(ierr);
1002     }
1003     break;
1004   }
1005   default:
1006     SETERRQ1(comm, PETSC_ERR_SUP, "Dimension not supported: %d", dim);
1007   }
1008   PetscFunctionReturn(0);
1009 }
1010 
1011 /*@C
1012   DMPlexSetOptionsPrefix - Sets the prefix used for searching for all DM options in the database.
1013 
1014   Logically Collective on DM
1015 
1016   Input Parameters:
1017 + dm - the DM context
1018 - prefix - the prefix to prepend to all option names
1019 
1020   Notes:
1021   A hyphen (-) must NOT be given at the beginning of the prefix name.
1022   The first character of all runtime options is AUTOMATICALLY the hyphen.
1023 
1024   Level: advanced
1025 
1026 .seealso: SNESSetFromOptions()
1027 @*/
1028 PetscErrorCode DMPlexSetOptionsPrefix(DM dm, const char prefix[])
1029 {
1030   DM_Plex       *mesh = (DM_Plex *) dm->data;
1031   PetscErrorCode ierr;
1032 
1033   PetscFunctionBegin;
1034   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1035   ierr = PetscObjectSetOptionsPrefix((PetscObject) dm, prefix);CHKERRQ(ierr);
1036   ierr = PetscObjectSetOptionsPrefix((PetscObject) mesh->partitioner, prefix);CHKERRQ(ierr);
1037   PetscFunctionReturn(0);
1038 }
1039 
1040 /*@
1041   DMPlexCreateHexCylinderMesh - Creates a mesh on the tensor product of the unit interval with the circle (cylinder) using hexahedra.
1042 
1043   Collective on MPI_Comm
1044 
1045   Input Parameters:
1046 + comm      - The communicator for the DM object
1047 . numRefine - The number of regular refinements to the basic 5 cell structure
1048 - periodicZ - The boundary type for the Z direction
1049 
1050   Output Parameter:
1051 . dm  - The DM object
1052 
1053   Note: Here is the output numbering looking from the bottom of the cylinder:
1054 $       17-----14
1055 $        |     |
1056 $        |  2  |
1057 $        |     |
1058 $ 17-----8-----7-----14
1059 $  |     |     |     |
1060 $  |  3  |  0  |  1  |
1061 $  |     |     |     |
1062 $ 19-----5-----6-----13
1063 $        |     |
1064 $        |  4  |
1065 $        |     |
1066 $       19-----13
1067 $
1068 $ and up through the top
1069 $
1070 $       18-----16
1071 $        |     |
1072 $        |  2  |
1073 $        |     |
1074 $ 18----10----11-----16
1075 $  |     |     |     |
1076 $  |  3  |  0  |  1  |
1077 $  |     |     |     |
1078 $ 20-----9----12-----15
1079 $        |     |
1080 $        |  4  |
1081 $        |     |
1082 $       20-----15
1083 
1084   Level: beginner
1085 
1086 .keywords: DM, create
1087 .seealso: DMPlexCreateHexBoxMesh(), DMPlexCreateBoxMesh(), DMSetType(), DMCreate()
1088 @*/
1089 PetscErrorCode DMPlexCreateHexCylinderMesh(MPI_Comm comm, PetscInt numRefine, DMBoundaryType periodicZ, DM *dm)
1090 {
1091   const PetscInt dim = 3;
1092   PetscInt       numCells, numVertices, r;
1093   PetscErrorCode ierr;
1094 
1095   PetscFunctionBegin;
1096   PetscValidPointer(dm, 4);
1097   if (numRefine < 0) SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Number of refinements %D cannot be negative", numRefine);
1098   ierr = DMCreate(comm, dm);CHKERRQ(ierr);
1099   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
1100   ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
1101   /* Create topology */
1102   {
1103     PetscInt cone[8], c;
1104 
1105     numCells    = 5;
1106     numVertices = 16;
1107     if (periodicZ == DM_BOUNDARY_PERIODIC) {
1108       numCells   *= 3;
1109       numVertices = 24;
1110     }
1111     ierr = DMPlexSetChart(*dm, 0, numCells+numVertices);CHKERRQ(ierr);
1112     for (c = 0; c < numCells; c++) {ierr = DMPlexSetConeSize(*dm, c, 8);CHKERRQ(ierr);}
1113     ierr = DMSetUp(*dm);CHKERRQ(ierr);
1114     if (periodicZ == DM_BOUNDARY_PERIODIC) {
1115       cone[0] = 15; cone[1] = 18; cone[2] = 17; cone[3] = 16;
1116       cone[4] = 31; cone[5] = 32; cone[6] = 33; cone[7] = 34;
1117       ierr = DMPlexSetCone(*dm, 0, cone);CHKERRQ(ierr);
1118       cone[0] = 16; cone[1] = 17; cone[2] = 24; cone[3] = 23;
1119       cone[4] = 32; cone[5] = 36; cone[6] = 37; cone[7] = 33; /* 22 25 26 21 */
1120       ierr = DMPlexSetCone(*dm, 1, cone);CHKERRQ(ierr);
1121       cone[0] = 18; cone[1] = 27; cone[2] = 24; cone[3] = 17;
1122       cone[4] = 34; cone[5] = 33; cone[6] = 37; cone[7] = 38;
1123       ierr = DMPlexSetCone(*dm, 2, cone);CHKERRQ(ierr);
1124       cone[0] = 29; cone[1] = 27; cone[2] = 18; cone[3] = 15;
1125       cone[4] = 35; cone[5] = 31; cone[6] = 34; cone[7] = 38;
1126       ierr = DMPlexSetCone(*dm, 3, cone);CHKERRQ(ierr);
1127       cone[0] = 29; cone[1] = 15; cone[2] = 16; cone[3] = 23;
1128       cone[4] = 35; cone[5] = 36; cone[6] = 32; cone[7] = 31;
1129       ierr = DMPlexSetCone(*dm, 4, cone);CHKERRQ(ierr);
1130 
1131       cone[0] = 31; cone[1] = 34; cone[2] = 33; cone[3] = 32;
1132       cone[4] = 19; cone[5] = 22; cone[6] = 21; cone[7] = 20;
1133       ierr = DMPlexSetCone(*dm, 5, cone);CHKERRQ(ierr);
1134       cone[0] = 32; cone[1] = 33; cone[2] = 37; cone[3] = 36;
1135       cone[4] = 22; cone[5] = 25; cone[6] = 26; cone[7] = 21;
1136       ierr = DMPlexSetCone(*dm, 6, cone);CHKERRQ(ierr);
1137       cone[0] = 34; cone[1] = 38; cone[2] = 37; cone[3] = 33;
1138       cone[4] = 20; cone[5] = 21; cone[6] = 26; cone[7] = 28;
1139       ierr = DMPlexSetCone(*dm, 7, cone);CHKERRQ(ierr);
1140       cone[0] = 35; cone[1] = 38; cone[2] = 34; cone[3] = 31;
1141       cone[4] = 30; cone[5] = 19; cone[6] = 20; cone[7] = 28;
1142       ierr = DMPlexSetCone(*dm, 8, cone);CHKERRQ(ierr);
1143       cone[0] = 35; cone[1] = 31; cone[2] = 32; cone[3] = 36;
1144       cone[4] = 30; cone[5] = 25; cone[6] = 22; cone[7] = 19;
1145       ierr = DMPlexSetCone(*dm, 9, cone);CHKERRQ(ierr);
1146 
1147       cone[0] = 19; cone[1] = 20; cone[2] = 21; cone[3] = 22;
1148       cone[4] = 15; cone[5] = 16; cone[6] = 17; cone[7] = 18;
1149       ierr = DMPlexSetCone(*dm, 10, cone);CHKERRQ(ierr);
1150       cone[0] = 22; cone[1] = 21; cone[2] = 26; cone[3] = 25;
1151       cone[4] = 16; cone[5] = 23; cone[6] = 24; cone[7] = 17;
1152       ierr = DMPlexSetCone(*dm, 11, cone);CHKERRQ(ierr);
1153       cone[0] = 20; cone[1] = 28; cone[2] = 26; cone[3] = 21;
1154       cone[4] = 18; cone[5] = 17; cone[6] = 24; cone[7] = 27;
1155       ierr = DMPlexSetCone(*dm, 12, cone);CHKERRQ(ierr);
1156       cone[0] = 30; cone[1] = 28; cone[2] = 20; cone[3] = 19;
1157       cone[4] = 29; cone[5] = 15; cone[6] = 18; cone[7] = 27;
1158       ierr = DMPlexSetCone(*dm, 13, cone);CHKERRQ(ierr);
1159       cone[0] = 30; cone[1] = 19; cone[2] = 22; cone[3] = 25;
1160       cone[4] = 29; cone[5] = 23; cone[6] = 16; cone[7] = 15;
1161       ierr = DMPlexSetCone(*dm, 14, cone);CHKERRQ(ierr);
1162     } else {
1163       cone[0] =  5; cone[1] =  8; cone[2] =  7; cone[3] =  6;
1164       cone[4] =  9; cone[5] = 12; cone[6] = 11; cone[7] = 10;
1165       ierr = DMPlexSetCone(*dm, 0, cone);CHKERRQ(ierr);
1166       cone[0] =  6; cone[1] =  7; cone[2] = 14; cone[3] = 13;
1167       cone[4] = 12; cone[5] = 15; cone[6] = 16; cone[7] = 11;
1168       ierr = DMPlexSetCone(*dm, 1, cone);CHKERRQ(ierr);
1169       cone[0] =  8; cone[1] = 17; cone[2] = 14; cone[3] =  7;
1170       cone[4] = 10; cone[5] = 11; cone[6] = 16; cone[7] = 18;
1171       ierr = DMPlexSetCone(*dm, 2, cone);CHKERRQ(ierr);
1172       cone[0] = 19; cone[1] = 17; cone[2] =  8; cone[3] =  5;
1173       cone[4] = 20; cone[5] =  9; cone[6] = 10; cone[7] = 18;
1174       ierr = DMPlexSetCone(*dm, 3, cone);CHKERRQ(ierr);
1175       cone[0] = 19; cone[1] =  5; cone[2] =  6; cone[3] = 13;
1176       cone[4] = 20; cone[5] = 15; cone[6] = 12; cone[7] =  9;
1177       ierr = DMPlexSetCone(*dm, 4, cone);CHKERRQ(ierr);
1178     }
1179     ierr = DMPlexSymmetrize(*dm);CHKERRQ(ierr);
1180     ierr = DMPlexStratify(*dm);CHKERRQ(ierr);
1181   }
1182   /* Interpolate */
1183   {
1184     DM idm = NULL;
1185 
1186     ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
1187     ierr = DMDestroy(dm);CHKERRQ(ierr);
1188     *dm  = idm;
1189   }
1190   /* Create cube geometry */
1191   {
1192     Vec             coordinates;
1193     PetscSection    coordSection;
1194     PetscScalar    *coords;
1195     PetscInt        coordSize, v;
1196     const PetscReal dis = 1.0/PetscSqrtReal(2.0);
1197     const PetscReal ds2 = dis/2.0;
1198 
1199     /* Build coordinates */
1200     ierr = DMGetCoordinateSection(*dm, &coordSection);CHKERRQ(ierr);
1201     ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
1202     ierr = PetscSectionSetFieldComponents(coordSection, 0, dim);CHKERRQ(ierr);
1203     ierr = PetscSectionSetChart(coordSection, numCells, numCells+numVertices);CHKERRQ(ierr);
1204     for (v = numCells; v < numCells+numVertices; ++v) {
1205       ierr = PetscSectionSetDof(coordSection, v, dim);CHKERRQ(ierr);
1206       ierr = PetscSectionSetFieldDof(coordSection, v, 0, dim);CHKERRQ(ierr);
1207     }
1208     ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
1209     ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
1210     ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
1211     ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
1212     ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
1213     ierr = VecSetBlockSize(coordinates, dim);CHKERRQ(ierr);
1214     ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
1215     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
1216     coords[0*dim+0] = -ds2; coords[0*dim+1] = -ds2; coords[0*dim+2] = 0.0;
1217     coords[1*dim+0] =  ds2; coords[1*dim+1] = -ds2; coords[1*dim+2] = 0.0;
1218     coords[2*dim+0] =  ds2; coords[2*dim+1] =  ds2; coords[2*dim+2] = 0.0;
1219     coords[3*dim+0] = -ds2; coords[3*dim+1] =  ds2; coords[3*dim+2] = 0.0;
1220     coords[4*dim+0] = -ds2; coords[4*dim+1] = -ds2; coords[4*dim+2] = 1.0;
1221     coords[5*dim+0] = -ds2; coords[5*dim+1] =  ds2; coords[5*dim+2] = 1.0;
1222     coords[6*dim+0] =  ds2; coords[6*dim+1] =  ds2; coords[6*dim+2] = 1.0;
1223     coords[7*dim+0] =  ds2; coords[7*dim+1] = -ds2; coords[7*dim+2] = 1.0;
1224     coords[ 8*dim+0] =  dis; coords[ 8*dim+1] = -dis; coords[ 8*dim+2] = 0.0;
1225     coords[ 9*dim+0] =  dis; coords[ 9*dim+1] =  dis; coords[ 9*dim+2] = 0.0;
1226     coords[10*dim+0] =  dis; coords[10*dim+1] = -dis; coords[10*dim+2] = 1.0;
1227     coords[11*dim+0] =  dis; coords[11*dim+1] =  dis; coords[11*dim+2] = 1.0;
1228     coords[12*dim+0] = -dis; coords[12*dim+1] =  dis; coords[12*dim+2] = 0.0;
1229     coords[13*dim+0] = -dis; coords[13*dim+1] =  dis; coords[13*dim+2] = 1.0;
1230     coords[14*dim+0] = -dis; coords[14*dim+1] = -dis; coords[14*dim+2] = 0.0;
1231     coords[15*dim+0] = -dis; coords[15*dim+1] = -dis; coords[15*dim+2] = 1.0;
1232     if (periodicZ == DM_BOUNDARY_PERIODIC) {
1233       /* 15 31 19 */ coords[16*dim+0] = -ds2; coords[16*dim+1] = -ds2; coords[16*dim+2] = 0.5;
1234       /* 16 32 22 */ coords[17*dim+0] =  ds2; coords[17*dim+1] = -ds2; coords[17*dim+2] = 0.5;
1235       /* 17 33 21 */ coords[18*dim+0] =  ds2; coords[18*dim+1] =  ds2; coords[18*dim+2] = 0.5;
1236       /* 18 34 20 */ coords[19*dim+0] = -ds2; coords[19*dim+1] =  ds2; coords[19*dim+2] = 0.5;
1237       /* 29 35 30 */ coords[20*dim+0] = -dis; coords[20*dim+1] = -dis; coords[20*dim+2] = 0.5;
1238       /* 23 36 25 */ coords[21*dim+0] =  dis; coords[21*dim+1] = -dis; coords[21*dim+2] = 0.5;
1239       /* 24 37 26 */ coords[22*dim+0] =  dis; coords[22*dim+1] =  dis; coords[22*dim+2] = 0.5;
1240       /* 27 38 28 */ coords[23*dim+0] = -dis; coords[23*dim+1] =  dis; coords[23*dim+2] = 0.5;
1241     }
1242     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
1243     ierr = DMSetCoordinatesLocal(*dm, coordinates);CHKERRQ(ierr);
1244     ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
1245   }
1246   /* Create periodicity */
1247   if (periodicZ == DM_BOUNDARY_PERIODIC || periodicZ == DM_BOUNDARY_TWIST) {
1248     PetscReal      L[3];
1249     PetscReal      maxCell[3];
1250     DMBoundaryType bdType[3];
1251     PetscReal      lower[3] = {0.0, 0.0, 0.0};
1252     PetscReal      upper[3] = {1.0, 1.0, 1.5};
1253     PetscInt       i, numZCells = 3;
1254 
1255     bdType[0] = DM_BOUNDARY_NONE;
1256     bdType[1] = DM_BOUNDARY_NONE;
1257     bdType[2] = periodicZ;
1258     for (i = 0; i < dim; i++) {
1259       L[i]       = upper[i] - lower[i];
1260       maxCell[i] = 1.1 * (L[i] / numZCells);
1261     }
1262     ierr = DMSetPeriodicity(*dm, PETSC_TRUE, maxCell, L, bdType);CHKERRQ(ierr);
1263   }
1264   /* Refine topology */
1265   for (r = 0; r < numRefine; ++r) {
1266     DM rdm = NULL;
1267 
1268     ierr = DMRefine(*dm, comm, &rdm);CHKERRQ(ierr);
1269     ierr = DMDestroy(dm);CHKERRQ(ierr);
1270     *dm  = rdm;
1271   }
1272   /* Remap geometry to cylinder
1273        Interior square: Linear interpolation is correct
1274        The other cells all have vertices on rays from the origin. We want to uniformly expand the spacing
1275        such that the last vertex is on the unit circle. So the closest and farthest vertices are at distance
1276 
1277          phi     = arctan(y/x)
1278          d_close = sqrt(1/8 + 1/4 sin^2(phi))
1279          d_far   = sqrt(1/2 + sin^2(phi))
1280 
1281        so we remap them using
1282 
1283          x_new = x_close + (x - x_close) (1 - d_close) / (d_far - d_close)
1284          y_new = y_close + (y - y_close) (1 - d_close) / (d_far - d_close)
1285 
1286        If pi/4 < phi < 3pi/4 or -3pi/4 < phi < -pi/4, then we switch x and y.
1287   */
1288   {
1289     Vec           coordinates;
1290     PetscSection  coordSection;
1291     PetscScalar  *coords;
1292     PetscInt      vStart, vEnd, v;
1293     const PetscReal dis = 1.0/PetscSqrtReal(2.0);
1294     const PetscReal ds2 = 0.5*dis;
1295 
1296     ierr = DMPlexGetDepthStratum(*dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
1297     ierr = DMGetCoordinateSection(*dm, &coordSection);CHKERRQ(ierr);
1298     ierr = DMGetCoordinatesLocal(*dm, &coordinates);CHKERRQ(ierr);
1299     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
1300     for (v = vStart; v < vEnd; ++v) {
1301       PetscReal phi, sinp, cosp, dc, df, x, y, xc, yc;
1302       PetscInt  off;
1303 
1304       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
1305       if ((PetscAbsScalar(coords[off+0]) <= ds2) && (PetscAbsScalar(coords[off+1]) <= ds2)) continue;
1306       x    = PetscRealPart(coords[off]);
1307       y    = PetscRealPart(coords[off+1]);
1308       phi  = PetscAtan2Real(y, x);
1309       sinp = PetscSinReal(phi);
1310       cosp = PetscCosReal(phi);
1311       if ((PetscAbsReal(phi) > PETSC_PI/4.0) && (PetscAbsReal(phi) < 3.0*PETSC_PI/4.0)) {
1312         dc = PetscAbsReal(ds2/sinp);
1313         df = PetscAbsReal(dis/sinp);
1314         xc = ds2*x/PetscAbsReal(y);
1315         yc = ds2*PetscSignReal(y);
1316       } else {
1317         dc = PetscAbsReal(ds2/cosp);
1318         df = PetscAbsReal(dis/cosp);
1319         xc = ds2*PetscSignReal(x);
1320         yc = ds2*y/PetscAbsReal(x);
1321       }
1322       coords[off+0] = xc + (coords[off+0] - xc)*(1.0 - dc)/(df - dc);
1323       coords[off+1] = yc + (coords[off+1] - yc)*(1.0 - dc)/(df - dc);
1324     }
1325     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
1326     if (periodicZ == DM_BOUNDARY_PERIODIC || periodicZ == DM_BOUNDARY_TWIST) {
1327       ierr = DMLocalizeCoordinates(*dm);CHKERRQ(ierr);
1328     }
1329   }
1330   PetscFunctionReturn(0);
1331 }
1332 
1333 /*@
1334   DMPlexCreateWedgeCylinderMesh - Creates a mesh on the tensor product of the unit interval with the circle (cylinder) using wedges.
1335 
1336   Collective on MPI_Comm
1337 
1338   Input Parameters:
1339 + comm - The communicator for the DM object
1340 . n    - The number of wedges around the origin
1341 - interpolate - Create edges and faces
1342 
1343   Output Parameter:
1344 . dm  - The DM object
1345 
1346   Level: beginner
1347 
1348 .keywords: DM, create
1349 .seealso: DMPlexCreateHexCylinderMesh(), DMPlexCreateHexBoxMesh(), DMPlexCreateBoxMesh(), DMSetType(), DMCreate()
1350 @*/
1351 PetscErrorCode DMPlexCreateWedgeCylinderMesh(MPI_Comm comm, PetscInt n, PetscBool interpolate, DM *dm)
1352 {
1353   const PetscInt dim = 3;
1354   PetscInt       numCells, numVertices;
1355   PetscErrorCode ierr;
1356 
1357   PetscFunctionBegin;
1358   PetscValidPointer(dm, 3);
1359   if (n < 0) SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Number of wedges %D cannot be negative", n);
1360   ierr = DMCreate(comm, dm);CHKERRQ(ierr);
1361   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
1362   ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
1363   /* Create topology */
1364   {
1365     PetscInt cone[6], c;
1366 
1367     numCells    = n;
1368     numVertices = 2*(n+1);
1369     ierr = DMPlexSetChart(*dm, 0, numCells+numVertices);CHKERRQ(ierr);
1370     for (c = 0; c < numCells; c++) {ierr = DMPlexSetConeSize(*dm, c, 6);CHKERRQ(ierr);}
1371     ierr = DMSetUp(*dm);CHKERRQ(ierr);
1372     for (c = 0; c < numCells; c++) {
1373       cone[0] =  c+n*1; cone[1] = (c+1)%n+n*1; cone[2] = 0+3*n;
1374       cone[3] =  c+n*2; cone[4] = (c+1)%n+n*2; cone[5] = 1+3*n;
1375       ierr = DMPlexSetCone(*dm, c, cone);CHKERRQ(ierr);
1376     }
1377     ierr = DMPlexSymmetrize(*dm);CHKERRQ(ierr);
1378     ierr = DMPlexStratify(*dm);CHKERRQ(ierr);
1379   }
1380   /* Interpolate */
1381   if (interpolate) {
1382     DM idm = NULL;
1383 
1384     ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
1385     ierr = DMDestroy(dm);CHKERRQ(ierr);
1386     *dm  = idm;
1387   }
1388   /* Create cylinder geometry */
1389   {
1390     Vec          coordinates;
1391     PetscSection coordSection;
1392     PetscScalar *coords;
1393     PetscInt     coordSize, v, c;
1394 
1395     /* Build coordinates */
1396     ierr = DMGetCoordinateSection(*dm, &coordSection);CHKERRQ(ierr);
1397     ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
1398     ierr = PetscSectionSetFieldComponents(coordSection, 0, dim);CHKERRQ(ierr);
1399     ierr = PetscSectionSetChart(coordSection, numCells, numCells+numVertices);CHKERRQ(ierr);
1400     for (v = numCells; v < numCells+numVertices; ++v) {
1401       ierr = PetscSectionSetDof(coordSection, v, dim);CHKERRQ(ierr);
1402       ierr = PetscSectionSetFieldDof(coordSection, v, 0, dim);CHKERRQ(ierr);
1403     }
1404     ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
1405     ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
1406     ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
1407     ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
1408     ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
1409     ierr = VecSetBlockSize(coordinates, dim);CHKERRQ(ierr);
1410     ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
1411     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
1412     for (c = 0; c < numCells; c++) {
1413       coords[(c+0*n)*dim+0] = PetscCosReal(2.0*c*PETSC_PI/n); coords[(c+0*n)*dim+1] = PetscSinReal(2.0*c*PETSC_PI/n); coords[(c+0*n)*dim+2] = 1.0;
1414       coords[(c+1*n)*dim+0] = PetscCosReal(2.0*c*PETSC_PI/n); coords[(c+1*n)*dim+1] = PetscSinReal(2.0*c*PETSC_PI/n); coords[(c+1*n)*dim+2] = 0.0;
1415     }
1416     coords[(2*n+0)*dim+0] = 0.0; coords[(2*n+0)*dim+1] = 0.0; coords[(2*n+0)*dim+2] = 1.0;
1417     coords[(2*n+1)*dim+0] = 0.0; coords[(2*n+1)*dim+1] = 0.0; coords[(2*n+1)*dim+2] = 0.0;
1418     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
1419     ierr = DMSetCoordinatesLocal(*dm, coordinates);CHKERRQ(ierr);
1420     ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
1421   }
1422   PetscFunctionReturn(0);
1423 }
1424 
1425 PETSC_STATIC_INLINE PetscReal DiffNormReal(PetscInt dim, const PetscReal x[], const PetscReal y[])
1426 {
1427   PetscReal prod = 0.0;
1428   PetscInt  i;
1429   for (i = 0; i < dim; ++i) prod += PetscSqr(x[i] - y[i]);
1430   return PetscSqrtReal(prod);
1431 }
1432 PETSC_STATIC_INLINE PetscReal DotReal(PetscInt dim, const PetscReal x[], const PetscReal y[])
1433 {
1434   PetscReal prod = 0.0;
1435   PetscInt  i;
1436   for (i = 0; i < dim; ++i) prod += x[i]*y[i];
1437   return prod;
1438 }
1439 
1440 #undef __FUNCT__
1441 #define __FUNCT__ "DMPlexCreateSphereMesh"
1442 /*@
1443   DMPlexCreateSphereMesh - Creates a mesh on the d-dimensional sphere, S^d.
1444 
1445   Collective on MPI_Comm
1446 
1447   Input Parameters:
1448 . comm  - The communicator for the DM object
1449 . dim   - The dimension
1450 - simplex - Use simplices, or tensor product cells
1451 
1452   Output Parameter:
1453 . dm  - The DM object
1454 
1455   Level: beginner
1456 
1457 .keywords: DM, create
1458 .seealso: DMPlexCreateBoxMesh(), DMPlexCreateHexBoxMesh(), DMSetType(), DMCreate()
1459 @*/
1460 PetscErrorCode DMPlexCreateSphereMesh(MPI_Comm comm, PetscInt dim, PetscBool simplex, DM *dm)
1461 {
1462   const PetscInt  embedDim = dim+1;
1463   PetscSection    coordSection;
1464   Vec             coordinates;
1465   PetscScalar    *coords;
1466   PetscReal      *coordsIn;
1467   PetscInt        numCells, numEdges, numVerts, firstVertex, v, firstEdge, coordSize, d, c, e;
1468   PetscMPIInt     rank;
1469   PetscErrorCode  ierr;
1470 
1471   PetscFunctionBegin;
1472   PetscValidPointer(dm, 4);
1473   ierr = DMCreate(comm, dm);CHKERRQ(ierr);
1474   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
1475   ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
1476   ierr = DMSetCoordinateDim(*dm, dim+1);CHKERRQ(ierr);
1477   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) *dm), &rank);CHKERRQ(ierr);
1478   switch (dim) {
1479   case 2:
1480     if (simplex) {
1481       DM              idm         = NULL;
1482       const PetscReal edgeLen     = 2.0/(1.0 + PETSC_PHI);
1483       const PetscReal vertex[3]   = {0.0, 1.0/(1.0 + PETSC_PHI), PETSC_PHI/(1.0 + PETSC_PHI)};
1484       const PetscInt  degree      = 5;
1485       PetscInt        s[3]        = {1, 1, 1};
1486       PetscInt        cone[3];
1487       PetscInt       *graph, p, i, j, k;
1488 
1489       numCells    = !rank ? 20 : 0;
1490       numVerts    = !rank ? 12 : 0;
1491       firstVertex = numCells;
1492       /* Use icosahedron, which for a unit sphere has coordinates which are all cyclic permutations of
1493 
1494            (0, \pm 1/\phi+1, \pm \phi/\phi+1)
1495 
1496          where \phi^2 - \phi - 1 = 0, meaning \phi is the golden ratio \frac{1 + \sqrt{5}}{2}. The edge
1497          length is then given by 2/\phi = 2 * 2.73606 = 5.47214.
1498        */
1499       /* Construct vertices */
1500       ierr = PetscCalloc1(numVerts * embedDim, &coordsIn);CHKERRQ(ierr);
1501       for (p = 0, i = 0; p < embedDim; ++p) {
1502         for (s[1] = -1; s[1] < 2; s[1] += 2) {
1503           for (s[2] = -1; s[2] < 2; s[2] += 2) {
1504             for (d = 0; d < embedDim; ++d) coordsIn[i*embedDim+d] = s[(d+p)%embedDim]*vertex[(d+p)%embedDim];
1505             ++i;
1506           }
1507         }
1508       }
1509       /* Construct graph */
1510       ierr = PetscCalloc1(numVerts * numVerts, &graph);CHKERRQ(ierr);
1511       for (i = 0; i < numVerts; ++i) {
1512         for (j = 0, k = 0; j < numVerts; ++j) {
1513           if (PetscAbsReal(DiffNormReal(embedDim, &coordsIn[i*embedDim], &coordsIn[j*embedDim]) - edgeLen) < PETSC_SMALL) {graph[i*numVerts+j] = 1; ++k;}
1514         }
1515         if (k != degree) SETERRQ3(comm, PETSC_ERR_PLIB, "Invalid icosahedron, vertex %D degree %D != %D", i, k, degree);
1516       }
1517       /* Build Topology */
1518       ierr = DMPlexSetChart(*dm, 0, numCells+numVerts);CHKERRQ(ierr);
1519       for (c = 0; c < numCells; c++) {
1520         ierr = DMPlexSetConeSize(*dm, c, embedDim);CHKERRQ(ierr);
1521       }
1522       ierr = DMSetUp(*dm);CHKERRQ(ierr); /* Allocate space for cones */
1523       /* Cells */
1524       for (i = 0, c = 0; i < numVerts; ++i) {
1525         for (j = 0; j < i; ++j) {
1526           for (k = 0; k < j; ++k) {
1527             if (graph[i*numVerts+j] && graph[j*numVerts+k] && graph[k*numVerts+i]) {
1528               cone[0] = firstVertex+i; cone[1] = firstVertex+j; cone[2] = firstVertex+k;
1529               /* Check orientation */
1530               {
1531                 const PetscInt epsilon[3][3][3] = {{{0, 0, 0}, {0, 0, 1}, {0, -1, 0}}, {{0, 0, -1}, {0, 0, 0}, {1, 0, 0}}, {{0, 1, 0}, {-1, 0, 0}, {0, 0, 0}}};
1532                 PetscReal normal[3];
1533                 PetscInt  e, f;
1534 
1535                 for (d = 0; d < embedDim; ++d) {
1536                   normal[d] = 0.0;
1537                   for (e = 0; e < embedDim; ++e) {
1538                     for (f = 0; f < embedDim; ++f) {
1539                       normal[d] += epsilon[d][e][f]*(coordsIn[j*embedDim+e] - coordsIn[i*embedDim+e])*(coordsIn[k*embedDim+f] - coordsIn[i*embedDim+f]);
1540                     }
1541                   }
1542                 }
1543                 if (DotReal(embedDim, normal, &coordsIn[i*embedDim]) < 0) {PetscInt tmp = cone[1]; cone[1] = cone[2]; cone[2] = tmp;}
1544               }
1545               ierr = DMPlexSetCone(*dm, c++, cone);CHKERRQ(ierr);
1546             }
1547           }
1548         }
1549       }
1550       ierr = DMPlexSymmetrize(*dm);CHKERRQ(ierr);
1551       ierr = DMPlexStratify(*dm);CHKERRQ(ierr);
1552       ierr = PetscFree(graph);CHKERRQ(ierr);
1553       /* Interpolate mesh */
1554       ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
1555       ierr = DMDestroy(dm);CHKERRQ(ierr);
1556       *dm  = idm;
1557     } else {
1558       /*
1559         12-21--13
1560          |     |
1561         25  4  24
1562          |     |
1563   12-25--9-16--8-24--13
1564    |     |     |     |
1565   23  5 17  0 15  3  22
1566    |     |     |     |
1567   10-20--6-14--7-19--11
1568          |     |
1569         20  1  19
1570          |     |
1571         10-18--11
1572          |     |
1573         23  2  22
1574          |     |
1575         12-21--13
1576        */
1577       const PetscReal dist = 1.0/PetscSqrtReal(3.0);
1578       PetscInt        cone[4], ornt[4];
1579 
1580       numCells    = !rank ?  6 : 0;
1581       numEdges    = !rank ? 12 : 0;
1582       numVerts    = !rank ?  8 : 0;
1583       firstVertex = numCells;
1584       firstEdge   = numCells + numVerts;
1585       /* Build Topology */
1586       ierr = DMPlexSetChart(*dm, 0, numCells+numEdges+numVerts);CHKERRQ(ierr);
1587       for (c = 0; c < numCells; c++) {
1588         ierr = DMPlexSetConeSize(*dm, c, 4);CHKERRQ(ierr);
1589       }
1590       for (e = firstEdge; e < firstEdge+numEdges; ++e) {
1591         ierr = DMPlexSetConeSize(*dm, e, 2);CHKERRQ(ierr);
1592       }
1593       ierr = DMSetUp(*dm);CHKERRQ(ierr); /* Allocate space for cones */
1594       /* Cell 0 */
1595       cone[0] = 14; cone[1] = 15; cone[2] = 16; cone[3] = 17;
1596       ierr = DMPlexSetCone(*dm, 0, cone);CHKERRQ(ierr);
1597       ornt[0] = 0; ornt[1] = 0; ornt[2] = 0; ornt[3] = 0;
1598       ierr = DMPlexSetConeOrientation(*dm, 0, ornt);CHKERRQ(ierr);
1599       /* Cell 1 */
1600       cone[0] = 18; cone[1] = 19; cone[2] = 14; cone[3] = 20;
1601       ierr = DMPlexSetCone(*dm, 1, cone);CHKERRQ(ierr);
1602       ornt[0] = 0; ornt[1] = 0; ornt[2] = -2; ornt[3] = 0;
1603       ierr = DMPlexSetConeOrientation(*dm, 1, ornt);CHKERRQ(ierr);
1604       /* Cell 2 */
1605       cone[0] = 21; cone[1] = 22; cone[2] = 18; cone[3] = 23;
1606       ierr = DMPlexSetCone(*dm, 2, cone);CHKERRQ(ierr);
1607       ornt[0] = 0; ornt[1] = 0; ornt[2] = -2; ornt[3] = 0;
1608       ierr = DMPlexSetConeOrientation(*dm, 2, ornt);CHKERRQ(ierr);
1609       /* Cell 3 */
1610       cone[0] = 19; cone[1] = 22; cone[2] = 24; cone[3] = 15;
1611       ierr = DMPlexSetCone(*dm, 3, cone);CHKERRQ(ierr);
1612       ornt[0] = -2; ornt[1] = -2; ornt[2] = 0; ornt[3] = -2;
1613       ierr = DMPlexSetConeOrientation(*dm, 3, ornt);CHKERRQ(ierr);
1614       /* Cell 4 */
1615       cone[0] = 16; cone[1] = 24; cone[2] = 21; cone[3] = 25;
1616       ierr = DMPlexSetCone(*dm, 4, cone);CHKERRQ(ierr);
1617       ornt[0] = -2; ornt[1] = -2; ornt[2] = -2; ornt[3] = 0;
1618       ierr = DMPlexSetConeOrientation(*dm, 4, ornt);CHKERRQ(ierr);
1619       /* Cell 5 */
1620       cone[0] = 20; cone[1] = 17; cone[2] = 25; cone[3] = 23;
1621       ierr = DMPlexSetCone(*dm, 5, cone);CHKERRQ(ierr);
1622       ornt[0] = -2; ornt[1] = -2; ornt[2] = -2; ornt[3] = -2;
1623       ierr = DMPlexSetConeOrientation(*dm, 5, ornt);CHKERRQ(ierr);
1624       /* Edges */
1625       cone[0] =  6; cone[1] =  7;
1626       ierr = DMPlexSetCone(*dm, 14, cone);CHKERRQ(ierr);
1627       cone[0] =  7; cone[1] =  8;
1628       ierr = DMPlexSetCone(*dm, 15, cone);CHKERRQ(ierr);
1629       cone[0] =  8; cone[1] =  9;
1630       ierr = DMPlexSetCone(*dm, 16, cone);CHKERRQ(ierr);
1631       cone[0] =  9; cone[1] =  6;
1632       ierr = DMPlexSetCone(*dm, 17, cone);CHKERRQ(ierr);
1633       cone[0] = 10; cone[1] = 11;
1634       ierr = DMPlexSetCone(*dm, 18, cone);CHKERRQ(ierr);
1635       cone[0] = 11; cone[1] =  7;
1636       ierr = DMPlexSetCone(*dm, 19, cone);CHKERRQ(ierr);
1637       cone[0] =  6; cone[1] = 10;
1638       ierr = DMPlexSetCone(*dm, 20, cone);CHKERRQ(ierr);
1639       cone[0] = 12; cone[1] = 13;
1640       ierr = DMPlexSetCone(*dm, 21, cone);CHKERRQ(ierr);
1641       cone[0] = 13; cone[1] = 11;
1642       ierr = DMPlexSetCone(*dm, 22, cone);CHKERRQ(ierr);
1643       cone[0] = 10; cone[1] = 12;
1644       ierr = DMPlexSetCone(*dm, 23, cone);CHKERRQ(ierr);
1645       cone[0] = 13; cone[1] =  8;
1646       ierr = DMPlexSetCone(*dm, 24, cone);CHKERRQ(ierr);
1647       cone[0] = 12; cone[1] =  9;
1648       ierr = DMPlexSetCone(*dm, 25, cone);CHKERRQ(ierr);
1649       ierr = DMPlexSymmetrize(*dm);CHKERRQ(ierr);
1650       ierr = DMPlexStratify(*dm);CHKERRQ(ierr);
1651       /* Build coordinates */
1652       ierr = PetscCalloc1(numVerts * embedDim, &coordsIn);CHKERRQ(ierr);
1653       coordsIn[0*embedDim+0] = -dist; coordsIn[0*embedDim+1] =  dist; coordsIn[0*embedDim+2] = -dist;
1654       coordsIn[1*embedDim+0] =  dist; coordsIn[1*embedDim+1] =  dist; coordsIn[1*embedDim+2] = -dist;
1655       coordsIn[2*embedDim+0] =  dist; coordsIn[2*embedDim+1] = -dist; coordsIn[2*embedDim+2] = -dist;
1656       coordsIn[3*embedDim+0] = -dist; coordsIn[3*embedDim+1] = -dist; coordsIn[3*embedDim+2] = -dist;
1657       coordsIn[4*embedDim+0] = -dist; coordsIn[4*embedDim+1] =  dist; coordsIn[4*embedDim+2] =  dist;
1658       coordsIn[5*embedDim+0] =  dist; coordsIn[5*embedDim+1] =  dist; coordsIn[5*embedDim+2] =  dist;
1659       coordsIn[6*embedDim+0] = -dist; coordsIn[6*embedDim+1] = -dist; coordsIn[6*embedDim+2] =  dist;
1660       coordsIn[7*embedDim+0] =  dist; coordsIn[7*embedDim+1] = -dist; coordsIn[7*embedDim+2] =  dist;
1661     }
1662     break;
1663   case 3:
1664     if (simplex) {
1665       DM              idm             = NULL;
1666       const PetscReal edgeLen         = 1.0/PETSC_PHI;
1667       const PetscReal vertexA[4]      = {0.5, 0.5, 0.5, 0.5};
1668       const PetscReal vertexB[4]      = {1.0, 0.0, 0.0, 0.0};
1669       const PetscReal vertexC[4]      = {0.5, 0.5*PETSC_PHI, 0.5/PETSC_PHI, 0.0};
1670       const PetscInt  degree          = 12;
1671       PetscInt        s[4]            = {1, 1, 1};
1672       PetscInt        evenPerm[12][4] = {{0, 1, 2, 3}, {0, 2, 3, 1}, {0, 3, 1, 2}, {1, 0, 3, 2}, {1, 2, 0, 3}, {1, 3, 2, 0},
1673                                          {2, 0, 1, 3}, {2, 1, 3, 0}, {2, 3, 0, 1}, {3, 0, 2, 1}, {3, 1, 0, 2}, {3, 2, 1, 0}};
1674       PetscInt        cone[4];
1675       PetscInt       *graph, p, i, j, k, l;
1676 
1677       numCells    = !rank ? 600 : 0;
1678       numVerts    = !rank ? 120 : 0;
1679       firstVertex = numCells;
1680       /* Use the 600-cell, which for a unit sphere has coordinates which are
1681 
1682            1/2 (\pm 1, \pm 1,    \pm 1, \pm 1)                          16
1683                (\pm 1,    0,       0,      0)  all cyclic permutations   8
1684            1/2 (\pm 1, \pm phi, \pm 1/phi, 0)  all even permutations    96
1685 
1686          where \phi^2 - \phi - 1 = 0, meaning \phi is the golden ratio \frac{1 + \sqrt{5}}{2}. The edge
1687          length is then given by 1/\phi = 2.73606.
1688 
1689          http://buzzard.pugetsound.edu/sage-practice/ch03s03.html
1690          http://mathworld.wolfram.com/600-Cell.html
1691        */
1692       /* Construct vertices */
1693       ierr = PetscCalloc1(numVerts * embedDim, &coordsIn);CHKERRQ(ierr);
1694       i    = 0;
1695       for (s[0] = -1; s[0] < 2; s[0] += 2) {
1696         for (s[1] = -1; s[1] < 2; s[1] += 2) {
1697           for (s[2] = -1; s[2] < 2; s[2] += 2) {
1698             for (s[3] = -1; s[3] < 2; s[3] += 2) {
1699               for (d = 0; d < embedDim; ++d) coordsIn[i*embedDim+d] = s[d]*vertexA[d];
1700               ++i;
1701             }
1702           }
1703         }
1704       }
1705       for (p = 0; p < embedDim; ++p) {
1706         s[1] = s[2] = s[3] = 1;
1707         for (s[0] = -1; s[0] < 2; s[0] += 2) {
1708           for (d = 0; d < embedDim; ++d) coordsIn[i*embedDim+d] = s[(d+p)%embedDim]*vertexB[(d+p)%embedDim];
1709           ++i;
1710         }
1711       }
1712       for (p = 0; p < 12; ++p) {
1713         s[3] = 1;
1714         for (s[0] = -1; s[0] < 2; s[0] += 2) {
1715           for (s[1] = -1; s[1] < 2; s[1] += 2) {
1716             for (s[2] = -1; s[2] < 2; s[2] += 2) {
1717               for (d = 0; d < embedDim; ++d) coordsIn[i*embedDim+d] = s[evenPerm[p][d]]*vertexC[evenPerm[p][d]];
1718               ++i;
1719             }
1720           }
1721         }
1722       }
1723       if (i != numVerts) SETERRQ2(comm, PETSC_ERR_PLIB, "Invalid 600-cell, vertices %D != %D", i, numVerts);
1724       /* Construct graph */
1725       ierr = PetscCalloc1(numVerts * numVerts, &graph);CHKERRQ(ierr);
1726       for (i = 0; i < numVerts; ++i) {
1727         for (j = 0, k = 0; j < numVerts; ++j) {
1728           if (PetscAbsReal(DiffNormReal(embedDim, &coordsIn[i*embedDim], &coordsIn[j*embedDim]) - edgeLen) < PETSC_SMALL) {graph[i*numVerts+j] = 1; ++k;}
1729         }
1730         if (k != degree) SETERRQ3(comm, PETSC_ERR_PLIB, "Invalid 600-cell, vertex %D degree %D != %D", i, k, degree);
1731       }
1732       /* Build Topology */
1733       ierr = DMPlexSetChart(*dm, 0, numCells+numVerts);CHKERRQ(ierr);
1734       for (c = 0; c < numCells; c++) {
1735         ierr = DMPlexSetConeSize(*dm, c, embedDim);CHKERRQ(ierr);
1736       }
1737       ierr = DMSetUp(*dm);CHKERRQ(ierr); /* Allocate space for cones */
1738       /* Cells */
1739       for (i = 0, c = 0; i < numVerts; ++i) {
1740         for (j = 0; j < i; ++j) {
1741           for (k = 0; k < j; ++k) {
1742             for (l = 0; l < k; ++l) {
1743               if (graph[i*numVerts+j] && graph[j*numVerts+k] && graph[k*numVerts+i] &&
1744                   graph[l*numVerts+i] && graph[l*numVerts+j] && graph[l*numVerts+k]) {
1745                 cone[0] = firstVertex+i; cone[1] = firstVertex+j; cone[2] = firstVertex+k; cone[3] = firstVertex+l;
1746                 /* Check orientation: https://ef.gy/linear-algebra:normal-vectors-in-higher-dimensional-spaces */
1747                 {
1748                   const PetscInt epsilon[4][4][4][4] = {{{{0,  0,  0,  0}, { 0, 0,  0,  0}, { 0,  0, 0,  0}, { 0,  0,  0, 0}},
1749                                                          {{0,  0,  0,  0}, { 0, 0,  0,  0}, { 0,  0, 0,  1}, { 0,  0, -1, 0}},
1750                                                          {{0,  0,  0,  0}, { 0, 0,  0, -1}, { 0,  0, 0,  0}, { 0,  1,  0, 0}},
1751                                                          {{0,  0,  0,  0}, { 0, 0,  1,  0}, { 0, -1, 0,  0}, { 0,  0,  0, 0}}},
1752 
1753                                                         {{{0,  0,  0,  0}, { 0, 0,  0,  0}, { 0,  0, 0, -1}, { 0,  0,  1, 0}},
1754                                                          {{0,  0,  0,  0}, { 0, 0,  0,  0}, { 0,  0, 0,  0}, { 0,  0,  0, 0}},
1755                                                          {{0,  0,  0,  1}, { 0, 0,  0,  0}, { 0,  0, 0,  0}, {-1,  0,  0, 0}},
1756                                                          {{0,  0, -1,  0}, { 0, 0,  0,  0}, { 1,  0, 0,  0}, { 0,  0,  0, 0}}},
1757 
1758                                                         {{{0,  0,  0,  0}, { 0, 0,  0,  1}, { 0,  0, 0,  0}, { 0, -1,  0, 0}},
1759                                                          {{0,  0,  0, -1}, { 0, 0,  0,  0}, { 0,  0, 0,  0}, { 1,  0,  0, 0}},
1760                                                          {{0,  0,  0,  0}, { 0, 0,  0,  0}, { 0,  0, 0,  0}, { 0,  0,  0, 0}},
1761                                                          {{0,  1,  0,  0}, {-1, 0,  0,  0}, { 0,  0, 0,  0}, { 0,  0,  0, 0}}},
1762 
1763                                                         {{{0,  0,  0,  0}, { 0, 0, -1,  0}, { 0,  1, 0,  0}, { 0,  0,  0, 0}},
1764                                                          {{0,  0,  1,  0}, { 0, 0,  0,  0}, {-1,  0, 0,  0}, { 0,  0,  0, 0}},
1765                                                          {{0, -1,  0,  0}, { 1, 0,  0,  0}, { 0,  0, 0,  0}, { 0,  0,  0, 0}},
1766                                                          {{0,  0,  0,  0}, { 0, 0,  0,  0}, { 0,  0, 0,  0}, { 0,  0,  0, 0}}}};
1767                   PetscReal normal[4];
1768                   PetscInt  e, f, g;
1769 
1770                   for (d = 0; d < embedDim; ++d) {
1771                     normal[d] = 0.0;
1772                     for (e = 0; e < embedDim; ++e) {
1773                       for (f = 0; f < embedDim; ++f) {
1774                         for (g = 0; g < embedDim; ++g) {
1775                           normal[d] += epsilon[d][e][f][g]*(coordsIn[j*embedDim+e] - coordsIn[i*embedDim+e])*(coordsIn[k*embedDim+f] - coordsIn[i*embedDim+f])*(coordsIn[l*embedDim+f] - coordsIn[i*embedDim+f]);
1776                         }
1777                       }
1778                     }
1779                   }
1780                   if (DotReal(embedDim, normal, &coordsIn[i*embedDim]) < 0) {PetscInt tmp = cone[1]; cone[1] = cone[2]; cone[2] = tmp;}
1781                 }
1782                 ierr = DMPlexSetCone(*dm, c++, cone);CHKERRQ(ierr);
1783               }
1784             }
1785           }
1786         }
1787       }
1788       ierr = DMPlexSymmetrize(*dm);CHKERRQ(ierr);
1789       ierr = DMPlexStratify(*dm);CHKERRQ(ierr);
1790       ierr = PetscFree(graph);CHKERRQ(ierr);
1791       /* Interpolate mesh */
1792       ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
1793       ierr = DMDestroy(dm);CHKERRQ(ierr);
1794       *dm  = idm;
1795       break;
1796     }
1797   default: SETERRQ1(comm, PETSC_ERR_SUP, "Unsupported dimension for sphere: %D", dim);
1798   }
1799   /* Create coordinates */
1800   ierr = DMGetCoordinateSection(*dm, &coordSection);CHKERRQ(ierr);
1801   ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
1802   ierr = PetscSectionSetFieldComponents(coordSection, 0, embedDim);CHKERRQ(ierr);
1803   ierr = PetscSectionSetChart(coordSection, firstVertex, firstVertex+numVerts);CHKERRQ(ierr);
1804   for (v = firstVertex; v < firstVertex+numVerts; ++v) {
1805     ierr = PetscSectionSetDof(coordSection, v, embedDim);CHKERRQ(ierr);
1806     ierr = PetscSectionSetFieldDof(coordSection, v, 0, embedDim);CHKERRQ(ierr);
1807   }
1808   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
1809   ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
1810   ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
1811   ierr = VecSetBlockSize(coordinates, embedDim);CHKERRQ(ierr);
1812   ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
1813   ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
1814   ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
1815   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
1816   for (v = 0; v < numVerts; ++v) for (d = 0; d < embedDim; ++d) {coords[v*embedDim+d] = coordsIn[v*embedDim+d];}
1817   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
1818   ierr = DMSetCoordinatesLocal(*dm, coordinates);CHKERRQ(ierr);
1819   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
1820   ierr = PetscFree(coordsIn);CHKERRQ(ierr);
1821   PetscFunctionReturn(0);
1822 }
1823 
1824 /* External function declarations here */
1825 extern PetscErrorCode DMCreateInterpolation_Plex(DM dmCoarse, DM dmFine, Mat *interpolation, Vec *scaling);
1826 extern PetscErrorCode DMCreateInjection_Plex(DM dmCoarse, DM dmFine, Mat *mat);
1827 extern PetscErrorCode DMCreateDefaultSection_Plex(DM dm);
1828 extern PetscErrorCode DMCreateDefaultConstraints_Plex(DM dm);
1829 extern PetscErrorCode DMCreateMatrix_Plex(DM dm,  Mat *J);
1830 extern PetscErrorCode DMCreateCoordinateDM_Plex(DM dm, DM *cdm);
1831 PETSC_INTERN PetscErrorCode DMClone_Plex(DM dm, DM *newdm);
1832 extern PetscErrorCode DMSetUp_Plex(DM dm);
1833 extern PetscErrorCode DMDestroy_Plex(DM dm);
1834 extern PetscErrorCode DMView_Plex(DM dm, PetscViewer viewer);
1835 extern PetscErrorCode DMLoad_Plex(DM dm, PetscViewer viewer);
1836 extern PetscErrorCode DMCreateSubDM_Plex(DM dm, PetscInt numFields, PetscInt fields[], IS *is, DM *subdm);
1837 static PetscErrorCode DMInitialize_Plex(DM dm);
1838 
1839 /* Replace dm with the contents of dmNew
1840    - Share the DM_Plex structure
1841    - Share the coordinates
1842    - Share the SF
1843 */
1844 static PetscErrorCode DMPlexReplace_Static(DM dm, DM dmNew)
1845 {
1846   PetscSF               sf;
1847   DM                    coordDM, coarseDM;
1848   Vec                   coords;
1849   PetscBool             isper;
1850   const PetscReal      *maxCell, *L;
1851   const DMBoundaryType *bd;
1852   PetscErrorCode        ierr;
1853 
1854   PetscFunctionBegin;
1855   ierr = DMGetPointSF(dmNew, &sf);CHKERRQ(ierr);
1856   ierr = DMSetPointSF(dm, sf);CHKERRQ(ierr);
1857   ierr = DMGetCoordinateDM(dmNew, &coordDM);CHKERRQ(ierr);
1858   ierr = DMGetCoordinatesLocal(dmNew, &coords);CHKERRQ(ierr);
1859   ierr = DMSetCoordinateDM(dm, coordDM);CHKERRQ(ierr);
1860   ierr = DMSetCoordinatesLocal(dm, coords);CHKERRQ(ierr);
1861   ierr = DMGetPeriodicity(dm, &isper, &maxCell, &L, &bd);CHKERRQ(ierr);
1862   ierr = DMSetPeriodicity(dmNew, isper, maxCell, L, bd);CHKERRQ(ierr);
1863   ierr = DMDestroy_Plex(dm);CHKERRQ(ierr);
1864   ierr = DMInitialize_Plex(dm);CHKERRQ(ierr);
1865   dm->data = dmNew->data;
1866   ((DM_Plex *) dmNew->data)->refct++;
1867   dmNew->labels->refct++;
1868   if (!--(dm->labels->refct)) {
1869     DMLabelLink next = dm->labels->next;
1870 
1871     /* destroy the labels */
1872     while (next) {
1873       DMLabelLink tmp = next->next;
1874 
1875       ierr = DMLabelDestroy(&next->label);CHKERRQ(ierr);
1876       ierr = PetscFree(next);CHKERRQ(ierr);
1877       next = tmp;
1878     }
1879     ierr = PetscFree(dm->labels);CHKERRQ(ierr);
1880   }
1881   dm->labels = dmNew->labels;
1882   dm->depthLabel = dmNew->depthLabel;
1883   ierr = DMGetCoarseDM(dmNew,&coarseDM);CHKERRQ(ierr);
1884   ierr = DMSetCoarseDM(dm,coarseDM);CHKERRQ(ierr);
1885   PetscFunctionReturn(0);
1886 }
1887 
1888 /* Swap dm with the contents of dmNew
1889    - Swap the DM_Plex structure
1890    - Swap the coordinates
1891    - Swap the point PetscSF
1892 */
1893 static PetscErrorCode DMPlexSwap_Static(DM dmA, DM dmB)
1894 {
1895   DM              coordDMA, coordDMB;
1896   Vec             coordsA,  coordsB;
1897   PetscSF         sfA,      sfB;
1898   void            *tmp;
1899   DMLabelLinkList listTmp;
1900   DMLabel         depthTmp;
1901   PetscErrorCode  ierr;
1902 
1903   PetscFunctionBegin;
1904   ierr = DMGetPointSF(dmA, &sfA);CHKERRQ(ierr);
1905   ierr = DMGetPointSF(dmB, &sfB);CHKERRQ(ierr);
1906   ierr = PetscObjectReference((PetscObject) sfA);CHKERRQ(ierr);
1907   ierr = DMSetPointSF(dmA, sfB);CHKERRQ(ierr);
1908   ierr = DMSetPointSF(dmB, sfA);CHKERRQ(ierr);
1909   ierr = PetscObjectDereference((PetscObject) sfA);CHKERRQ(ierr);
1910 
1911   ierr = DMGetCoordinateDM(dmA, &coordDMA);CHKERRQ(ierr);
1912   ierr = DMGetCoordinateDM(dmB, &coordDMB);CHKERRQ(ierr);
1913   ierr = PetscObjectReference((PetscObject) coordDMA);CHKERRQ(ierr);
1914   ierr = DMSetCoordinateDM(dmA, coordDMB);CHKERRQ(ierr);
1915   ierr = DMSetCoordinateDM(dmB, coordDMA);CHKERRQ(ierr);
1916   ierr = PetscObjectDereference((PetscObject) coordDMA);CHKERRQ(ierr);
1917 
1918   ierr = DMGetCoordinatesLocal(dmA, &coordsA);CHKERRQ(ierr);
1919   ierr = DMGetCoordinatesLocal(dmB, &coordsB);CHKERRQ(ierr);
1920   ierr = PetscObjectReference((PetscObject) coordsA);CHKERRQ(ierr);
1921   ierr = DMSetCoordinatesLocal(dmA, coordsB);CHKERRQ(ierr);
1922   ierr = DMSetCoordinatesLocal(dmB, coordsA);CHKERRQ(ierr);
1923   ierr = PetscObjectDereference((PetscObject) coordsA);CHKERRQ(ierr);
1924 
1925   tmp       = dmA->data;
1926   dmA->data = dmB->data;
1927   dmB->data = tmp;
1928   listTmp   = dmA->labels;
1929   dmA->labels = dmB->labels;
1930   dmB->labels = listTmp;
1931   depthTmp  = dmA->depthLabel;
1932   dmA->depthLabel = dmB->depthLabel;
1933   dmB->depthLabel = depthTmp;
1934   PetscFunctionReturn(0);
1935 }
1936 
1937 PetscErrorCode DMSetFromOptions_NonRefinement_Plex(PetscOptionItems *PetscOptionsObject,DM dm)
1938 {
1939   DM_Plex       *mesh = (DM_Plex*) dm->data;
1940   PetscErrorCode ierr;
1941 
1942   PetscFunctionBegin;
1943   /* Handle viewing */
1944   ierr = PetscOptionsBool("-dm_plex_print_set_values", "Output all set values info", "DMView", PETSC_FALSE, &mesh->printSetValues, NULL);CHKERRQ(ierr);
1945   ierr = PetscOptionsInt("-dm_plex_print_fem", "Debug output level all fem computations", "DMView", 0, &mesh->printFEM, NULL);CHKERRQ(ierr);
1946   ierr = PetscOptionsReal("-dm_plex_print_tol", "Tolerance for FEM output", "DMView", mesh->printTol, &mesh->printTol, NULL);CHKERRQ(ierr);
1947   /* Point Location */
1948   ierr = PetscOptionsBool("-dm_plex_hash_location", "Use grid hashing for point location", "DMView", PETSC_FALSE, &mesh->useHashLocation, NULL);CHKERRQ(ierr);
1949   /* Generation and remeshing */
1950   ierr = PetscOptionsBool("-dm_plex_remesh_bd", "Allow changes to the boundary on remeshing", "DMView", PETSC_FALSE, &mesh->remeshBd, NULL);CHKERRQ(ierr);
1951   /* Projection behavior */
1952   ierr = PetscOptionsInt("-dm_plex_max_projection_height", "Maxmimum mesh point height used to project locally", "DMPlexSetMaxProjectionHeight", 0, &mesh->maxProjectionHeight, NULL);CHKERRQ(ierr);
1953   ierr = PetscOptionsBool("-dm_plex_regular_refinement", "Use special nested projection algorithm for regular refinement", "DMPlexSetRegularRefinement", mesh->regularRefinement, &mesh->regularRefinement, NULL);CHKERRQ(ierr);
1954 
1955   ierr = PetscPartitionerSetFromOptions(mesh->partitioner);CHKERRQ(ierr);
1956   PetscFunctionReturn(0);
1957 }
1958 
1959 static PetscErrorCode DMSetFromOptions_Plex(PetscOptionItems *PetscOptionsObject,DM dm)
1960 {
1961   PetscInt       refine = 0, coarsen = 0, r;
1962   PetscBool      isHierarchy;
1963   PetscErrorCode ierr;
1964 
1965   PetscFunctionBegin;
1966   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1967   ierr = PetscOptionsHead(PetscOptionsObject,"DMPlex Options");CHKERRQ(ierr);
1968   /* Handle DMPlex refinement */
1969   ierr = PetscOptionsInt("-dm_refine", "The number of uniform refinements", "DMCreate", refine, &refine, NULL);CHKERRQ(ierr);
1970   ierr = PetscOptionsInt("-dm_refine_hierarchy", "The number of uniform refinements", "DMCreate", refine, &refine, &isHierarchy);CHKERRQ(ierr);
1971   if (refine) {ierr = DMPlexSetRefinementUniform(dm, PETSC_TRUE);CHKERRQ(ierr);}
1972   if (refine && isHierarchy) {
1973     DM *dms, coarseDM;
1974 
1975     ierr = DMGetCoarseDM(dm, &coarseDM);CHKERRQ(ierr);
1976     ierr = PetscObjectReference((PetscObject)coarseDM);CHKERRQ(ierr);
1977     ierr = PetscMalloc1(refine,&dms);CHKERRQ(ierr);
1978     ierr = DMRefineHierarchy(dm, refine, dms);CHKERRQ(ierr);
1979     /* Total hack since we do not pass in a pointer */
1980     ierr = DMPlexSwap_Static(dm, dms[refine-1]);CHKERRQ(ierr);
1981     if (refine == 1) {
1982       ierr = DMSetCoarseDM(dm, dms[0]);CHKERRQ(ierr);
1983       ierr = DMPlexSetRegularRefinement(dm, PETSC_TRUE);CHKERRQ(ierr);
1984     } else {
1985       ierr = DMSetCoarseDM(dm, dms[refine-2]);CHKERRQ(ierr);
1986       ierr = DMPlexSetRegularRefinement(dm, PETSC_TRUE);CHKERRQ(ierr);
1987       ierr = DMSetCoarseDM(dms[0], dms[refine-1]);CHKERRQ(ierr);
1988       ierr = DMPlexSetRegularRefinement(dms[0], PETSC_TRUE);CHKERRQ(ierr);
1989     }
1990     ierr = DMSetCoarseDM(dms[refine-1], coarseDM);CHKERRQ(ierr);
1991     ierr = PetscObjectDereference((PetscObject)coarseDM);CHKERRQ(ierr);
1992     /* Free DMs */
1993     for (r = 0; r < refine; ++r) {
1994       ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dms[r]);CHKERRQ(ierr);
1995       ierr = DMDestroy(&dms[r]);CHKERRQ(ierr);
1996     }
1997     ierr = PetscFree(dms);CHKERRQ(ierr);
1998   } else {
1999     for (r = 0; r < refine; ++r) {
2000       DM refinedMesh;
2001 
2002       ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dm);CHKERRQ(ierr);
2003       ierr = DMRefine(dm, PetscObjectComm((PetscObject) dm), &refinedMesh);CHKERRQ(ierr);
2004       /* Total hack since we do not pass in a pointer */
2005       ierr = DMPlexReplace_Static(dm, refinedMesh);CHKERRQ(ierr);
2006       ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dm);CHKERRQ(ierr);
2007       ierr = DMDestroy(&refinedMesh);CHKERRQ(ierr);
2008     }
2009   }
2010   /* Handle DMPlex coarsening */
2011   ierr = PetscOptionsInt("-dm_coarsen", "Coarsen the mesh", "DMCreate", coarsen, &coarsen, NULL);CHKERRQ(ierr);
2012   ierr = PetscOptionsInt("-dm_coarsen_hierarchy", "The number of coarsenings", "DMCreate", coarsen, &coarsen, &isHierarchy);CHKERRQ(ierr);
2013   if (coarsen && isHierarchy) {
2014     DM *dms;
2015 
2016     ierr = PetscMalloc1(coarsen, &dms);CHKERRQ(ierr);
2017     ierr = DMCoarsenHierarchy(dm, coarsen, dms);CHKERRQ(ierr);
2018     /* Free DMs */
2019     for (r = 0; r < coarsen; ++r) {
2020       ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dms[r]);CHKERRQ(ierr);
2021       ierr = DMDestroy(&dms[r]);CHKERRQ(ierr);
2022     }
2023     ierr = PetscFree(dms);CHKERRQ(ierr);
2024   } else {
2025     for (r = 0; r < coarsen; ++r) {
2026       DM coarseMesh;
2027 
2028       ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dm);CHKERRQ(ierr);
2029       ierr = DMCoarsen(dm, PetscObjectComm((PetscObject) dm), &coarseMesh);CHKERRQ(ierr);
2030       /* Total hack since we do not pass in a pointer */
2031       ierr = DMPlexReplace_Static(dm, coarseMesh);CHKERRQ(ierr);
2032       ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dm);CHKERRQ(ierr);
2033       ierr = DMDestroy(&coarseMesh);CHKERRQ(ierr);
2034     }
2035   }
2036   /* Handle */
2037   ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dm);CHKERRQ(ierr);
2038   ierr = PetscOptionsTail();CHKERRQ(ierr);
2039   PetscFunctionReturn(0);
2040 }
2041 
2042 static PetscErrorCode DMCreateGlobalVector_Plex(DM dm,Vec *vec)
2043 {
2044   PetscErrorCode ierr;
2045 
2046   PetscFunctionBegin;
2047   ierr = DMCreateGlobalVector_Section_Private(dm,vec);CHKERRQ(ierr);
2048   /* ierr = VecSetOperation(*vec, VECOP_DUPLICATE, (void(*)(void)) VecDuplicate_MPI_DM);CHKERRQ(ierr); */
2049   ierr = VecSetOperation(*vec, VECOP_VIEW, (void (*)(void)) VecView_Plex);CHKERRQ(ierr);
2050   ierr = VecSetOperation(*vec, VECOP_VIEWNATIVE, (void (*)(void)) VecView_Plex_Native);CHKERRQ(ierr);
2051   ierr = VecSetOperation(*vec, VECOP_LOAD, (void (*)(void)) VecLoad_Plex);CHKERRQ(ierr);
2052   ierr = VecSetOperation(*vec, VECOP_LOADNATIVE, (void (*)(void)) VecLoad_Plex_Native);CHKERRQ(ierr);
2053   PetscFunctionReturn(0);
2054 }
2055 
2056 static PetscErrorCode DMCreateLocalVector_Plex(DM dm,Vec *vec)
2057 {
2058   PetscErrorCode ierr;
2059 
2060   PetscFunctionBegin;
2061   ierr = DMCreateLocalVector_Section_Private(dm,vec);CHKERRQ(ierr);
2062   ierr = VecSetOperation(*vec, VECOP_VIEW, (void (*)(void)) VecView_Plex_Local);CHKERRQ(ierr);
2063   ierr = VecSetOperation(*vec, VECOP_LOAD, (void (*)(void)) VecLoad_Plex_Local);CHKERRQ(ierr);
2064   PetscFunctionReturn(0);
2065 }
2066 
2067 static PetscErrorCode DMGetDimPoints_Plex(DM dm, PetscInt dim, PetscInt *pStart, PetscInt *pEnd)
2068 {
2069   PetscInt       depth, d;
2070   PetscErrorCode ierr;
2071 
2072   PetscFunctionBegin;
2073   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
2074   if (depth == 1) {
2075     ierr = DMGetDimension(dm, &d);CHKERRQ(ierr);
2076     if (dim == 0)      {ierr = DMPlexGetDepthStratum(dm, dim, pStart, pEnd);CHKERRQ(ierr);}
2077     else if (dim == d) {ierr = DMPlexGetDepthStratum(dm, 1, pStart, pEnd);CHKERRQ(ierr);}
2078     else               {*pStart = 0; *pEnd = 0;}
2079   } else {
2080     ierr = DMPlexGetDepthStratum(dm, dim, pStart, pEnd);CHKERRQ(ierr);
2081   }
2082   PetscFunctionReturn(0);
2083 }
2084 
2085 static PetscErrorCode DMGetNeighors_Plex(DM dm, PetscInt *nranks, const PetscMPIInt *ranks[])
2086 {
2087   PetscSF        sf;
2088   PetscErrorCode ierr;
2089 
2090   PetscFunctionBegin;
2091   ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr);
2092   ierr = PetscSFGetRanks(sf, nranks, ranks, NULL, NULL, NULL);CHKERRQ(ierr);
2093   PetscFunctionReturn(0);
2094 }
2095 
2096 static PetscErrorCode DMInitialize_Plex(DM dm)
2097 {
2098   PetscErrorCode ierr;
2099 
2100   PetscFunctionBegin;
2101   dm->ops->view                            = DMView_Plex;
2102   dm->ops->load                            = DMLoad_Plex;
2103   dm->ops->setfromoptions                  = DMSetFromOptions_Plex;
2104   dm->ops->clone                           = DMClone_Plex;
2105   dm->ops->setup                           = DMSetUp_Plex;
2106   dm->ops->createdefaultsection            = DMCreateDefaultSection_Plex;
2107   dm->ops->createdefaultconstraints        = DMCreateDefaultConstraints_Plex;
2108   dm->ops->createglobalvector              = DMCreateGlobalVector_Plex;
2109   dm->ops->createlocalvector               = DMCreateLocalVector_Plex;
2110   dm->ops->getlocaltoglobalmapping         = NULL;
2111   dm->ops->createfieldis                   = NULL;
2112   dm->ops->createcoordinatedm              = DMCreateCoordinateDM_Plex;
2113   dm->ops->getcoloring                     = NULL;
2114   dm->ops->creatematrix                    = DMCreateMatrix_Plex;
2115   dm->ops->createinterpolation             = DMCreateInterpolation_Plex;
2116   dm->ops->getaggregates                   = NULL;
2117   dm->ops->getinjection                    = DMCreateInjection_Plex;
2118   dm->ops->refine                          = DMRefine_Plex;
2119   dm->ops->coarsen                         = DMCoarsen_Plex;
2120   dm->ops->refinehierarchy                 = DMRefineHierarchy_Plex;
2121   dm->ops->coarsenhierarchy                = DMCoarsenHierarchy_Plex;
2122   dm->ops->adaptlabel                      = DMAdaptLabel_Plex;
2123   dm->ops->adaptmetric                     = DMAdaptMetric_Plex;
2124   dm->ops->globaltolocalbegin              = NULL;
2125   dm->ops->globaltolocalend                = NULL;
2126   dm->ops->localtoglobalbegin              = NULL;
2127   dm->ops->localtoglobalend                = NULL;
2128   dm->ops->destroy                         = DMDestroy_Plex;
2129   dm->ops->createsubdm                     = DMCreateSubDM_Plex;
2130   dm->ops->getdimpoints                    = DMGetDimPoints_Plex;
2131   dm->ops->locatepoints                    = DMLocatePoints_Plex;
2132   dm->ops->projectfunctionlocal            = DMProjectFunctionLocal_Plex;
2133   dm->ops->projectfunctionlabellocal       = DMProjectFunctionLabelLocal_Plex;
2134   dm->ops->projectfieldlocal               = DMProjectFieldLocal_Plex;
2135   dm->ops->projectfieldlabellocal          = DMProjectFieldLabelLocal_Plex;
2136   dm->ops->computel2diff                   = DMComputeL2Diff_Plex;
2137   dm->ops->computel2gradientdiff           = DMComputeL2GradientDiff_Plex;
2138   dm->ops->computel2fielddiff              = DMComputeL2FieldDiff_Plex;
2139   dm->ops->getneighbors                    = DMGetNeighors_Plex;
2140   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMPlexInsertBoundaryValues_C",DMPlexInsertBoundaryValues_Plex);CHKERRQ(ierr);
2141   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMSetUpGLVisViewer_C",DMSetUpGLVisViewer_Plex);CHKERRQ(ierr);
2142   PetscFunctionReturn(0);
2143 }
2144 
2145 PETSC_INTERN PetscErrorCode DMClone_Plex(DM dm, DM *newdm)
2146 {
2147   DM_Plex        *mesh = (DM_Plex *) dm->data;
2148   PetscErrorCode ierr;
2149 
2150   PetscFunctionBegin;
2151   mesh->refct++;
2152   (*newdm)->data = mesh;
2153   ierr = PetscObjectChangeTypeName((PetscObject) *newdm, DMPLEX);CHKERRQ(ierr);
2154   ierr = DMInitialize_Plex(*newdm);CHKERRQ(ierr);
2155   PetscFunctionReturn(0);
2156 }
2157 
2158 /*MC
2159   DMPLEX = "plex" - A DM object that encapsulates an unstructured mesh, or CW Complex, which can be expressed using a Hasse Diagram.
2160                     In the local representation, Vecs contain all unknowns in the interior and shared boundary. This is
2161                     specified by a PetscSection object. Ownership in the global representation is determined by
2162                     ownership of the underlying DMPlex points. This is specified by another PetscSection object.
2163 
2164   Level: intermediate
2165 
2166 .seealso: DMType, DMPlexCreate(), DMCreate(), DMSetType()
2167 M*/
2168 
2169 PETSC_EXTERN PetscErrorCode DMCreate_Plex(DM dm)
2170 {
2171   DM_Plex        *mesh;
2172   PetscInt       unit, d;
2173   PetscErrorCode ierr;
2174 
2175   PetscFunctionBegin;
2176   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2177   ierr     = PetscNewLog(dm,&mesh);CHKERRQ(ierr);
2178   dm->dim  = 0;
2179   dm->data = mesh;
2180 
2181   mesh->refct             = 1;
2182   ierr                    = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &mesh->coneSection);CHKERRQ(ierr);
2183   mesh->maxConeSize       = 0;
2184   mesh->cones             = NULL;
2185   mesh->coneOrientations  = NULL;
2186   ierr                    = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &mesh->supportSection);CHKERRQ(ierr);
2187   mesh->maxSupportSize    = 0;
2188   mesh->supports          = NULL;
2189   mesh->refinementUniform = PETSC_TRUE;
2190   mesh->refinementLimit   = -1.0;
2191 
2192   mesh->facesTmp = NULL;
2193 
2194   mesh->tetgenOpts   = NULL;
2195   mesh->triangleOpts = NULL;
2196   ierr = PetscPartitionerCreate(PetscObjectComm((PetscObject)dm), &mesh->partitioner);CHKERRQ(ierr);
2197   mesh->remeshBd     = PETSC_FALSE;
2198 
2199   mesh->subpointMap = NULL;
2200 
2201   for (unit = 0; unit < NUM_PETSC_UNITS; ++unit) mesh->scale[unit] = 1.0;
2202 
2203   mesh->regularRefinement   = PETSC_FALSE;
2204   mesh->depthState          = -1;
2205   mesh->globalVertexNumbers = NULL;
2206   mesh->globalCellNumbers   = NULL;
2207   mesh->anchorSection       = NULL;
2208   mesh->anchorIS            = NULL;
2209   mesh->createanchors       = NULL;
2210   mesh->computeanchormatrix = NULL;
2211   mesh->parentSection       = NULL;
2212   mesh->parents             = NULL;
2213   mesh->childIDs            = NULL;
2214   mesh->childSection        = NULL;
2215   mesh->children            = NULL;
2216   mesh->referenceTree       = NULL;
2217   mesh->getchildsymmetry    = NULL;
2218   for (d = 0; d < 8; ++d) mesh->hybridPointMax[d] = PETSC_DETERMINE;
2219   mesh->vtkCellHeight       = 0;
2220   mesh->useCone             = PETSC_FALSE;
2221   mesh->useClosure          = PETSC_TRUE;
2222   mesh->useAnchors          = PETSC_FALSE;
2223 
2224   mesh->maxProjectionHeight = 0;
2225 
2226   mesh->printSetValues = PETSC_FALSE;
2227   mesh->printFEM       = 0;
2228   mesh->printTol       = 1.0e-10;
2229 
2230   ierr = DMInitialize_Plex(dm);CHKERRQ(ierr);
2231   PetscFunctionReturn(0);
2232 }
2233 
2234 /*@
2235   DMPlexCreate - Creates a DMPlex object, which encapsulates an unstructured mesh, or CW complex, which can be expressed using a Hasse Diagram.
2236 
2237   Collective on MPI_Comm
2238 
2239   Input Parameter:
2240 . comm - The communicator for the DMPlex object
2241 
2242   Output Parameter:
2243 . mesh  - The DMPlex object
2244 
2245   Level: beginner
2246 
2247 .keywords: DMPlex, create
2248 @*/
2249 PetscErrorCode DMPlexCreate(MPI_Comm comm, DM *mesh)
2250 {
2251   PetscErrorCode ierr;
2252 
2253   PetscFunctionBegin;
2254   PetscValidPointer(mesh,2);
2255   ierr = DMCreate(comm, mesh);CHKERRQ(ierr);
2256   ierr = DMSetType(*mesh, DMPLEX);CHKERRQ(ierr);
2257   PetscFunctionReturn(0);
2258 }
2259 
2260 /*
2261   This takes as input the common mesh generator output, a list of the vertices for each cell, but vertex numbers are global and an SF is built for them
2262 */
2263 static PetscErrorCode DMPlexBuildFromCellList_Parallel_Private(DM dm, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, const int cells[], PetscSF *sfVert)
2264 {
2265   PetscSF         sfPoint;
2266   PetscLayout     vLayout;
2267   PetscHashI      vhash;
2268   PetscSFNode    *remoteVerticesAdj, *vertexLocal, *vertexOwner, *remoteVertex;
2269   const PetscInt *vrange;
2270   PetscInt        numVerticesAdj, off, *verticesAdj, numVerticesGhost = 0, *localVertex, *cone, c, p, v, g;
2271   PETSC_UNUSED PetscHashIIter ret, iter;
2272   PetscMPIInt     rank, size;
2273   PetscErrorCode  ierr;
2274 
2275   PetscFunctionBegin;
2276   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRQ(ierr);
2277   ierr = MPI_Comm_size(PetscObjectComm((PetscObject) dm), &size);CHKERRQ(ierr);
2278   /* Partition vertices */
2279   ierr = PetscLayoutCreate(PetscObjectComm((PetscObject) dm), &vLayout);CHKERRQ(ierr);
2280   ierr = PetscLayoutSetLocalSize(vLayout, numVertices);CHKERRQ(ierr);
2281   ierr = PetscLayoutSetBlockSize(vLayout, 1);CHKERRQ(ierr);
2282   ierr = PetscLayoutSetUp(vLayout);CHKERRQ(ierr);
2283   ierr = PetscLayoutGetRanges(vLayout, &vrange);CHKERRQ(ierr);
2284   /* Count vertices and map them to procs */
2285   PetscHashICreate(vhash);
2286   for (c = 0; c < numCells; ++c) {
2287     for (p = 0; p < numCorners; ++p) {
2288       PetscHashIPut(vhash, cells[c*numCorners+p], ret, iter);
2289     }
2290   }
2291   PetscHashISize(vhash, numVerticesAdj);
2292   ierr = PetscMalloc1(numVerticesAdj, &verticesAdj);CHKERRQ(ierr);
2293   off = 0; ierr = PetscHashIGetKeys(vhash, &off, verticesAdj);CHKERRQ(ierr);
2294   if (off != numVerticesAdj) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid number of local vertices %D should be %D", off, numVerticesAdj);
2295   ierr = PetscSortInt(numVerticesAdj, verticesAdj);CHKERRQ(ierr);
2296   ierr = PetscMalloc1(numVerticesAdj, &remoteVerticesAdj);CHKERRQ(ierr);
2297   for (v = 0; v < numVerticesAdj; ++v) {
2298     const PetscInt gv = verticesAdj[v];
2299     PetscInt       vrank;
2300 
2301     ierr = PetscFindInt(gv, size+1, vrange, &vrank);CHKERRQ(ierr);
2302     vrank = vrank < 0 ? -(vrank+2) : vrank;
2303     remoteVerticesAdj[v].index = gv - vrange[vrank];
2304     remoteVerticesAdj[v].rank  = vrank;
2305   }
2306   /* Create cones */
2307   ierr = DMPlexSetChart(dm, 0, numCells+numVerticesAdj);CHKERRQ(ierr);
2308   for (c = 0; c < numCells; ++c) {ierr = DMPlexSetConeSize(dm, c, numCorners);CHKERRQ(ierr);}
2309   ierr = DMSetUp(dm);CHKERRQ(ierr);
2310   ierr = DMGetWorkArray(dm, numCorners, PETSC_INT, &cone);CHKERRQ(ierr);
2311   for (c = 0; c < numCells; ++c) {
2312     for (p = 0; p < numCorners; ++p) {
2313       const PetscInt gv = cells[c*numCorners+p];
2314       PetscInt       lv;
2315 
2316       ierr = PetscFindInt(gv, numVerticesAdj, verticesAdj, &lv);CHKERRQ(ierr);
2317       if (lv < 0) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Could not find global vertex %D in local connectivity", gv);
2318       cone[p] = lv+numCells;
2319     }
2320     ierr = DMPlexSetCone(dm, c, cone);CHKERRQ(ierr);
2321   }
2322   ierr = DMRestoreWorkArray(dm, numCorners, PETSC_INT, &cone);CHKERRQ(ierr);
2323   /* Create SF for vertices */
2324   ierr = PetscSFCreate(PetscObjectComm((PetscObject)dm), sfVert);CHKERRQ(ierr);
2325   ierr = PetscObjectSetName((PetscObject) *sfVert, "Vertex Ownership SF");CHKERRQ(ierr);
2326   ierr = PetscSFSetFromOptions(*sfVert);CHKERRQ(ierr);
2327   ierr = PetscSFSetGraph(*sfVert, numVertices, numVerticesAdj, NULL, PETSC_OWN_POINTER, remoteVerticesAdj, PETSC_OWN_POINTER);CHKERRQ(ierr);
2328   ierr = PetscFree(verticesAdj);CHKERRQ(ierr);
2329   /* Build pointSF */
2330   ierr = PetscMalloc2(numVerticesAdj, &vertexLocal, numVertices, &vertexOwner);CHKERRQ(ierr);
2331   for (v = 0; v < numVerticesAdj; ++v) {vertexLocal[v].index = v+numCells; vertexLocal[v].rank = rank;}
2332   for (v = 0; v < numVertices;    ++v) {vertexOwner[v].index = -1;         vertexOwner[v].rank = -1;}
2333   ierr = PetscSFReduceBegin(*sfVert, MPIU_2INT, vertexLocal, vertexOwner, MPI_MAXLOC);CHKERRQ(ierr);
2334   ierr = PetscSFReduceEnd(*sfVert, MPIU_2INT, vertexLocal, vertexOwner, MPI_MAXLOC);CHKERRQ(ierr);
2335   for (v = 0; v < numVertices;    ++v) if (vertexOwner[v].rank < 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Global vertex %d on rank %d was unclaimed", v + vrange[rank], rank);
2336   ierr = PetscSFBcastBegin(*sfVert, MPIU_2INT, vertexOwner, vertexLocal);CHKERRQ(ierr);
2337   ierr = PetscSFBcastEnd(*sfVert, MPIU_2INT, vertexOwner, vertexLocal);CHKERRQ(ierr);
2338   for (v = 0; v < numVerticesAdj; ++v) if (vertexLocal[v].rank != rank) ++numVerticesGhost;
2339   ierr = PetscMalloc1(numVerticesGhost, &localVertex);CHKERRQ(ierr);
2340   ierr = PetscMalloc1(numVerticesGhost, &remoteVertex);CHKERRQ(ierr);
2341   for (v = 0, g = 0; v < numVerticesAdj; ++v) {
2342     if (vertexLocal[v].rank != rank) {
2343       localVertex[g]        = v+numCells;
2344       remoteVertex[g].index = vertexLocal[v].index;
2345       remoteVertex[g].rank  = vertexLocal[v].rank;
2346       ++g;
2347     }
2348   }
2349   ierr = PetscFree2(vertexLocal, vertexOwner);CHKERRQ(ierr);
2350   if (g != numVerticesGhost) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid number of vertex ghosts %D should be %D", g, numVerticesGhost);
2351   ierr = DMGetPointSF(dm, &sfPoint);CHKERRQ(ierr);
2352   ierr = PetscObjectSetName((PetscObject) sfPoint, "point SF");CHKERRQ(ierr);
2353   ierr = PetscSFSetGraph(sfPoint, numCells+numVerticesAdj, numVerticesGhost, localVertex, PETSC_OWN_POINTER, remoteVertex, PETSC_OWN_POINTER);CHKERRQ(ierr);
2354   ierr = PetscLayoutDestroy(&vLayout);CHKERRQ(ierr);
2355   PetscHashIDestroy(vhash);
2356   /* Fill in the rest of the topology structure */
2357   ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
2358   ierr = DMPlexStratify(dm);CHKERRQ(ierr);
2359   PetscFunctionReturn(0);
2360 }
2361 
2362 /*
2363   This takes as input the coordinates for each owned vertex
2364 */
2365 static PetscErrorCode DMPlexBuildCoordinates_Parallel_Private(DM dm, PetscInt spaceDim, PetscInt numCells, PetscInt numV, PetscSF sfVert, const PetscReal vertexCoords[])
2366 {
2367   PetscSection   coordSection;
2368   Vec            coordinates;
2369   PetscScalar   *coords;
2370   PetscInt       numVertices, numVerticesAdj, coordSize, v;
2371   PetscErrorCode ierr;
2372 
2373   PetscFunctionBegin;
2374   ierr = DMSetCoordinateDim(dm, spaceDim);CHKERRQ(ierr);
2375   ierr = PetscSFGetGraph(sfVert, &numVertices, &numVerticesAdj, NULL, NULL);CHKERRQ(ierr);
2376   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
2377   ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
2378   ierr = PetscSectionSetFieldComponents(coordSection, 0, spaceDim);CHKERRQ(ierr);
2379   ierr = PetscSectionSetChart(coordSection, numCells, numCells + numVerticesAdj);CHKERRQ(ierr);
2380   for (v = numCells; v < numCells+numVerticesAdj; ++v) {
2381     ierr = PetscSectionSetDof(coordSection, v, spaceDim);CHKERRQ(ierr);
2382     ierr = PetscSectionSetFieldDof(coordSection, v, 0, spaceDim);CHKERRQ(ierr);
2383   }
2384   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
2385   ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
2386   ierr = VecCreate(PetscObjectComm((PetscObject)dm), &coordinates);CHKERRQ(ierr);
2387   ierr = VecSetBlockSize(coordinates, spaceDim);CHKERRQ(ierr);
2388   ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
2389   ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
2390   ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
2391   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
2392   {
2393     MPI_Datatype coordtype;
2394 
2395     /* Need a temp buffer for coords if we have complex/single */
2396     ierr = MPI_Type_contiguous(spaceDim, MPIU_SCALAR, &coordtype);CHKERRQ(ierr);
2397     ierr = MPI_Type_commit(&coordtype);CHKERRQ(ierr);
2398 #if defined(PETSC_USE_COMPLEX)
2399     {
2400     PetscScalar *svertexCoords;
2401     PetscInt    i;
2402     ierr = PetscMalloc1(numV*spaceDim,&svertexCoords);CHKERRQ(ierr);
2403     for (i=0; i<numV*spaceDim; i++) svertexCoords[i] = vertexCoords[i];
2404     ierr = PetscSFBcastBegin(sfVert, coordtype, svertexCoords, coords);CHKERRQ(ierr);
2405     ierr = PetscSFBcastEnd(sfVert, coordtype, svertexCoords, coords);CHKERRQ(ierr);
2406     ierr = PetscFree(svertexCoords);CHKERRQ(ierr);
2407     }
2408 #else
2409     ierr = PetscSFBcastBegin(sfVert, coordtype, vertexCoords, coords);CHKERRQ(ierr);
2410     ierr = PetscSFBcastEnd(sfVert, coordtype, vertexCoords, coords);CHKERRQ(ierr);
2411 #endif
2412     ierr = MPI_Type_free(&coordtype);CHKERRQ(ierr);
2413   }
2414   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
2415   ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
2416   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
2417   PetscFunctionReturn(0);
2418 }
2419 
2420 /*@C
2421   DMPlexCreateFromCellListParallel - This takes as input common mesh generator output, a list of the vertices for each cell, and produces a DM
2422 
2423   Input Parameters:
2424 + comm - The communicator
2425 . dim - The topological dimension of the mesh
2426 . numCells - The number of cells owned by this process
2427 . numVertices - The number of vertices owned by this process
2428 . numCorners - The number of vertices for each cell
2429 . interpolate - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
2430 . cells - An array of numCells*numCorners numbers, the global vertex numbers for each cell
2431 . spaceDim - The spatial dimension used for coordinates
2432 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
2433 
2434   Output Parameter:
2435 + dm - The DM
2436 - vertexSF - Optional, SF describing complete vertex ownership
2437 
2438   Note: Two triangles sharing a face
2439 $
2440 $        2
2441 $      / | \
2442 $     /  |  \
2443 $    /   |   \
2444 $   0  0 | 1  3
2445 $    \   |   /
2446 $     \  |  /
2447 $      \ | /
2448 $        1
2449 would have input
2450 $  numCells = 2, numVertices = 4
2451 $  cells = [0 1 2  1 3 2]
2452 $
2453 which would result in the DMPlex
2454 $
2455 $        4
2456 $      / | \
2457 $     /  |  \
2458 $    /   |   \
2459 $   2  0 | 1  5
2460 $    \   |   /
2461 $     \  |  /
2462 $      \ | /
2463 $        3
2464 
2465   Level: beginner
2466 
2467 .seealso: DMPlexCreateFromCellList(), DMPlexCreateFromDAG(), DMPlexCreate()
2468 @*/
2469 PetscErrorCode DMPlexCreateFromCellListParallel(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, PetscBool interpolate, const int cells[], PetscInt spaceDim, const PetscReal vertexCoords[], PetscSF *vertexSF, DM *dm)
2470 {
2471   PetscSF        sfVert;
2472   PetscErrorCode ierr;
2473 
2474   PetscFunctionBegin;
2475   ierr = DMCreate(comm, dm);CHKERRQ(ierr);
2476   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
2477   PetscValidLogicalCollectiveInt(*dm, dim, 2);
2478   PetscValidLogicalCollectiveInt(*dm, spaceDim, 8);
2479   ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
2480   ierr = DMPlexBuildFromCellList_Parallel_Private(*dm, numCells, numVertices, numCorners, cells, &sfVert);CHKERRQ(ierr);
2481   if (interpolate) {
2482     DM idm = NULL;
2483 
2484     ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
2485     ierr = DMDestroy(dm);CHKERRQ(ierr);
2486     *dm  = idm;
2487   }
2488   ierr = DMPlexBuildCoordinates_Parallel_Private(*dm, spaceDim, numCells, numVertices,sfVert, vertexCoords);CHKERRQ(ierr);
2489   if (vertexSF) *vertexSF = sfVert;
2490   else {ierr = PetscSFDestroy(&sfVert);CHKERRQ(ierr);}
2491   PetscFunctionReturn(0);
2492 }
2493 
2494 /*
2495   This takes as input the common mesh generator output, a list of the vertices for each cell
2496 */
2497 static PetscErrorCode DMPlexBuildFromCellList_Private(DM dm, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, const int cells[])
2498 {
2499   PetscInt      *cone, c, p;
2500   PetscErrorCode ierr;
2501 
2502   PetscFunctionBegin;
2503   ierr = DMPlexSetChart(dm, 0, numCells+numVertices);CHKERRQ(ierr);
2504   for (c = 0; c < numCells; ++c) {
2505     ierr = DMPlexSetConeSize(dm, c, numCorners);CHKERRQ(ierr);
2506   }
2507   ierr = DMSetUp(dm);CHKERRQ(ierr);
2508   ierr = DMGetWorkArray(dm, numCorners, PETSC_INT, &cone);CHKERRQ(ierr);
2509   for (c = 0; c < numCells; ++c) {
2510     for (p = 0; p < numCorners; ++p) {
2511       cone[p] = cells[c*numCorners+p]+numCells;
2512     }
2513     ierr = DMPlexSetCone(dm, c, cone);CHKERRQ(ierr);
2514   }
2515   ierr = DMRestoreWorkArray(dm, numCorners, PETSC_INT, &cone);CHKERRQ(ierr);
2516   ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
2517   ierr = DMPlexStratify(dm);CHKERRQ(ierr);
2518   PetscFunctionReturn(0);
2519 }
2520 
2521 /*
2522   This takes as input the coordinates for each vertex
2523 */
2524 static PetscErrorCode DMPlexBuildCoordinates_Private(DM dm, PetscInt spaceDim, PetscInt numCells, PetscInt numVertices, const double vertexCoords[])
2525 {
2526   PetscSection   coordSection;
2527   Vec            coordinates;
2528   DM             cdm;
2529   PetscScalar   *coords;
2530   PetscInt       v, d;
2531   PetscErrorCode ierr;
2532 
2533   PetscFunctionBegin;
2534   ierr = DMSetCoordinateDim(dm, spaceDim);CHKERRQ(ierr);
2535   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
2536   ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
2537   ierr = PetscSectionSetFieldComponents(coordSection, 0, spaceDim);CHKERRQ(ierr);
2538   ierr = PetscSectionSetChart(coordSection, numCells, numCells + numVertices);CHKERRQ(ierr);
2539   for (v = numCells; v < numCells+numVertices; ++v) {
2540     ierr = PetscSectionSetDof(coordSection, v, spaceDim);CHKERRQ(ierr);
2541     ierr = PetscSectionSetFieldDof(coordSection, v, 0, spaceDim);CHKERRQ(ierr);
2542   }
2543   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
2544 
2545   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
2546   ierr = DMCreateLocalVector(cdm, &coordinates);CHKERRQ(ierr);
2547   ierr = VecSetBlockSize(coordinates, spaceDim);CHKERRQ(ierr);
2548   ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
2549   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
2550   for (v = 0; v < numVertices; ++v) {
2551     for (d = 0; d < spaceDim; ++d) {
2552       coords[v*spaceDim+d] = vertexCoords[v*spaceDim+d];
2553     }
2554   }
2555   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
2556   ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
2557   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
2558   PetscFunctionReturn(0);
2559 }
2560 
2561 /*@C
2562   DMPlexCreateFromCellList - This takes as input common mesh generator output, a list of the vertices for each cell, and produces a DM
2563 
2564   Input Parameters:
2565 + comm - The communicator
2566 . dim - The topological dimension of the mesh
2567 . numCells - The number of cells
2568 . numVertices - The number of vertices
2569 . numCorners - The number of vertices for each cell
2570 . interpolate - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
2571 . cells - An array of numCells*numCorners numbers, the vertices for each cell
2572 . spaceDim - The spatial dimension used for coordinates
2573 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
2574 
2575   Output Parameter:
2576 . dm - The DM
2577 
2578   Note: Two triangles sharing a face
2579 $
2580 $        2
2581 $      / | \
2582 $     /  |  \
2583 $    /   |   \
2584 $   0  0 | 1  3
2585 $    \   |   /
2586 $     \  |  /
2587 $      \ | /
2588 $        1
2589 would have input
2590 $  numCells = 2, numVertices = 4
2591 $  cells = [0 1 2  1 3 2]
2592 $
2593 which would result in the DMPlex
2594 $
2595 $        4
2596 $      / | \
2597 $     /  |  \
2598 $    /   |   \
2599 $   2  0 | 1  5
2600 $    \   |   /
2601 $     \  |  /
2602 $      \ | /
2603 $        3
2604 
2605   Level: beginner
2606 
2607 .seealso: DMPlexCreateFromDAG(), DMPlexCreate()
2608 @*/
2609 PetscErrorCode DMPlexCreateFromCellList(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, PetscBool interpolate, const int cells[], PetscInt spaceDim, const double vertexCoords[], DM *dm)
2610 {
2611   PetscErrorCode ierr;
2612 
2613   PetscFunctionBegin;
2614   ierr = DMCreate(comm, dm);CHKERRQ(ierr);
2615   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
2616   ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
2617   ierr = DMPlexBuildFromCellList_Private(*dm, numCells, numVertices, numCorners, cells);CHKERRQ(ierr);
2618   if (interpolate) {
2619     DM idm = NULL;
2620 
2621     ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
2622     ierr = DMDestroy(dm);CHKERRQ(ierr);
2623     *dm  = idm;
2624   }
2625   ierr = DMPlexBuildCoordinates_Private(*dm, spaceDim, numCells, numVertices, vertexCoords);CHKERRQ(ierr);
2626   PetscFunctionReturn(0);
2627 }
2628 
2629 /*@
2630   DMPlexCreateFromDAG - This takes as input the adjacency-list representation of the Directed Acyclic Graph (Hasse Diagram) encoding a mesh, and produces a DM
2631 
2632   Input Parameters:
2633 + dm - The empty DM object, usually from DMCreate() and DMSetDimension()
2634 . depth - The depth of the DAG
2635 . numPoints - The number of points at each depth
2636 . coneSize - The cone size of each point
2637 . cones - The concatenation of the cone points for each point, the cone list must be oriented correctly for each point
2638 . coneOrientations - The orientation of each cone point
2639 - vertexCoords - An array of numVertices*dim numbers, the coordinates of each vertex
2640 
2641   Output Parameter:
2642 . dm - The DM
2643 
2644   Note: Two triangles sharing a face would have input
2645 $  depth = 1, numPoints = [4 2], coneSize = [3 3 0 0 0 0]
2646 $  cones = [2 3 4  3 5 4], coneOrientations = [0 0 0  0 0 0]
2647 $ vertexCoords = [-1.0 0.0  0.0 -1.0  0.0 1.0  1.0 0.0]
2648 $
2649 which would result in the DMPlex
2650 $
2651 $        4
2652 $      / | \
2653 $     /  |  \
2654 $    /   |   \
2655 $   2  0 | 1  5
2656 $    \   |   /
2657 $     \  |  /
2658 $      \ | /
2659 $        3
2660 $
2661 $ Notice that all points are numbered consecutively, unlikely DMPlexCreateFromCellList()
2662 
2663   Level: advanced
2664 
2665 .seealso: DMPlexCreateFromCellList(), DMPlexCreate()
2666 @*/
2667 PetscErrorCode DMPlexCreateFromDAG(DM dm, PetscInt depth, const PetscInt numPoints[], const PetscInt coneSize[], const PetscInt cones[], const PetscInt coneOrientations[], const PetscScalar vertexCoords[])
2668 {
2669   Vec            coordinates;
2670   PetscSection   coordSection;
2671   PetscScalar    *coords;
2672   PetscInt       coordSize, firstVertex = -1, pStart = 0, pEnd = 0, p, v, dim, dimEmbed, d, off;
2673   PetscErrorCode ierr;
2674 
2675   PetscFunctionBegin;
2676   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
2677   ierr = DMGetCoordinateDim(dm, &dimEmbed);CHKERRQ(ierr);
2678   if (dimEmbed < dim) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Embedding dimension %d cannot be less than intrinsic dimension %d",dimEmbed,dim);
2679   for (d = 0; d <= depth; ++d) pEnd += numPoints[d];
2680   ierr = DMPlexSetChart(dm, pStart, pEnd);CHKERRQ(ierr);
2681   for (p = pStart; p < pEnd; ++p) {
2682     ierr = DMPlexSetConeSize(dm, p, coneSize[p-pStart]);CHKERRQ(ierr);
2683     if (firstVertex < 0 && !coneSize[p - pStart]) {
2684       firstVertex = p - pStart;
2685     }
2686   }
2687   if (firstVertex < 0 && numPoints[0]) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONG,"Expected %d vertices but could not find any", numPoints[0]);
2688   ierr = DMSetUp(dm);CHKERRQ(ierr); /* Allocate space for cones */
2689   for (p = pStart, off = 0; p < pEnd; off += coneSize[p-pStart], ++p) {
2690     ierr = DMPlexSetCone(dm, p, &cones[off]);CHKERRQ(ierr);
2691     ierr = DMPlexSetConeOrientation(dm, p, &coneOrientations[off]);CHKERRQ(ierr);
2692   }
2693   ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
2694   ierr = DMPlexStratify(dm);CHKERRQ(ierr);
2695   /* Build coordinates */
2696   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
2697   ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
2698   ierr = PetscSectionSetFieldComponents(coordSection, 0, dimEmbed);CHKERRQ(ierr);
2699   ierr = PetscSectionSetChart(coordSection, firstVertex, firstVertex+numPoints[0]);CHKERRQ(ierr);
2700   for (v = firstVertex; v < firstVertex+numPoints[0]; ++v) {
2701     ierr = PetscSectionSetDof(coordSection, v, dimEmbed);CHKERRQ(ierr);
2702     ierr = PetscSectionSetFieldDof(coordSection, v, 0, dimEmbed);CHKERRQ(ierr);
2703   }
2704   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
2705   ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
2706   ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
2707   ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
2708   ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
2709   ierr = VecSetBlockSize(coordinates, dimEmbed);CHKERRQ(ierr);
2710   ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
2711   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
2712   for (v = 0; v < numPoints[0]; ++v) {
2713     PetscInt off;
2714 
2715     ierr = PetscSectionGetOffset(coordSection, v+firstVertex, &off);CHKERRQ(ierr);
2716     for (d = 0; d < dimEmbed; ++d) {
2717       coords[off+d] = vertexCoords[v*dimEmbed+d];
2718     }
2719   }
2720   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
2721   ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
2722   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
2723   PetscFunctionReturn(0);
2724 }
2725 
2726 /*@C
2727   DMPlexCreateFromFile - This takes a filename and produces a DM
2728 
2729   Input Parameters:
2730 + comm - The communicator
2731 . filename - A file name
2732 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
2733 
2734   Output Parameter:
2735 . dm - The DM
2736 
2737   Level: beginner
2738 
2739 .seealso: DMPlexCreateFromDAG(), DMPlexCreateFromCellList(), DMPlexCreate()
2740 @*/
2741 PetscErrorCode DMPlexCreateFromFile(MPI_Comm comm, const char filename[], PetscBool interpolate, DM *dm)
2742 {
2743   const char    *extGmsh    = ".msh";
2744   const char    *extCGNS    = ".cgns";
2745   const char    *extExodus  = ".exo";
2746   const char    *extGenesis = ".gen";
2747   const char    *extFluent  = ".cas";
2748   const char    *extHDF5    = ".h5";
2749   const char    *extMed     = ".med";
2750   const char    *extPLY     = ".ply";
2751   size_t         len;
2752   PetscBool      isGmsh, isCGNS, isExodus, isGenesis, isFluent, isHDF5, isMed, isPLY;
2753   PetscMPIInt    rank;
2754   PetscErrorCode ierr;
2755 
2756   PetscFunctionBegin;
2757   PetscValidPointer(filename, 2);
2758   PetscValidPointer(dm, 4);
2759   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
2760   ierr = PetscStrlen(filename, &len);CHKERRQ(ierr);
2761   if (!len) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Filename must be a valid path");
2762   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)], extGmsh,    4, &isGmsh);CHKERRQ(ierr);
2763   ierr = PetscStrncmp(&filename[PetscMax(0,len-5)], extCGNS,    5, &isCGNS);CHKERRQ(ierr);
2764   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)], extExodus,  4, &isExodus);CHKERRQ(ierr);
2765   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)], extGenesis, 4, &isGenesis);CHKERRQ(ierr);
2766   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)], extFluent,  4, &isFluent);CHKERRQ(ierr);
2767   ierr = PetscStrncmp(&filename[PetscMax(0,len-3)], extHDF5,    3, &isHDF5);CHKERRQ(ierr);
2768   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)], extMed,     4, &isMed);CHKERRQ(ierr);
2769   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)], extPLY,     4, &isPLY);CHKERRQ(ierr);
2770   if (isGmsh) {
2771     ierr = DMPlexCreateGmshFromFile(comm, filename, interpolate, dm);CHKERRQ(ierr);
2772   } else if (isCGNS) {
2773     ierr = DMPlexCreateCGNSFromFile(comm, filename, interpolate, dm);CHKERRQ(ierr);
2774   } else if (isExodus || isGenesis) {
2775     ierr = DMPlexCreateExodusFromFile(comm, filename, interpolate, dm);CHKERRQ(ierr);
2776   } else if (isFluent) {
2777     ierr = DMPlexCreateFluentFromFile(comm, filename, interpolate, dm);CHKERRQ(ierr);
2778   } else if (isHDF5) {
2779     PetscViewer viewer;
2780 
2781     ierr = PetscViewerCreate(comm, &viewer);CHKERRQ(ierr);
2782     ierr = PetscViewerSetType(viewer, PETSCVIEWERHDF5);CHKERRQ(ierr);
2783     ierr = PetscViewerFileSetMode(viewer, FILE_MODE_READ);CHKERRQ(ierr);
2784     ierr = PetscViewerFileSetName(viewer, filename);CHKERRQ(ierr);
2785     ierr = DMCreate(comm, dm);CHKERRQ(ierr);
2786     ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
2787     ierr = DMLoad(*dm, viewer);CHKERRQ(ierr);
2788     ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
2789   } else if (isMed) {
2790     ierr = DMPlexCreateMedFromFile(comm, filename, interpolate, dm);CHKERRQ(ierr);
2791   } else if (isPLY) {
2792     ierr = DMPlexCreatePLYFromFile(comm, filename, interpolate, dm);CHKERRQ(ierr);
2793   } else SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cannot load file %s: unrecognized extension", filename);
2794   PetscFunctionReturn(0);
2795 }
2796 
2797 /*@
2798   DMPlexCreateReferenceCell - Create a DMPLEX with the appropriate FEM reference cell
2799 
2800   Collective on comm
2801 
2802   Input Parameters:
2803 + comm    - The communicator
2804 . dim     - The spatial dimension
2805 - simplex - Flag for simplex, otherwise use a tensor-product cell
2806 
2807   Output Parameter:
2808 . refdm - The reference cell
2809 
2810   Level: intermediate
2811 
2812 .keywords: reference cell
2813 .seealso:
2814 @*/
2815 PetscErrorCode DMPlexCreateReferenceCell(MPI_Comm comm, PetscInt dim, PetscBool simplex, DM *refdm)
2816 {
2817   DM             rdm;
2818   Vec            coords;
2819   PetscErrorCode ierr;
2820 
2821   PetscFunctionBeginUser;
2822   ierr = DMCreate(comm, &rdm);CHKERRQ(ierr);
2823   ierr = DMSetType(rdm, DMPLEX);CHKERRQ(ierr);
2824   ierr = DMSetDimension(rdm, dim);CHKERRQ(ierr);
2825   switch (dim) {
2826   case 0:
2827   {
2828     PetscInt    numPoints[1]        = {1};
2829     PetscInt    coneSize[1]         = {0};
2830     PetscInt    cones[1]            = {0};
2831     PetscInt    coneOrientations[1] = {0};
2832     PetscScalar vertexCoords[1]     = {0.0};
2833 
2834     ierr = DMPlexCreateFromDAG(rdm, 0, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
2835   }
2836   break;
2837   case 1:
2838   {
2839     PetscInt    numPoints[2]        = {2, 1};
2840     PetscInt    coneSize[3]         = {2, 0, 0};
2841     PetscInt    cones[2]            = {1, 2};
2842     PetscInt    coneOrientations[2] = {0, 0};
2843     PetscScalar vertexCoords[2]     = {-1.0,  1.0};
2844 
2845     ierr = DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
2846   }
2847   break;
2848   case 2:
2849     if (simplex) {
2850       PetscInt    numPoints[2]        = {3, 1};
2851       PetscInt    coneSize[4]         = {3, 0, 0, 0};
2852       PetscInt    cones[3]            = {1, 2, 3};
2853       PetscInt    coneOrientations[3] = {0, 0, 0};
2854       PetscScalar vertexCoords[6]     = {-1.0, -1.0,  1.0, -1.0,  -1.0, 1.0};
2855 
2856       ierr = DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
2857     } else {
2858       PetscInt    numPoints[2]        = {4, 1};
2859       PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
2860       PetscInt    cones[4]            = {1, 2, 3, 4};
2861       PetscInt    coneOrientations[4] = {0, 0, 0, 0};
2862       PetscScalar vertexCoords[8]     = {-1.0, -1.0,  1.0, -1.0,  1.0, 1.0,  -1.0, 1.0};
2863 
2864       ierr = DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
2865     }
2866   break;
2867   case 3:
2868     if (simplex) {
2869       PetscInt    numPoints[2]        = {4, 1};
2870       PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
2871       PetscInt    cones[4]            = {1, 3, 2, 4};
2872       PetscInt    coneOrientations[4] = {0, 0, 0, 0};
2873       PetscScalar vertexCoords[12]    = {-1.0, -1.0, -1.0,  1.0, -1.0, -1.0,  -1.0, 1.0, -1.0,  -1.0, -1.0, 1.0};
2874 
2875       ierr = DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
2876     } else {
2877       PetscInt    numPoints[2]        = {8, 1};
2878       PetscInt    coneSize[9]         = {8, 0, 0, 0, 0, 0, 0, 0, 0};
2879       PetscInt    cones[8]            = {1, 4, 3, 2, 5, 6, 7, 8};
2880       PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
2881       PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0,  1.0, -1.0, -1.0,  1.0, 1.0, -1.0,  -1.0, 1.0, -1.0,
2882                                          -1.0, -1.0,  1.0,  1.0, -1.0,  1.0,  1.0, 1.0,  1.0,  -1.0, 1.0,  1.0};
2883 
2884       ierr = DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
2885     }
2886   break;
2887   default:
2888     SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Cannot create reference cell for dimension %d", dim);
2889   }
2890   *refdm = NULL;
2891   ierr = DMPlexInterpolate(rdm, refdm);CHKERRQ(ierr);
2892   if (rdm->coordinateDM) {
2893     DM           ncdm;
2894     PetscSection cs;
2895     PetscInt     pEnd = -1;
2896 
2897     ierr = DMGetDefaultSection(rdm->coordinateDM, &cs);CHKERRQ(ierr);
2898     if (cs) {ierr = PetscSectionGetChart(cs, NULL, &pEnd);CHKERRQ(ierr);}
2899     if (pEnd >= 0) {
2900       ierr = DMClone(rdm->coordinateDM, &ncdm);CHKERRQ(ierr);
2901       ierr = DMSetDefaultSection(ncdm, cs);CHKERRQ(ierr);
2902       ierr = DMSetCoordinateDM(*refdm, ncdm);CHKERRQ(ierr);
2903       ierr = DMDestroy(&ncdm);CHKERRQ(ierr);
2904     }
2905   }
2906   ierr = DMGetCoordinatesLocal(rdm, &coords);CHKERRQ(ierr);
2907   if (coords) {
2908     ierr = DMSetCoordinatesLocal(*refdm, coords);CHKERRQ(ierr);
2909   } else {
2910     ierr = DMGetCoordinates(rdm, &coords);CHKERRQ(ierr);
2911     if (coords) {ierr = DMSetCoordinates(*refdm, coords);CHKERRQ(ierr);}
2912   }
2913   ierr = DMDestroy(&rdm);CHKERRQ(ierr);
2914   PetscFunctionReturn(0);
2915 }
2916