xref: /petsc/src/dm/impls/plex/plexcreate.c (revision 55e7fe800d976e85ed2b5cd8bfdef564daa37bd9)
1 #define PETSCDM_DLL
2 #include <petsc/private/dmpleximpl.h>    /*I   "petscdmplex.h"   I*/
3 #include <petsc/private/hashseti.h>          /*I   "petscdmplex.h"   I*/
4 #include <petscsf.h>
5 
6 /*@
7   DMPlexCreateDoublet - Creates a mesh of two cells of the specified type, optionally with later refinement.
8 
9   Collective on MPI_Comm
10 
11   Input Parameters:
12 + comm - The communicator for the DM object
13 . dim - The spatial dimension
14 . simplex - Flag for simplicial cells, otherwise they are tensor product cells
15 . interpolate - Flag to create intermediate mesh pieces (edges, faces)
16 . refinementUniform - Flag for uniform parallel refinement
17 - refinementLimit - A nonzero number indicates the largest admissible volume for a refined cell
18 
19   Output Parameter:
20 . dm - The DM object
21 
22   Level: beginner
23 
24 .keywords: DM, create
25 .seealso: DMSetType(), DMCreate()
26 @*/
27 PetscErrorCode DMPlexCreateDoublet(MPI_Comm comm, PetscInt dim, PetscBool simplex, PetscBool interpolate, PetscBool refinementUniform, PetscReal refinementLimit, DM *newdm)
28 {
29   DM             dm;
30   PetscInt       p;
31   PetscMPIInt    rank;
32   PetscErrorCode ierr;
33 
34   PetscFunctionBegin;
35   ierr = DMCreate(comm, &dm);CHKERRQ(ierr);
36   ierr = DMSetType(dm, DMPLEX);CHKERRQ(ierr);
37   ierr = DMSetDimension(dm, dim);CHKERRQ(ierr);
38   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
39   switch (dim) {
40   case 2:
41     if (simplex) {ierr = PetscObjectSetName((PetscObject) dm, "triangular");CHKERRQ(ierr);}
42     else         {ierr = PetscObjectSetName((PetscObject) dm, "quadrilateral");CHKERRQ(ierr);}
43     break;
44   case 3:
45     if (simplex) {ierr = PetscObjectSetName((PetscObject) dm, "tetrahedral");CHKERRQ(ierr);}
46     else         {ierr = PetscObjectSetName((PetscObject) dm, "hexahedral");CHKERRQ(ierr);}
47     break;
48   default:
49     SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make meshes for dimension %d", dim);
50   }
51   if (rank) {
52     PetscInt numPoints[2] = {0, 0};
53     ierr = DMPlexCreateFromDAG(dm, 1, numPoints, NULL, NULL, NULL, NULL);CHKERRQ(ierr);
54   } else {
55     switch (dim) {
56     case 2:
57       if (simplex) {
58         PetscInt    numPoints[2]        = {4, 2};
59         PetscInt    coneSize[6]         = {3, 3, 0, 0, 0, 0};
60         PetscInt    cones[6]            = {2, 3, 4,  5, 4, 3};
61         PetscInt    coneOrientations[6] = {0, 0, 0,  0, 0, 0};
62         PetscScalar vertexCoords[8]     = {-0.5, 0.5, 0.0, 0.0, 0.0, 1.0, 0.5, 0.5};
63         PetscInt    markerPoints[8]     = {2, 1, 3, 1, 4, 1, 5, 1};
64 
65         ierr = DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
66         for (p = 0; p < 4; ++p) {ierr = DMSetLabelValue(dm, "marker", markerPoints[p*2], markerPoints[p*2+1]);CHKERRQ(ierr);}
67       } else {
68         PetscInt    numPoints[2]        = {6, 2};
69         PetscInt    coneSize[8]         = {4, 4, 0, 0, 0, 0, 0, 0};
70         PetscInt    cones[8]            = {2, 3, 4, 5,  3, 6, 7, 4};
71         PetscInt    coneOrientations[8] = {0, 0, 0, 0,  0, 0, 0, 0};
72         PetscScalar vertexCoords[12]    = {-1.0, -0.5,  0.0, -0.5,  0.0, 0.5,  -1.0, 0.5,  1.0, -0.5,  1.0, 0.5};
73 
74         ierr = DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
75       }
76       break;
77     case 3:
78       if (simplex) {
79         PetscInt    numPoints[2]        = {5, 2};
80         PetscInt    coneSize[7]         = {4, 4, 0, 0, 0, 0, 0};
81         PetscInt    cones[8]            = {4, 3, 5, 2,  5, 3, 4, 6};
82         PetscInt    coneOrientations[8] = {0, 0, 0, 0,  0, 0, 0, 0};
83         PetscScalar vertexCoords[15]    = {-1.0, 0.0, 0.0,  0.0, -1.0, 0.0,  0.0, 0.0, 1.0,  0.0, 1.0, 0.0,  1.0, 0.0, 0.0};
84         PetscInt    markerPoints[10]    = {2, 1, 3, 1, 4, 1, 5, 1, 6, 1};
85 
86         ierr = DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
87         for (p = 0; p < 5; ++p) {ierr = DMSetLabelValue(dm, "marker", markerPoints[p*2], markerPoints[p*2+1]);CHKERRQ(ierr);}
88       } else {
89         PetscInt    numPoints[2]         = {12, 2};
90         PetscInt    coneSize[14]         = {8, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
91         PetscInt    cones[16]            = {2, 3, 4, 5, 6, 7, 8, 9,  5, 4, 10, 11, 7, 12, 13, 8};
92         PetscInt    coneOrientations[16] = {0, 0, 0, 0, 0, 0, 0, 0,  0, 0,  0,  0, 0,  0,  0, 0};
93         PetscScalar vertexCoords[36]     = {-1.0, -0.5, -0.5,  -1.0,  0.5, -0.5,  0.0,  0.5, -0.5,   0.0, -0.5, -0.5,
94                                             -1.0, -0.5,  0.5,   0.0, -0.5,  0.5,  0.0,  0.5,  0.5,  -1.0,  0.5,  0.5,
95                                              1.0,  0.5, -0.5,   1.0, -0.5, -0.5,  1.0, -0.5,  0.5,   1.0,  0.5,  0.5};
96 
97         ierr = DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
98       }
99       break;
100     default:
101       SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make meshes for dimension %d", dim);
102     }
103   }
104   *newdm = dm;
105   if (refinementLimit > 0.0) {
106     DM rdm;
107     const char *name;
108 
109     ierr = DMPlexSetRefinementUniform(*newdm, PETSC_FALSE);CHKERRQ(ierr);
110     ierr = DMPlexSetRefinementLimit(*newdm, refinementLimit);CHKERRQ(ierr);
111     ierr = DMRefine(*newdm, comm, &rdm);CHKERRQ(ierr);
112     ierr = PetscObjectGetName((PetscObject) *newdm, &name);CHKERRQ(ierr);
113     ierr = PetscObjectSetName((PetscObject)    rdm,  name);CHKERRQ(ierr);
114     ierr = DMDestroy(newdm);CHKERRQ(ierr);
115     *newdm = rdm;
116   }
117   if (interpolate) {
118     DM idm;
119 
120     ierr = DMPlexInterpolate(*newdm, &idm);CHKERRQ(ierr);
121     ierr = DMDestroy(newdm);CHKERRQ(ierr);
122     *newdm = idm;
123   }
124   {
125     DM refinedMesh     = NULL;
126     DM distributedMesh = NULL;
127 
128     /* Distribute mesh over processes */
129     ierr = DMPlexDistribute(*newdm, 0, NULL, &distributedMesh);CHKERRQ(ierr);
130     if (distributedMesh) {
131       ierr = DMDestroy(newdm);CHKERRQ(ierr);
132       *newdm = distributedMesh;
133     }
134     if (refinementUniform) {
135       ierr = DMPlexSetRefinementUniform(*newdm, refinementUniform);CHKERRQ(ierr);
136       ierr = DMRefine(*newdm, comm, &refinedMesh);CHKERRQ(ierr);
137       if (refinedMesh) {
138         ierr = DMDestroy(newdm);CHKERRQ(ierr);
139         *newdm = refinedMesh;
140       }
141     }
142   }
143   PetscFunctionReturn(0);
144 }
145 
146 /*@
147   DMPlexCreateSquareBoundary - Creates a 1D mesh the is the boundary of a square lattice.
148 
149   Collective on MPI_Comm
150 
151   Input Parameters:
152 + comm  - The communicator for the DM object
153 . lower - The lower left corner coordinates
154 . upper - The upper right corner coordinates
155 - edges - The number of cells in each direction
156 
157   Output Parameter:
158 . dm  - The DM object
159 
160   Note: Here is the numbering returned for 2 cells in each direction:
161 $ 18--5-17--4--16
162 $  |     |     |
163 $  6    10     3
164 $  |     |     |
165 $ 19-11-20--9--15
166 $  |     |     |
167 $  7     8     2
168 $  |     |     |
169 $ 12--0-13--1--14
170 
171   Level: beginner
172 
173 .keywords: DM, create
174 .seealso: DMPlexCreateBoxMesh(), DMPlexCreateCubeBoundary(), DMSetType(), DMCreate()
175 @*/
176 PetscErrorCode DMPlexCreateSquareBoundary(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[])
177 {
178   const PetscInt numVertices    = (edges[0]+1)*(edges[1]+1);
179   const PetscInt numEdges       = edges[0]*(edges[1]+1) + (edges[0]+1)*edges[1];
180   PetscInt       markerTop      = 1;
181   PetscInt       markerBottom   = 1;
182   PetscInt       markerRight    = 1;
183   PetscInt       markerLeft     = 1;
184   PetscBool      markerSeparate = PETSC_FALSE;
185   Vec            coordinates;
186   PetscSection   coordSection;
187   PetscScalar    *coords;
188   PetscInt       coordSize;
189   PetscMPIInt    rank;
190   PetscInt       v, vx, vy;
191   PetscErrorCode ierr;
192 
193   PetscFunctionBegin;
194   ierr = PetscOptionsGetBool(((PetscObject) dm)->options,((PetscObject) dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL);CHKERRQ(ierr);
195   if (markerSeparate) {
196     markerTop    = 3;
197     markerBottom = 1;
198     markerRight  = 2;
199     markerLeft   = 4;
200   }
201   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRQ(ierr);
202   if (!rank) {
203     PetscInt e, ex, ey;
204 
205     ierr = DMPlexSetChart(dm, 0, numEdges+numVertices);CHKERRQ(ierr);
206     for (e = 0; e < numEdges; ++e) {
207       ierr = DMPlexSetConeSize(dm, e, 2);CHKERRQ(ierr);
208     }
209     ierr = DMSetUp(dm);CHKERRQ(ierr); /* Allocate space for cones */
210     for (vx = 0; vx <= edges[0]; vx++) {
211       for (ey = 0; ey < edges[1]; ey++) {
212         PetscInt edge   = vx*edges[1] + ey + edges[0]*(edges[1]+1);
213         PetscInt vertex = ey*(edges[0]+1) + vx + numEdges;
214         PetscInt cone[2];
215 
216         cone[0] = vertex; cone[1] = vertex+edges[0]+1;
217         ierr    = DMPlexSetCone(dm, edge, cone);CHKERRQ(ierr);
218         if (vx == edges[0]) {
219           ierr = DMSetLabelValue(dm, "marker", edge,    markerRight);CHKERRQ(ierr);
220           ierr = DMSetLabelValue(dm, "marker", cone[0], markerRight);CHKERRQ(ierr);
221           if (ey == edges[1]-1) {
222             ierr = DMSetLabelValue(dm, "marker", cone[1], markerRight);CHKERRQ(ierr);
223             ierr = DMSetLabelValue(dm, "Face Sets", cone[1], markerRight);CHKERRQ(ierr);
224           }
225         } else if (vx == 0) {
226           ierr = DMSetLabelValue(dm, "marker", edge,    markerLeft);CHKERRQ(ierr);
227           ierr = DMSetLabelValue(dm, "marker", cone[0], markerLeft);CHKERRQ(ierr);
228           if (ey == edges[1]-1) {
229             ierr = DMSetLabelValue(dm, "marker", cone[1], markerLeft);CHKERRQ(ierr);
230             ierr = DMSetLabelValue(dm, "Face Sets", cone[1], markerLeft);CHKERRQ(ierr);
231           }
232         }
233       }
234     }
235     for (vy = 0; vy <= edges[1]; vy++) {
236       for (ex = 0; ex < edges[0]; ex++) {
237         PetscInt edge   = vy*edges[0]     + ex;
238         PetscInt vertex = vy*(edges[0]+1) + ex + numEdges;
239         PetscInt cone[2];
240 
241         cone[0] = vertex; cone[1] = vertex+1;
242         ierr    = DMPlexSetCone(dm, edge, cone);CHKERRQ(ierr);
243         if (vy == edges[1]) {
244           ierr = DMSetLabelValue(dm, "marker", edge,    markerTop);CHKERRQ(ierr);
245           ierr = DMSetLabelValue(dm, "marker", cone[0], markerTop);CHKERRQ(ierr);
246           if (ex == edges[0]-1) {
247             ierr = DMSetLabelValue(dm, "marker", cone[1], markerTop);CHKERRQ(ierr);
248             ierr = DMSetLabelValue(dm, "Face Sets", cone[1], markerTop);CHKERRQ(ierr);
249           }
250         } else if (vy == 0) {
251           ierr = DMSetLabelValue(dm, "marker", edge,    markerBottom);CHKERRQ(ierr);
252           ierr = DMSetLabelValue(dm, "marker", cone[0], markerBottom);CHKERRQ(ierr);
253           if (ex == edges[0]-1) {
254             ierr = DMSetLabelValue(dm, "marker", cone[1], markerBottom);CHKERRQ(ierr);
255             ierr = DMSetLabelValue(dm, "Face Sets", cone[1], markerBottom);CHKERRQ(ierr);
256           }
257         }
258       }
259     }
260   }
261   ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
262   ierr = DMPlexStratify(dm);CHKERRQ(ierr);
263   /* Build coordinates */
264   ierr = DMSetCoordinateDim(dm, 2);CHKERRQ(ierr);
265   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
266   ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
267   ierr = PetscSectionSetChart(coordSection, numEdges, numEdges + numVertices);CHKERRQ(ierr);
268   ierr = PetscSectionSetFieldComponents(coordSection, 0, 2);CHKERRQ(ierr);
269   for (v = numEdges; v < numEdges+numVertices; ++v) {
270     ierr = PetscSectionSetDof(coordSection, v, 2);CHKERRQ(ierr);
271     ierr = PetscSectionSetFieldDof(coordSection, v, 0, 2);CHKERRQ(ierr);
272   }
273   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
274   ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
275   ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
276   ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
277   ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
278   ierr = VecSetBlockSize(coordinates, 2);CHKERRQ(ierr);
279   ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
280   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
281   for (vy = 0; vy <= edges[1]; ++vy) {
282     for (vx = 0; vx <= edges[0]; ++vx) {
283       coords[(vy*(edges[0]+1)+vx)*2+0] = lower[0] + ((upper[0] - lower[0])/edges[0])*vx;
284       coords[(vy*(edges[0]+1)+vx)*2+1] = lower[1] + ((upper[1] - lower[1])/edges[1])*vy;
285     }
286   }
287   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
288   ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
289   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
290   PetscFunctionReturn(0);
291 }
292 
293 /*@
294   DMPlexCreateCubeBoundary - Creates a 2D mesh that is the boundary of a cubic lattice.
295 
296   Collective on MPI_Comm
297 
298   Input Parameters:
299 + comm  - The communicator for the DM object
300 . lower - The lower left front corner coordinates
301 . upper - The upper right back corner coordinates
302 - edges - The number of cells in each direction
303 
304   Output Parameter:
305 . dm  - The DM object
306 
307   Level: beginner
308 
309 .keywords: DM, create
310 .seealso: DMPlexCreateBoxMesh(), DMPlexCreateSquareBoundary(), DMSetType(), DMCreate()
311 @*/
312 PetscErrorCode DMPlexCreateCubeBoundary(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt faces[])
313 {
314   PetscInt       vertices[3], numVertices;
315   PetscInt       numFaces    = 2*faces[0]*faces[1] + 2*faces[1]*faces[2] + 2*faces[0]*faces[2];
316   Vec            coordinates;
317   PetscSection   coordSection;
318   PetscScalar    *coords;
319   PetscInt       coordSize;
320   PetscMPIInt    rank;
321   PetscInt       v, vx, vy, vz;
322   PetscInt       voffset, iface=0, cone[4];
323   PetscErrorCode ierr;
324 
325   PetscFunctionBegin;
326   if ((faces[0] < 1) || (faces[1] < 1) || (faces[2] < 1)) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Must have at least 1 face per side");
327   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRQ(ierr);
328   vertices[0] = faces[0]+1; vertices[1] = faces[1]+1; vertices[2] = faces[2]+1;
329   numVertices = vertices[0]*vertices[1]*vertices[2];
330   if (!rank) {
331     PetscInt f;
332 
333     ierr = DMPlexSetChart(dm, 0, numFaces+numVertices);CHKERRQ(ierr);
334     for (f = 0; f < numFaces; ++f) {
335       ierr = DMPlexSetConeSize(dm, f, 4);CHKERRQ(ierr);
336     }
337     ierr = DMSetUp(dm);CHKERRQ(ierr); /* Allocate space for cones */
338     for (v = 0; v < numFaces+numVertices; ++v) {
339       ierr = DMSetLabelValue(dm, "marker", v, 1);CHKERRQ(ierr);
340     }
341 
342     /* Side 0 (Top) */
343     for (vy = 0; vy < faces[1]; vy++) {
344       for (vx = 0; vx < faces[0]; vx++) {
345         voffset = numFaces + vertices[0]*vertices[1]*(vertices[2]-1) + vy*vertices[0] + vx;
346         cone[0] = voffset; cone[1] = voffset+1; cone[2] = voffset+vertices[0]+1; cone[3] = voffset+vertices[0];
347         ierr    = DMPlexSetCone(dm, iface, cone);CHKERRQ(ierr);
348         iface++;
349       }
350     }
351 
352     /* Side 1 (Bottom) */
353     for (vy = 0; vy < faces[1]; vy++) {
354       for (vx = 0; vx < faces[0]; vx++) {
355         voffset = numFaces + vy*(faces[0]+1) + vx;
356         cone[0] = voffset+1; cone[1] = voffset; cone[2] = voffset+vertices[0]; cone[3] = voffset+vertices[0]+1;
357         ierr    = DMPlexSetCone(dm, iface, cone);CHKERRQ(ierr);
358         iface++;
359       }
360     }
361 
362     /* Side 2 (Front) */
363     for (vz = 0; vz < faces[2]; vz++) {
364       for (vx = 0; vx < faces[0]; vx++) {
365         voffset = numFaces + vz*vertices[0]*vertices[1] + vx;
366         cone[0] = voffset; cone[1] = voffset+1; cone[2] = voffset+vertices[0]*vertices[1]+1; cone[3] = voffset+vertices[0]*vertices[1];
367         ierr    = DMPlexSetCone(dm, iface, cone);CHKERRQ(ierr);
368         iface++;
369       }
370     }
371 
372     /* Side 3 (Back) */
373     for (vz = 0; vz < faces[2]; vz++) {
374       for (vx = 0; vx < faces[0]; vx++) {
375         voffset = numFaces + vz*vertices[0]*vertices[1] + vertices[0]*(vertices[1]-1) + vx;
376         cone[0] = voffset+vertices[0]*vertices[1]; cone[1] = voffset+vertices[0]*vertices[1]+1;
377         cone[2] = voffset+1; cone[3] = voffset;
378         ierr    = DMPlexSetCone(dm, iface, cone);CHKERRQ(ierr);
379         iface++;
380       }
381     }
382 
383     /* Side 4 (Left) */
384     for (vz = 0; vz < faces[2]; vz++) {
385       for (vy = 0; vy < faces[1]; vy++) {
386         voffset = numFaces + vz*vertices[0]*vertices[1] + vy*vertices[0];
387         cone[0] = voffset; cone[1] = voffset+vertices[0]*vertices[1];
388         cone[2] = voffset+vertices[0]*vertices[1]+vertices[0]; cone[3] = voffset+vertices[0];
389         ierr    = DMPlexSetCone(dm, iface, cone);CHKERRQ(ierr);
390         iface++;
391       }
392     }
393 
394     /* Side 5 (Right) */
395     for (vz = 0; vz < faces[2]; vz++) {
396       for (vy = 0; vy < faces[1]; vy++) {
397         voffset = numFaces + vz*vertices[0]*vertices[1] + vy*vertices[0] + faces[0];
398         cone[0] = voffset+vertices[0]*vertices[1]; cone[1] = voffset;
399         cone[2] = voffset+vertices[0]; cone[3] = voffset+vertices[0]*vertices[1]+vertices[0];
400         ierr    = DMPlexSetCone(dm, iface, cone);CHKERRQ(ierr);
401         iface++;
402       }
403     }
404   }
405   ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
406   ierr = DMPlexStratify(dm);CHKERRQ(ierr);
407   /* Build coordinates */
408   ierr = DMSetCoordinateDim(dm, 3);CHKERRQ(ierr);
409   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
410   ierr = PetscSectionSetChart(coordSection, numFaces, numFaces + numVertices);CHKERRQ(ierr);
411   for (v = numFaces; v < numFaces+numVertices; ++v) {
412     ierr = PetscSectionSetDof(coordSection, v, 3);CHKERRQ(ierr);
413   }
414   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
415   ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
416   ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
417   ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
418   ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
419   ierr = VecSetBlockSize(coordinates, 3);CHKERRQ(ierr);
420   ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
421   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
422   for (vz = 0; vz <= faces[2]; ++vz) {
423     for (vy = 0; vy <= faces[1]; ++vy) {
424       for (vx = 0; vx <= faces[0]; ++vx) {
425         coords[((vz*(faces[1]+1)+vy)*(faces[0]+1)+vx)*3+0] = lower[0] + ((upper[0] - lower[0])/faces[0])*vx;
426         coords[((vz*(faces[1]+1)+vy)*(faces[0]+1)+vx)*3+1] = lower[1] + ((upper[1] - lower[1])/faces[1])*vy;
427         coords[((vz*(faces[1]+1)+vy)*(faces[0]+1)+vx)*3+2] = lower[2] + ((upper[2] - lower[2])/faces[2])*vz;
428       }
429     }
430   }
431   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
432   ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
433   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
434   PetscFunctionReturn(0);
435 }
436 
437 static PetscErrorCode DMPlexCreateLineMesh_Internal(MPI_Comm comm,PetscInt segments,PetscReal lower,PetscReal upper,DMBoundaryType bd,DM *dm)
438 {
439   PetscInt       i,fStart,fEnd,numCells = 0,numVerts = 0;
440   PetscInt       numPoints[2],*coneSize,*cones,*coneOrientations;
441   PetscScalar    *vertexCoords;
442   PetscReal      L,maxCell;
443   PetscBool      markerSeparate = PETSC_FALSE;
444   PetscInt       markerLeft  = 1, faceMarkerLeft  = 1;
445   PetscInt       markerRight = 1, faceMarkerRight = 2;
446   PetscBool      wrap = (bd == DM_BOUNDARY_PERIODIC || bd == DM_BOUNDARY_TWIST) ? PETSC_TRUE : PETSC_FALSE;
447   PetscMPIInt    rank;
448   PetscErrorCode ierr;
449 
450   PetscFunctionBegin;
451   PetscValidPointer(dm,4);
452 
453   ierr = DMCreate(comm,dm);CHKERRQ(ierr);
454   ierr = DMSetType(*dm,DMPLEX);CHKERRQ(ierr);
455   ierr = DMSetDimension(*dm,1);CHKERRQ(ierr);
456   ierr = DMCreateLabel(*dm,"marker");CHKERRQ(ierr);
457   ierr = DMCreateLabel(*dm,"Face Sets");CHKERRQ(ierr);
458 
459   ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr);
460   if (!rank) numCells = segments;
461   if (!rank) numVerts = segments + (wrap ? 0 : 1);
462 
463   numPoints[0] = numVerts ; numPoints[1] = numCells;
464   ierr = PetscMalloc4(numCells+numVerts,&coneSize,numCells*2,&cones,numCells+numVerts,&coneOrientations,numVerts,&vertexCoords);CHKERRQ(ierr);
465   ierr = PetscMemzero(coneOrientations,(numCells+numVerts)*sizeof(PetscInt));CHKERRQ(ierr);
466   for (i = 0; i < numCells; ++i) { coneSize[i] = 2; }
467   for (i = 0; i < numVerts; ++i) { coneSize[numCells+i] = 0; }
468   for (i = 0; i < numCells; ++i) { cones[2*i] = numCells + i%numVerts; cones[2*i+1] = numCells + (i+1)%numVerts; }
469   for (i = 0; i < numVerts; ++i) { vertexCoords[i] = lower + (upper-lower)*((PetscReal)i/(PetscReal)numCells); }
470   ierr = DMPlexCreateFromDAG(*dm,1,numPoints,coneSize,cones,coneOrientations,vertexCoords);CHKERRQ(ierr);
471   ierr = PetscFree4(coneSize,cones,coneOrientations,vertexCoords);CHKERRQ(ierr);
472 
473   ierr = PetscOptionsGetBool(((PetscObject)*dm)->options,((PetscObject)*dm)->prefix,"-dm_plex_separate_marker",&markerSeparate,NULL);CHKERRQ(ierr);
474   if (markerSeparate) { markerLeft = faceMarkerLeft; markerRight = faceMarkerRight;}
475   if (!wrap && !rank) {
476     ierr = DMPlexGetHeightStratum(*dm,1,&fStart,&fEnd);CHKERRQ(ierr);
477     ierr = DMSetLabelValue(*dm,"marker",fStart,markerLeft);CHKERRQ(ierr);
478     ierr = DMSetLabelValue(*dm,"marker",fEnd-1,markerRight);CHKERRQ(ierr);
479     ierr = DMSetLabelValue(*dm,"Face Sets",fStart,faceMarkerLeft);CHKERRQ(ierr);
480     ierr = DMSetLabelValue(*dm,"Face Sets",fEnd-1,faceMarkerRight);CHKERRQ(ierr);
481   }
482   if (wrap) {
483     L       = upper - lower;
484     maxCell = (PetscReal)1.1*(L/(PetscReal)PetscMax(1,segments));
485     ierr = DMSetPeriodicity(*dm,PETSC_TRUE,&maxCell,&L,&bd);CHKERRQ(ierr);
486   }
487   PetscFunctionReturn(0);
488 }
489 
490 static PetscErrorCode DMPlexCreateBoxMesh_Simplex_Internal(MPI_Comm comm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate, DM *dm)
491 {
492   DM             boundary;
493   PetscInt       i;
494   PetscErrorCode ierr;
495 
496   PetscFunctionBegin;
497   PetscValidPointer(dm, 4);
498   for (i = 0; i < dim; ++i) if (periodicity[i] != DM_BOUNDARY_NONE) SETERRQ(comm, PETSC_ERR_SUP, "Periodicity is not supported for simplex meshes");
499   ierr = DMCreate(comm, &boundary);CHKERRQ(ierr);
500   PetscValidLogicalCollectiveInt(boundary,dim,2);
501   ierr = DMSetType(boundary, DMPLEX);CHKERRQ(ierr);
502   ierr = DMSetDimension(boundary, dim-1);CHKERRQ(ierr);
503   ierr = DMSetCoordinateDim(boundary, dim);CHKERRQ(ierr);
504   switch (dim) {
505   case 2: ierr = DMPlexCreateSquareBoundary(boundary, lower, upper, faces);CHKERRQ(ierr);break;
506   case 3: ierr = DMPlexCreateCubeBoundary(boundary, lower, upper, faces);CHKERRQ(ierr);break;
507   default: SETERRQ1(comm, PETSC_ERR_SUP, "Dimension not supported: %d", dim);
508   }
509   ierr = DMPlexGenerate(boundary, NULL, interpolate, dm);CHKERRQ(ierr);
510   ierr = DMDestroy(&boundary);CHKERRQ(ierr);
511   PetscFunctionReturn(0);
512 }
513 
514 static PetscErrorCode DMPlexCreateCubeMesh_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[], DMBoundaryType bdX, DMBoundaryType bdY, DMBoundaryType bdZ)
515 {
516   DMLabel        cutLabel = NULL;
517   PetscInt       markerTop      = 1, faceMarkerTop      = 1;
518   PetscInt       markerBottom   = 1, faceMarkerBottom   = 1;
519   PetscInt       markerFront    = 1, faceMarkerFront    = 1;
520   PetscInt       markerBack     = 1, faceMarkerBack     = 1;
521   PetscInt       markerRight    = 1, faceMarkerRight    = 1;
522   PetscInt       markerLeft     = 1, faceMarkerLeft     = 1;
523   PetscInt       dim;
524   PetscBool      markerSeparate = PETSC_FALSE, cutMarker = PETSC_FALSE;
525   PetscMPIInt    rank;
526   PetscErrorCode ierr;
527 
528   PetscFunctionBegin;
529   ierr = DMGetDimension(dm,&dim);CHKERRQ(ierr);
530   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRQ(ierr);
531   ierr = DMCreateLabel(dm,"marker");CHKERRQ(ierr);
532   ierr = DMCreateLabel(dm,"Face Sets");CHKERRQ(ierr);
533   ierr = PetscOptionsGetBool(((PetscObject) dm)->options,((PetscObject) dm)->prefix, "-dm_plex_periodic_cut", &cutMarker, NULL);CHKERRQ(ierr);
534   if (bdX == DM_BOUNDARY_PERIODIC || bdX == DM_BOUNDARY_TWIST ||
535       bdY == DM_BOUNDARY_PERIODIC || bdY == DM_BOUNDARY_TWIST ||
536       bdZ == DM_BOUNDARY_PERIODIC || bdZ == DM_BOUNDARY_TWIST) {
537 
538     if (cutMarker) {ierr = DMCreateLabel(dm, "periodic_cut");CHKERRQ(ierr); ierr = DMGetLabel(dm, "periodic_cut", &cutLabel);CHKERRQ(ierr);}
539   }
540   switch (dim) {
541   case 2:
542     faceMarkerTop    = 3;
543     faceMarkerBottom = 1;
544     faceMarkerRight  = 2;
545     faceMarkerLeft   = 4;
546     break;
547   case 3:
548     faceMarkerBottom = 1;
549     faceMarkerTop    = 2;
550     faceMarkerFront  = 3;
551     faceMarkerBack   = 4;
552     faceMarkerRight  = 5;
553     faceMarkerLeft   = 6;
554     break;
555   default:
556     SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SUP,"Dimension %d not supported",dim);
557     break;
558   }
559   ierr = PetscOptionsGetBool(((PetscObject) dm)->options,((PetscObject) dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL);CHKERRQ(ierr);
560   if (markerSeparate) {
561     markerBottom = faceMarkerBottom;
562     markerTop    = faceMarkerTop;
563     markerFront  = faceMarkerFront;
564     markerBack   = faceMarkerBack;
565     markerRight  = faceMarkerRight;
566     markerLeft   = faceMarkerLeft;
567   }
568   {
569     const PetscInt numXEdges    = !rank ? edges[0] : 0;
570     const PetscInt numYEdges    = !rank ? edges[1] : 0;
571     const PetscInt numZEdges    = !rank ? edges[2] : 0;
572     const PetscInt numXVertices = !rank ? (bdX == DM_BOUNDARY_PERIODIC || bdX == DM_BOUNDARY_TWIST ? edges[0] : edges[0]+1) : 0;
573     const PetscInt numYVertices = !rank ? (bdY == DM_BOUNDARY_PERIODIC || bdY == DM_BOUNDARY_TWIST ? edges[1] : edges[1]+1) : 0;
574     const PetscInt numZVertices = !rank ? (bdZ == DM_BOUNDARY_PERIODIC || bdZ == DM_BOUNDARY_TWIST ? edges[2] : edges[2]+1) : 0;
575     const PetscInt numCells     = numXEdges*numYEdges*numZEdges;
576     const PetscInt numXFaces    = numYEdges*numZEdges;
577     const PetscInt numYFaces    = numXEdges*numZEdges;
578     const PetscInt numZFaces    = numXEdges*numYEdges;
579     const PetscInt numTotXFaces = numXVertices*numXFaces;
580     const PetscInt numTotYFaces = numYVertices*numYFaces;
581     const PetscInt numTotZFaces = numZVertices*numZFaces;
582     const PetscInt numFaces     = numTotXFaces + numTotYFaces + numTotZFaces;
583     const PetscInt numTotXEdges = numXEdges*numYVertices*numZVertices;
584     const PetscInt numTotYEdges = numYEdges*numXVertices*numZVertices;
585     const PetscInt numTotZEdges = numZEdges*numXVertices*numYVertices;
586     const PetscInt numVertices  = numXVertices*numYVertices*numZVertices;
587     const PetscInt numEdges     = numTotXEdges + numTotYEdges + numTotZEdges;
588     const PetscInt firstVertex  = (dim == 2) ? numFaces : numCells;
589     const PetscInt firstXFace   = (dim == 2) ? 0 : numCells + numVertices;
590     const PetscInt firstYFace   = firstXFace + numTotXFaces;
591     const PetscInt firstZFace   = firstYFace + numTotYFaces;
592     const PetscInt firstXEdge   = numCells + numFaces + numVertices;
593     const PetscInt firstYEdge   = firstXEdge + numTotXEdges;
594     const PetscInt firstZEdge   = firstYEdge + numTotYEdges;
595     Vec            coordinates;
596     PetscSection   coordSection;
597     PetscScalar   *coords;
598     PetscInt       coordSize;
599     PetscInt       v, vx, vy, vz;
600     PetscInt       c, f, fx, fy, fz, e, ex, ey, ez;
601 
602     ierr = DMPlexSetChart(dm, 0, numCells+numFaces+numEdges+numVertices);CHKERRQ(ierr);
603     for (c = 0; c < numCells; c++) {
604       ierr = DMPlexSetConeSize(dm, c, 6);CHKERRQ(ierr);
605     }
606     for (f = firstXFace; f < firstXFace+numFaces; ++f) {
607       ierr = DMPlexSetConeSize(dm, f, 4);CHKERRQ(ierr);
608     }
609     for (e = firstXEdge; e < firstXEdge+numEdges; ++e) {
610       ierr = DMPlexSetConeSize(dm, e, 2);CHKERRQ(ierr);
611     }
612     ierr = DMSetUp(dm);CHKERRQ(ierr); /* Allocate space for cones */
613     /* Build cells */
614     for (fz = 0; fz < numZEdges; ++fz) {
615       for (fy = 0; fy < numYEdges; ++fy) {
616         for (fx = 0; fx < numXEdges; ++fx) {
617           PetscInt cell    = (fz*numYEdges + fy)*numXEdges + fx;
618           PetscInt faceB   = firstZFace + (fy*numXEdges+fx)*numZVertices +   fz;
619           PetscInt faceT   = firstZFace + (fy*numXEdges+fx)*numZVertices + ((fz+1)%numZVertices);
620           PetscInt faceF   = firstYFace + (fz*numXEdges+fx)*numYVertices +   fy;
621           PetscInt faceK   = firstYFace + (fz*numXEdges+fx)*numYVertices + ((fy+1)%numYVertices);
622           PetscInt faceL   = firstXFace + (fz*numYEdges+fy)*numXVertices +   fx;
623           PetscInt faceR   = firstXFace + (fz*numYEdges+fy)*numXVertices + ((fx+1)%numXVertices);
624                             /* B,  T,  F,  K,  R,  L */
625           PetscInt ornt[6] = {-4,  0,  0, -1,  0, -4}; /* ??? */
626           PetscInt cone[6];
627 
628           /* no boundary twisting in 3D */
629           cone[0] = faceB; cone[1] = faceT; cone[2] = faceF; cone[3] = faceK; cone[4] = faceR; cone[5] = faceL;
630           ierr    = DMPlexSetCone(dm, cell, cone);CHKERRQ(ierr);
631           ierr    = DMPlexSetConeOrientation(dm, cell, ornt);CHKERRQ(ierr);
632           if (bdX != DM_BOUNDARY_NONE && fx == numXEdges-1 && cutLabel) {ierr = DMLabelSetValue(cutLabel, cell, 2);CHKERRQ(ierr);}
633           if (bdY != DM_BOUNDARY_NONE && fy == numYEdges-1 && cutLabel) {ierr = DMLabelSetValue(cutLabel, cell, 2);CHKERRQ(ierr);}
634           if (bdZ != DM_BOUNDARY_NONE && fz == numZEdges-1 && cutLabel) {ierr = DMLabelSetValue(cutLabel, cell, 2);CHKERRQ(ierr);}
635         }
636       }
637     }
638     /* Build x faces */
639     for (fz = 0; fz < numZEdges; ++fz) {
640       for (fy = 0; fy < numYEdges; ++fy) {
641         for (fx = 0; fx < numXVertices; ++fx) {
642           PetscInt face    = firstXFace + (fz*numYEdges+fy)*numXVertices + fx;
643           PetscInt edgeL   = firstZEdge + (  fy*                 numXVertices+fx)*numZEdges + fz;
644           PetscInt edgeR   = firstZEdge + (((fy+1)%numYVertices)*numXVertices+fx)*numZEdges + fz;
645           PetscInt edgeB   = firstYEdge + (  fz*                 numXVertices+fx)*numYEdges + fy;
646           PetscInt edgeT   = firstYEdge + (((fz+1)%numZVertices)*numXVertices+fx)*numYEdges + fy;
647           PetscInt ornt[4] = {0, 0, -2, -2};
648           PetscInt cone[4];
649 
650           if (dim == 3) {
651             /* markers */
652             if (bdX != DM_BOUNDARY_PERIODIC) {
653               if (fx == numXVertices-1) {
654                 ierr = DMSetLabelValue(dm, "Face Sets", face, faceMarkerRight);CHKERRQ(ierr);
655                 ierr = DMSetLabelValue(dm, "marker", face, markerRight);CHKERRQ(ierr);
656               }
657               else if (fx == 0) {
658                 ierr = DMSetLabelValue(dm, "Face Sets", face, faceMarkerLeft);CHKERRQ(ierr);
659                 ierr = DMSetLabelValue(dm, "marker", face, markerLeft);CHKERRQ(ierr);
660               }
661             }
662           }
663           cone[0] = edgeB; cone[1] = edgeR; cone[2] = edgeT; cone[3] = edgeL;
664           ierr    = DMPlexSetCone(dm, face, cone);CHKERRQ(ierr);
665           ierr    = DMPlexSetConeOrientation(dm, face, ornt);CHKERRQ(ierr);
666         }
667       }
668     }
669     /* Build y faces */
670     for (fz = 0; fz < numZEdges; ++fz) {
671       for (fx = 0; fx < numXEdges; ++fx) {
672         for (fy = 0; fy < numYVertices; ++fy) {
673           PetscInt face    = firstYFace + (fz*numXEdges+fx)*numYVertices + fy;
674           PetscInt edgeL   = firstZEdge + (fy*numXVertices+  fx                 )*numZEdges + fz;
675           PetscInt edgeR   = firstZEdge + (fy*numXVertices+((fx+1)%numXVertices))*numZEdges + fz;
676           PetscInt edgeB   = firstXEdge + (  fz                 *numYVertices+fy)*numXEdges + fx;
677           PetscInt edgeT   = firstXEdge + (((fz+1)%numZVertices)*numYVertices+fy)*numXEdges + fx;
678           PetscInt ornt[4] = {0, 0, -2, -2};
679           PetscInt cone[4];
680 
681           if (dim == 3) {
682             /* markers */
683             if (bdY != DM_BOUNDARY_PERIODIC) {
684               if (fy == numYVertices-1) {
685                 ierr = DMSetLabelValue(dm, "Face Sets", face, faceMarkerBack);CHKERRQ(ierr);
686                 ierr = DMSetLabelValue(dm, "marker", face, markerBack);CHKERRQ(ierr);
687               }
688               else if (fy == 0) {
689                 ierr = DMSetLabelValue(dm, "Face Sets", face, faceMarkerFront);CHKERRQ(ierr);
690                 ierr = DMSetLabelValue(dm, "marker", face, markerFront);CHKERRQ(ierr);
691               }
692             }
693           }
694           cone[0] = edgeB; cone[1] = edgeR; cone[2] = edgeT; cone[3] = edgeL;
695           ierr    = DMPlexSetCone(dm, face, cone);CHKERRQ(ierr);
696           ierr    = DMPlexSetConeOrientation(dm, face, ornt);CHKERRQ(ierr);
697         }
698       }
699     }
700     /* Build z faces */
701     for (fy = 0; fy < numYEdges; ++fy) {
702       for (fx = 0; fx < numXEdges; ++fx) {
703         for (fz = 0; fz < numZVertices; fz++) {
704           PetscInt face    = firstZFace + (fy*numXEdges+fx)*numZVertices + fz;
705           PetscInt edgeL   = firstYEdge + (fz*numXVertices+  fx                 )*numYEdges + fy;
706           PetscInt edgeR   = firstYEdge + (fz*numXVertices+((fx+1)%numXVertices))*numYEdges + fy;
707           PetscInt edgeB   = firstXEdge + (fz*numYVertices+  fy                 )*numXEdges + fx;
708           PetscInt edgeT   = firstXEdge + (fz*numYVertices+((fy+1)%numYVertices))*numXEdges + fx;
709           PetscInt ornt[4] = {0, 0, -2, -2};
710           PetscInt cone[4];
711 
712           if (dim == 2) {
713             if (bdX == DM_BOUNDARY_TWIST && fx == numXEdges-1) {edgeR += numYEdges-1-2*fy; ornt[1] = -2;}
714             if (bdY == DM_BOUNDARY_TWIST && fy == numYEdges-1) {edgeT += numXEdges-1-2*fx; ornt[2] =  0;}
715             if (bdX != DM_BOUNDARY_NONE && fx == numXEdges-1 && cutLabel) {ierr = DMLabelSetValue(cutLabel, face, 2);CHKERRQ(ierr);}
716             if (bdY != DM_BOUNDARY_NONE && fy == numYEdges-1 && cutLabel) {ierr = DMLabelSetValue(cutLabel, face, 2);CHKERRQ(ierr);}
717           } else {
718             /* markers */
719             if (bdZ != DM_BOUNDARY_PERIODIC) {
720               if (fz == numZVertices-1) {
721                 ierr = DMSetLabelValue(dm, "Face Sets", face, faceMarkerTop);CHKERRQ(ierr);
722                 ierr = DMSetLabelValue(dm, "marker", face, markerTop);CHKERRQ(ierr);
723               }
724               else if (fz == 0) {
725                 ierr = DMSetLabelValue(dm, "Face Sets", face, faceMarkerBottom);CHKERRQ(ierr);
726                 ierr = DMSetLabelValue(dm, "marker", face, markerBottom);CHKERRQ(ierr);
727               }
728             }
729           }
730           cone[0] = edgeB; cone[1] = edgeR; cone[2] = edgeT; cone[3] = edgeL;
731           ierr    = DMPlexSetCone(dm, face, cone);CHKERRQ(ierr);
732           ierr    = DMPlexSetConeOrientation(dm, face, ornt);CHKERRQ(ierr);
733         }
734       }
735     }
736     /* Build Z edges*/
737     for (vy = 0; vy < numYVertices; vy++) {
738       for (vx = 0; vx < numXVertices; vx++) {
739         for (ez = 0; ez < numZEdges; ez++) {
740           const PetscInt edge    = firstZEdge  + (vy*numXVertices+vx)*numZEdges + ez;
741           const PetscInt vertexB = firstVertex + (  ez                 *numYVertices+vy)*numXVertices + vx;
742           const PetscInt vertexT = firstVertex + (((ez+1)%numZVertices)*numYVertices+vy)*numXVertices + vx;
743           PetscInt       cone[2];
744 
745           if (dim == 3) {
746             if (bdX != DM_BOUNDARY_PERIODIC) {
747               if (vx == numXVertices-1) {
748                 ierr = DMSetLabelValue(dm, "marker", edge, markerRight);CHKERRQ(ierr);
749               }
750               else if (vx == 0) {
751                 ierr = DMSetLabelValue(dm, "marker", edge, markerLeft);CHKERRQ(ierr);
752               }
753             }
754             if (bdY != DM_BOUNDARY_PERIODIC) {
755               if (vy == numYVertices-1) {
756                 ierr = DMSetLabelValue(dm, "marker", edge, markerBack);CHKERRQ(ierr);
757               }
758               else if (vy == 0) {
759                 ierr = DMSetLabelValue(dm, "marker", edge, markerFront);CHKERRQ(ierr);
760               }
761             }
762           }
763           cone[0] = vertexB; cone[1] = vertexT;
764           ierr = DMPlexSetCone(dm, edge, cone);CHKERRQ(ierr);
765         }
766       }
767     }
768     /* Build Y edges*/
769     for (vz = 0; vz < numZVertices; vz++) {
770       for (vx = 0; vx < numXVertices; vx++) {
771         for (ey = 0; ey < numYEdges; ey++) {
772           const PetscInt nextv   = (dim == 2 && bdY == DM_BOUNDARY_TWIST && ey == numYEdges-1) ? (numXVertices-vx-1) : (vz*numYVertices+((ey+1)%numYVertices))*numXVertices + vx;
773           const PetscInt edge    = firstYEdge  + (vz*numXVertices+vx)*numYEdges + ey;
774           const PetscInt vertexF = firstVertex + (vz*numYVertices+ey)*numXVertices + vx;
775           const PetscInt vertexK = firstVertex + nextv;
776           PetscInt       cone[2];
777 
778           cone[0] = vertexF; cone[1] = vertexK;
779           ierr = DMPlexSetCone(dm, edge, cone);CHKERRQ(ierr);
780           if (dim == 2) {
781             if ((bdX != DM_BOUNDARY_PERIODIC) && (bdX != DM_BOUNDARY_TWIST)) {
782               if (vx == numXVertices-1) {
783                 ierr = DMSetLabelValue(dm, "Face Sets", edge, faceMarkerRight);CHKERRQ(ierr);
784                 ierr = DMSetLabelValue(dm, "marker", edge,    markerRight);CHKERRQ(ierr);
785                 ierr = DMSetLabelValue(dm, "marker", cone[0], markerRight);CHKERRQ(ierr);
786                 if (ey == numYEdges-1) {
787                   ierr = DMSetLabelValue(dm, "marker", cone[1], markerRight);CHKERRQ(ierr);
788                 }
789               } else if (vx == 0) {
790                 ierr = DMSetLabelValue(dm, "Face Sets", edge, faceMarkerLeft);CHKERRQ(ierr);
791                 ierr = DMSetLabelValue(dm, "marker", edge,    markerLeft);CHKERRQ(ierr);
792                 ierr = DMSetLabelValue(dm, "marker", cone[0], markerLeft);CHKERRQ(ierr);
793                 if (ey == numYEdges-1) {
794                   ierr = DMSetLabelValue(dm, "marker", cone[1], markerLeft);CHKERRQ(ierr);
795                 }
796               }
797             } else {
798               if (vx == 0 && cutLabel) {
799                 ierr = DMLabelSetValue(cutLabel, edge,    1);CHKERRQ(ierr);
800                 ierr = DMLabelSetValue(cutLabel, cone[0], 1);CHKERRQ(ierr);
801                 if (ey == numYEdges-1) {
802                   ierr = DMLabelSetValue(cutLabel, cone[1], 1);CHKERRQ(ierr);
803                 }
804               }
805             }
806           } else {
807             if (bdX != DM_BOUNDARY_PERIODIC) {
808               if (vx == numXVertices-1) {
809                 ierr = DMSetLabelValue(dm, "marker", edge, markerRight);CHKERRQ(ierr);
810               } else if (vx == 0) {
811                 ierr = DMSetLabelValue(dm, "marker", edge, markerLeft);CHKERRQ(ierr);
812               }
813             }
814             if (bdZ != DM_BOUNDARY_PERIODIC) {
815               if (vz == numZVertices-1) {
816                 ierr = DMSetLabelValue(dm, "marker", edge, markerTop);CHKERRQ(ierr);
817               } else if (vz == 0) {
818                 ierr = DMSetLabelValue(dm, "marker", edge, markerBottom);CHKERRQ(ierr);
819               }
820             }
821           }
822         }
823       }
824     }
825     /* Build X edges*/
826     for (vz = 0; vz < numZVertices; vz++) {
827       for (vy = 0; vy < numYVertices; vy++) {
828         for (ex = 0; ex < numXEdges; ex++) {
829           const PetscInt nextv   = (dim == 2 && bdX == DM_BOUNDARY_TWIST && ex == numXEdges-1) ? (numYVertices-vy-1)*numXVertices : (vz*numYVertices+vy)*numXVertices + (ex+1)%numXVertices;
830           const PetscInt edge    = firstXEdge  + (vz*numYVertices+vy)*numXEdges + ex;
831           const PetscInt vertexL = firstVertex + (vz*numYVertices+vy)*numXVertices + ex;
832           const PetscInt vertexR = firstVertex + nextv;
833           PetscInt       cone[2];
834 
835           cone[0] = vertexL; cone[1] = vertexR;
836           ierr = DMPlexSetCone(dm, edge, cone);CHKERRQ(ierr);
837           if (dim == 2) {
838             if ((bdY != DM_BOUNDARY_PERIODIC) && (bdY != DM_BOUNDARY_TWIST)) {
839               if (vy == numYVertices-1) {
840                 ierr = DMSetLabelValue(dm, "Face Sets", edge, faceMarkerTop);CHKERRQ(ierr);
841                 ierr = DMSetLabelValue(dm, "marker", edge,    markerTop);CHKERRQ(ierr);
842                 ierr = DMSetLabelValue(dm, "marker", cone[0], markerTop);CHKERRQ(ierr);
843                 if (ex == numXEdges-1) {
844                   ierr = DMSetLabelValue(dm, "marker", cone[1], markerTop);CHKERRQ(ierr);
845                 }
846               } else if (vy == 0) {
847                 ierr = DMSetLabelValue(dm, "Face Sets", edge, faceMarkerBottom);CHKERRQ(ierr);
848                 ierr = DMSetLabelValue(dm, "marker", edge,    markerBottom);CHKERRQ(ierr);
849                 ierr = DMSetLabelValue(dm, "marker", cone[0], markerBottom);CHKERRQ(ierr);
850                 if (ex == numXEdges-1) {
851                   ierr = DMSetLabelValue(dm, "marker", cone[1], markerBottom);CHKERRQ(ierr);
852                 }
853               }
854             } else {
855               if (vy == 0 && cutLabel) {
856                 ierr = DMLabelSetValue(cutLabel, edge,    1);CHKERRQ(ierr);
857                 ierr = DMLabelSetValue(cutLabel, cone[0], 1);CHKERRQ(ierr);
858                 if (ex == numXEdges-1) {
859                   ierr = DMLabelSetValue(cutLabel, cone[1], 1);CHKERRQ(ierr);
860                 }
861               }
862             }
863           } else {
864             if (bdY != DM_BOUNDARY_PERIODIC) {
865               if (vy == numYVertices-1) {
866                 ierr = DMSetLabelValue(dm, "marker", edge, markerBack);CHKERRQ(ierr);
867               }
868               else if (vy == 0) {
869                 ierr = DMSetLabelValue(dm, "marker", edge, markerFront);CHKERRQ(ierr);
870               }
871             }
872             if (bdZ != DM_BOUNDARY_PERIODIC) {
873               if (vz == numZVertices-1) {
874                 ierr = DMSetLabelValue(dm, "marker", edge, markerTop);CHKERRQ(ierr);
875               }
876               else if (vz == 0) {
877                 ierr = DMSetLabelValue(dm, "marker", edge, markerBottom);CHKERRQ(ierr);
878               }
879             }
880           }
881         }
882       }
883     }
884     ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
885     ierr = DMPlexStratify(dm);CHKERRQ(ierr);
886     /* Build coordinates */
887     ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
888     ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
889     ierr = PetscSectionSetFieldComponents(coordSection, 0, dim);CHKERRQ(ierr);
890     ierr = PetscSectionSetChart(coordSection, firstVertex, firstVertex+numVertices);CHKERRQ(ierr);
891     for (v = firstVertex; v < firstVertex+numVertices; ++v) {
892       ierr = PetscSectionSetDof(coordSection, v, dim);CHKERRQ(ierr);
893       ierr = PetscSectionSetFieldDof(coordSection, v, 0, dim);CHKERRQ(ierr);
894     }
895     ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
896     ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
897     ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
898     ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
899     ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
900     ierr = VecSetBlockSize(coordinates, dim);CHKERRQ(ierr);
901     ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
902     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
903     for (vz = 0; vz < numZVertices; ++vz) {
904       for (vy = 0; vy < numYVertices; ++vy) {
905         for (vx = 0; vx < numXVertices; ++vx) {
906           coords[((vz*numYVertices+vy)*numXVertices+vx)*dim+0] = lower[0] + ((upper[0] - lower[0])/numXEdges)*vx;
907           coords[((vz*numYVertices+vy)*numXVertices+vx)*dim+1] = lower[1] + ((upper[1] - lower[1])/numYEdges)*vy;
908           if (dim == 3) {
909             coords[((vz*numYVertices+vy)*numXVertices+vx)*dim+2] = lower[2] + ((upper[2] - lower[2])/numZEdges)*vz;
910           }
911         }
912       }
913     }
914     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
915     ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
916     ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
917   }
918   PetscFunctionReturn(0);
919 }
920 
921 static PetscErrorCode DMPlexCreateBoxMesh_Tensor_Internal(MPI_Comm comm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate, DM *dm)
922 {
923   PetscInt       i;
924   PetscErrorCode ierr;
925 
926   PetscFunctionBegin;
927   PetscValidPointer(dm, 7);
928   ierr = DMCreate(comm, dm);CHKERRQ(ierr);
929   PetscValidLogicalCollectiveInt(*dm,dim,2);
930   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
931   ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
932   switch (dim) {
933   case 2: {ierr = DMPlexCreateCubeMesh_Internal(*dm, lower, upper, faces, periodicity[0], periodicity[1], DM_BOUNDARY_NONE);CHKERRQ(ierr);break;}
934   case 3: {ierr = DMPlexCreateCubeMesh_Internal(*dm, lower, upper, faces, periodicity[0], periodicity[1], periodicity[2]);CHKERRQ(ierr);break;}
935   default: SETERRQ1(comm, PETSC_ERR_SUP, "Dimension not supported: %d", dim);
936   }
937   if (periodicity[0] == DM_BOUNDARY_PERIODIC || periodicity[0] == DM_BOUNDARY_TWIST ||
938       periodicity[1] == DM_BOUNDARY_PERIODIC || periodicity[1] == DM_BOUNDARY_TWIST ||
939       (dim > 2 && (periodicity[2] == DM_BOUNDARY_PERIODIC || periodicity[2] == DM_BOUNDARY_TWIST))) {
940     PetscReal L[3];
941     PetscReal maxCell[3];
942 
943     for (i = 0; i < dim; i++) {
944       L[i]       = upper[i] - lower[i];
945       maxCell[i] = 1.1 * (L[i] / PetscMax(1,faces[i]));
946     }
947     ierr = DMSetPeriodicity(*dm,PETSC_TRUE,maxCell,L,periodicity);CHKERRQ(ierr);
948   }
949   if (!interpolate) {
950     DM udm;
951 
952     ierr = DMPlexUninterpolate(*dm, &udm);CHKERRQ(ierr);
953     ierr = DMPlexCopyCoordinates(*dm, udm);CHKERRQ(ierr);
954     ierr = DMDestroy(dm);CHKERRQ(ierr);
955     *dm  = udm;
956   }
957   PetscFunctionReturn(0);
958 }
959 
960 /*@C
961   DMPlexCreateBoxMesh - Creates a mesh on the tensor product of unit intervals (box) using simplices or tensor cells (hexahedra).
962 
963   Collective on MPI_Comm
964 
965   Input Parameters:
966 + comm        - The communicator for the DM object
967 . dim         - The spatial dimension
968 . simplex     - PETSC_TRUE for simplices, PETSC_FALSE for tensor cells
969 . faces       - Number of faces per dimension, or NULL for (1,) in 1D and (2, 2) in 2D and (1, 1, 1) in 3D
970 . lower       - The lower left corner, or NULL for (0, 0, 0)
971 . upper       - The upper right corner, or NULL for (1, 1, 1)
972 . periodicity - The boundary type for the X,Y,Z direction, or NULL for DM_BOUNDARY_NONE
973 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
974 
975   Output Parameter:
976 . dm  - The DM object
977 
978   Note: Here is the numbering returned for 2 faces in each direction for tensor cells:
979 $ 10---17---11---18----12
980 $  |         |         |
981 $  |         |         |
982 $ 20    2   22    3    24
983 $  |         |         |
984 $  |         |         |
985 $  7---15----8---16----9
986 $  |         |         |
987 $  |         |         |
988 $ 19    0   21    1   23
989 $  |         |         |
990 $  |         |         |
991 $  4---13----5---14----6
992 
993 and for simplicial cells
994 
995 $ 14----8---15----9----16
996 $  |\     5  |\      7 |
997 $  | \       | \       |
998 $ 13   2    14    3    15
999 $  | 4   \   | 6   \   |
1000 $  |       \ |       \ |
1001 $ 11----6---12----7----13
1002 $  |\        |\        |
1003 $  | \    1  | \     3 |
1004 $ 10   0    11    1    12
1005 $  | 0   \   | 2   \   |
1006 $  |       \ |       \ |
1007 $  8----4----9----5----10
1008 
1009   Level: beginner
1010 
1011 .keywords: DM, create
1012 .seealso: DMPlexCreateFromFile(), DMPlexCreateHexCylinderMesh(), DMSetType(), DMCreate()
1013 @*/
1014 PetscErrorCode DMPlexCreateBoxMesh(MPI_Comm comm, PetscInt dim, PetscBool simplex, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate, DM *dm)
1015 {
1016   PetscInt       i;
1017   PetscInt       fac[3] = {0, 0, 0};
1018   PetscReal      low[3] = {0, 0, 0};
1019   PetscReal      upp[3] = {1, 1, 1};
1020   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1021   PetscErrorCode ierr;
1022 
1023   PetscFunctionBegin;
1024   for (i = 0; i < dim; ++i) fac[i] = faces ? faces[i] : (dim == 1 ? 1 : 4-dim);
1025   if (lower) for (i = 0; i < dim; ++i) low[i] = lower[i];
1026   if (upper) for (i = 0; i < dim; ++i) upp[i] = upper[i];
1027   if (periodicity) for (i = 0; i < dim; ++i) bdt[i] = periodicity[i];
1028 
1029   if (dim == 1)      {ierr = DMPlexCreateLineMesh_Internal(comm, fac[0], low[0], upp[0], bdt[0], dm);CHKERRQ(ierr);}
1030   else if (simplex)  {ierr = DMPlexCreateBoxMesh_Simplex_Internal(comm, dim, fac, low, upp, bdt, interpolate, dm);CHKERRQ(ierr);}
1031   else               {ierr = DMPlexCreateBoxMesh_Tensor_Internal(comm, dim, fac, low, upp, bdt, interpolate, dm);CHKERRQ(ierr);}
1032   PetscFunctionReturn(0);
1033 }
1034 
1035 /*@
1036   DMPlexCreateWedgeBoxMesh - Creates a 3-D mesh tesselating the (x,y) plane and extruding in the third direction using wedge cells.
1037 
1038   Collective on MPI_Comm
1039 
1040   Input Parameters:
1041 + comm        - The communicator for the DM object
1042 . faces       - Number of faces per dimension, or NULL for (1, 1, 1)
1043 . lower       - The lower left corner, or NULL for (0, 0, 0)
1044 . upper       - The upper right corner, or NULL for (1, 1, 1)
1045 . periodicity - The boundary type for the X,Y,Z direction, or NULL for DM_BOUNDARY_NONE
1046 . ordExt      - If PETSC_TRUE, orders the extruded cells in the height first. Otherwise, orders the cell on the layers first
1047 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
1048 
1049   Output Parameter:
1050 . dm  - The DM object
1051 
1052   Level: beginner
1053 
1054 .keywords: DM, create
1055 .seealso: DMPlexCreateHexCylinderMesh(), DMPlexCreateWedgeCylinderMesh(), DMPlexExtrude(), DMPlexCreateBoxMesh(), DMSetType(), DMCreate()
1056 @*/
1057 PetscErrorCode DMPlexCreateWedgeBoxMesh(MPI_Comm comm, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool ordExt, PetscBool interpolate, DM *dm)
1058 {
1059   DM             bdm, botdm;
1060   PetscInt       i;
1061   PetscInt       fac[3] = {0, 0, 0};
1062   PetscReal      low[3] = {0, 0, 0};
1063   PetscReal      upp[3] = {1, 1, 1};
1064   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1065   PetscErrorCode ierr;
1066 
1067   PetscFunctionBegin;
1068   for (i = 0; i < 3; ++i) fac[i] = faces ? (faces[i] > 0 ? faces[i] : 1) : 1;
1069   if (lower) for (i = 0; i < 3; ++i) low[i] = lower[i];
1070   if (upper) for (i = 0; i < 3; ++i) upp[i] = upper[i];
1071   if (periodicity) for (i = 0; i < 3; ++i) bdt[i] = periodicity[i];
1072   for (i = 0; i < 3; ++i) if (bdt[i] != DM_BOUNDARY_NONE) SETERRQ(comm, PETSC_ERR_SUP, "Periodicity not yet supported");
1073 
1074   ierr = DMCreate(comm, &bdm);CHKERRQ(ierr);
1075   ierr = DMSetType(bdm, DMPLEX);CHKERRQ(ierr);
1076   ierr = DMSetDimension(bdm, 1);CHKERRQ(ierr);
1077   ierr = DMSetCoordinateDim(bdm, 2);CHKERRQ(ierr);
1078   ierr = DMPlexCreateSquareBoundary(bdm, low, upp, fac);CHKERRQ(ierr);
1079   ierr = DMPlexGenerate(bdm, NULL, PETSC_FALSE, &botdm);CHKERRQ(ierr);
1080   ierr = DMDestroy(&bdm);CHKERRQ(ierr);
1081   ierr = DMPlexExtrude(botdm, fac[2], upp[2] - low[2], ordExt, interpolate, dm);CHKERRQ(ierr);
1082   if (low[2] != 0.0) {
1083     Vec         v;
1084     PetscScalar *x;
1085     PetscInt    cDim, n;
1086 
1087     ierr = DMGetCoordinatesLocal(*dm, &v);CHKERRQ(ierr);
1088     ierr = VecGetBlockSize(v, &cDim);CHKERRQ(ierr);
1089     ierr = VecGetLocalSize(v, &n);CHKERRQ(ierr);
1090     ierr = VecGetArray(v, &x);CHKERRQ(ierr);
1091     x   += cDim;
1092     for (i=0; i<n; i+=cDim) x[i] += low[2];
1093     ierr = VecRestoreArray(v,&x);CHKERRQ(ierr);
1094     ierr = DMSetCoordinatesLocal(*dm, v);CHKERRQ(ierr);
1095   }
1096   ierr = DMDestroy(&botdm);CHKERRQ(ierr);
1097   PetscFunctionReturn(0);
1098 }
1099 
1100 /*@
1101   DMPlexExtrude - Creates a (d+1)-D mesh by extruding a d-D mesh in the normal direction using prismatic cells.
1102 
1103   Collective on idm
1104 
1105   Input Parameters:
1106 + idm - The mesh to be extruted
1107 . layers - The number of layers
1108 . height - The height of the extruded layer
1109 . ordExt - If PETSC_TRUE, orders the extruded cells in the height first. Otherwise, orders the cell on the layers first
1110 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
1111 
1112   Output Parameter:
1113 . dm  - The DM object
1114 
1115   Notes: The object created is an hybrid mesh, the vertex ordering in the cone of the cell is that of the prismatic cells
1116 
1117   Level: advanced
1118 
1119 .keywords: DM, create
1120 .seealso: DMPlexCreateWedgeCylinderMesh(), DMPlexCreateWedgeBoxMesh(), DMPlexSetHybridBounds(), DMSetType(), DMCreate()
1121 @*/
1122 PetscErrorCode DMPlexExtrude(DM idm, PetscInt layers, PetscReal height, PetscBool ordExt, PetscBool interpolate, DM* dm)
1123 {
1124   PetscScalar       *coordsB;
1125   const PetscScalar *coordsA;
1126   PetscReal         *normals = NULL;
1127   Vec               coordinatesA, coordinatesB;
1128   PetscSection      coordSectionA, coordSectionB;
1129   PetscInt          dim, cDim, cDimB, c, l, v, coordSize, *newCone;
1130   PetscInt          cStart, cEnd, vStart, vEnd, cellV, numCells, numVertices;
1131   PetscErrorCode    ierr;
1132 
1133   PetscFunctionBegin;
1134   PetscValidHeaderSpecific(idm, DM_CLASSID, 1);
1135   PetscValidLogicalCollectiveInt(idm, layers, 2);
1136   PetscValidLogicalCollectiveReal(idm, height, 3);
1137   PetscValidLogicalCollectiveBool(idm, interpolate, 4);
1138   ierr = DMGetDimension(idm, &dim);CHKERRQ(ierr);
1139   if (dim < 1 || dim > 3) SETERRQ1(PetscObjectComm((PetscObject)idm), PETSC_ERR_SUP, "Support for dimension %D not coded", dim);
1140 
1141   ierr = DMPlexGetHeightStratum(idm, 0, &cStart, &cEnd);CHKERRQ(ierr);
1142   ierr = DMPlexGetDepthStratum(idm, 0, &vStart, &vEnd);CHKERRQ(ierr);
1143   numCells = (cEnd - cStart)*layers;
1144   numVertices = (vEnd - vStart)*(layers+1);
1145   ierr = DMCreate(PetscObjectComm((PetscObject)idm), dm);CHKERRQ(ierr);
1146   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
1147   ierr = DMSetDimension(*dm, dim+1);CHKERRQ(ierr);
1148   ierr = DMPlexSetChart(*dm, 0, numCells+numVertices);CHKERRQ(ierr);
1149   for (c = cStart, cellV = 0; c < cEnd; ++c) {
1150     PetscInt *closure = NULL;
1151     PetscInt closureSize, numCorners = 0;
1152 
1153     ierr = DMPlexGetTransitiveClosure(idm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1154     for (v = 0; v < closureSize*2; v += 2) if ((closure[v] >= vStart) && (closure[v] < vEnd)) numCorners++;
1155     ierr = DMPlexRestoreTransitiveClosure(idm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1156     for (l = 0; l < layers; ++l) {
1157       ierr = DMPlexSetConeSize(*dm, ordExt ? layers*(c - cStart) + l : l*(cEnd - cStart) + c - cStart, 2*numCorners);CHKERRQ(ierr);
1158     }
1159     cellV = PetscMax(numCorners,cellV);
1160   }
1161   ierr = DMPlexSetHybridBounds(*dm, 0, PETSC_DETERMINE, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
1162   ierr = DMSetUp(*dm);CHKERRQ(ierr);
1163 
1164   ierr = DMGetCoordinateDim(idm, &cDim);CHKERRQ(ierr);
1165   if (dim != cDim) {
1166     ierr = PetscCalloc1(cDim*(vEnd - vStart), &normals);CHKERRQ(ierr);
1167   }
1168   ierr = PetscMalloc1(3*cellV,&newCone);CHKERRQ(ierr);
1169   for (c = cStart; c < cEnd; ++c) {
1170     PetscInt *closure = NULL;
1171     PetscInt closureSize, numCorners = 0, l;
1172     PetscReal normal[3] = {0, 0, 0};
1173 
1174     if (normals) {
1175       ierr = DMPlexComputeCellGeometryFVM(idm, c, NULL, NULL, normal);CHKERRQ(ierr);
1176     }
1177     ierr = DMPlexGetTransitiveClosure(idm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1178     for (v = 0; v < closureSize*2; v += 2) {
1179       if ((closure[v] >= vStart) && (closure[v] < vEnd)) {
1180         PetscInt d;
1181 
1182         newCone[numCorners++] = closure[v] - vStart;
1183         if (normals) { for (d = 0; d < cDim; ++d) normals[cDim*(closure[v]-vStart)+d] += normal[d]; }
1184       }
1185     }
1186     switch (numCorners) {
1187     case 4: /* do nothing */
1188     case 2: /* do nothing */
1189       break;
1190     case 3: /* from counter-clockwise to wedge ordering */
1191       l = newCone[1];
1192       newCone[1] = newCone[2];
1193       newCone[2] = l;
1194       break;
1195     default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_SUP, "Unsupported number of corners: %D", numCorners);
1196     }
1197     ierr = DMPlexRestoreTransitiveClosure(idm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1198     for (l = 0; l < layers; ++l) {
1199       PetscInt i;
1200 
1201       for (i = 0; i < numCorners; ++i) {
1202         newCone[  numCorners + i] = ordExt ? (layers+1)*newCone[i] + l     + numCells :     l*(vEnd - vStart) + newCone[i] + numCells;
1203         newCone[2*numCorners + i] = ordExt ? (layers+1)*newCone[i] + l + 1 + numCells : (l+1)*(vEnd - vStart) + newCone[i] + numCells;
1204       }
1205       ierr = DMPlexSetCone(*dm, ordExt ? layers*(c - cStart) + l : l*(cEnd - cStart) + c - cStart, newCone + numCorners);CHKERRQ(ierr);
1206     }
1207   }
1208   ierr = DMPlexSymmetrize(*dm);CHKERRQ(ierr);
1209   ierr = DMPlexStratify(*dm);CHKERRQ(ierr);
1210   ierr = PetscFree(newCone);CHKERRQ(ierr);
1211 
1212   cDimB = cDim == dim ? cDim+1 : cDim;
1213   ierr = DMGetCoordinateSection(*dm, &coordSectionB);CHKERRQ(ierr);
1214   ierr = PetscSectionSetNumFields(coordSectionB, 1);CHKERRQ(ierr);
1215   ierr = PetscSectionSetFieldComponents(coordSectionB, 0, cDimB);CHKERRQ(ierr);
1216   ierr = PetscSectionSetChart(coordSectionB, numCells, numCells+numVertices);CHKERRQ(ierr);
1217   for (v = numCells; v < numCells+numVertices; ++v) {
1218     ierr = PetscSectionSetDof(coordSectionB, v, cDimB);CHKERRQ(ierr);
1219     ierr = PetscSectionSetFieldDof(coordSectionB, v, 0, cDimB);CHKERRQ(ierr);
1220   }
1221   ierr = PetscSectionSetUp(coordSectionB);CHKERRQ(ierr);
1222   ierr = PetscSectionGetStorageSize(coordSectionB, &coordSize);CHKERRQ(ierr);
1223   ierr = VecCreate(PETSC_COMM_SELF, &coordinatesB);CHKERRQ(ierr);
1224   ierr = PetscObjectSetName((PetscObject) coordinatesB, "coordinates");CHKERRQ(ierr);
1225   ierr = VecSetSizes(coordinatesB, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
1226   ierr = VecSetBlockSize(coordinatesB, cDimB);CHKERRQ(ierr);
1227   ierr = VecSetType(coordinatesB,VECSTANDARD);CHKERRQ(ierr);
1228 
1229   ierr = DMGetCoordinateSection(idm, &coordSectionA);CHKERRQ(ierr);
1230   ierr = DMGetCoordinatesLocal(idm, &coordinatesA);CHKERRQ(ierr);
1231   ierr = VecGetArray(coordinatesB, &coordsB);CHKERRQ(ierr);
1232   ierr = VecGetArrayRead(coordinatesA, &coordsA);CHKERRQ(ierr);
1233   for (v = vStart; v < vEnd; ++v) {
1234     const PetscScalar *cptr;
1235     PetscReal         ones2[2] = { 0., 1.}, ones3[3] = { 0., 0., 1.};
1236     PetscReal         *normal, norm, h = height/layers;
1237     PetscInt          offA, d, cDimA = cDim;
1238 
1239     normal = normals ? normals + cDimB*(v - vStart) : (cDim > 1 ? ones3 : ones2);
1240     if (normals) {
1241       for (d = 0, norm = 0.0; d < cDimB; ++d) norm += normal[d]*normal[d];
1242       for (d = 0; d < cDimB; ++d) normal[d] *= 1./PetscSqrtReal(norm);
1243     }
1244 
1245     ierr = PetscSectionGetOffset(coordSectionA, v, &offA);CHKERRQ(ierr);
1246     cptr = coordsA + offA;
1247     for (l = 0; l < layers+1; ++l) {
1248       PetscInt offB, d, newV;
1249 
1250       newV = ordExt ? (layers+1)*(v -vStart) + l + numCells : (vEnd -vStart)*l + (v -vStart) + numCells;
1251       ierr = PetscSectionGetOffset(coordSectionB, newV, &offB);CHKERRQ(ierr);
1252       for (d = 0; d < cDimA; ++d) { coordsB[offB+d]  = cptr[d]; }
1253       for (d = 0; d < cDimB; ++d) { coordsB[offB+d] += l ? normal[d]*h : 0.0; }
1254       cptr    = coordsB + offB;
1255       cDimA   = cDimB;
1256     }
1257   }
1258   ierr = VecRestoreArrayRead(coordinatesA, &coordsA);CHKERRQ(ierr);
1259   ierr = VecRestoreArray(coordinatesB, &coordsB);CHKERRQ(ierr);
1260   ierr = DMSetCoordinatesLocal(*dm, coordinatesB);CHKERRQ(ierr);
1261   ierr = VecDestroy(&coordinatesB);CHKERRQ(ierr);
1262   ierr = PetscFree(normals);CHKERRQ(ierr);
1263   if (interpolate) {
1264     DM idm;
1265 
1266     ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
1267     ierr = DMPlexCopyCoordinates(*dm, idm);CHKERRQ(ierr);
1268     ierr = DMDestroy(dm);CHKERRQ(ierr);
1269     *dm  = idm;
1270   }
1271   PetscFunctionReturn(0);
1272 }
1273 
1274 /*@C
1275   DMPlexSetOptionsPrefix - Sets the prefix used for searching for all DM options in the database.
1276 
1277   Logically Collective on DM
1278 
1279   Input Parameters:
1280 + dm - the DM context
1281 - prefix - the prefix to prepend to all option names
1282 
1283   Notes:
1284   A hyphen (-) must NOT be given at the beginning of the prefix name.
1285   The first character of all runtime options is AUTOMATICALLY the hyphen.
1286 
1287   Level: advanced
1288 
1289 .seealso: SNESSetFromOptions()
1290 @*/
1291 PetscErrorCode DMPlexSetOptionsPrefix(DM dm, const char prefix[])
1292 {
1293   DM_Plex       *mesh = (DM_Plex *) dm->data;
1294   PetscErrorCode ierr;
1295 
1296   PetscFunctionBegin;
1297   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1298   ierr = PetscObjectSetOptionsPrefix((PetscObject) dm, prefix);CHKERRQ(ierr);
1299   ierr = PetscObjectSetOptionsPrefix((PetscObject) mesh->partitioner, prefix);CHKERRQ(ierr);
1300   PetscFunctionReturn(0);
1301 }
1302 
1303 /*@
1304   DMPlexCreateHexCylinderMesh - Creates a mesh on the tensor product of the unit interval with the circle (cylinder) using hexahedra.
1305 
1306   Collective on MPI_Comm
1307 
1308   Input Parameters:
1309 + comm      - The communicator for the DM object
1310 . numRefine - The number of regular refinements to the basic 5 cell structure
1311 - periodicZ - The boundary type for the Z direction
1312 
1313   Output Parameter:
1314 . dm  - The DM object
1315 
1316   Note: Here is the output numbering looking from the bottom of the cylinder:
1317 $       17-----14
1318 $        |     |
1319 $        |  2  |
1320 $        |     |
1321 $ 17-----8-----7-----14
1322 $  |     |     |     |
1323 $  |  3  |  0  |  1  |
1324 $  |     |     |     |
1325 $ 19-----5-----6-----13
1326 $        |     |
1327 $        |  4  |
1328 $        |     |
1329 $       19-----13
1330 $
1331 $ and up through the top
1332 $
1333 $       18-----16
1334 $        |     |
1335 $        |  2  |
1336 $        |     |
1337 $ 18----10----11-----16
1338 $  |     |     |     |
1339 $  |  3  |  0  |  1  |
1340 $  |     |     |     |
1341 $ 20-----9----12-----15
1342 $        |     |
1343 $        |  4  |
1344 $        |     |
1345 $       20-----15
1346 
1347   Level: beginner
1348 
1349 .keywords: DM, create
1350 .seealso: DMPlexCreateBoxMesh(), DMSetType(), DMCreate()
1351 @*/
1352 PetscErrorCode DMPlexCreateHexCylinderMesh(MPI_Comm comm, PetscInt numRefine, DMBoundaryType periodicZ, DM *dm)
1353 {
1354   const PetscInt dim = 3;
1355   PetscInt       numCells, numVertices, r;
1356   PetscMPIInt    rank;
1357   PetscErrorCode ierr;
1358 
1359   PetscFunctionBegin;
1360   PetscValidPointer(dm, 4);
1361   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
1362   if (numRefine < 0) SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Number of refinements %D cannot be negative", numRefine);
1363   ierr = DMCreate(comm, dm);CHKERRQ(ierr);
1364   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
1365   ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
1366   /* Create topology */
1367   {
1368     PetscInt cone[8], c;
1369 
1370     numCells    = !rank ?  5 : 0;
1371     numVertices = !rank ? 16 : 0;
1372     if (periodicZ == DM_BOUNDARY_PERIODIC) {
1373       numCells   *= 3;
1374       numVertices = !rank ? 24 : 0;
1375     }
1376     ierr = DMPlexSetChart(*dm, 0, numCells+numVertices);CHKERRQ(ierr);
1377     for (c = 0; c < numCells; c++) {ierr = DMPlexSetConeSize(*dm, c, 8);CHKERRQ(ierr);}
1378     ierr = DMSetUp(*dm);CHKERRQ(ierr);
1379     if (!rank) {
1380       if (periodicZ == DM_BOUNDARY_PERIODIC) {
1381         cone[0] = 15; cone[1] = 18; cone[2] = 17; cone[3] = 16;
1382         cone[4] = 31; cone[5] = 32; cone[6] = 33; cone[7] = 34;
1383         ierr = DMPlexSetCone(*dm, 0, cone);CHKERRQ(ierr);
1384         cone[0] = 16; cone[1] = 17; cone[2] = 24; cone[3] = 23;
1385         cone[4] = 32; cone[5] = 36; cone[6] = 37; cone[7] = 33; /* 22 25 26 21 */
1386         ierr = DMPlexSetCone(*dm, 1, cone);CHKERRQ(ierr);
1387         cone[0] = 18; cone[1] = 27; cone[2] = 24; cone[3] = 17;
1388         cone[4] = 34; cone[5] = 33; cone[6] = 37; cone[7] = 38;
1389         ierr = DMPlexSetCone(*dm, 2, cone);CHKERRQ(ierr);
1390         cone[0] = 29; cone[1] = 27; cone[2] = 18; cone[3] = 15;
1391         cone[4] = 35; cone[5] = 31; cone[6] = 34; cone[7] = 38;
1392         ierr = DMPlexSetCone(*dm, 3, cone);CHKERRQ(ierr);
1393         cone[0] = 29; cone[1] = 15; cone[2] = 16; cone[3] = 23;
1394         cone[4] = 35; cone[5] = 36; cone[6] = 32; cone[7] = 31;
1395         ierr = DMPlexSetCone(*dm, 4, cone);CHKERRQ(ierr);
1396 
1397         cone[0] = 31; cone[1] = 34; cone[2] = 33; cone[3] = 32;
1398         cone[4] = 19; cone[5] = 22; cone[6] = 21; cone[7] = 20;
1399         ierr = DMPlexSetCone(*dm, 5, cone);CHKERRQ(ierr);
1400         cone[0] = 32; cone[1] = 33; cone[2] = 37; cone[3] = 36;
1401         cone[4] = 22; cone[5] = 25; cone[6] = 26; cone[7] = 21;
1402         ierr = DMPlexSetCone(*dm, 6, cone);CHKERRQ(ierr);
1403         cone[0] = 34; cone[1] = 38; cone[2] = 37; cone[3] = 33;
1404         cone[4] = 20; cone[5] = 21; cone[6] = 26; cone[7] = 28;
1405         ierr = DMPlexSetCone(*dm, 7, cone);CHKERRQ(ierr);
1406         cone[0] = 35; cone[1] = 38; cone[2] = 34; cone[3] = 31;
1407         cone[4] = 30; cone[5] = 19; cone[6] = 20; cone[7] = 28;
1408         ierr = DMPlexSetCone(*dm, 8, cone);CHKERRQ(ierr);
1409         cone[0] = 35; cone[1] = 31; cone[2] = 32; cone[3] = 36;
1410         cone[4] = 30; cone[5] = 25; cone[6] = 22; cone[7] = 19;
1411         ierr = DMPlexSetCone(*dm, 9, cone);CHKERRQ(ierr);
1412 
1413         cone[0] = 19; cone[1] = 20; cone[2] = 21; cone[3] = 22;
1414         cone[4] = 15; cone[5] = 16; cone[6] = 17; cone[7] = 18;
1415         ierr = DMPlexSetCone(*dm, 10, cone);CHKERRQ(ierr);
1416         cone[0] = 22; cone[1] = 21; cone[2] = 26; cone[3] = 25;
1417         cone[4] = 16; cone[5] = 23; cone[6] = 24; cone[7] = 17;
1418         ierr = DMPlexSetCone(*dm, 11, cone);CHKERRQ(ierr);
1419         cone[0] = 20; cone[1] = 28; cone[2] = 26; cone[3] = 21;
1420         cone[4] = 18; cone[5] = 17; cone[6] = 24; cone[7] = 27;
1421         ierr = DMPlexSetCone(*dm, 12, cone);CHKERRQ(ierr);
1422         cone[0] = 30; cone[1] = 28; cone[2] = 20; cone[3] = 19;
1423         cone[4] = 29; cone[5] = 15; cone[6] = 18; cone[7] = 27;
1424         ierr = DMPlexSetCone(*dm, 13, cone);CHKERRQ(ierr);
1425         cone[0] = 30; cone[1] = 19; cone[2] = 22; cone[3] = 25;
1426         cone[4] = 29; cone[5] = 23; cone[6] = 16; cone[7] = 15;
1427         ierr = DMPlexSetCone(*dm, 14, cone);CHKERRQ(ierr);
1428       } else {
1429         cone[0] =  5; cone[1] =  8; cone[2] =  7; cone[3] =  6;
1430         cone[4] =  9; cone[5] = 12; cone[6] = 11; cone[7] = 10;
1431         ierr = DMPlexSetCone(*dm, 0, cone);CHKERRQ(ierr);
1432         cone[0] =  6; cone[1] =  7; cone[2] = 14; cone[3] = 13;
1433         cone[4] = 12; cone[5] = 15; cone[6] = 16; cone[7] = 11;
1434         ierr = DMPlexSetCone(*dm, 1, cone);CHKERRQ(ierr);
1435         cone[0] =  8; cone[1] = 17; cone[2] = 14; cone[3] =  7;
1436         cone[4] = 10; cone[5] = 11; cone[6] = 16; cone[7] = 18;
1437         ierr = DMPlexSetCone(*dm, 2, cone);CHKERRQ(ierr);
1438         cone[0] = 19; cone[1] = 17; cone[2] =  8; cone[3] =  5;
1439         cone[4] = 20; cone[5] =  9; cone[6] = 10; cone[7] = 18;
1440         ierr = DMPlexSetCone(*dm, 3, cone);CHKERRQ(ierr);
1441         cone[0] = 19; cone[1] =  5; cone[2] =  6; cone[3] = 13;
1442         cone[4] = 20; cone[5] = 15; cone[6] = 12; cone[7] =  9;
1443         ierr = DMPlexSetCone(*dm, 4, cone);CHKERRQ(ierr);
1444       }
1445     }
1446     ierr = DMPlexSymmetrize(*dm);CHKERRQ(ierr);
1447     ierr = DMPlexStratify(*dm);CHKERRQ(ierr);
1448   }
1449   /* Interpolate */
1450   {
1451     DM idm;
1452 
1453     ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
1454     ierr = DMDestroy(dm);CHKERRQ(ierr);
1455     *dm  = idm;
1456   }
1457   /* Create cube geometry */
1458   {
1459     Vec             coordinates;
1460     PetscSection    coordSection;
1461     PetscScalar    *coords;
1462     PetscInt        coordSize, v;
1463     const PetscReal dis = 1.0/PetscSqrtReal(2.0);
1464     const PetscReal ds2 = dis/2.0;
1465 
1466     /* Build coordinates */
1467     ierr = DMGetCoordinateSection(*dm, &coordSection);CHKERRQ(ierr);
1468     ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
1469     ierr = PetscSectionSetFieldComponents(coordSection, 0, dim);CHKERRQ(ierr);
1470     ierr = PetscSectionSetChart(coordSection, numCells, numCells+numVertices);CHKERRQ(ierr);
1471     for (v = numCells; v < numCells+numVertices; ++v) {
1472       ierr = PetscSectionSetDof(coordSection, v, dim);CHKERRQ(ierr);
1473       ierr = PetscSectionSetFieldDof(coordSection, v, 0, dim);CHKERRQ(ierr);
1474     }
1475     ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
1476     ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
1477     ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
1478     ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
1479     ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
1480     ierr = VecSetBlockSize(coordinates, dim);CHKERRQ(ierr);
1481     ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
1482     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
1483     if (!rank) {
1484       coords[0*dim+0] = -ds2; coords[0*dim+1] = -ds2; coords[0*dim+2] = 0.0;
1485       coords[1*dim+0] =  ds2; coords[1*dim+1] = -ds2; coords[1*dim+2] = 0.0;
1486       coords[2*dim+0] =  ds2; coords[2*dim+1] =  ds2; coords[2*dim+2] = 0.0;
1487       coords[3*dim+0] = -ds2; coords[3*dim+1] =  ds2; coords[3*dim+2] = 0.0;
1488       coords[4*dim+0] = -ds2; coords[4*dim+1] = -ds2; coords[4*dim+2] = 1.0;
1489       coords[5*dim+0] = -ds2; coords[5*dim+1] =  ds2; coords[5*dim+2] = 1.0;
1490       coords[6*dim+0] =  ds2; coords[6*dim+1] =  ds2; coords[6*dim+2] = 1.0;
1491       coords[7*dim+0] =  ds2; coords[7*dim+1] = -ds2; coords[7*dim+2] = 1.0;
1492       coords[ 8*dim+0] =  dis; coords[ 8*dim+1] = -dis; coords[ 8*dim+2] = 0.0;
1493       coords[ 9*dim+0] =  dis; coords[ 9*dim+1] =  dis; coords[ 9*dim+2] = 0.0;
1494       coords[10*dim+0] =  dis; coords[10*dim+1] = -dis; coords[10*dim+2] = 1.0;
1495       coords[11*dim+0] =  dis; coords[11*dim+1] =  dis; coords[11*dim+2] = 1.0;
1496       coords[12*dim+0] = -dis; coords[12*dim+1] =  dis; coords[12*dim+2] = 0.0;
1497       coords[13*dim+0] = -dis; coords[13*dim+1] =  dis; coords[13*dim+2] = 1.0;
1498       coords[14*dim+0] = -dis; coords[14*dim+1] = -dis; coords[14*dim+2] = 0.0;
1499       coords[15*dim+0] = -dis; coords[15*dim+1] = -dis; coords[15*dim+2] = 1.0;
1500       if (periodicZ == DM_BOUNDARY_PERIODIC) {
1501         /* 15 31 19 */ coords[16*dim+0] = -ds2; coords[16*dim+1] = -ds2; coords[16*dim+2] = 0.5;
1502         /* 16 32 22 */ coords[17*dim+0] =  ds2; coords[17*dim+1] = -ds2; coords[17*dim+2] = 0.5;
1503         /* 17 33 21 */ coords[18*dim+0] =  ds2; coords[18*dim+1] =  ds2; coords[18*dim+2] = 0.5;
1504         /* 18 34 20 */ coords[19*dim+0] = -ds2; coords[19*dim+1] =  ds2; coords[19*dim+2] = 0.5;
1505         /* 29 35 30 */ coords[20*dim+0] = -dis; coords[20*dim+1] = -dis; coords[20*dim+2] = 0.5;
1506         /* 23 36 25 */ coords[21*dim+0] =  dis; coords[21*dim+1] = -dis; coords[21*dim+2] = 0.5;
1507         /* 24 37 26 */ coords[22*dim+0] =  dis; coords[22*dim+1] =  dis; coords[22*dim+2] = 0.5;
1508         /* 27 38 28 */ coords[23*dim+0] = -dis; coords[23*dim+1] =  dis; coords[23*dim+2] = 0.5;
1509       }
1510     }
1511     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
1512     ierr = DMSetCoordinatesLocal(*dm, coordinates);CHKERRQ(ierr);
1513     ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
1514   }
1515   /* Create periodicity */
1516   if (periodicZ == DM_BOUNDARY_PERIODIC || periodicZ == DM_BOUNDARY_TWIST) {
1517     PetscReal      L[3];
1518     PetscReal      maxCell[3];
1519     DMBoundaryType bdType[3];
1520     PetscReal      lower[3] = {0.0, 0.0, 0.0};
1521     PetscReal      upper[3] = {1.0, 1.0, 1.5};
1522     PetscInt       i, numZCells = 3;
1523 
1524     bdType[0] = DM_BOUNDARY_NONE;
1525     bdType[1] = DM_BOUNDARY_NONE;
1526     bdType[2] = periodicZ;
1527     for (i = 0; i < dim; i++) {
1528       L[i]       = upper[i] - lower[i];
1529       maxCell[i] = 1.1 * (L[i] / numZCells);
1530     }
1531     ierr = DMSetPeriodicity(*dm, PETSC_TRUE, maxCell, L, bdType);CHKERRQ(ierr);
1532   }
1533   /* Refine topology */
1534   for (r = 0; r < numRefine; ++r) {
1535     DM rdm = NULL;
1536 
1537     ierr = DMRefine(*dm, comm, &rdm);CHKERRQ(ierr);
1538     ierr = DMDestroy(dm);CHKERRQ(ierr);
1539     *dm  = rdm;
1540   }
1541   /* Remap geometry to cylinder
1542        Interior square: Linear interpolation is correct
1543        The other cells all have vertices on rays from the origin. We want to uniformly expand the spacing
1544        such that the last vertex is on the unit circle. So the closest and farthest vertices are at distance
1545 
1546          phi     = arctan(y/x)
1547          d_close = sqrt(1/8 + 1/4 sin^2(phi))
1548          d_far   = sqrt(1/2 + sin^2(phi))
1549 
1550        so we remap them using
1551 
1552          x_new = x_close + (x - x_close) (1 - d_close) / (d_far - d_close)
1553          y_new = y_close + (y - y_close) (1 - d_close) / (d_far - d_close)
1554 
1555        If pi/4 < phi < 3pi/4 or -3pi/4 < phi < -pi/4, then we switch x and y.
1556   */
1557   {
1558     Vec           coordinates;
1559     PetscSection  coordSection;
1560     PetscScalar  *coords;
1561     PetscInt      vStart, vEnd, v;
1562     const PetscReal dis = 1.0/PetscSqrtReal(2.0);
1563     const PetscReal ds2 = 0.5*dis;
1564 
1565     ierr = DMPlexGetDepthStratum(*dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
1566     ierr = DMGetCoordinateSection(*dm, &coordSection);CHKERRQ(ierr);
1567     ierr = DMGetCoordinatesLocal(*dm, &coordinates);CHKERRQ(ierr);
1568     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
1569     for (v = vStart; v < vEnd; ++v) {
1570       PetscReal phi, sinp, cosp, dc, df, x, y, xc, yc;
1571       PetscInt  off;
1572 
1573       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
1574       if ((PetscAbsScalar(coords[off+0]) <= ds2) && (PetscAbsScalar(coords[off+1]) <= ds2)) continue;
1575       x    = PetscRealPart(coords[off]);
1576       y    = PetscRealPart(coords[off+1]);
1577       phi  = PetscAtan2Real(y, x);
1578       sinp = PetscSinReal(phi);
1579       cosp = PetscCosReal(phi);
1580       if ((PetscAbsReal(phi) > PETSC_PI/4.0) && (PetscAbsReal(phi) < 3.0*PETSC_PI/4.0)) {
1581         dc = PetscAbsReal(ds2/sinp);
1582         df = PetscAbsReal(dis/sinp);
1583         xc = ds2*x/PetscAbsReal(y);
1584         yc = ds2*PetscSignReal(y);
1585       } else {
1586         dc = PetscAbsReal(ds2/cosp);
1587         df = PetscAbsReal(dis/cosp);
1588         xc = ds2*PetscSignReal(x);
1589         yc = ds2*y/PetscAbsReal(x);
1590       }
1591       coords[off+0] = xc + (coords[off+0] - xc)*(1.0 - dc)/(df - dc);
1592       coords[off+1] = yc + (coords[off+1] - yc)*(1.0 - dc)/(df - dc);
1593     }
1594     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
1595     if (periodicZ == DM_BOUNDARY_PERIODIC || periodicZ == DM_BOUNDARY_TWIST) {
1596       ierr = DMLocalizeCoordinates(*dm);CHKERRQ(ierr);
1597     }
1598   }
1599   PetscFunctionReturn(0);
1600 }
1601 
1602 /*@
1603   DMPlexCreateWedgeCylinderMesh - Creates a mesh on the tensor product of the unit interval with the circle (cylinder) using wedges.
1604 
1605   Collective on MPI_Comm
1606 
1607   Input Parameters:
1608 + comm - The communicator for the DM object
1609 . n    - The number of wedges around the origin
1610 - interpolate - Create edges and faces
1611 
1612   Output Parameter:
1613 . dm  - The DM object
1614 
1615   Level: beginner
1616 
1617 .keywords: DM, create
1618 .seealso: DMPlexCreateHexCylinderMesh(), DMPlexCreateBoxMesh(), DMSetType(), DMCreate()
1619 @*/
1620 PetscErrorCode DMPlexCreateWedgeCylinderMesh(MPI_Comm comm, PetscInt n, PetscBool interpolate, DM *dm)
1621 {
1622   const PetscInt dim = 3;
1623   PetscInt       numCells, numVertices;
1624   PetscMPIInt    rank;
1625   PetscErrorCode ierr;
1626 
1627   PetscFunctionBegin;
1628   PetscValidPointer(dm, 3);
1629   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
1630   if (n < 0) SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Number of wedges %D cannot be negative", n);
1631   ierr = DMCreate(comm, dm);CHKERRQ(ierr);
1632   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
1633   ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
1634   /* Create topology */
1635   {
1636     PetscInt cone[6], c;
1637 
1638     numCells    = !rank ?        n : 0;
1639     numVertices = !rank ?  2*(n+1) : 0;
1640     ierr = DMPlexSetChart(*dm, 0, numCells+numVertices);CHKERRQ(ierr);
1641     ierr = DMPlexSetHybridBounds(*dm, 0, PETSC_DETERMINE, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
1642     for (c = 0; c < numCells; c++) {ierr = DMPlexSetConeSize(*dm, c, 6);CHKERRQ(ierr);}
1643     ierr = DMSetUp(*dm);CHKERRQ(ierr);
1644     for (c = 0; c < numCells; c++) {
1645       cone[0] =  c+n*1; cone[1] = (c+1)%n+n*1; cone[2] = 0+3*n;
1646       cone[3] =  c+n*2; cone[4] = (c+1)%n+n*2; cone[5] = 1+3*n;
1647       ierr = DMPlexSetCone(*dm, c, cone);CHKERRQ(ierr);
1648     }
1649     ierr = DMPlexSymmetrize(*dm);CHKERRQ(ierr);
1650     ierr = DMPlexStratify(*dm);CHKERRQ(ierr);
1651   }
1652   /* Interpolate */
1653   if (interpolate) {
1654     DM idm;
1655 
1656     ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
1657     ierr = DMDestroy(dm);CHKERRQ(ierr);
1658     *dm  = idm;
1659   }
1660   /* Create cylinder geometry */
1661   {
1662     Vec          coordinates;
1663     PetscSection coordSection;
1664     PetscScalar *coords;
1665     PetscInt     coordSize, v, c;
1666 
1667     /* Build coordinates */
1668     ierr = DMGetCoordinateSection(*dm, &coordSection);CHKERRQ(ierr);
1669     ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
1670     ierr = PetscSectionSetFieldComponents(coordSection, 0, dim);CHKERRQ(ierr);
1671     ierr = PetscSectionSetChart(coordSection, numCells, numCells+numVertices);CHKERRQ(ierr);
1672     for (v = numCells; v < numCells+numVertices; ++v) {
1673       ierr = PetscSectionSetDof(coordSection, v, dim);CHKERRQ(ierr);
1674       ierr = PetscSectionSetFieldDof(coordSection, v, 0, dim);CHKERRQ(ierr);
1675     }
1676     ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
1677     ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
1678     ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
1679     ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
1680     ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
1681     ierr = VecSetBlockSize(coordinates, dim);CHKERRQ(ierr);
1682     ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
1683     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
1684     for (c = 0; c < numCells; c++) {
1685       coords[(c+0*n)*dim+0] = PetscCosReal(2.0*c*PETSC_PI/n); coords[(c+0*n)*dim+1] = PetscSinReal(2.0*c*PETSC_PI/n); coords[(c+0*n)*dim+2] = 1.0;
1686       coords[(c+1*n)*dim+0] = PetscCosReal(2.0*c*PETSC_PI/n); coords[(c+1*n)*dim+1] = PetscSinReal(2.0*c*PETSC_PI/n); coords[(c+1*n)*dim+2] = 0.0;
1687     }
1688     if (!rank) {
1689       coords[(2*n+0)*dim+0] = 0.0; coords[(2*n+0)*dim+1] = 0.0; coords[(2*n+0)*dim+2] = 1.0;
1690       coords[(2*n+1)*dim+0] = 0.0; coords[(2*n+1)*dim+1] = 0.0; coords[(2*n+1)*dim+2] = 0.0;
1691     }
1692     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
1693     ierr = DMSetCoordinatesLocal(*dm, coordinates);CHKERRQ(ierr);
1694     ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
1695   }
1696   PetscFunctionReturn(0);
1697 }
1698 
1699 PETSC_STATIC_INLINE PetscReal DiffNormReal(PetscInt dim, const PetscReal x[], const PetscReal y[])
1700 {
1701   PetscReal prod = 0.0;
1702   PetscInt  i;
1703   for (i = 0; i < dim; ++i) prod += PetscSqr(x[i] - y[i]);
1704   return PetscSqrtReal(prod);
1705 }
1706 PETSC_STATIC_INLINE PetscReal DotReal(PetscInt dim, const PetscReal x[], const PetscReal y[])
1707 {
1708   PetscReal prod = 0.0;
1709   PetscInt  i;
1710   for (i = 0; i < dim; ++i) prod += x[i]*y[i];
1711   return prod;
1712 }
1713 
1714 /*@
1715   DMPlexCreateSphereMesh - Creates a mesh on the d-dimensional sphere, S^d.
1716 
1717   Collective on MPI_Comm
1718 
1719   Input Parameters:
1720 . comm  - The communicator for the DM object
1721 . dim   - The dimension
1722 - simplex - Use simplices, or tensor product cells
1723 
1724   Output Parameter:
1725 . dm  - The DM object
1726 
1727   Level: beginner
1728 
1729 .keywords: DM, create
1730 .seealso: DMPlexCreateBoxMesh(), DMSetType(), DMCreate()
1731 @*/
1732 PetscErrorCode DMPlexCreateSphereMesh(MPI_Comm comm, PetscInt dim, PetscBool simplex, DM *dm)
1733 {
1734   const PetscInt  embedDim = dim+1;
1735   PetscSection    coordSection;
1736   Vec             coordinates;
1737   PetscScalar    *coords;
1738   PetscReal      *coordsIn;
1739   PetscInt        numCells, numEdges, numVerts, firstVertex, v, firstEdge, coordSize, d, c, e;
1740   PetscMPIInt     rank;
1741   PetscErrorCode  ierr;
1742 
1743   PetscFunctionBegin;
1744   PetscValidPointer(dm, 4);
1745   ierr = DMCreate(comm, dm);CHKERRQ(ierr);
1746   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
1747   ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
1748   ierr = DMSetCoordinateDim(*dm, dim+1);CHKERRQ(ierr);
1749   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) *dm), &rank);CHKERRQ(ierr);
1750   switch (dim) {
1751   case 2:
1752     if (simplex) {
1753       DM              idm;
1754       const PetscReal edgeLen     = 2.0/(1.0 + PETSC_PHI);
1755       const PetscReal vertex[3]   = {0.0, 1.0/(1.0 + PETSC_PHI), PETSC_PHI/(1.0 + PETSC_PHI)};
1756       const PetscInt  degree      = 5;
1757       PetscInt        s[3]        = {1, 1, 1};
1758       PetscInt        cone[3];
1759       PetscInt       *graph, p, i, j, k;
1760 
1761       numCells    = !rank ? 20 : 0;
1762       numVerts    = !rank ? 12 : 0;
1763       firstVertex = numCells;
1764       /* Use icosahedron, which for a unit sphere has coordinates which are all cyclic permutations of
1765 
1766            (0, \pm 1/\phi+1, \pm \phi/\phi+1)
1767 
1768          where \phi^2 - \phi - 1 = 0, meaning \phi is the golden ratio \frac{1 + \sqrt{5}}{2}. The edge
1769          length is then given by 2/\phi = 2 * 2.73606 = 5.47214.
1770        */
1771       /* Construct vertices */
1772       ierr = PetscCalloc1(numVerts * embedDim, &coordsIn);CHKERRQ(ierr);
1773       for (p = 0, i = 0; p < embedDim; ++p) {
1774         for (s[1] = -1; s[1] < 2; s[1] += 2) {
1775           for (s[2] = -1; s[2] < 2; s[2] += 2) {
1776             for (d = 0; d < embedDim; ++d) coordsIn[i*embedDim+d] = s[(d+p)%embedDim]*vertex[(d+p)%embedDim];
1777             ++i;
1778           }
1779         }
1780       }
1781       /* Construct graph */
1782       ierr = PetscCalloc1(numVerts * numVerts, &graph);CHKERRQ(ierr);
1783       for (i = 0; i < numVerts; ++i) {
1784         for (j = 0, k = 0; j < numVerts; ++j) {
1785           if (PetscAbsReal(DiffNormReal(embedDim, &coordsIn[i*embedDim], &coordsIn[j*embedDim]) - edgeLen) < PETSC_SMALL) {graph[i*numVerts+j] = 1; ++k;}
1786         }
1787         if (k != degree) SETERRQ3(comm, PETSC_ERR_PLIB, "Invalid icosahedron, vertex %D degree %D != %D", i, k, degree);
1788       }
1789       /* Build Topology */
1790       ierr = DMPlexSetChart(*dm, 0, numCells+numVerts);CHKERRQ(ierr);
1791       for (c = 0; c < numCells; c++) {
1792         ierr = DMPlexSetConeSize(*dm, c, embedDim);CHKERRQ(ierr);
1793       }
1794       ierr = DMSetUp(*dm);CHKERRQ(ierr); /* Allocate space for cones */
1795       /* Cells */
1796       for (i = 0, c = 0; i < numVerts; ++i) {
1797         for (j = 0; j < i; ++j) {
1798           for (k = 0; k < j; ++k) {
1799             if (graph[i*numVerts+j] && graph[j*numVerts+k] && graph[k*numVerts+i]) {
1800               cone[0] = firstVertex+i; cone[1] = firstVertex+j; cone[2] = firstVertex+k;
1801               /* Check orientation */
1802               {
1803                 const PetscInt epsilon[3][3][3] = {{{0, 0, 0}, {0, 0, 1}, {0, -1, 0}}, {{0, 0, -1}, {0, 0, 0}, {1, 0, 0}}, {{0, 1, 0}, {-1, 0, 0}, {0, 0, 0}}};
1804                 PetscReal normal[3];
1805                 PetscInt  e, f;
1806 
1807                 for (d = 0; d < embedDim; ++d) {
1808                   normal[d] = 0.0;
1809                   for (e = 0; e < embedDim; ++e) {
1810                     for (f = 0; f < embedDim; ++f) {
1811                       normal[d] += epsilon[d][e][f]*(coordsIn[j*embedDim+e] - coordsIn[i*embedDim+e])*(coordsIn[k*embedDim+f] - coordsIn[i*embedDim+f]);
1812                     }
1813                   }
1814                 }
1815                 if (DotReal(embedDim, normal, &coordsIn[i*embedDim]) < 0) {PetscInt tmp = cone[1]; cone[1] = cone[2]; cone[2] = tmp;}
1816               }
1817               ierr = DMPlexSetCone(*dm, c++, cone);CHKERRQ(ierr);
1818             }
1819           }
1820         }
1821       }
1822       ierr = DMPlexSymmetrize(*dm);CHKERRQ(ierr);
1823       ierr = DMPlexStratify(*dm);CHKERRQ(ierr);
1824       ierr = PetscFree(graph);CHKERRQ(ierr);
1825       /* Interpolate mesh */
1826       ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
1827       ierr = DMDestroy(dm);CHKERRQ(ierr);
1828       *dm  = idm;
1829     } else {
1830       /*
1831         12-21--13
1832          |     |
1833         25  4  24
1834          |     |
1835   12-25--9-16--8-24--13
1836    |     |     |     |
1837   23  5 17  0 15  3  22
1838    |     |     |     |
1839   10-20--6-14--7-19--11
1840          |     |
1841         20  1  19
1842          |     |
1843         10-18--11
1844          |     |
1845         23  2  22
1846          |     |
1847         12-21--13
1848        */
1849       const PetscReal dist = 1.0/PetscSqrtReal(3.0);
1850       PetscInt        cone[4], ornt[4];
1851 
1852       numCells    = !rank ?  6 : 0;
1853       numEdges    = !rank ? 12 : 0;
1854       numVerts    = !rank ?  8 : 0;
1855       firstVertex = numCells;
1856       firstEdge   = numCells + numVerts;
1857       /* Build Topology */
1858       ierr = DMPlexSetChart(*dm, 0, numCells+numEdges+numVerts);CHKERRQ(ierr);
1859       for (c = 0; c < numCells; c++) {
1860         ierr = DMPlexSetConeSize(*dm, c, 4);CHKERRQ(ierr);
1861       }
1862       for (e = firstEdge; e < firstEdge+numEdges; ++e) {
1863         ierr = DMPlexSetConeSize(*dm, e, 2);CHKERRQ(ierr);
1864       }
1865       ierr = DMSetUp(*dm);CHKERRQ(ierr); /* Allocate space for cones */
1866       /* Cell 0 */
1867       cone[0] = 14; cone[1] = 15; cone[2] = 16; cone[3] = 17;
1868       ierr = DMPlexSetCone(*dm, 0, cone);CHKERRQ(ierr);
1869       ornt[0] = 0; ornt[1] = 0; ornt[2] = 0; ornt[3] = 0;
1870       ierr = DMPlexSetConeOrientation(*dm, 0, ornt);CHKERRQ(ierr);
1871       /* Cell 1 */
1872       cone[0] = 18; cone[1] = 19; cone[2] = 14; cone[3] = 20;
1873       ierr = DMPlexSetCone(*dm, 1, cone);CHKERRQ(ierr);
1874       ornt[0] = 0; ornt[1] = 0; ornt[2] = -2; ornt[3] = 0;
1875       ierr = DMPlexSetConeOrientation(*dm, 1, ornt);CHKERRQ(ierr);
1876       /* Cell 2 */
1877       cone[0] = 21; cone[1] = 22; cone[2] = 18; cone[3] = 23;
1878       ierr = DMPlexSetCone(*dm, 2, cone);CHKERRQ(ierr);
1879       ornt[0] = 0; ornt[1] = 0; ornt[2] = -2; ornt[3] = 0;
1880       ierr = DMPlexSetConeOrientation(*dm, 2, ornt);CHKERRQ(ierr);
1881       /* Cell 3 */
1882       cone[0] = 19; cone[1] = 22; cone[2] = 24; cone[3] = 15;
1883       ierr = DMPlexSetCone(*dm, 3, cone);CHKERRQ(ierr);
1884       ornt[0] = -2; ornt[1] = -2; ornt[2] = 0; ornt[3] = -2;
1885       ierr = DMPlexSetConeOrientation(*dm, 3, ornt);CHKERRQ(ierr);
1886       /* Cell 4 */
1887       cone[0] = 16; cone[1] = 24; cone[2] = 21; cone[3] = 25;
1888       ierr = DMPlexSetCone(*dm, 4, cone);CHKERRQ(ierr);
1889       ornt[0] = -2; ornt[1] = -2; ornt[2] = -2; ornt[3] = 0;
1890       ierr = DMPlexSetConeOrientation(*dm, 4, ornt);CHKERRQ(ierr);
1891       /* Cell 5 */
1892       cone[0] = 20; cone[1] = 17; cone[2] = 25; cone[3] = 23;
1893       ierr = DMPlexSetCone(*dm, 5, cone);CHKERRQ(ierr);
1894       ornt[0] = -2; ornt[1] = -2; ornt[2] = -2; ornt[3] = -2;
1895       ierr = DMPlexSetConeOrientation(*dm, 5, ornt);CHKERRQ(ierr);
1896       /* Edges */
1897       cone[0] =  6; cone[1] =  7;
1898       ierr = DMPlexSetCone(*dm, 14, cone);CHKERRQ(ierr);
1899       cone[0] =  7; cone[1] =  8;
1900       ierr = DMPlexSetCone(*dm, 15, cone);CHKERRQ(ierr);
1901       cone[0] =  8; cone[1] =  9;
1902       ierr = DMPlexSetCone(*dm, 16, cone);CHKERRQ(ierr);
1903       cone[0] =  9; cone[1] =  6;
1904       ierr = DMPlexSetCone(*dm, 17, cone);CHKERRQ(ierr);
1905       cone[0] = 10; cone[1] = 11;
1906       ierr = DMPlexSetCone(*dm, 18, cone);CHKERRQ(ierr);
1907       cone[0] = 11; cone[1] =  7;
1908       ierr = DMPlexSetCone(*dm, 19, cone);CHKERRQ(ierr);
1909       cone[0] =  6; cone[1] = 10;
1910       ierr = DMPlexSetCone(*dm, 20, cone);CHKERRQ(ierr);
1911       cone[0] = 12; cone[1] = 13;
1912       ierr = DMPlexSetCone(*dm, 21, cone);CHKERRQ(ierr);
1913       cone[0] = 13; cone[1] = 11;
1914       ierr = DMPlexSetCone(*dm, 22, cone);CHKERRQ(ierr);
1915       cone[0] = 10; cone[1] = 12;
1916       ierr = DMPlexSetCone(*dm, 23, cone);CHKERRQ(ierr);
1917       cone[0] = 13; cone[1] =  8;
1918       ierr = DMPlexSetCone(*dm, 24, cone);CHKERRQ(ierr);
1919       cone[0] = 12; cone[1] =  9;
1920       ierr = DMPlexSetCone(*dm, 25, cone);CHKERRQ(ierr);
1921       ierr = DMPlexSymmetrize(*dm);CHKERRQ(ierr);
1922       ierr = DMPlexStratify(*dm);CHKERRQ(ierr);
1923       /* Build coordinates */
1924       ierr = PetscCalloc1(numVerts * embedDim, &coordsIn);CHKERRQ(ierr);
1925       coordsIn[0*embedDim+0] = -dist; coordsIn[0*embedDim+1] =  dist; coordsIn[0*embedDim+2] = -dist;
1926       coordsIn[1*embedDim+0] =  dist; coordsIn[1*embedDim+1] =  dist; coordsIn[1*embedDim+2] = -dist;
1927       coordsIn[2*embedDim+0] =  dist; coordsIn[2*embedDim+1] = -dist; coordsIn[2*embedDim+2] = -dist;
1928       coordsIn[3*embedDim+0] = -dist; coordsIn[3*embedDim+1] = -dist; coordsIn[3*embedDim+2] = -dist;
1929       coordsIn[4*embedDim+0] = -dist; coordsIn[4*embedDim+1] =  dist; coordsIn[4*embedDim+2] =  dist;
1930       coordsIn[5*embedDim+0] =  dist; coordsIn[5*embedDim+1] =  dist; coordsIn[5*embedDim+2] =  dist;
1931       coordsIn[6*embedDim+0] = -dist; coordsIn[6*embedDim+1] = -dist; coordsIn[6*embedDim+2] =  dist;
1932       coordsIn[7*embedDim+0] =  dist; coordsIn[7*embedDim+1] = -dist; coordsIn[7*embedDim+2] =  dist;
1933     }
1934     break;
1935   case 3:
1936     if (simplex) {
1937       DM              idm;
1938       const PetscReal edgeLen         = 1.0/PETSC_PHI;
1939       const PetscReal vertexA[4]      = {0.5, 0.5, 0.5, 0.5};
1940       const PetscReal vertexB[4]      = {1.0, 0.0, 0.0, 0.0};
1941       const PetscReal vertexC[4]      = {0.5, 0.5*PETSC_PHI, 0.5/PETSC_PHI, 0.0};
1942       const PetscInt  degree          = 12;
1943       PetscInt        s[4]            = {1, 1, 1};
1944       PetscInt        evenPerm[12][4] = {{0, 1, 2, 3}, {0, 2, 3, 1}, {0, 3, 1, 2}, {1, 0, 3, 2}, {1, 2, 0, 3}, {1, 3, 2, 0},
1945                                          {2, 0, 1, 3}, {2, 1, 3, 0}, {2, 3, 0, 1}, {3, 0, 2, 1}, {3, 1, 0, 2}, {3, 2, 1, 0}};
1946       PetscInt        cone[4];
1947       PetscInt       *graph, p, i, j, k, l;
1948 
1949       numCells    = !rank ? 600 : 0;
1950       numVerts    = !rank ? 120 : 0;
1951       firstVertex = numCells;
1952       /* Use the 600-cell, which for a unit sphere has coordinates which are
1953 
1954            1/2 (\pm 1, \pm 1,    \pm 1, \pm 1)                          16
1955                (\pm 1,    0,       0,      0)  all cyclic permutations   8
1956            1/2 (\pm 1, \pm phi, \pm 1/phi, 0)  all even permutations    96
1957 
1958          where \phi^2 - \phi - 1 = 0, meaning \phi is the golden ratio \frac{1 + \sqrt{5}}{2}. The edge
1959          length is then given by 1/\phi = 2.73606.
1960 
1961          http://buzzard.pugetsound.edu/sage-practice/ch03s03.html
1962          http://mathworld.wolfram.com/600-Cell.html
1963        */
1964       /* Construct vertices */
1965       ierr = PetscCalloc1(numVerts * embedDim, &coordsIn);CHKERRQ(ierr);
1966       i    = 0;
1967       for (s[0] = -1; s[0] < 2; s[0] += 2) {
1968         for (s[1] = -1; s[1] < 2; s[1] += 2) {
1969           for (s[2] = -1; s[2] < 2; s[2] += 2) {
1970             for (s[3] = -1; s[3] < 2; s[3] += 2) {
1971               for (d = 0; d < embedDim; ++d) coordsIn[i*embedDim+d] = s[d]*vertexA[d];
1972               ++i;
1973             }
1974           }
1975         }
1976       }
1977       for (p = 0; p < embedDim; ++p) {
1978         s[1] = s[2] = s[3] = 1;
1979         for (s[0] = -1; s[0] < 2; s[0] += 2) {
1980           for (d = 0; d < embedDim; ++d) coordsIn[i*embedDim+d] = s[(d+p)%embedDim]*vertexB[(d+p)%embedDim];
1981           ++i;
1982         }
1983       }
1984       for (p = 0; p < 12; ++p) {
1985         s[3] = 1;
1986         for (s[0] = -1; s[0] < 2; s[0] += 2) {
1987           for (s[1] = -1; s[1] < 2; s[1] += 2) {
1988             for (s[2] = -1; s[2] < 2; s[2] += 2) {
1989               for (d = 0; d < embedDim; ++d) coordsIn[i*embedDim+d] = s[evenPerm[p][d]]*vertexC[evenPerm[p][d]];
1990               ++i;
1991             }
1992           }
1993         }
1994       }
1995       if (i != numVerts) SETERRQ2(comm, PETSC_ERR_PLIB, "Invalid 600-cell, vertices %D != %D", i, numVerts);
1996       /* Construct graph */
1997       ierr = PetscCalloc1(numVerts * numVerts, &graph);CHKERRQ(ierr);
1998       for (i = 0; i < numVerts; ++i) {
1999         for (j = 0, k = 0; j < numVerts; ++j) {
2000           if (PetscAbsReal(DiffNormReal(embedDim, &coordsIn[i*embedDim], &coordsIn[j*embedDim]) - edgeLen) < PETSC_SMALL) {graph[i*numVerts+j] = 1; ++k;}
2001         }
2002         if (k != degree) SETERRQ3(comm, PETSC_ERR_PLIB, "Invalid 600-cell, vertex %D degree %D != %D", i, k, degree);
2003       }
2004       /* Build Topology */
2005       ierr = DMPlexSetChart(*dm, 0, numCells+numVerts);CHKERRQ(ierr);
2006       for (c = 0; c < numCells; c++) {
2007         ierr = DMPlexSetConeSize(*dm, c, embedDim);CHKERRQ(ierr);
2008       }
2009       ierr = DMSetUp(*dm);CHKERRQ(ierr); /* Allocate space for cones */
2010       /* Cells */
2011       for (i = 0, c = 0; i < numVerts; ++i) {
2012         for (j = 0; j < i; ++j) {
2013           for (k = 0; k < j; ++k) {
2014             for (l = 0; l < k; ++l) {
2015               if (graph[i*numVerts+j] && graph[j*numVerts+k] && graph[k*numVerts+i] &&
2016                   graph[l*numVerts+i] && graph[l*numVerts+j] && graph[l*numVerts+k]) {
2017                 cone[0] = firstVertex+i; cone[1] = firstVertex+j; cone[2] = firstVertex+k; cone[3] = firstVertex+l;
2018                 /* Check orientation: https://ef.gy/linear-algebra:normal-vectors-in-higher-dimensional-spaces */
2019                 {
2020                   const PetscInt epsilon[4][4][4][4] = {{{{0,  0,  0,  0}, { 0, 0,  0,  0}, { 0,  0, 0,  0}, { 0,  0,  0, 0}},
2021                                                          {{0,  0,  0,  0}, { 0, 0,  0,  0}, { 0,  0, 0,  1}, { 0,  0, -1, 0}},
2022                                                          {{0,  0,  0,  0}, { 0, 0,  0, -1}, { 0,  0, 0,  0}, { 0,  1,  0, 0}},
2023                                                          {{0,  0,  0,  0}, { 0, 0,  1,  0}, { 0, -1, 0,  0}, { 0,  0,  0, 0}}},
2024 
2025                                                         {{{0,  0,  0,  0}, { 0, 0,  0,  0}, { 0,  0, 0, -1}, { 0,  0,  1, 0}},
2026                                                          {{0,  0,  0,  0}, { 0, 0,  0,  0}, { 0,  0, 0,  0}, { 0,  0,  0, 0}},
2027                                                          {{0,  0,  0,  1}, { 0, 0,  0,  0}, { 0,  0, 0,  0}, {-1,  0,  0, 0}},
2028                                                          {{0,  0, -1,  0}, { 0, 0,  0,  0}, { 1,  0, 0,  0}, { 0,  0,  0, 0}}},
2029 
2030                                                         {{{0,  0,  0,  0}, { 0, 0,  0,  1}, { 0,  0, 0,  0}, { 0, -1,  0, 0}},
2031                                                          {{0,  0,  0, -1}, { 0, 0,  0,  0}, { 0,  0, 0,  0}, { 1,  0,  0, 0}},
2032                                                          {{0,  0,  0,  0}, { 0, 0,  0,  0}, { 0,  0, 0,  0}, { 0,  0,  0, 0}},
2033                                                          {{0,  1,  0,  0}, {-1, 0,  0,  0}, { 0,  0, 0,  0}, { 0,  0,  0, 0}}},
2034 
2035                                                         {{{0,  0,  0,  0}, { 0, 0, -1,  0}, { 0,  1, 0,  0}, { 0,  0,  0, 0}},
2036                                                          {{0,  0,  1,  0}, { 0, 0,  0,  0}, {-1,  0, 0,  0}, { 0,  0,  0, 0}},
2037                                                          {{0, -1,  0,  0}, { 1, 0,  0,  0}, { 0,  0, 0,  0}, { 0,  0,  0, 0}},
2038                                                          {{0,  0,  0,  0}, { 0, 0,  0,  0}, { 0,  0, 0,  0}, { 0,  0,  0, 0}}}};
2039                   PetscReal normal[4];
2040                   PetscInt  e, f, g;
2041 
2042                   for (d = 0; d < embedDim; ++d) {
2043                     normal[d] = 0.0;
2044                     for (e = 0; e < embedDim; ++e) {
2045                       for (f = 0; f < embedDim; ++f) {
2046                         for (g = 0; g < embedDim; ++g) {
2047                           normal[d] += epsilon[d][e][f][g]*(coordsIn[j*embedDim+e] - coordsIn[i*embedDim+e])*(coordsIn[k*embedDim+f] - coordsIn[i*embedDim+f])*(coordsIn[l*embedDim+f] - coordsIn[i*embedDim+f]);
2048                         }
2049                       }
2050                     }
2051                   }
2052                   if (DotReal(embedDim, normal, &coordsIn[i*embedDim]) < 0) {PetscInt tmp = cone[1]; cone[1] = cone[2]; cone[2] = tmp;}
2053                 }
2054                 ierr = DMPlexSetCone(*dm, c++, cone);CHKERRQ(ierr);
2055               }
2056             }
2057           }
2058         }
2059       }
2060       ierr = DMPlexSymmetrize(*dm);CHKERRQ(ierr);
2061       ierr = DMPlexStratify(*dm);CHKERRQ(ierr);
2062       ierr = PetscFree(graph);CHKERRQ(ierr);
2063       /* Interpolate mesh */
2064       ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
2065       ierr = DMDestroy(dm);CHKERRQ(ierr);
2066       *dm  = idm;
2067       break;
2068     }
2069   default: SETERRQ1(comm, PETSC_ERR_SUP, "Unsupported dimension for sphere: %D", dim);
2070   }
2071   /* Create coordinates */
2072   ierr = DMGetCoordinateSection(*dm, &coordSection);CHKERRQ(ierr);
2073   ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
2074   ierr = PetscSectionSetFieldComponents(coordSection, 0, embedDim);CHKERRQ(ierr);
2075   ierr = PetscSectionSetChart(coordSection, firstVertex, firstVertex+numVerts);CHKERRQ(ierr);
2076   for (v = firstVertex; v < firstVertex+numVerts; ++v) {
2077     ierr = PetscSectionSetDof(coordSection, v, embedDim);CHKERRQ(ierr);
2078     ierr = PetscSectionSetFieldDof(coordSection, v, 0, embedDim);CHKERRQ(ierr);
2079   }
2080   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
2081   ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
2082   ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
2083   ierr = VecSetBlockSize(coordinates, embedDim);CHKERRQ(ierr);
2084   ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
2085   ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
2086   ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
2087   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
2088   for (v = 0; v < numVerts; ++v) for (d = 0; d < embedDim; ++d) {coords[v*embedDim+d] = coordsIn[v*embedDim+d];}
2089   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
2090   ierr = DMSetCoordinatesLocal(*dm, coordinates);CHKERRQ(ierr);
2091   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
2092   ierr = PetscFree(coordsIn);CHKERRQ(ierr);
2093   PetscFunctionReturn(0);
2094 }
2095 
2096 /* External function declarations here */
2097 extern PetscErrorCode DMCreateInterpolation_Plex(DM dmCoarse, DM dmFine, Mat *interpolation, Vec *scaling);
2098 extern PetscErrorCode DMCreateInjection_Plex(DM dmCoarse, DM dmFine, Mat *mat);
2099 extern PetscErrorCode DMCreateMassMatrix_Plex(DM dmCoarse, DM dmFine, Mat *mat);
2100 extern PetscErrorCode DMCreateDefaultSection_Plex(DM dm);
2101 extern PetscErrorCode DMCreateDefaultConstraints_Plex(DM dm);
2102 extern PetscErrorCode DMCreateMatrix_Plex(DM dm,  Mat *J);
2103 extern PetscErrorCode DMCreateCoordinateDM_Plex(DM dm, DM *cdm);
2104 extern PetscErrorCode DMCreateCoordinateField_Plex(DM dm, DMField *field);
2105 PETSC_INTERN PetscErrorCode DMClone_Plex(DM dm, DM *newdm);
2106 extern PetscErrorCode DMSetUp_Plex(DM dm);
2107 extern PetscErrorCode DMDestroy_Plex(DM dm);
2108 extern PetscErrorCode DMView_Plex(DM dm, PetscViewer viewer);
2109 extern PetscErrorCode DMLoad_Plex(DM dm, PetscViewer viewer);
2110 extern PetscErrorCode DMCreateSubDM_Plex(DM dm, PetscInt numFields, const PetscInt fields[], IS *is, DM *subdm);
2111 extern PetscErrorCode DMCreateSuperDM_Plex(DM dms[], PetscInt len, IS **is, DM *superdm);
2112 static PetscErrorCode DMInitialize_Plex(DM dm);
2113 
2114 /* Replace dm with the contents of dmNew
2115    - Share the DM_Plex structure
2116    - Share the coordinates
2117    - Share the SF
2118 */
2119 static PetscErrorCode DMPlexReplace_Static(DM dm, DM dmNew)
2120 {
2121   PetscSF               sf;
2122   DM                    coordDM, coarseDM;
2123   Vec                   coords;
2124   PetscBool             isper;
2125   const PetscReal      *maxCell, *L;
2126   const DMBoundaryType *bd;
2127   PetscErrorCode        ierr;
2128 
2129   PetscFunctionBegin;
2130   ierr = DMGetPointSF(dmNew, &sf);CHKERRQ(ierr);
2131   ierr = DMSetPointSF(dm, sf);CHKERRQ(ierr);
2132   ierr = DMGetCoordinateDM(dmNew, &coordDM);CHKERRQ(ierr);
2133   ierr = DMGetCoordinatesLocal(dmNew, &coords);CHKERRQ(ierr);
2134   ierr = DMSetCoordinateDM(dm, coordDM);CHKERRQ(ierr);
2135   ierr = DMSetCoordinatesLocal(dm, coords);CHKERRQ(ierr);
2136   ierr = DMGetPeriodicity(dm, &isper, &maxCell, &L, &bd);CHKERRQ(ierr);
2137   ierr = DMSetPeriodicity(dmNew, isper, maxCell, L, bd);CHKERRQ(ierr);
2138   ierr = DMDestroy_Plex(dm);CHKERRQ(ierr);
2139   ierr = DMInitialize_Plex(dm);CHKERRQ(ierr);
2140   dm->data = dmNew->data;
2141   ((DM_Plex *) dmNew->data)->refct++;
2142   dmNew->labels->refct++;
2143   if (!--(dm->labels->refct)) {
2144     DMLabelLink next = dm->labels->next;
2145 
2146     /* destroy the labels */
2147     while (next) {
2148       DMLabelLink tmp = next->next;
2149 
2150       ierr = DMLabelDestroy(&next->label);CHKERRQ(ierr);
2151       ierr = PetscFree(next);CHKERRQ(ierr);
2152       next = tmp;
2153     }
2154     ierr = PetscFree(dm->labels);CHKERRQ(ierr);
2155   }
2156   dm->labels = dmNew->labels;
2157   dm->depthLabel = dmNew->depthLabel;
2158   ierr = DMGetCoarseDM(dmNew,&coarseDM);CHKERRQ(ierr);
2159   ierr = DMSetCoarseDM(dm,coarseDM);CHKERRQ(ierr);
2160   PetscFunctionReturn(0);
2161 }
2162 
2163 /* Swap dm with the contents of dmNew
2164    - Swap the DM_Plex structure
2165    - Swap the coordinates
2166    - Swap the point PetscSF
2167 */
2168 static PetscErrorCode DMPlexSwap_Static(DM dmA, DM dmB)
2169 {
2170   DM              coordDMA, coordDMB;
2171   Vec             coordsA,  coordsB;
2172   PetscSF         sfA,      sfB;
2173   void            *tmp;
2174   DMLabelLinkList listTmp;
2175   DMLabel         depthTmp;
2176   PetscErrorCode  ierr;
2177 
2178   PetscFunctionBegin;
2179   ierr = DMGetPointSF(dmA, &sfA);CHKERRQ(ierr);
2180   ierr = DMGetPointSF(dmB, &sfB);CHKERRQ(ierr);
2181   ierr = PetscObjectReference((PetscObject) sfA);CHKERRQ(ierr);
2182   ierr = DMSetPointSF(dmA, sfB);CHKERRQ(ierr);
2183   ierr = DMSetPointSF(dmB, sfA);CHKERRQ(ierr);
2184   ierr = PetscObjectDereference((PetscObject) sfA);CHKERRQ(ierr);
2185 
2186   ierr = DMGetCoordinateDM(dmA, &coordDMA);CHKERRQ(ierr);
2187   ierr = DMGetCoordinateDM(dmB, &coordDMB);CHKERRQ(ierr);
2188   ierr = PetscObjectReference((PetscObject) coordDMA);CHKERRQ(ierr);
2189   ierr = DMSetCoordinateDM(dmA, coordDMB);CHKERRQ(ierr);
2190   ierr = DMSetCoordinateDM(dmB, coordDMA);CHKERRQ(ierr);
2191   ierr = PetscObjectDereference((PetscObject) coordDMA);CHKERRQ(ierr);
2192 
2193   ierr = DMGetCoordinatesLocal(dmA, &coordsA);CHKERRQ(ierr);
2194   ierr = DMGetCoordinatesLocal(dmB, &coordsB);CHKERRQ(ierr);
2195   ierr = PetscObjectReference((PetscObject) coordsA);CHKERRQ(ierr);
2196   ierr = DMSetCoordinatesLocal(dmA, coordsB);CHKERRQ(ierr);
2197   ierr = DMSetCoordinatesLocal(dmB, coordsA);CHKERRQ(ierr);
2198   ierr = PetscObjectDereference((PetscObject) coordsA);CHKERRQ(ierr);
2199 
2200   tmp       = dmA->data;
2201   dmA->data = dmB->data;
2202   dmB->data = tmp;
2203   listTmp   = dmA->labels;
2204   dmA->labels = dmB->labels;
2205   dmB->labels = listTmp;
2206   depthTmp  = dmA->depthLabel;
2207   dmA->depthLabel = dmB->depthLabel;
2208   dmB->depthLabel = depthTmp;
2209   PetscFunctionReturn(0);
2210 }
2211 
2212 PetscErrorCode DMSetFromOptions_NonRefinement_Plex(PetscOptionItems *PetscOptionsObject,DM dm)
2213 {
2214   DM_Plex       *mesh = (DM_Plex*) dm->data;
2215   PetscErrorCode ierr;
2216 
2217   PetscFunctionBegin;
2218   /* Handle viewing */
2219   ierr = PetscOptionsBool("-dm_plex_print_set_values", "Output all set values info", "DMPlexMatSetClosure", PETSC_FALSE, &mesh->printSetValues, NULL);CHKERRQ(ierr);
2220   ierr = PetscOptionsInt("-dm_plex_print_fem", "Debug output level all fem computations", "DMPlexSNESComputeResidualFEM", 0, &mesh->printFEM, NULL);CHKERRQ(ierr);
2221   ierr = PetscOptionsReal("-dm_plex_print_tol", "Tolerance for FEM output", "DMPlexSNESComputeResidualFEM", mesh->printTol, &mesh->printTol, NULL);CHKERRQ(ierr);
2222   ierr = PetscOptionsInt("-dm_plex_print_l2", "Debug output level all L2 diff computations", "DMComputeL2Diff", 0, &mesh->printL2, NULL);CHKERRQ(ierr);
2223   /* Point Location */
2224   ierr = PetscOptionsBool("-dm_plex_hash_location", "Use grid hashing for point location", "DMInterpolate", PETSC_FALSE, &mesh->useHashLocation, NULL);CHKERRQ(ierr);
2225   /* Partitioning and distribution */
2226   ierr = PetscOptionsBool("-dm_plex_partition_balance", "Attempt to evenly divide points on partition boundary between processes", "DMPlexSetPartitionBalance", PETSC_FALSE, &mesh->partitionBalance, NULL);CHKERRQ(ierr);
2227   /* Generation and remeshing */
2228   ierr = PetscOptionsBool("-dm_plex_remesh_bd", "Allow changes to the boundary on remeshing", "DMAdapt", PETSC_FALSE, &mesh->remeshBd, NULL);CHKERRQ(ierr);
2229   /* Projection behavior */
2230   ierr = PetscOptionsInt("-dm_plex_max_projection_height", "Maxmimum mesh point height used to project locally", "DMPlexSetMaxProjectionHeight", 0, &mesh->maxProjectionHeight, NULL);CHKERRQ(ierr);
2231   ierr = PetscOptionsBool("-dm_plex_regular_refinement", "Use special nested projection algorithm for regular refinement", "DMPlexSetRegularRefinement", mesh->regularRefinement, &mesh->regularRefinement, NULL);CHKERRQ(ierr);
2232 
2233   ierr = PetscPartitionerSetFromOptions(mesh->partitioner);CHKERRQ(ierr);
2234   PetscFunctionReturn(0);
2235 }
2236 
2237 static PetscErrorCode DMSetFromOptions_Plex(PetscOptionItems *PetscOptionsObject,DM dm)
2238 {
2239   PetscInt       refine = 0, coarsen = 0, r;
2240   PetscBool      isHierarchy;
2241   PetscErrorCode ierr;
2242 
2243   PetscFunctionBegin;
2244   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2245   ierr = PetscOptionsHead(PetscOptionsObject,"DMPlex Options");CHKERRQ(ierr);
2246   /* Handle DMPlex refinement */
2247   ierr = PetscOptionsInt("-dm_refine", "The number of uniform refinements", "DMCreate", refine, &refine, NULL);CHKERRQ(ierr);
2248   ierr = PetscOptionsInt("-dm_refine_hierarchy", "The number of uniform refinements", "DMCreate", refine, &refine, &isHierarchy);CHKERRQ(ierr);
2249   if (refine) {ierr = DMPlexSetRefinementUniform(dm, PETSC_TRUE);CHKERRQ(ierr);}
2250   if (refine && isHierarchy) {
2251     DM *dms, coarseDM;
2252 
2253     ierr = DMGetCoarseDM(dm, &coarseDM);CHKERRQ(ierr);
2254     ierr = PetscObjectReference((PetscObject)coarseDM);CHKERRQ(ierr);
2255     ierr = PetscMalloc1(refine,&dms);CHKERRQ(ierr);
2256     ierr = DMRefineHierarchy(dm, refine, dms);CHKERRQ(ierr);
2257     /* Total hack since we do not pass in a pointer */
2258     ierr = DMPlexSwap_Static(dm, dms[refine-1]);CHKERRQ(ierr);
2259     if (refine == 1) {
2260       ierr = DMSetCoarseDM(dm, dms[0]);CHKERRQ(ierr);
2261       ierr = DMPlexSetRegularRefinement(dm, PETSC_TRUE);CHKERRQ(ierr);
2262     } else {
2263       ierr = DMSetCoarseDM(dm, dms[refine-2]);CHKERRQ(ierr);
2264       ierr = DMPlexSetRegularRefinement(dm, PETSC_TRUE);CHKERRQ(ierr);
2265       ierr = DMSetCoarseDM(dms[0], dms[refine-1]);CHKERRQ(ierr);
2266       ierr = DMPlexSetRegularRefinement(dms[0], PETSC_TRUE);CHKERRQ(ierr);
2267     }
2268     ierr = DMSetCoarseDM(dms[refine-1], coarseDM);CHKERRQ(ierr);
2269     ierr = PetscObjectDereference((PetscObject)coarseDM);CHKERRQ(ierr);
2270     /* Free DMs */
2271     for (r = 0; r < refine; ++r) {
2272       ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dms[r]);CHKERRQ(ierr);
2273       ierr = DMDestroy(&dms[r]);CHKERRQ(ierr);
2274     }
2275     ierr = PetscFree(dms);CHKERRQ(ierr);
2276   } else {
2277     for (r = 0; r < refine; ++r) {
2278       DM refinedMesh;
2279 
2280       ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dm);CHKERRQ(ierr);
2281       ierr = DMRefine(dm, PetscObjectComm((PetscObject) dm), &refinedMesh);CHKERRQ(ierr);
2282       /* Total hack since we do not pass in a pointer */
2283       ierr = DMPlexReplace_Static(dm, refinedMesh);CHKERRQ(ierr);
2284       ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dm);CHKERRQ(ierr);
2285       ierr = DMDestroy(&refinedMesh);CHKERRQ(ierr);
2286     }
2287   }
2288   /* Handle DMPlex coarsening */
2289   ierr = PetscOptionsInt("-dm_coarsen", "Coarsen the mesh", "DMCreate", coarsen, &coarsen, NULL);CHKERRQ(ierr);
2290   ierr = PetscOptionsInt("-dm_coarsen_hierarchy", "The number of coarsenings", "DMCreate", coarsen, &coarsen, &isHierarchy);CHKERRQ(ierr);
2291   if (coarsen && isHierarchy) {
2292     DM *dms;
2293 
2294     ierr = PetscMalloc1(coarsen, &dms);CHKERRQ(ierr);
2295     ierr = DMCoarsenHierarchy(dm, coarsen, dms);CHKERRQ(ierr);
2296     /* Free DMs */
2297     for (r = 0; r < coarsen; ++r) {
2298       ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dms[r]);CHKERRQ(ierr);
2299       ierr = DMDestroy(&dms[r]);CHKERRQ(ierr);
2300     }
2301     ierr = PetscFree(dms);CHKERRQ(ierr);
2302   } else {
2303     for (r = 0; r < coarsen; ++r) {
2304       DM coarseMesh;
2305 
2306       ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dm);CHKERRQ(ierr);
2307       ierr = DMCoarsen(dm, PetscObjectComm((PetscObject) dm), &coarseMesh);CHKERRQ(ierr);
2308       /* Total hack since we do not pass in a pointer */
2309       ierr = DMPlexReplace_Static(dm, coarseMesh);CHKERRQ(ierr);
2310       ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dm);CHKERRQ(ierr);
2311       ierr = DMDestroy(&coarseMesh);CHKERRQ(ierr);
2312     }
2313   }
2314   /* Handle */
2315   ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dm);CHKERRQ(ierr);
2316   ierr = PetscOptionsTail();CHKERRQ(ierr);
2317   PetscFunctionReturn(0);
2318 }
2319 
2320 static PetscErrorCode DMCreateGlobalVector_Plex(DM dm,Vec *vec)
2321 {
2322   PetscErrorCode ierr;
2323 
2324   PetscFunctionBegin;
2325   ierr = DMCreateGlobalVector_Section_Private(dm,vec);CHKERRQ(ierr);
2326   /* ierr = VecSetOperation(*vec, VECOP_DUPLICATE, (void(*)(void)) VecDuplicate_MPI_DM);CHKERRQ(ierr); */
2327   ierr = VecSetOperation(*vec, VECOP_VIEW, (void (*)(void)) VecView_Plex);CHKERRQ(ierr);
2328   ierr = VecSetOperation(*vec, VECOP_VIEWNATIVE, (void (*)(void)) VecView_Plex_Native);CHKERRQ(ierr);
2329   ierr = VecSetOperation(*vec, VECOP_LOAD, (void (*)(void)) VecLoad_Plex);CHKERRQ(ierr);
2330   ierr = VecSetOperation(*vec, VECOP_LOADNATIVE, (void (*)(void)) VecLoad_Plex_Native);CHKERRQ(ierr);
2331   PetscFunctionReturn(0);
2332 }
2333 
2334 static PetscErrorCode DMCreateLocalVector_Plex(DM dm,Vec *vec)
2335 {
2336   PetscErrorCode ierr;
2337 
2338   PetscFunctionBegin;
2339   ierr = DMCreateLocalVector_Section_Private(dm,vec);CHKERRQ(ierr);
2340   ierr = VecSetOperation(*vec, VECOP_VIEW, (void (*)(void)) VecView_Plex_Local);CHKERRQ(ierr);
2341   ierr = VecSetOperation(*vec, VECOP_LOAD, (void (*)(void)) VecLoad_Plex_Local);CHKERRQ(ierr);
2342   PetscFunctionReturn(0);
2343 }
2344 
2345 static PetscErrorCode DMGetDimPoints_Plex(DM dm, PetscInt dim, PetscInt *pStart, PetscInt *pEnd)
2346 {
2347   PetscInt       depth, d;
2348   PetscErrorCode ierr;
2349 
2350   PetscFunctionBegin;
2351   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
2352   if (depth == 1) {
2353     ierr = DMGetDimension(dm, &d);CHKERRQ(ierr);
2354     if (dim == 0)      {ierr = DMPlexGetDepthStratum(dm, dim, pStart, pEnd);CHKERRQ(ierr);}
2355     else if (dim == d) {ierr = DMPlexGetDepthStratum(dm, 1, pStart, pEnd);CHKERRQ(ierr);}
2356     else               {*pStart = 0; *pEnd = 0;}
2357   } else {
2358     ierr = DMPlexGetDepthStratum(dm, dim, pStart, pEnd);CHKERRQ(ierr);
2359   }
2360   PetscFunctionReturn(0);
2361 }
2362 
2363 static PetscErrorCode DMGetNeighors_Plex(DM dm, PetscInt *nranks, const PetscMPIInt *ranks[])
2364 {
2365   PetscSF        sf;
2366   PetscErrorCode ierr;
2367 
2368   PetscFunctionBegin;
2369   ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr);
2370   ierr = PetscSFGetRanks(sf, nranks, ranks, NULL, NULL, NULL);CHKERRQ(ierr);
2371   PetscFunctionReturn(0);
2372 }
2373 
2374 static PetscErrorCode DMHasCreateInjection_Plex(DM dm, PetscBool *flg)
2375 {
2376   PetscFunctionBegin;
2377   PetscValidHeaderSpecific(dm,DM_CLASSID,1);
2378   PetscValidPointer(flg,2);
2379   *flg = PETSC_TRUE;
2380   PetscFunctionReturn(0);
2381 }
2382 
2383 static PetscErrorCode DMInitialize_Plex(DM dm)
2384 {
2385   PetscErrorCode ierr;
2386 
2387   PetscFunctionBegin;
2388   dm->ops->view                            = DMView_Plex;
2389   dm->ops->load                            = DMLoad_Plex;
2390   dm->ops->setfromoptions                  = DMSetFromOptions_Plex;
2391   dm->ops->clone                           = DMClone_Plex;
2392   dm->ops->setup                           = DMSetUp_Plex;
2393   dm->ops->createdefaultsection            = DMCreateDefaultSection_Plex;
2394   dm->ops->createdefaultconstraints        = DMCreateDefaultConstraints_Plex;
2395   dm->ops->createglobalvector              = DMCreateGlobalVector_Plex;
2396   dm->ops->createlocalvector               = DMCreateLocalVector_Plex;
2397   dm->ops->getlocaltoglobalmapping         = NULL;
2398   dm->ops->createfieldis                   = NULL;
2399   dm->ops->createcoordinatedm              = DMCreateCoordinateDM_Plex;
2400   dm->ops->createcoordinatefield           = DMCreateCoordinateField_Plex;
2401   dm->ops->getcoloring                     = NULL;
2402   dm->ops->creatematrix                    = DMCreateMatrix_Plex;
2403   dm->ops->createinterpolation             = DMCreateInterpolation_Plex;
2404   dm->ops->createmassmatrix                = DMCreateMassMatrix_Plex;
2405   dm->ops->getaggregates                   = NULL;
2406   dm->ops->getinjection                    = DMCreateInjection_Plex;
2407   dm->ops->hascreateinjection              = DMHasCreateInjection_Plex;
2408   dm->ops->refine                          = DMRefine_Plex;
2409   dm->ops->coarsen                         = DMCoarsen_Plex;
2410   dm->ops->refinehierarchy                 = DMRefineHierarchy_Plex;
2411   dm->ops->coarsenhierarchy                = DMCoarsenHierarchy_Plex;
2412   dm->ops->adaptlabel                      = DMAdaptLabel_Plex;
2413   dm->ops->adaptmetric                     = DMAdaptMetric_Plex;
2414   dm->ops->globaltolocalbegin              = NULL;
2415   dm->ops->globaltolocalend                = NULL;
2416   dm->ops->localtoglobalbegin              = NULL;
2417   dm->ops->localtoglobalend                = NULL;
2418   dm->ops->destroy                         = DMDestroy_Plex;
2419   dm->ops->createsubdm                     = DMCreateSubDM_Plex;
2420   dm->ops->createsuperdm                   = DMCreateSuperDM_Plex;
2421   dm->ops->getdimpoints                    = DMGetDimPoints_Plex;
2422   dm->ops->locatepoints                    = DMLocatePoints_Plex;
2423   dm->ops->projectfunctionlocal            = DMProjectFunctionLocal_Plex;
2424   dm->ops->projectfunctionlabellocal       = DMProjectFunctionLabelLocal_Plex;
2425   dm->ops->projectfieldlocal               = DMProjectFieldLocal_Plex;
2426   dm->ops->projectfieldlabellocal          = DMProjectFieldLabelLocal_Plex;
2427   dm->ops->computel2diff                   = DMComputeL2Diff_Plex;
2428   dm->ops->computel2gradientdiff           = DMComputeL2GradientDiff_Plex;
2429   dm->ops->computel2fielddiff              = DMComputeL2FieldDiff_Plex;
2430   dm->ops->getneighbors                    = DMGetNeighors_Plex;
2431   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMPlexInsertBoundaryValues_C",DMPlexInsertBoundaryValues_Plex);CHKERRQ(ierr);
2432   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMSetUpGLVisViewer_C",DMSetUpGLVisViewer_Plex);CHKERRQ(ierr);
2433   PetscFunctionReturn(0);
2434 }
2435 
2436 PETSC_INTERN PetscErrorCode DMClone_Plex(DM dm, DM *newdm)
2437 {
2438   DM_Plex        *mesh = (DM_Plex *) dm->data;
2439   PetscErrorCode ierr;
2440 
2441   PetscFunctionBegin;
2442   mesh->refct++;
2443   (*newdm)->data = mesh;
2444   ierr = PetscObjectChangeTypeName((PetscObject) *newdm, DMPLEX);CHKERRQ(ierr);
2445   ierr = DMInitialize_Plex(*newdm);CHKERRQ(ierr);
2446   PetscFunctionReturn(0);
2447 }
2448 
2449 /*MC
2450   DMPLEX = "plex" - A DM object that encapsulates an unstructured mesh, or CW Complex, which can be expressed using a Hasse Diagram.
2451                     In the local representation, Vecs contain all unknowns in the interior and shared boundary. This is
2452                     specified by a PetscSection object. Ownership in the global representation is determined by
2453                     ownership of the underlying DMPlex points. This is specified by another PetscSection object.
2454 
2455   Options Database Keys:
2456 + -dm_view :mesh.tex:ascii_latex - View the mesh in LaTeX/TikZ
2457 . -dm_plex_view_scale <num>      - Scale the TikZ
2458 - -dm_plex_print_fem <num>       - View FEM assembly information, such as element vectors and matrices
2459 
2460 
2461   Level: intermediate
2462 
2463 .seealso: DMType, DMPlexCreate(), DMCreate(), DMSetType()
2464 M*/
2465 
2466 PETSC_EXTERN PetscErrorCode DMCreate_Plex(DM dm)
2467 {
2468   DM_Plex        *mesh;
2469   PetscInt       unit, d;
2470   PetscErrorCode ierr;
2471 
2472   PetscFunctionBegin;
2473   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2474   ierr     = PetscNewLog(dm,&mesh);CHKERRQ(ierr);
2475   dm->dim  = 0;
2476   dm->data = mesh;
2477 
2478   mesh->refct             = 1;
2479   ierr                    = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &mesh->coneSection);CHKERRQ(ierr);
2480   mesh->maxConeSize       = 0;
2481   mesh->cones             = NULL;
2482   mesh->coneOrientations  = NULL;
2483   ierr                    = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &mesh->supportSection);CHKERRQ(ierr);
2484   mesh->maxSupportSize    = 0;
2485   mesh->supports          = NULL;
2486   mesh->refinementUniform = PETSC_TRUE;
2487   mesh->refinementLimit   = -1.0;
2488 
2489   mesh->facesTmp = NULL;
2490 
2491   mesh->tetgenOpts   = NULL;
2492   mesh->triangleOpts = NULL;
2493   ierr = PetscPartitionerCreate(PetscObjectComm((PetscObject)dm), &mesh->partitioner);CHKERRQ(ierr);
2494   mesh->remeshBd     = PETSC_FALSE;
2495 
2496   mesh->subpointMap = NULL;
2497 
2498   for (unit = 0; unit < NUM_PETSC_UNITS; ++unit) mesh->scale[unit] = 1.0;
2499 
2500   mesh->regularRefinement   = PETSC_FALSE;
2501   mesh->depthState          = -1;
2502   mesh->globalVertexNumbers = NULL;
2503   mesh->globalCellNumbers   = NULL;
2504   mesh->anchorSection       = NULL;
2505   mesh->anchorIS            = NULL;
2506   mesh->createanchors       = NULL;
2507   mesh->computeanchormatrix = NULL;
2508   mesh->parentSection       = NULL;
2509   mesh->parents             = NULL;
2510   mesh->childIDs            = NULL;
2511   mesh->childSection        = NULL;
2512   mesh->children            = NULL;
2513   mesh->referenceTree       = NULL;
2514   mesh->getchildsymmetry    = NULL;
2515   for (d = 0; d < 8; ++d) mesh->hybridPointMax[d] = PETSC_DETERMINE;
2516   mesh->vtkCellHeight       = 0;
2517   mesh->useAnchors          = PETSC_FALSE;
2518 
2519   mesh->maxProjectionHeight = 0;
2520 
2521   mesh->printSetValues = PETSC_FALSE;
2522   mesh->printFEM       = 0;
2523   mesh->printTol       = 1.0e-10;
2524 
2525   ierr = DMInitialize_Plex(dm);CHKERRQ(ierr);
2526   PetscFunctionReturn(0);
2527 }
2528 
2529 /*@
2530   DMPlexCreate - Creates a DMPlex object, which encapsulates an unstructured mesh, or CW complex, which can be expressed using a Hasse Diagram.
2531 
2532   Collective on MPI_Comm
2533 
2534   Input Parameter:
2535 . comm - The communicator for the DMPlex object
2536 
2537   Output Parameter:
2538 . mesh  - The DMPlex object
2539 
2540   Level: beginner
2541 
2542 .keywords: DMPlex, create
2543 @*/
2544 PetscErrorCode DMPlexCreate(MPI_Comm comm, DM *mesh)
2545 {
2546   PetscErrorCode ierr;
2547 
2548   PetscFunctionBegin;
2549   PetscValidPointer(mesh,2);
2550   ierr = DMCreate(comm, mesh);CHKERRQ(ierr);
2551   ierr = DMSetType(*mesh, DMPLEX);CHKERRQ(ierr);
2552   PetscFunctionReturn(0);
2553 }
2554 
2555 /*
2556   This takes as input the common mesh generator output, a list of the vertices for each cell, but vertex numbers are global and an SF is built for them
2557 */
2558 /* TODO: invertCells and spaceDim arguments could be added also to to DMPlexCreateFromCellListParallel(), DMPlexBuildFromCellList_Internal() and DMPlexCreateFromCellList() */
2559 PetscErrorCode DMPlexBuildFromCellList_Parallel_Internal(DM dm, PetscInt spaceDim, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, const int cells[], PetscBool invertCells, PetscSF *sfVert)
2560 {
2561   PetscSF         sfPoint;
2562   PetscLayout     vLayout;
2563   PetscHSetI      vhash;
2564   PetscSFNode    *remoteVerticesAdj, *vertexLocal, *vertexOwner, *remoteVertex;
2565   const PetscInt *vrange;
2566   PetscInt        numVerticesAdj, off = 0, *verticesAdj, numVerticesGhost = 0, *localVertex, *cone, c, p, v, g;
2567   PetscMPIInt     rank, size;
2568   PetscErrorCode  ierr;
2569 
2570   PetscFunctionBegin;
2571   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRQ(ierr);
2572   ierr = MPI_Comm_size(PetscObjectComm((PetscObject) dm), &size);CHKERRQ(ierr);
2573   /* Partition vertices */
2574   ierr = PetscLayoutCreate(PetscObjectComm((PetscObject) dm), &vLayout);CHKERRQ(ierr);
2575   ierr = PetscLayoutSetLocalSize(vLayout, numVertices);CHKERRQ(ierr);
2576   ierr = PetscLayoutSetBlockSize(vLayout, 1);CHKERRQ(ierr);
2577   ierr = PetscLayoutSetUp(vLayout);CHKERRQ(ierr);
2578   ierr = PetscLayoutGetRanges(vLayout, &vrange);CHKERRQ(ierr);
2579   /* Count vertices and map them to procs */
2580   ierr = PetscHSetICreate(&vhash);CHKERRQ(ierr);
2581   for (c = 0; c < numCells; ++c) {
2582     for (p = 0; p < numCorners; ++p) {
2583       ierr = PetscHSetIAdd(vhash, cells[c*numCorners+p]);CHKERRQ(ierr);
2584     }
2585   }
2586   ierr = PetscHSetIGetSize(vhash, &numVerticesAdj);CHKERRQ(ierr);
2587   ierr = PetscMalloc1(numVerticesAdj, &verticesAdj);CHKERRQ(ierr);
2588   ierr = PetscHSetIGetElems(vhash, &off, verticesAdj);CHKERRQ(ierr);
2589   ierr = PetscHSetIDestroy(&vhash);CHKERRQ(ierr);
2590   if (off != numVerticesAdj) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid number of local vertices %D should be %D", off, numVerticesAdj);
2591   ierr = PetscSortInt(numVerticesAdj, verticesAdj);CHKERRQ(ierr);
2592   ierr = PetscMalloc1(numVerticesAdj, &remoteVerticesAdj);CHKERRQ(ierr);
2593   for (v = 0; v < numVerticesAdj; ++v) {
2594     const PetscInt gv = verticesAdj[v];
2595     PetscInt       vrank;
2596 
2597     ierr = PetscFindInt(gv, size+1, vrange, &vrank);CHKERRQ(ierr);
2598     vrank = vrank < 0 ? -(vrank+2) : vrank;
2599     remoteVerticesAdj[v].index = gv - vrange[vrank];
2600     remoteVerticesAdj[v].rank  = vrank;
2601   }
2602   /* Create cones */
2603   ierr = DMPlexSetChart(dm, 0, numCells+numVerticesAdj);CHKERRQ(ierr);
2604   for (c = 0; c < numCells; ++c) {ierr = DMPlexSetConeSize(dm, c, numCorners);CHKERRQ(ierr);}
2605   ierr = DMSetUp(dm);CHKERRQ(ierr);
2606   ierr = DMGetWorkArray(dm, numCorners, MPIU_INT, &cone);CHKERRQ(ierr);
2607   for (c = 0; c < numCells; ++c) {
2608     for (p = 0; p < numCorners; ++p) {
2609       const PetscInt gv = cells[c*numCorners+p];
2610       PetscInt       lv;
2611 
2612       ierr = PetscFindInt(gv, numVerticesAdj, verticesAdj, &lv);CHKERRQ(ierr);
2613       if (lv < 0) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Could not find global vertex %D in local connectivity", gv);
2614       cone[p] = lv+numCells;
2615     }
2616     if (invertCells) { ierr = DMPlexInvertCell_Internal(spaceDim, numCorners, cone);CHKERRQ(ierr); }
2617     ierr = DMPlexSetCone(dm, c, cone);CHKERRQ(ierr);
2618   }
2619   ierr = DMRestoreWorkArray(dm, numCorners, MPIU_INT, &cone);CHKERRQ(ierr);
2620   /* Create SF for vertices */
2621   ierr = PetscSFCreate(PetscObjectComm((PetscObject)dm), sfVert);CHKERRQ(ierr);
2622   ierr = PetscObjectSetName((PetscObject) *sfVert, "Vertex Ownership SF");CHKERRQ(ierr);
2623   ierr = PetscSFSetFromOptions(*sfVert);CHKERRQ(ierr);
2624   ierr = PetscSFSetGraph(*sfVert, numVertices, numVerticesAdj, NULL, PETSC_OWN_POINTER, remoteVerticesAdj, PETSC_OWN_POINTER);CHKERRQ(ierr);
2625   ierr = PetscFree(verticesAdj);CHKERRQ(ierr);
2626   /* Build pointSF */
2627   ierr = PetscMalloc2(numVerticesAdj, &vertexLocal, numVertices, &vertexOwner);CHKERRQ(ierr);
2628   for (v = 0; v < numVerticesAdj; ++v) {vertexLocal[v].index = v+numCells; vertexLocal[v].rank = rank;}
2629   for (v = 0; v < numVertices;    ++v) {vertexOwner[v].index = -1;         vertexOwner[v].rank = -1;}
2630   ierr = PetscSFReduceBegin(*sfVert, MPIU_2INT, vertexLocal, vertexOwner, MPI_MAXLOC);CHKERRQ(ierr);
2631   ierr = PetscSFReduceEnd(*sfVert, MPIU_2INT, vertexLocal, vertexOwner, MPI_MAXLOC);CHKERRQ(ierr);
2632   for (v = 0; v < numVertices;    ++v) if (vertexOwner[v].rank < 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Global vertex %d on rank %d was unclaimed", v + vrange[rank], rank);
2633   ierr = PetscSFBcastBegin(*sfVert, MPIU_2INT, vertexOwner, vertexLocal);CHKERRQ(ierr);
2634   ierr = PetscSFBcastEnd(*sfVert, MPIU_2INT, vertexOwner, vertexLocal);CHKERRQ(ierr);
2635   for (v = 0; v < numVerticesAdj; ++v) if (vertexLocal[v].rank != rank) ++numVerticesGhost;
2636   ierr = PetscMalloc1(numVerticesGhost, &localVertex);CHKERRQ(ierr);
2637   ierr = PetscMalloc1(numVerticesGhost, &remoteVertex);CHKERRQ(ierr);
2638   for (v = 0, g = 0; v < numVerticesAdj; ++v) {
2639     if (vertexLocal[v].rank != rank) {
2640       localVertex[g]        = v+numCells;
2641       remoteVertex[g].index = vertexLocal[v].index;
2642       remoteVertex[g].rank  = vertexLocal[v].rank;
2643       ++g;
2644     }
2645   }
2646   ierr = PetscFree2(vertexLocal, vertexOwner);CHKERRQ(ierr);
2647   if (g != numVerticesGhost) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid number of vertex ghosts %D should be %D", g, numVerticesGhost);
2648   ierr = DMGetPointSF(dm, &sfPoint);CHKERRQ(ierr);
2649   ierr = PetscObjectSetName((PetscObject) sfPoint, "point SF");CHKERRQ(ierr);
2650   ierr = PetscSFSetGraph(sfPoint, numCells+numVerticesAdj, numVerticesGhost, localVertex, PETSC_OWN_POINTER, remoteVertex, PETSC_OWN_POINTER);CHKERRQ(ierr);
2651   ierr = PetscLayoutDestroy(&vLayout);CHKERRQ(ierr);
2652   /* Fill in the rest of the topology structure */
2653   ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
2654   ierr = DMPlexStratify(dm);CHKERRQ(ierr);
2655   PetscFunctionReturn(0);
2656 }
2657 
2658 /*
2659   This takes as input the coordinates for each owned vertex
2660 */
2661 PetscErrorCode DMPlexBuildCoordinates_Parallel_Internal(DM dm, PetscInt spaceDim, PetscInt numCells, PetscInt numV, PetscSF sfVert, const PetscReal vertexCoords[])
2662 {
2663   PetscSection   coordSection;
2664   Vec            coordinates;
2665   PetscScalar   *coords;
2666   PetscInt       numVertices, numVerticesAdj, coordSize, v;
2667   PetscErrorCode ierr;
2668 
2669   PetscFunctionBegin;
2670   ierr = DMSetCoordinateDim(dm, spaceDim);CHKERRQ(ierr);
2671   ierr = PetscSFGetGraph(sfVert, &numVertices, &numVerticesAdj, NULL, NULL);CHKERRQ(ierr);
2672   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
2673   ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
2674   ierr = PetscSectionSetFieldComponents(coordSection, 0, spaceDim);CHKERRQ(ierr);
2675   ierr = PetscSectionSetChart(coordSection, numCells, numCells + numVerticesAdj);CHKERRQ(ierr);
2676   for (v = numCells; v < numCells+numVerticesAdj; ++v) {
2677     ierr = PetscSectionSetDof(coordSection, v, spaceDim);CHKERRQ(ierr);
2678     ierr = PetscSectionSetFieldDof(coordSection, v, 0, spaceDim);CHKERRQ(ierr);
2679   }
2680   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
2681   ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
2682   ierr = VecCreate(PetscObjectComm((PetscObject)dm), &coordinates);CHKERRQ(ierr);
2683   ierr = VecSetBlockSize(coordinates, spaceDim);CHKERRQ(ierr);
2684   ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
2685   ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
2686   ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
2687   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
2688   {
2689     MPI_Datatype coordtype;
2690 
2691     /* Need a temp buffer for coords if we have complex/single */
2692     ierr = MPI_Type_contiguous(spaceDim, MPIU_SCALAR, &coordtype);CHKERRQ(ierr);
2693     ierr = MPI_Type_commit(&coordtype);CHKERRQ(ierr);
2694 #if defined(PETSC_USE_COMPLEX)
2695     {
2696     PetscScalar *svertexCoords;
2697     PetscInt    i;
2698     ierr = PetscMalloc1(numV*spaceDim,&svertexCoords);CHKERRQ(ierr);
2699     for (i=0; i<numV*spaceDim; i++) svertexCoords[i] = vertexCoords[i];
2700     ierr = PetscSFBcastBegin(sfVert, coordtype, svertexCoords, coords);CHKERRQ(ierr);
2701     ierr = PetscSFBcastEnd(sfVert, coordtype, svertexCoords, coords);CHKERRQ(ierr);
2702     ierr = PetscFree(svertexCoords);CHKERRQ(ierr);
2703     }
2704 #else
2705     ierr = PetscSFBcastBegin(sfVert, coordtype, vertexCoords, coords);CHKERRQ(ierr);
2706     ierr = PetscSFBcastEnd(sfVert, coordtype, vertexCoords, coords);CHKERRQ(ierr);
2707 #endif
2708     ierr = MPI_Type_free(&coordtype);CHKERRQ(ierr);
2709   }
2710   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
2711   ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
2712   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
2713   PetscFunctionReturn(0);
2714 }
2715 
2716 /*@C
2717   DMPlexCreateFromCellListParallel - This takes as input common mesh generator output, a list of the vertices for each cell, and produces a DM
2718 
2719   Input Parameters:
2720 + comm - The communicator
2721 . dim - The topological dimension of the mesh
2722 . numCells - The number of cells owned by this process
2723 . numVertices - The number of vertices owned by this process
2724 . numCorners - The number of vertices for each cell
2725 . interpolate - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
2726 . cells - An array of numCells*numCorners numbers, the global vertex numbers for each cell
2727 . spaceDim - The spatial dimension used for coordinates
2728 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
2729 
2730   Output Parameter:
2731 + dm - The DM
2732 - vertexSF - Optional, SF describing complete vertex ownership
2733 
2734   Note: Two triangles sharing a face
2735 $
2736 $        2
2737 $      / | \
2738 $     /  |  \
2739 $    /   |   \
2740 $   0  0 | 1  3
2741 $    \   |   /
2742 $     \  |  /
2743 $      \ | /
2744 $        1
2745 would have input
2746 $  numCells = 2, numVertices = 4
2747 $  cells = [0 1 2  1 3 2]
2748 $
2749 which would result in the DMPlex
2750 $
2751 $        4
2752 $      / | \
2753 $     /  |  \
2754 $    /   |   \
2755 $   2  0 | 1  5
2756 $    \   |   /
2757 $     \  |  /
2758 $      \ | /
2759 $        3
2760 
2761   Level: beginner
2762 
2763 .seealso: DMPlexCreateFromCellList(), DMPlexCreateFromDAG(), DMPlexCreate()
2764 @*/
2765 PetscErrorCode DMPlexCreateFromCellListParallel(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, PetscBool interpolate, const int cells[], PetscInt spaceDim, const PetscReal vertexCoords[], PetscSF *vertexSF, DM *dm)
2766 {
2767   PetscSF        sfVert;
2768   PetscErrorCode ierr;
2769 
2770   PetscFunctionBegin;
2771   ierr = DMCreate(comm, dm);CHKERRQ(ierr);
2772   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
2773   PetscValidLogicalCollectiveInt(*dm, dim, 2);
2774   PetscValidLogicalCollectiveInt(*dm, spaceDim, 8);
2775   ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
2776   ierr = DMPlexBuildFromCellList_Parallel_Internal(*dm, spaceDim, numCells, numVertices, numCorners, cells, PETSC_FALSE, &sfVert);CHKERRQ(ierr);
2777   if (interpolate) {
2778     DM idm;
2779 
2780     ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
2781     ierr = DMDestroy(dm);CHKERRQ(ierr);
2782     *dm  = idm;
2783   }
2784   ierr = DMPlexBuildCoordinates_Parallel_Internal(*dm, spaceDim, numCells, numVertices, sfVert, vertexCoords);CHKERRQ(ierr);
2785   if (vertexSF) *vertexSF = sfVert;
2786   else {ierr = PetscSFDestroy(&sfVert);CHKERRQ(ierr);}
2787   PetscFunctionReturn(0);
2788 }
2789 
2790 /*
2791   This takes as input the common mesh generator output, a list of the vertices for each cell
2792 */
2793 PetscErrorCode DMPlexBuildFromCellList_Internal(DM dm, PetscInt spaceDim, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, const int cells[], PetscBool invertCells)
2794 {
2795   PetscInt      *cone, c, p;
2796   PetscErrorCode ierr;
2797 
2798   PetscFunctionBegin;
2799   ierr = DMPlexSetChart(dm, 0, numCells+numVertices);CHKERRQ(ierr);
2800   for (c = 0; c < numCells; ++c) {
2801     ierr = DMPlexSetConeSize(dm, c, numCorners);CHKERRQ(ierr);
2802   }
2803   ierr = DMSetUp(dm);CHKERRQ(ierr);
2804   ierr = DMGetWorkArray(dm, numCorners, MPIU_INT, &cone);CHKERRQ(ierr);
2805   for (c = 0; c < numCells; ++c) {
2806     for (p = 0; p < numCorners; ++p) {
2807       cone[p] = cells[c*numCorners+p]+numCells;
2808     }
2809     if (invertCells) { ierr = DMPlexInvertCell_Internal(spaceDim, numCorners, cone);CHKERRQ(ierr); }
2810     ierr = DMPlexSetCone(dm, c, cone);CHKERRQ(ierr);
2811   }
2812   ierr = DMRestoreWorkArray(dm, numCorners, MPIU_INT, &cone);CHKERRQ(ierr);
2813   ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
2814   ierr = DMPlexStratify(dm);CHKERRQ(ierr);
2815   PetscFunctionReturn(0);
2816 }
2817 
2818 /*
2819   This takes as input the coordinates for each vertex
2820 */
2821 PetscErrorCode DMPlexBuildCoordinates_Internal(DM dm, PetscInt spaceDim, PetscInt numCells, PetscInt numVertices, const double vertexCoords[])
2822 {
2823   PetscSection   coordSection;
2824   Vec            coordinates;
2825   DM             cdm;
2826   PetscScalar   *coords;
2827   PetscInt       v, d;
2828   PetscErrorCode ierr;
2829 
2830   PetscFunctionBegin;
2831   ierr = DMSetCoordinateDim(dm, spaceDim);CHKERRQ(ierr);
2832   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
2833   ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
2834   ierr = PetscSectionSetFieldComponents(coordSection, 0, spaceDim);CHKERRQ(ierr);
2835   ierr = PetscSectionSetChart(coordSection, numCells, numCells + numVertices);CHKERRQ(ierr);
2836   for (v = numCells; v < numCells+numVertices; ++v) {
2837     ierr = PetscSectionSetDof(coordSection, v, spaceDim);CHKERRQ(ierr);
2838     ierr = PetscSectionSetFieldDof(coordSection, v, 0, spaceDim);CHKERRQ(ierr);
2839   }
2840   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
2841 
2842   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
2843   ierr = DMCreateLocalVector(cdm, &coordinates);CHKERRQ(ierr);
2844   ierr = VecSetBlockSize(coordinates, spaceDim);CHKERRQ(ierr);
2845   ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
2846   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
2847   for (v = 0; v < numVertices; ++v) {
2848     for (d = 0; d < spaceDim; ++d) {
2849       coords[v*spaceDim+d] = vertexCoords[v*spaceDim+d];
2850     }
2851   }
2852   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
2853   ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
2854   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
2855   PetscFunctionReturn(0);
2856 }
2857 
2858 /*@C
2859   DMPlexCreateFromCellList - This takes as input common mesh generator output, a list of the vertices for each cell, and produces a DM
2860 
2861   Input Parameters:
2862 + comm - The communicator
2863 . dim - The topological dimension of the mesh
2864 . numCells - The number of cells
2865 . numVertices - The number of vertices
2866 . numCorners - The number of vertices for each cell
2867 . interpolate - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
2868 . cells - An array of numCells*numCorners numbers, the vertices for each cell
2869 . spaceDim - The spatial dimension used for coordinates
2870 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
2871 
2872   Output Parameter:
2873 . dm - The DM
2874 
2875   Note: Two triangles sharing a face
2876 $
2877 $        2
2878 $      / | \
2879 $     /  |  \
2880 $    /   |   \
2881 $   0  0 | 1  3
2882 $    \   |   /
2883 $     \  |  /
2884 $      \ | /
2885 $        1
2886 would have input
2887 $  numCells = 2, numVertices = 4
2888 $  cells = [0 1 2  1 3 2]
2889 $
2890 which would result in the DMPlex
2891 $
2892 $        4
2893 $      / | \
2894 $     /  |  \
2895 $    /   |   \
2896 $   2  0 | 1  5
2897 $    \   |   /
2898 $     \  |  /
2899 $      \ | /
2900 $        3
2901 
2902   Level: beginner
2903 
2904 .seealso: DMPlexCreateFromDAG(), DMPlexCreate()
2905 @*/
2906 PetscErrorCode DMPlexCreateFromCellList(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, PetscBool interpolate, const int cells[], PetscInt spaceDim, const double vertexCoords[], DM *dm)
2907 {
2908   PetscErrorCode ierr;
2909 
2910   PetscFunctionBegin;
2911   ierr = DMCreate(comm, dm);CHKERRQ(ierr);
2912   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
2913   ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
2914   ierr = DMPlexBuildFromCellList_Internal(*dm, spaceDim, numCells, numVertices, numCorners, cells, PETSC_FALSE);CHKERRQ(ierr);
2915   if (interpolate) {
2916     DM idm;
2917 
2918     ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
2919     ierr = DMDestroy(dm);CHKERRQ(ierr);
2920     *dm  = idm;
2921   }
2922   ierr = DMPlexBuildCoordinates_Internal(*dm, spaceDim, numCells, numVertices, vertexCoords);CHKERRQ(ierr);
2923   PetscFunctionReturn(0);
2924 }
2925 
2926 /*@
2927   DMPlexCreateFromDAG - This takes as input the adjacency-list representation of the Directed Acyclic Graph (Hasse Diagram) encoding a mesh, and produces a DM
2928 
2929   Input Parameters:
2930 + dm - The empty DM object, usually from DMCreate() and DMSetDimension()
2931 . depth - The depth of the DAG
2932 . numPoints - Array of size depth + 1 containing the number of points at each depth
2933 . coneSize - The cone size of each point
2934 . cones - The concatenation of the cone points for each point, the cone list must be oriented correctly for each point
2935 . coneOrientations - The orientation of each cone point
2936 - vertexCoords - An array of numPoints[0]*spacedim numbers representing the coordinates of each vertex, with spacedim the value set via DMSetCoordinateDim()
2937 
2938   Output Parameter:
2939 . dm - The DM
2940 
2941   Note: Two triangles sharing a face would have input
2942 $  depth = 1, numPoints = [4 2], coneSize = [3 3 0 0 0 0]
2943 $  cones = [2 3 4  3 5 4], coneOrientations = [0 0 0  0 0 0]
2944 $ vertexCoords = [-1.0 0.0  0.0 -1.0  0.0 1.0  1.0 0.0]
2945 $
2946 which would result in the DMPlex
2947 $
2948 $        4
2949 $      / | \
2950 $     /  |  \
2951 $    /   |   \
2952 $   2  0 | 1  5
2953 $    \   |   /
2954 $     \  |  /
2955 $      \ | /
2956 $        3
2957 $
2958 $ Notice that all points are numbered consecutively, unlikely DMPlexCreateFromCellList()
2959 
2960   Level: advanced
2961 
2962 .seealso: DMPlexCreateFromCellList(), DMPlexCreate()
2963 @*/
2964 PetscErrorCode DMPlexCreateFromDAG(DM dm, PetscInt depth, const PetscInt numPoints[], const PetscInt coneSize[], const PetscInt cones[], const PetscInt coneOrientations[], const PetscScalar vertexCoords[])
2965 {
2966   Vec            coordinates;
2967   PetscSection   coordSection;
2968   PetscScalar    *coords;
2969   PetscInt       coordSize, firstVertex = -1, pStart = 0, pEnd = 0, p, v, dim, dimEmbed, d, off;
2970   PetscErrorCode ierr;
2971 
2972   PetscFunctionBegin;
2973   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
2974   ierr = DMGetCoordinateDim(dm, &dimEmbed);CHKERRQ(ierr);
2975   if (dimEmbed < dim) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Embedding dimension %d cannot be less than intrinsic dimension %d",dimEmbed,dim);
2976   for (d = 0; d <= depth; ++d) pEnd += numPoints[d];
2977   ierr = DMPlexSetChart(dm, pStart, pEnd);CHKERRQ(ierr);
2978   for (p = pStart; p < pEnd; ++p) {
2979     ierr = DMPlexSetConeSize(dm, p, coneSize[p-pStart]);CHKERRQ(ierr);
2980     if (firstVertex < 0 && !coneSize[p - pStart]) {
2981       firstVertex = p - pStart;
2982     }
2983   }
2984   if (firstVertex < 0 && numPoints[0]) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONG,"Expected %d vertices but could not find any", numPoints[0]);
2985   ierr = DMSetUp(dm);CHKERRQ(ierr); /* Allocate space for cones */
2986   for (p = pStart, off = 0; p < pEnd; off += coneSize[p-pStart], ++p) {
2987     ierr = DMPlexSetCone(dm, p, &cones[off]);CHKERRQ(ierr);
2988     ierr = DMPlexSetConeOrientation(dm, p, &coneOrientations[off]);CHKERRQ(ierr);
2989   }
2990   ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
2991   ierr = DMPlexStratify(dm);CHKERRQ(ierr);
2992   /* Build coordinates */
2993   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
2994   ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
2995   ierr = PetscSectionSetFieldComponents(coordSection, 0, dimEmbed);CHKERRQ(ierr);
2996   ierr = PetscSectionSetChart(coordSection, firstVertex, firstVertex+numPoints[0]);CHKERRQ(ierr);
2997   for (v = firstVertex; v < firstVertex+numPoints[0]; ++v) {
2998     ierr = PetscSectionSetDof(coordSection, v, dimEmbed);CHKERRQ(ierr);
2999     ierr = PetscSectionSetFieldDof(coordSection, v, 0, dimEmbed);CHKERRQ(ierr);
3000   }
3001   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
3002   ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
3003   ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
3004   ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
3005   ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
3006   ierr = VecSetBlockSize(coordinates, dimEmbed);CHKERRQ(ierr);
3007   ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
3008   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
3009   for (v = 0; v < numPoints[0]; ++v) {
3010     PetscInt off;
3011 
3012     ierr = PetscSectionGetOffset(coordSection, v+firstVertex, &off);CHKERRQ(ierr);
3013     for (d = 0; d < dimEmbed; ++d) {
3014       coords[off+d] = vertexCoords[v*dimEmbed+d];
3015     }
3016   }
3017   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
3018   ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
3019   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
3020   PetscFunctionReturn(0);
3021 }
3022 
3023 /*@C
3024   DMPlexCreateCellVertexFromFile - Create a DMPlex mesh from a simple cell-vertex file.
3025 
3026 + comm        - The MPI communicator
3027 . filename    - Name of the .dat file
3028 - interpolate - Create faces and edges in the mesh
3029 
3030   Output Parameter:
3031 . dm  - The DM object representing the mesh
3032 
3033   Note: The format is the simplest possible:
3034 $ Ne
3035 $ v0 v1 ... vk
3036 $ Nv
3037 $ x y z marker
3038 
3039   Level: beginner
3040 
3041 .seealso: DMPlexCreateFromFile(), DMPlexCreateMedFromFile(), DMPlexCreateGmsh(), DMPlexCreate()
3042 @*/
3043 PetscErrorCode DMPlexCreateCellVertexFromFile(MPI_Comm comm, const char filename[], PetscBool interpolate, DM *dm)
3044 {
3045   DMLabel         marker;
3046   PetscViewer     viewer;
3047   Vec             coordinates;
3048   PetscSection    coordSection;
3049   PetscScalar    *coords;
3050   char            line[PETSC_MAX_PATH_LEN];
3051   PetscInt        dim = 3, cdim = 3, coordSize, v, c, d;
3052   PetscMPIInt     rank;
3053   int             snum, Nv, Nc;
3054   PetscErrorCode  ierr;
3055 
3056   PetscFunctionBegin;
3057   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
3058   ierr = PetscViewerCreate(comm, &viewer);CHKERRQ(ierr);
3059   ierr = PetscViewerSetType(viewer, PETSCVIEWERASCII);CHKERRQ(ierr);
3060   ierr = PetscViewerFileSetMode(viewer, FILE_MODE_READ);CHKERRQ(ierr);
3061   ierr = PetscViewerFileSetName(viewer, filename);CHKERRQ(ierr);
3062   if (!rank) {
3063     ierr = PetscViewerRead(viewer, line, 2, NULL, PETSC_STRING);CHKERRQ(ierr);
3064     snum = sscanf(line, "%d %d", &Nc, &Nv);
3065     if (snum != 2) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
3066   } else {
3067     Nc = Nv = 0;
3068   }
3069   ierr = DMCreate(comm, dm);CHKERRQ(ierr);
3070   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
3071   ierr = DMPlexSetChart(*dm, 0, Nc+Nv);CHKERRQ(ierr);
3072   ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
3073   ierr = DMSetCoordinateDim(*dm, cdim);CHKERRQ(ierr);
3074   /* Read topology */
3075   if (!rank) {
3076     PetscInt cone[8], corners = 8;
3077     int      vbuf[8], v;
3078 
3079     for (c = 0; c < Nc; ++c) {ierr = DMPlexSetConeSize(*dm, c, corners);CHKERRQ(ierr);}
3080     ierr = DMSetUp(*dm);CHKERRQ(ierr);
3081     for (c = 0; c < Nc; ++c) {
3082       ierr = PetscViewerRead(viewer, line, corners, NULL, PETSC_STRING);CHKERRQ(ierr);
3083       snum = sscanf(line, "%d %d %d %d %d %d %d %d", &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3], &vbuf[4], &vbuf[5], &vbuf[6], &vbuf[7]);
3084       if (snum != corners) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
3085       for (v = 0; v < corners; ++v) cone[v] = vbuf[v] + Nc;
3086       /* Hexahedra are inverted */
3087       {
3088         PetscInt tmp = cone[1];
3089         cone[1] = cone[3];
3090         cone[3] = tmp;
3091       }
3092       ierr = DMPlexSetCone(*dm, c, cone);CHKERRQ(ierr);
3093     }
3094   }
3095   ierr = DMPlexSymmetrize(*dm);CHKERRQ(ierr);
3096   ierr = DMPlexStratify(*dm);CHKERRQ(ierr);
3097   /* Read coordinates */
3098   ierr = DMGetCoordinateSection(*dm, &coordSection);CHKERRQ(ierr);
3099   ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
3100   ierr = PetscSectionSetFieldComponents(coordSection, 0, cdim);CHKERRQ(ierr);
3101   ierr = PetscSectionSetChart(coordSection, Nc, Nc + Nv);CHKERRQ(ierr);
3102   for (v = Nc; v < Nc+Nv; ++v) {
3103     ierr = PetscSectionSetDof(coordSection, v, cdim);CHKERRQ(ierr);
3104     ierr = PetscSectionSetFieldDof(coordSection, v, 0, cdim);CHKERRQ(ierr);
3105   }
3106   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
3107   ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
3108   ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
3109   ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
3110   ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
3111   ierr = VecSetBlockSize(coordinates, cdim);CHKERRQ(ierr);
3112   ierr = VecSetType(coordinates, VECSTANDARD);CHKERRQ(ierr);
3113   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
3114   if (!rank) {
3115     double x[3];
3116     int    val;
3117 
3118     ierr = DMCreateLabel(*dm, "marker");CHKERRQ(ierr);
3119     ierr = DMGetLabel(*dm, "marker", &marker);CHKERRQ(ierr);
3120     for (v = 0; v < Nv; ++v) {
3121       ierr = PetscViewerRead(viewer, line, 4, NULL, PETSC_STRING);CHKERRQ(ierr);
3122       snum = sscanf(line, "%lg %lg %lg %d", &x[0], &x[1], &x[2], &val);
3123       if (snum != 4) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
3124       for (d = 0; d < cdim; ++d) coords[v*cdim+d] = x[d];
3125       if (val) {ierr = DMLabelSetValue(marker, v+Nc, val);CHKERRQ(ierr);}
3126     }
3127   }
3128   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
3129   ierr = DMSetCoordinatesLocal(*dm, coordinates);CHKERRQ(ierr);
3130   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
3131   ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
3132   if (interpolate) {
3133     DM      idm;
3134     DMLabel bdlabel;
3135 
3136     ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
3137     ierr = DMDestroy(dm);CHKERRQ(ierr);
3138     *dm  = idm;
3139 
3140     ierr = DMGetLabel(*dm, "marker", &bdlabel);CHKERRQ(ierr);
3141     ierr = DMPlexMarkBoundaryFaces(*dm, PETSC_DETERMINE, bdlabel);CHKERRQ(ierr);
3142     ierr = DMPlexLabelComplete(*dm, bdlabel);CHKERRQ(ierr);
3143   }
3144   PetscFunctionReturn(0);
3145 }
3146 
3147 /*@C
3148   DMPlexCreateFromFile - This takes a filename and produces a DM
3149 
3150   Input Parameters:
3151 + comm - The communicator
3152 . filename - A file name
3153 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
3154 
3155   Output Parameter:
3156 . dm - The DM
3157 
3158   Options Database Keys:
3159 . -dm_plex_create_from_hdf5_xdmf - use the PETSC_VIEWER_HDF5_XDMF format for reading HDF5
3160 
3161   Level: beginner
3162 
3163 .seealso: DMPlexCreateFromDAG(), DMPlexCreateFromCellList(), DMPlexCreate()
3164 @*/
3165 PetscErrorCode DMPlexCreateFromFile(MPI_Comm comm, const char filename[], PetscBool interpolate, DM *dm)
3166 {
3167   const char    *extGmsh    = ".msh";
3168   const char    *extCGNS    = ".cgns";
3169   const char    *extExodus  = ".exo";
3170   const char    *extGenesis = ".gen";
3171   const char    *extFluent  = ".cas";
3172   const char    *extHDF5    = ".h5";
3173   const char    *extMed     = ".med";
3174   const char    *extPLY     = ".ply";
3175   const char    *extCV      = ".dat";
3176   size_t         len;
3177   PetscBool      isGmsh, isCGNS, isExodus, isGenesis, isFluent, isHDF5, isMed, isPLY, isCV;
3178   PetscMPIInt    rank;
3179   PetscErrorCode ierr;
3180 
3181   PetscFunctionBegin;
3182   PetscValidPointer(filename, 2);
3183   PetscValidPointer(dm, 4);
3184   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
3185   ierr = PetscStrlen(filename, &len);CHKERRQ(ierr);
3186   if (!len) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Filename must be a valid path");
3187   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)], extGmsh,    4, &isGmsh);CHKERRQ(ierr);
3188   ierr = PetscStrncmp(&filename[PetscMax(0,len-5)], extCGNS,    5, &isCGNS);CHKERRQ(ierr);
3189   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)], extExodus,  4, &isExodus);CHKERRQ(ierr);
3190   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)], extGenesis, 4, &isGenesis);CHKERRQ(ierr);
3191   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)], extFluent,  4, &isFluent);CHKERRQ(ierr);
3192   ierr = PetscStrncmp(&filename[PetscMax(0,len-3)], extHDF5,    3, &isHDF5);CHKERRQ(ierr);
3193   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)], extMed,     4, &isMed);CHKERRQ(ierr);
3194   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)], extPLY,     4, &isPLY);CHKERRQ(ierr);
3195   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)], extCV,      4, &isCV);CHKERRQ(ierr);
3196   if (isGmsh) {
3197     ierr = DMPlexCreateGmshFromFile(comm, filename, interpolate, dm);CHKERRQ(ierr);
3198   } else if (isCGNS) {
3199     ierr = DMPlexCreateCGNSFromFile(comm, filename, interpolate, dm);CHKERRQ(ierr);
3200   } else if (isExodus || isGenesis) {
3201     ierr = DMPlexCreateExodusFromFile(comm, filename, interpolate, dm);CHKERRQ(ierr);
3202   } else if (isFluent) {
3203     ierr = DMPlexCreateFluentFromFile(comm, filename, interpolate, dm);CHKERRQ(ierr);
3204   } else if (isHDF5) {
3205     PetscBool      load_hdf5_xdmf = PETSC_FALSE;
3206     PetscViewer viewer;
3207 
3208     ierr = PetscOptionsGetBool(NULL, NULL, "-dm_plex_create_from_hdf5_xdmf", &load_hdf5_xdmf, NULL);CHKERRQ(ierr);
3209     ierr = PetscViewerCreate(comm, &viewer);CHKERRQ(ierr);
3210     ierr = PetscViewerSetType(viewer, PETSCVIEWERHDF5);CHKERRQ(ierr);
3211     ierr = PetscViewerFileSetMode(viewer, FILE_MODE_READ);CHKERRQ(ierr);
3212     ierr = PetscViewerFileSetName(viewer, filename);CHKERRQ(ierr);
3213     ierr = DMCreate(comm, dm);CHKERRQ(ierr);
3214     ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
3215     if (load_hdf5_xdmf) {ierr = PetscViewerPushFormat(viewer, PETSC_VIEWER_HDF5_XDMF);CHKERRQ(ierr);}
3216     ierr = DMLoad(*dm, viewer);CHKERRQ(ierr);
3217     if (load_hdf5_xdmf) {ierr = PetscViewerPopFormat(viewer);CHKERRQ(ierr);}
3218     ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
3219 
3220     if (interpolate) {
3221       DM idm;
3222 
3223       ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
3224       ierr = DMDestroy(dm);CHKERRQ(ierr);
3225       *dm  = idm;
3226     }
3227   } else if (isMed) {
3228     ierr = DMPlexCreateMedFromFile(comm, filename, interpolate, dm);CHKERRQ(ierr);
3229   } else if (isPLY) {
3230     ierr = DMPlexCreatePLYFromFile(comm, filename, interpolate, dm);CHKERRQ(ierr);
3231   } else if (isCV) {
3232     ierr = DMPlexCreateCellVertexFromFile(comm, filename, interpolate, dm);CHKERRQ(ierr);
3233   } else SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cannot load file %s: unrecognized extension", filename);
3234   PetscFunctionReturn(0);
3235 }
3236 
3237 /*@
3238   DMPlexCreateReferenceCell - Create a DMPLEX with the appropriate FEM reference cell
3239 
3240   Collective on comm
3241 
3242   Input Parameters:
3243 + comm    - The communicator
3244 . dim     - The spatial dimension
3245 - simplex - Flag for simplex, otherwise use a tensor-product cell
3246 
3247   Output Parameter:
3248 . refdm - The reference cell
3249 
3250   Level: intermediate
3251 
3252 .keywords: reference cell
3253 .seealso:
3254 @*/
3255 PetscErrorCode DMPlexCreateReferenceCell(MPI_Comm comm, PetscInt dim, PetscBool simplex, DM *refdm)
3256 {
3257   DM             rdm;
3258   Vec            coords;
3259   PetscErrorCode ierr;
3260 
3261   PetscFunctionBeginUser;
3262   ierr = DMCreate(comm, &rdm);CHKERRQ(ierr);
3263   ierr = DMSetType(rdm, DMPLEX);CHKERRQ(ierr);
3264   ierr = DMSetDimension(rdm, dim);CHKERRQ(ierr);
3265   switch (dim) {
3266   case 0:
3267   {
3268     PetscInt    numPoints[1]        = {1};
3269     PetscInt    coneSize[1]         = {0};
3270     PetscInt    cones[1]            = {0};
3271     PetscInt    coneOrientations[1] = {0};
3272     PetscScalar vertexCoords[1]     = {0.0};
3273 
3274     ierr = DMPlexCreateFromDAG(rdm, 0, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
3275   }
3276   break;
3277   case 1:
3278   {
3279     PetscInt    numPoints[2]        = {2, 1};
3280     PetscInt    coneSize[3]         = {2, 0, 0};
3281     PetscInt    cones[2]            = {1, 2};
3282     PetscInt    coneOrientations[2] = {0, 0};
3283     PetscScalar vertexCoords[2]     = {-1.0,  1.0};
3284 
3285     ierr = DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
3286   }
3287   break;
3288   case 2:
3289     if (simplex) {
3290       PetscInt    numPoints[2]        = {3, 1};
3291       PetscInt    coneSize[4]         = {3, 0, 0, 0};
3292       PetscInt    cones[3]            = {1, 2, 3};
3293       PetscInt    coneOrientations[3] = {0, 0, 0};
3294       PetscScalar vertexCoords[6]     = {-1.0, -1.0,  1.0, -1.0,  -1.0, 1.0};
3295 
3296       ierr = DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
3297     } else {
3298       PetscInt    numPoints[2]        = {4, 1};
3299       PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
3300       PetscInt    cones[4]            = {1, 2, 3, 4};
3301       PetscInt    coneOrientations[4] = {0, 0, 0, 0};
3302       PetscScalar vertexCoords[8]     = {-1.0, -1.0,  1.0, -1.0,  1.0, 1.0,  -1.0, 1.0};
3303 
3304       ierr = DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
3305     }
3306   break;
3307   case 3:
3308     if (simplex) {
3309       PetscInt    numPoints[2]        = {4, 1};
3310       PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
3311       PetscInt    cones[4]            = {1, 3, 2, 4};
3312       PetscInt    coneOrientations[4] = {0, 0, 0, 0};
3313       PetscScalar vertexCoords[12]    = {-1.0, -1.0, -1.0,  1.0, -1.0, -1.0,  -1.0, 1.0, -1.0,  -1.0, -1.0, 1.0};
3314 
3315       ierr = DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
3316     } else {
3317       PetscInt    numPoints[2]        = {8, 1};
3318       PetscInt    coneSize[9]         = {8, 0, 0, 0, 0, 0, 0, 0, 0};
3319       PetscInt    cones[8]            = {1, 4, 3, 2, 5, 6, 7, 8};
3320       PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
3321       PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0,  1.0, -1.0, -1.0,  1.0, 1.0, -1.0,  -1.0, 1.0, -1.0,
3322                                          -1.0, -1.0,  1.0,  1.0, -1.0,  1.0,  1.0, 1.0,  1.0,  -1.0, 1.0,  1.0};
3323 
3324       ierr = DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
3325     }
3326   break;
3327   default:
3328     SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Cannot create reference cell for dimension %d", dim);
3329   }
3330   ierr = DMPlexInterpolate(rdm, refdm);CHKERRQ(ierr);
3331   if (rdm->coordinateDM) {
3332     DM           ncdm;
3333     PetscSection cs;
3334     PetscInt     pEnd = -1;
3335 
3336     ierr = DMGetSection(rdm->coordinateDM, &cs);CHKERRQ(ierr);
3337     if (cs) {ierr = PetscSectionGetChart(cs, NULL, &pEnd);CHKERRQ(ierr);}
3338     if (pEnd >= 0) {
3339       ierr = DMClone(rdm->coordinateDM, &ncdm);CHKERRQ(ierr);
3340       ierr = DMSetSection(ncdm, cs);CHKERRQ(ierr);
3341       ierr = DMSetCoordinateDM(*refdm, ncdm);CHKERRQ(ierr);
3342       ierr = DMDestroy(&ncdm);CHKERRQ(ierr);
3343     }
3344   }
3345   ierr = DMGetCoordinatesLocal(rdm, &coords);CHKERRQ(ierr);
3346   if (coords) {
3347     ierr = DMSetCoordinatesLocal(*refdm, coords);CHKERRQ(ierr);
3348   } else {
3349     ierr = DMGetCoordinates(rdm, &coords);CHKERRQ(ierr);
3350     if (coords) {ierr = DMSetCoordinates(*refdm, coords);CHKERRQ(ierr);}
3351   }
3352   ierr = DMDestroy(&rdm);CHKERRQ(ierr);
3353   PetscFunctionReturn(0);
3354 }
3355