xref: /petsc/src/dm/impls/plex/plexcreate.c (revision 87050cfd2c0579ea516b8fcfef1cf8502bc01bdc)
1 #define PETSCDM_DLL
2 #include <petsc/private/dmpleximpl.h>    /*I   "petscdmplex.h"   I*/
3 #include <petsc/private/hashseti.h>          /*I   "petscdmplex.h"   I*/
4 #include <petscsf.h>
5 
6 /*@
7   DMPlexCreateDoublet - Creates a mesh of two cells of the specified type, optionally with later refinement.
8 
9   Collective on MPI_Comm
10 
11   Input Parameters:
12 + comm - The communicator for the DM object
13 . dim - The spatial dimension
14 . simplex - Flag for simplicial cells, otherwise they are tensor product cells
15 . interpolate - Flag to create intermediate mesh pieces (edges, faces)
16 . refinementUniform - Flag for uniform parallel refinement
17 - refinementLimit - A nonzero number indicates the largest admissible volume for a refined cell
18 
19   Output Parameter:
20 . dm - The DM object
21 
22   Level: beginner
23 
24 .keywords: DM, create
25 .seealso: DMSetType(), DMCreate()
26 @*/
27 PetscErrorCode DMPlexCreateDoublet(MPI_Comm comm, PetscInt dim, PetscBool simplex, PetscBool interpolate, PetscBool refinementUniform, PetscReal refinementLimit, DM *newdm)
28 {
29   DM             dm;
30   PetscInt       p;
31   PetscMPIInt    rank;
32   PetscErrorCode ierr;
33 
34   PetscFunctionBegin;
35   ierr = DMCreate(comm, &dm);CHKERRQ(ierr);
36   ierr = DMSetType(dm, DMPLEX);CHKERRQ(ierr);
37   ierr = DMSetDimension(dm, dim);CHKERRQ(ierr);
38   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
39   switch (dim) {
40   case 2:
41     if (simplex) {ierr = PetscObjectSetName((PetscObject) dm, "triangular");CHKERRQ(ierr);}
42     else         {ierr = PetscObjectSetName((PetscObject) dm, "quadrilateral");CHKERRQ(ierr);}
43     break;
44   case 3:
45     if (simplex) {ierr = PetscObjectSetName((PetscObject) dm, "tetrahedral");CHKERRQ(ierr);}
46     else         {ierr = PetscObjectSetName((PetscObject) dm, "hexahedral");CHKERRQ(ierr);}
47     break;
48   default:
49     SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make meshes for dimension %d", dim);
50   }
51   if (rank) {
52     PetscInt numPoints[2] = {0, 0};
53     ierr = DMPlexCreateFromDAG(dm, 1, numPoints, NULL, NULL, NULL, NULL);CHKERRQ(ierr);
54   } else {
55     switch (dim) {
56     case 2:
57       if (simplex) {
58         PetscInt    numPoints[2]        = {4, 2};
59         PetscInt    coneSize[6]         = {3, 3, 0, 0, 0, 0};
60         PetscInt    cones[6]            = {2, 3, 4,  5, 4, 3};
61         PetscInt    coneOrientations[6] = {0, 0, 0,  0, 0, 0};
62         PetscScalar vertexCoords[8]     = {-0.5, 0.5, 0.0, 0.0, 0.0, 1.0, 0.5, 0.5};
63         PetscInt    markerPoints[8]     = {2, 1, 3, 1, 4, 1, 5, 1};
64 
65         ierr = DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
66         for (p = 0; p < 4; ++p) {ierr = DMSetLabelValue(dm, "marker", markerPoints[p*2], markerPoints[p*2+1]);CHKERRQ(ierr);}
67       } else {
68         PetscInt    numPoints[2]        = {6, 2};
69         PetscInt    coneSize[8]         = {4, 4, 0, 0, 0, 0, 0, 0};
70         PetscInt    cones[8]            = {2, 3, 4, 5,  3, 6, 7, 4};
71         PetscInt    coneOrientations[8] = {0, 0, 0, 0,  0, 0, 0, 0};
72         PetscScalar vertexCoords[12]    = {-1.0, -0.5,  0.0, -0.5,  0.0, 0.5,  -1.0, 0.5,  1.0, -0.5,  1.0, 0.5};
73 
74         ierr = DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
75       }
76       break;
77     case 3:
78       if (simplex) {
79         PetscInt    numPoints[2]        = {5, 2};
80         PetscInt    coneSize[7]         = {4, 4, 0, 0, 0, 0, 0};
81         PetscInt    cones[8]            = {4, 3, 5, 2,  5, 3, 4, 6};
82         PetscInt    coneOrientations[8] = {0, 0, 0, 0,  0, 0, 0, 0};
83         PetscScalar vertexCoords[15]    = {-1.0, 0.0, 0.0,  0.0, -1.0, 0.0,  0.0, 0.0, 1.0,  0.0, 1.0, 0.0,  1.0, 0.0, 0.0};
84         PetscInt    markerPoints[10]    = {2, 1, 3, 1, 4, 1, 5, 1, 6, 1};
85 
86         ierr = DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
87         for (p = 0; p < 5; ++p) {ierr = DMSetLabelValue(dm, "marker", markerPoints[p*2], markerPoints[p*2+1]);CHKERRQ(ierr);}
88       } else {
89         PetscInt    numPoints[2]         = {12, 2};
90         PetscInt    coneSize[14]         = {8, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
91         PetscInt    cones[16]            = {2, 3, 4, 5, 6, 7, 8, 9,  5, 4, 10, 11, 7, 12, 13, 8};
92         PetscInt    coneOrientations[16] = {0, 0, 0, 0, 0, 0, 0, 0,  0, 0,  0,  0, 0,  0,  0, 0};
93         PetscScalar vertexCoords[36]     = {-1.0, -0.5, -0.5,  -1.0,  0.5, -0.5,  0.0,  0.5, -0.5,   0.0, -0.5, -0.5,
94                                             -1.0, -0.5,  0.5,   0.0, -0.5,  0.5,  0.0,  0.5,  0.5,  -1.0,  0.5,  0.5,
95                                              1.0,  0.5, -0.5,   1.0, -0.5, -0.5,  1.0, -0.5,  0.5,   1.0,  0.5,  0.5};
96 
97         ierr = DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
98       }
99       break;
100     default:
101       SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make meshes for dimension %d", dim);
102     }
103   }
104   *newdm = dm;
105   if (refinementLimit > 0.0) {
106     DM rdm;
107     const char *name;
108 
109     ierr = DMPlexSetRefinementUniform(*newdm, PETSC_FALSE);CHKERRQ(ierr);
110     ierr = DMPlexSetRefinementLimit(*newdm, refinementLimit);CHKERRQ(ierr);
111     ierr = DMRefine(*newdm, comm, &rdm);CHKERRQ(ierr);
112     ierr = PetscObjectGetName((PetscObject) *newdm, &name);CHKERRQ(ierr);
113     ierr = PetscObjectSetName((PetscObject)    rdm,  name);CHKERRQ(ierr);
114     ierr = DMDestroy(newdm);CHKERRQ(ierr);
115     *newdm = rdm;
116   }
117   if (interpolate) {
118     DM idm;
119 
120     ierr = DMPlexInterpolate(*newdm, &idm);CHKERRQ(ierr);
121     ierr = DMDestroy(newdm);CHKERRQ(ierr);
122     *newdm = idm;
123   }
124   {
125     DM refinedMesh     = NULL;
126     DM distributedMesh = NULL;
127 
128     /* Distribute mesh over processes */
129     ierr = DMPlexDistribute(*newdm, 0, NULL, &distributedMesh);CHKERRQ(ierr);
130     if (distributedMesh) {
131       ierr = DMDestroy(newdm);CHKERRQ(ierr);
132       *newdm = distributedMesh;
133     }
134     if (refinementUniform) {
135       ierr = DMPlexSetRefinementUniform(*newdm, refinementUniform);CHKERRQ(ierr);
136       ierr = DMRefine(*newdm, comm, &refinedMesh);CHKERRQ(ierr);
137       if (refinedMesh) {
138         ierr = DMDestroy(newdm);CHKERRQ(ierr);
139         *newdm = refinedMesh;
140       }
141     }
142   }
143   PetscFunctionReturn(0);
144 }
145 
146 /*@
147   DMPlexCreateSquareBoundary - Creates a 1D mesh the is the boundary of a square lattice.
148 
149   Collective on MPI_Comm
150 
151   Input Parameters:
152 + comm  - The communicator for the DM object
153 . lower - The lower left corner coordinates
154 . upper - The upper right corner coordinates
155 - edges - The number of cells in each direction
156 
157   Output Parameter:
158 . dm  - The DM object
159 
160   Note: Here is the numbering returned for 2 cells in each direction:
161 $ 18--5-17--4--16
162 $  |     |     |
163 $  6    10     3
164 $  |     |     |
165 $ 19-11-20--9--15
166 $  |     |     |
167 $  7     8     2
168 $  |     |     |
169 $ 12--0-13--1--14
170 
171   Level: beginner
172 
173 .keywords: DM, create
174 .seealso: DMPlexCreateBoxMesh(), DMPlexCreateCubeBoundary(), DMSetType(), DMCreate()
175 @*/
176 PetscErrorCode DMPlexCreateSquareBoundary(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[])
177 {
178   const PetscInt numVertices    = (edges[0]+1)*(edges[1]+1);
179   const PetscInt numEdges       = edges[0]*(edges[1]+1) + (edges[0]+1)*edges[1];
180   PetscInt       markerTop      = 1;
181   PetscInt       markerBottom   = 1;
182   PetscInt       markerRight    = 1;
183   PetscInt       markerLeft     = 1;
184   PetscBool      markerSeparate = PETSC_FALSE;
185   Vec            coordinates;
186   PetscSection   coordSection;
187   PetscScalar    *coords;
188   PetscInt       coordSize;
189   PetscMPIInt    rank;
190   PetscInt       v, vx, vy;
191   PetscErrorCode ierr;
192 
193   PetscFunctionBegin;
194   ierr = PetscOptionsGetBool(((PetscObject) dm)->options,((PetscObject) dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL);CHKERRQ(ierr);
195   if (markerSeparate) {
196     markerTop    = 3;
197     markerBottom = 1;
198     markerRight  = 2;
199     markerLeft   = 4;
200   }
201   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRQ(ierr);
202   if (!rank) {
203     PetscInt e, ex, ey;
204 
205     ierr = DMPlexSetChart(dm, 0, numEdges+numVertices);CHKERRQ(ierr);
206     for (e = 0; e < numEdges; ++e) {
207       ierr = DMPlexSetConeSize(dm, e, 2);CHKERRQ(ierr);
208     }
209     ierr = DMSetUp(dm);CHKERRQ(ierr); /* Allocate space for cones */
210     for (vx = 0; vx <= edges[0]; vx++) {
211       for (ey = 0; ey < edges[1]; ey++) {
212         PetscInt edge   = vx*edges[1] + ey + edges[0]*(edges[1]+1);
213         PetscInt vertex = ey*(edges[0]+1) + vx + numEdges;
214         PetscInt cone[2];
215 
216         cone[0] = vertex; cone[1] = vertex+edges[0]+1;
217         ierr    = DMPlexSetCone(dm, edge, cone);CHKERRQ(ierr);
218         if (vx == edges[0]) {
219           ierr = DMSetLabelValue(dm, "marker", edge,    markerRight);CHKERRQ(ierr);
220           ierr = DMSetLabelValue(dm, "marker", cone[0], markerRight);CHKERRQ(ierr);
221           if (ey == edges[1]-1) {
222             ierr = DMSetLabelValue(dm, "marker", cone[1], markerRight);CHKERRQ(ierr);
223             ierr = DMSetLabelValue(dm, "Face Sets", cone[1], markerRight);CHKERRQ(ierr);
224           }
225         } else if (vx == 0) {
226           ierr = DMSetLabelValue(dm, "marker", edge,    markerLeft);CHKERRQ(ierr);
227           ierr = DMSetLabelValue(dm, "marker", cone[0], markerLeft);CHKERRQ(ierr);
228           if (ey == edges[1]-1) {
229             ierr = DMSetLabelValue(dm, "marker", cone[1], markerLeft);CHKERRQ(ierr);
230             ierr = DMSetLabelValue(dm, "Face Sets", cone[1], markerLeft);CHKERRQ(ierr);
231           }
232         }
233       }
234     }
235     for (vy = 0; vy <= edges[1]; vy++) {
236       for (ex = 0; ex < edges[0]; ex++) {
237         PetscInt edge   = vy*edges[0]     + ex;
238         PetscInt vertex = vy*(edges[0]+1) + ex + numEdges;
239         PetscInt cone[2];
240 
241         cone[0] = vertex; cone[1] = vertex+1;
242         ierr    = DMPlexSetCone(dm, edge, cone);CHKERRQ(ierr);
243         if (vy == edges[1]) {
244           ierr = DMSetLabelValue(dm, "marker", edge,    markerTop);CHKERRQ(ierr);
245           ierr = DMSetLabelValue(dm, "marker", cone[0], markerTop);CHKERRQ(ierr);
246           if (ex == edges[0]-1) {
247             ierr = DMSetLabelValue(dm, "marker", cone[1], markerTop);CHKERRQ(ierr);
248             ierr = DMSetLabelValue(dm, "Face Sets", cone[1], markerTop);CHKERRQ(ierr);
249           }
250         } else if (vy == 0) {
251           ierr = DMSetLabelValue(dm, "marker", edge,    markerBottom);CHKERRQ(ierr);
252           ierr = DMSetLabelValue(dm, "marker", cone[0], markerBottom);CHKERRQ(ierr);
253           if (ex == edges[0]-1) {
254             ierr = DMSetLabelValue(dm, "marker", cone[1], markerBottom);CHKERRQ(ierr);
255             ierr = DMSetLabelValue(dm, "Face Sets", cone[1], markerBottom);CHKERRQ(ierr);
256           }
257         }
258       }
259     }
260   }
261   ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
262   ierr = DMPlexStratify(dm);CHKERRQ(ierr);
263   /* Build coordinates */
264   ierr = DMSetCoordinateDim(dm, 2);CHKERRQ(ierr);
265   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
266   ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
267   ierr = PetscSectionSetChart(coordSection, numEdges, numEdges + numVertices);CHKERRQ(ierr);
268   ierr = PetscSectionSetFieldComponents(coordSection, 0, 2);CHKERRQ(ierr);
269   for (v = numEdges; v < numEdges+numVertices; ++v) {
270     ierr = PetscSectionSetDof(coordSection, v, 2);CHKERRQ(ierr);
271     ierr = PetscSectionSetFieldDof(coordSection, v, 0, 2);CHKERRQ(ierr);
272   }
273   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
274   ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
275   ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
276   ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
277   ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
278   ierr = VecSetBlockSize(coordinates, 2);CHKERRQ(ierr);
279   ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
280   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
281   for (vy = 0; vy <= edges[1]; ++vy) {
282     for (vx = 0; vx <= edges[0]; ++vx) {
283       coords[(vy*(edges[0]+1)+vx)*2+0] = lower[0] + ((upper[0] - lower[0])/edges[0])*vx;
284       coords[(vy*(edges[0]+1)+vx)*2+1] = lower[1] + ((upper[1] - lower[1])/edges[1])*vy;
285     }
286   }
287   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
288   ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
289   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
290   PetscFunctionReturn(0);
291 }
292 
293 /*@
294   DMPlexCreateCubeBoundary - Creates a 2D mesh that is the boundary of a cubic lattice.
295 
296   Collective on MPI_Comm
297 
298   Input Parameters:
299 + comm  - The communicator for the DM object
300 . lower - The lower left front corner coordinates
301 . upper - The upper right back corner coordinates
302 - edges - The number of cells in each direction
303 
304   Output Parameter:
305 . dm  - The DM object
306 
307   Level: beginner
308 
309 .keywords: DM, create
310 .seealso: DMPlexCreateBoxMesh(), DMPlexCreateSquareBoundary(), DMSetType(), DMCreate()
311 @*/
312 PetscErrorCode DMPlexCreateCubeBoundary(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt faces[])
313 {
314   PetscInt       vertices[3], numVertices;
315   PetscInt       numFaces    = 2*faces[0]*faces[1] + 2*faces[1]*faces[2] + 2*faces[0]*faces[2];
316   Vec            coordinates;
317   PetscSection   coordSection;
318   PetscScalar    *coords;
319   PetscInt       coordSize;
320   PetscMPIInt    rank;
321   PetscInt       v, vx, vy, vz;
322   PetscInt       voffset, iface=0, cone[4];
323   PetscErrorCode ierr;
324 
325   PetscFunctionBegin;
326   if ((faces[0] < 1) || (faces[1] < 1) || (faces[2] < 1)) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Must have at least 1 face per side");
327   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRQ(ierr);
328   vertices[0] = faces[0]+1; vertices[1] = faces[1]+1; vertices[2] = faces[2]+1;
329   numVertices = vertices[0]*vertices[1]*vertices[2];
330   if (!rank) {
331     PetscInt f;
332 
333     ierr = DMPlexSetChart(dm, 0, numFaces+numVertices);CHKERRQ(ierr);
334     for (f = 0; f < numFaces; ++f) {
335       ierr = DMPlexSetConeSize(dm, f, 4);CHKERRQ(ierr);
336     }
337     ierr = DMSetUp(dm);CHKERRQ(ierr); /* Allocate space for cones */
338 
339     /* Side 0 (Top) */
340     for (vy = 0; vy < faces[1]; vy++) {
341       for (vx = 0; vx < faces[0]; vx++) {
342         voffset = numFaces + vertices[0]*vertices[1]*(vertices[2]-1) + vy*vertices[0] + vx;
343         cone[0] = voffset; cone[1] = voffset+1; cone[2] = voffset+vertices[0]+1; cone[3] = voffset+vertices[0];
344         ierr    = DMPlexSetCone(dm, iface, cone);CHKERRQ(ierr);
345         ierr    = DMSetLabelValue(dm, "marker", iface, 1);CHKERRQ(ierr);
346         ierr    = DMSetLabelValue(dm, "marker", voffset+0, 1);CHKERRQ(ierr);
347         ierr    = DMSetLabelValue(dm, "marker", voffset+1, 1);CHKERRQ(ierr);
348         ierr    = DMSetLabelValue(dm, "marker", voffset+vertices[0]+0, 1);CHKERRQ(ierr);
349         ierr    = DMSetLabelValue(dm, "marker", voffset+vertices[0]+1, 1);CHKERRQ(ierr);
350         iface++;
351       }
352     }
353 
354     /* Side 1 (Bottom) */
355     for (vy = 0; vy < faces[1]; vy++) {
356       for (vx = 0; vx < faces[0]; vx++) {
357         voffset = numFaces + vy*(faces[0]+1) + vx;
358         cone[0] = voffset+1; cone[1] = voffset; cone[2] = voffset+vertices[0]; cone[3] = voffset+vertices[0]+1;
359         ierr    = DMPlexSetCone(dm, iface, cone);CHKERRQ(ierr);
360         ierr    = DMSetLabelValue(dm, "marker", iface, 1);CHKERRQ(ierr);
361         ierr    = DMSetLabelValue(dm, "marker", voffset+0, 1);CHKERRQ(ierr);
362         ierr    = DMSetLabelValue(dm, "marker", voffset+1, 1);CHKERRQ(ierr);
363         ierr    = DMSetLabelValue(dm, "marker", voffset+vertices[0]+0, 1);CHKERRQ(ierr);
364         ierr    = DMSetLabelValue(dm, "marker", voffset+vertices[0]+1, 1);CHKERRQ(ierr);
365         iface++;
366       }
367     }
368 
369     /* Side 2 (Front) */
370     for (vz = 0; vz < faces[2]; vz++) {
371       for (vx = 0; vx < faces[0]; vx++) {
372         voffset = numFaces + vz*vertices[0]*vertices[1] + vx;
373         cone[0] = voffset; cone[1] = voffset+1; cone[2] = voffset+vertices[0]*vertices[1]+1; cone[3] = voffset+vertices[0]*vertices[1];
374         ierr    = DMPlexSetCone(dm, iface, cone);CHKERRQ(ierr);
375         ierr    = DMSetLabelValue(dm, "marker", iface, 1);CHKERRQ(ierr);
376         ierr    = DMSetLabelValue(dm, "marker", voffset+0, 1);CHKERRQ(ierr);
377         ierr    = DMSetLabelValue(dm, "marker", voffset+1, 1);CHKERRQ(ierr);
378         ierr    = DMSetLabelValue(dm, "marker", voffset+vertices[0]*vertices[1]+0, 1);CHKERRQ(ierr);
379         ierr    = DMSetLabelValue(dm, "marker", voffset+vertices[0]*vertices[1]+1, 1);CHKERRQ(ierr);
380         iface++;
381       }
382     }
383 
384     /* Side 3 (Back) */
385     for (vz = 0; vz < faces[2]; vz++) {
386       for (vx = 0; vx < faces[0]; vx++) {
387         voffset = numFaces + vz*vertices[0]*vertices[1] + vertices[0]*(vertices[1]-1) + vx;
388         cone[0] = voffset+vertices[0]*vertices[1]; cone[1] = voffset+vertices[0]*vertices[1]+1;
389         cone[2] = voffset+1; cone[3] = voffset;
390         ierr    = DMPlexSetCone(dm, iface, cone);CHKERRQ(ierr);
391         ierr    = DMSetLabelValue(dm, "marker", iface, 1);CHKERRQ(ierr);
392         ierr    = DMSetLabelValue(dm, "marker", voffset+0, 1);CHKERRQ(ierr);
393         ierr    = DMSetLabelValue(dm, "marker", voffset+1, 1);CHKERRQ(ierr);
394         ierr    = DMSetLabelValue(dm, "marker", voffset+vertices[0]*vertices[1]+0, 1);CHKERRQ(ierr);
395         ierr    = DMSetLabelValue(dm, "marker", voffset+vertices[0]*vertices[1]+1, 1);CHKERRQ(ierr);
396         iface++;
397       }
398     }
399 
400     /* Side 4 (Left) */
401     for (vz = 0; vz < faces[2]; vz++) {
402       for (vy = 0; vy < faces[1]; vy++) {
403         voffset = numFaces + vz*vertices[0]*vertices[1] + vy*vertices[0];
404         cone[0] = voffset; cone[1] = voffset+vertices[0]*vertices[1];
405         cone[2] = voffset+vertices[0]*vertices[1]+vertices[0]; cone[3] = voffset+vertices[0];
406         ierr    = DMPlexSetCone(dm, iface, cone);CHKERRQ(ierr);
407         ierr    = DMSetLabelValue(dm, "marker", iface, 1);CHKERRQ(ierr);
408         ierr    = DMSetLabelValue(dm, "marker", voffset+0, 1);CHKERRQ(ierr);
409         ierr    = DMSetLabelValue(dm, "marker", voffset+vertices[0]+0, 1);CHKERRQ(ierr);
410         ierr    = DMSetLabelValue(dm, "marker", voffset+vertices[1]+0, 1);CHKERRQ(ierr);
411         ierr    = DMSetLabelValue(dm, "marker", voffset+vertices[0]*vertices[1]+vertices[0], 1);CHKERRQ(ierr);
412         iface++;
413       }
414     }
415 
416     /* Side 5 (Right) */
417     for (vz = 0; vz < faces[2]; vz++) {
418       for (vy = 0; vy < faces[1]; vy++) {
419         voffset = numFaces + vz*vertices[0]*vertices[1] + vy*vertices[0] + faces[0];
420         cone[0] = voffset+vertices[0]*vertices[1]; cone[1] = voffset;
421         cone[2] = voffset+vertices[0]; cone[3] = voffset+vertices[0]*vertices[1]+vertices[0];
422         ierr    = DMPlexSetCone(dm, iface, cone);CHKERRQ(ierr);
423         ierr    = DMSetLabelValue(dm, "marker", iface, 1);CHKERRQ(ierr);
424         ierr    = DMSetLabelValue(dm, "marker", voffset+0, 1);CHKERRQ(ierr);
425         ierr    = DMSetLabelValue(dm, "marker", voffset+vertices[0]+0, 1);CHKERRQ(ierr);
426         ierr    = DMSetLabelValue(dm, "marker", voffset+vertices[0]*vertices[1]+0, 1);CHKERRQ(ierr);
427         ierr    = DMSetLabelValue(dm, "marker", voffset+vertices[0]*vertices[1]+vertices[0], 1);CHKERRQ(ierr);
428         iface++;
429       }
430     }
431   }
432   ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
433   ierr = DMPlexStratify(dm);CHKERRQ(ierr);
434   /* Build coordinates */
435   ierr = DMSetCoordinateDim(dm, 3);CHKERRQ(ierr);
436   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
437   ierr = PetscSectionSetChart(coordSection, numFaces, numFaces + numVertices);CHKERRQ(ierr);
438   for (v = numFaces; v < numFaces+numVertices; ++v) {
439     ierr = PetscSectionSetDof(coordSection, v, 3);CHKERRQ(ierr);
440   }
441   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
442   ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
443   ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
444   ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
445   ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
446   ierr = VecSetBlockSize(coordinates, 3);CHKERRQ(ierr);
447   ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
448   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
449   for (vz = 0; vz <= faces[2]; ++vz) {
450     for (vy = 0; vy <= faces[1]; ++vy) {
451       for (vx = 0; vx <= faces[0]; ++vx) {
452         coords[((vz*(faces[1]+1)+vy)*(faces[0]+1)+vx)*3+0] = lower[0] + ((upper[0] - lower[0])/faces[0])*vx;
453         coords[((vz*(faces[1]+1)+vy)*(faces[0]+1)+vx)*3+1] = lower[1] + ((upper[1] - lower[1])/faces[1])*vy;
454         coords[((vz*(faces[1]+1)+vy)*(faces[0]+1)+vx)*3+2] = lower[2] + ((upper[2] - lower[2])/faces[2])*vz;
455       }
456     }
457   }
458   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
459   ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
460   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
461   PetscFunctionReturn(0);
462 }
463 
464 static PetscErrorCode DMPlexCreateLineMesh_Internal(MPI_Comm comm,PetscInt segments,PetscReal lower,PetscReal upper,DMBoundaryType bd,DM *dm)
465 {
466   PetscInt       i,fStart,fEnd,numCells = 0,numVerts = 0;
467   PetscInt       numPoints[2],*coneSize,*cones,*coneOrientations;
468   PetscScalar    *vertexCoords;
469   PetscReal      L,maxCell;
470   PetscBool      markerSeparate = PETSC_FALSE;
471   PetscInt       markerLeft  = 1, faceMarkerLeft  = 1;
472   PetscInt       markerRight = 1, faceMarkerRight = 2;
473   PetscBool      wrap = (bd == DM_BOUNDARY_PERIODIC || bd == DM_BOUNDARY_TWIST) ? PETSC_TRUE : PETSC_FALSE;
474   PetscMPIInt    rank;
475   PetscErrorCode ierr;
476 
477   PetscFunctionBegin;
478   PetscValidPointer(dm,4);
479 
480   ierr = DMCreate(comm,dm);CHKERRQ(ierr);
481   ierr = DMSetType(*dm,DMPLEX);CHKERRQ(ierr);
482   ierr = DMSetDimension(*dm,1);CHKERRQ(ierr);
483   ierr = DMCreateLabel(*dm,"marker");CHKERRQ(ierr);
484   ierr = DMCreateLabel(*dm,"Face Sets");CHKERRQ(ierr);
485 
486   ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr);
487   if (!rank) numCells = segments;
488   if (!rank) numVerts = segments + (wrap ? 0 : 1);
489 
490   numPoints[0] = numVerts ; numPoints[1] = numCells;
491   ierr = PetscMalloc4(numCells+numVerts,&coneSize,numCells*2,&cones,numCells+numVerts,&coneOrientations,numVerts,&vertexCoords);CHKERRQ(ierr);
492   ierr = PetscMemzero(coneOrientations,(numCells+numVerts)*sizeof(PetscInt));CHKERRQ(ierr);
493   for (i = 0; i < numCells; ++i) { coneSize[i] = 2; }
494   for (i = 0; i < numVerts; ++i) { coneSize[numCells+i] = 0; }
495   for (i = 0; i < numCells; ++i) { cones[2*i] = numCells + i%numVerts; cones[2*i+1] = numCells + (i+1)%numVerts; }
496   for (i = 0; i < numVerts; ++i) { vertexCoords[i] = lower + (upper-lower)*((PetscReal)i/(PetscReal)numCells); }
497   ierr = DMPlexCreateFromDAG(*dm,1,numPoints,coneSize,cones,coneOrientations,vertexCoords);CHKERRQ(ierr);
498   ierr = PetscFree4(coneSize,cones,coneOrientations,vertexCoords);CHKERRQ(ierr);
499 
500   ierr = PetscOptionsGetBool(((PetscObject)*dm)->options,((PetscObject)*dm)->prefix,"-dm_plex_separate_marker",&markerSeparate,NULL);CHKERRQ(ierr);
501   if (markerSeparate) { markerLeft = faceMarkerLeft; markerRight = faceMarkerRight;}
502   if (!wrap && !rank) {
503     ierr = DMPlexGetHeightStratum(*dm,1,&fStart,&fEnd);CHKERRQ(ierr);
504     ierr = DMSetLabelValue(*dm,"marker",fStart,markerLeft);CHKERRQ(ierr);
505     ierr = DMSetLabelValue(*dm,"marker",fEnd-1,markerRight);CHKERRQ(ierr);
506     ierr = DMSetLabelValue(*dm,"Face Sets",fStart,faceMarkerLeft);CHKERRQ(ierr);
507     ierr = DMSetLabelValue(*dm,"Face Sets",fEnd-1,faceMarkerRight);CHKERRQ(ierr);
508   }
509   if (wrap) {
510     L       = upper - lower;
511     maxCell = (PetscReal)1.1*(L/(PetscReal)PetscMax(1,segments));
512     ierr = DMSetPeriodicity(*dm,PETSC_TRUE,&maxCell,&L,&bd);CHKERRQ(ierr);
513   }
514   PetscFunctionReturn(0);
515 }
516 
517 static PetscErrorCode DMPlexCreateBoxMesh_Simplex_Internal(MPI_Comm comm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate, DM *dm)
518 {
519   DM             boundary;
520   PetscInt       i;
521   PetscErrorCode ierr;
522 
523   PetscFunctionBegin;
524   PetscValidPointer(dm, 4);
525   for (i = 0; i < dim; ++i) if (periodicity[i] != DM_BOUNDARY_NONE) SETERRQ(comm, PETSC_ERR_SUP, "Periodicity is not supported for simplex meshes");
526   ierr = DMCreate(comm, &boundary);CHKERRQ(ierr);
527   PetscValidLogicalCollectiveInt(boundary,dim,2);
528   ierr = DMSetType(boundary, DMPLEX);CHKERRQ(ierr);
529   ierr = DMSetDimension(boundary, dim-1);CHKERRQ(ierr);
530   ierr = DMSetCoordinateDim(boundary, dim);CHKERRQ(ierr);
531   switch (dim) {
532   case 2: ierr = DMPlexCreateSquareBoundary(boundary, lower, upper, faces);CHKERRQ(ierr);break;
533   case 3: ierr = DMPlexCreateCubeBoundary(boundary, lower, upper, faces);CHKERRQ(ierr);break;
534   default: SETERRQ1(comm, PETSC_ERR_SUP, "Dimension not supported: %d", dim);
535   }
536   ierr = DMPlexGenerate(boundary, NULL, interpolate, dm);CHKERRQ(ierr);
537   ierr = DMDestroy(&boundary);CHKERRQ(ierr);
538   PetscFunctionReturn(0);
539 }
540 
541 static PetscErrorCode DMPlexCreateCubeMesh_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[], DMBoundaryType bdX, DMBoundaryType bdY, DMBoundaryType bdZ)
542 {
543   DMLabel        cutLabel = NULL;
544   PetscInt       markerTop      = 1, faceMarkerTop      = 1;
545   PetscInt       markerBottom   = 1, faceMarkerBottom   = 1;
546   PetscInt       markerFront    = 1, faceMarkerFront    = 1;
547   PetscInt       markerBack     = 1, faceMarkerBack     = 1;
548   PetscInt       markerRight    = 1, faceMarkerRight    = 1;
549   PetscInt       markerLeft     = 1, faceMarkerLeft     = 1;
550   PetscInt       dim;
551   PetscBool      markerSeparate = PETSC_FALSE, cutMarker = PETSC_FALSE;
552   PetscMPIInt    rank;
553   PetscErrorCode ierr;
554 
555   PetscFunctionBegin;
556   ierr = DMGetDimension(dm,&dim);CHKERRQ(ierr);
557   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRQ(ierr);
558   ierr = DMCreateLabel(dm,"marker");CHKERRQ(ierr);
559   ierr = DMCreateLabel(dm,"Face Sets");CHKERRQ(ierr);
560   ierr = PetscOptionsGetBool(((PetscObject) dm)->options,((PetscObject) dm)->prefix, "-dm_plex_periodic_cut", &cutMarker, NULL);CHKERRQ(ierr);
561   if (bdX == DM_BOUNDARY_PERIODIC || bdX == DM_BOUNDARY_TWIST ||
562       bdY == DM_BOUNDARY_PERIODIC || bdY == DM_BOUNDARY_TWIST ||
563       bdZ == DM_BOUNDARY_PERIODIC || bdZ == DM_BOUNDARY_TWIST) {
564 
565     if (cutMarker) {ierr = DMCreateLabel(dm, "periodic_cut");CHKERRQ(ierr); ierr = DMGetLabel(dm, "periodic_cut", &cutLabel);CHKERRQ(ierr);}
566   }
567   switch (dim) {
568   case 2:
569     faceMarkerTop    = 3;
570     faceMarkerBottom = 1;
571     faceMarkerRight  = 2;
572     faceMarkerLeft   = 4;
573     break;
574   case 3:
575     faceMarkerBottom = 1;
576     faceMarkerTop    = 2;
577     faceMarkerFront  = 3;
578     faceMarkerBack   = 4;
579     faceMarkerRight  = 5;
580     faceMarkerLeft   = 6;
581     break;
582   default:
583     SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SUP,"Dimension %d not supported",dim);
584     break;
585   }
586   ierr = PetscOptionsGetBool(((PetscObject) dm)->options,((PetscObject) dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL);CHKERRQ(ierr);
587   if (markerSeparate) {
588     markerBottom = faceMarkerBottom;
589     markerTop    = faceMarkerTop;
590     markerFront  = faceMarkerFront;
591     markerBack   = faceMarkerBack;
592     markerRight  = faceMarkerRight;
593     markerLeft   = faceMarkerLeft;
594   }
595   {
596     const PetscInt numXEdges    = !rank ? edges[0] : 0;
597     const PetscInt numYEdges    = !rank ? edges[1] : 0;
598     const PetscInt numZEdges    = !rank ? edges[2] : 0;
599     const PetscInt numXVertices = !rank ? (bdX == DM_BOUNDARY_PERIODIC || bdX == DM_BOUNDARY_TWIST ? edges[0] : edges[0]+1) : 0;
600     const PetscInt numYVertices = !rank ? (bdY == DM_BOUNDARY_PERIODIC || bdY == DM_BOUNDARY_TWIST ? edges[1] : edges[1]+1) : 0;
601     const PetscInt numZVertices = !rank ? (bdZ == DM_BOUNDARY_PERIODIC || bdZ == DM_BOUNDARY_TWIST ? edges[2] : edges[2]+1) : 0;
602     const PetscInt numCells     = numXEdges*numYEdges*numZEdges;
603     const PetscInt numXFaces    = numYEdges*numZEdges;
604     const PetscInt numYFaces    = numXEdges*numZEdges;
605     const PetscInt numZFaces    = numXEdges*numYEdges;
606     const PetscInt numTotXFaces = numXVertices*numXFaces;
607     const PetscInt numTotYFaces = numYVertices*numYFaces;
608     const PetscInt numTotZFaces = numZVertices*numZFaces;
609     const PetscInt numFaces     = numTotXFaces + numTotYFaces + numTotZFaces;
610     const PetscInt numTotXEdges = numXEdges*numYVertices*numZVertices;
611     const PetscInt numTotYEdges = numYEdges*numXVertices*numZVertices;
612     const PetscInt numTotZEdges = numZEdges*numXVertices*numYVertices;
613     const PetscInt numVertices  = numXVertices*numYVertices*numZVertices;
614     const PetscInt numEdges     = numTotXEdges + numTotYEdges + numTotZEdges;
615     const PetscInt firstVertex  = (dim == 2) ? numFaces : numCells;
616     const PetscInt firstXFace   = (dim == 2) ? 0 : numCells + numVertices;
617     const PetscInt firstYFace   = firstXFace + numTotXFaces;
618     const PetscInt firstZFace   = firstYFace + numTotYFaces;
619     const PetscInt firstXEdge   = numCells + numFaces + numVertices;
620     const PetscInt firstYEdge   = firstXEdge + numTotXEdges;
621     const PetscInt firstZEdge   = firstYEdge + numTotYEdges;
622     Vec            coordinates;
623     PetscSection   coordSection;
624     PetscScalar   *coords;
625     PetscInt       coordSize;
626     PetscInt       v, vx, vy, vz;
627     PetscInt       c, f, fx, fy, fz, e, ex, ey, ez;
628 
629     ierr = DMPlexSetChart(dm, 0, numCells+numFaces+numEdges+numVertices);CHKERRQ(ierr);
630     for (c = 0; c < numCells; c++) {
631       ierr = DMPlexSetConeSize(dm, c, 6);CHKERRQ(ierr);
632     }
633     for (f = firstXFace; f < firstXFace+numFaces; ++f) {
634       ierr = DMPlexSetConeSize(dm, f, 4);CHKERRQ(ierr);
635     }
636     for (e = firstXEdge; e < firstXEdge+numEdges; ++e) {
637       ierr = DMPlexSetConeSize(dm, e, 2);CHKERRQ(ierr);
638     }
639     ierr = DMSetUp(dm);CHKERRQ(ierr); /* Allocate space for cones */
640     /* Build cells */
641     for (fz = 0; fz < numZEdges; ++fz) {
642       for (fy = 0; fy < numYEdges; ++fy) {
643         for (fx = 0; fx < numXEdges; ++fx) {
644           PetscInt cell    = (fz*numYEdges + fy)*numXEdges + fx;
645           PetscInt faceB   = firstZFace + (fy*numXEdges+fx)*numZVertices +   fz;
646           PetscInt faceT   = firstZFace + (fy*numXEdges+fx)*numZVertices + ((fz+1)%numZVertices);
647           PetscInt faceF   = firstYFace + (fz*numXEdges+fx)*numYVertices +   fy;
648           PetscInt faceK   = firstYFace + (fz*numXEdges+fx)*numYVertices + ((fy+1)%numYVertices);
649           PetscInt faceL   = firstXFace + (fz*numYEdges+fy)*numXVertices +   fx;
650           PetscInt faceR   = firstXFace + (fz*numYEdges+fy)*numXVertices + ((fx+1)%numXVertices);
651                             /* B,  T,  F,  K,  R,  L */
652           PetscInt ornt[6] = {-4,  0,  0, -1,  0, -4}; /* ??? */
653           PetscInt cone[6];
654 
655           /* no boundary twisting in 3D */
656           cone[0] = faceB; cone[1] = faceT; cone[2] = faceF; cone[3] = faceK; cone[4] = faceR; cone[5] = faceL;
657           ierr    = DMPlexSetCone(dm, cell, cone);CHKERRQ(ierr);
658           ierr    = DMPlexSetConeOrientation(dm, cell, ornt);CHKERRQ(ierr);
659           if (bdX != DM_BOUNDARY_NONE && fx == numXEdges-1 && cutLabel) {ierr = DMLabelSetValue(cutLabel, cell, 2);CHKERRQ(ierr);}
660           if (bdY != DM_BOUNDARY_NONE && fy == numYEdges-1 && cutLabel) {ierr = DMLabelSetValue(cutLabel, cell, 2);CHKERRQ(ierr);}
661           if (bdZ != DM_BOUNDARY_NONE && fz == numZEdges-1 && cutLabel) {ierr = DMLabelSetValue(cutLabel, cell, 2);CHKERRQ(ierr);}
662         }
663       }
664     }
665     /* Build x faces */
666     for (fz = 0; fz < numZEdges; ++fz) {
667       for (fy = 0; fy < numYEdges; ++fy) {
668         for (fx = 0; fx < numXVertices; ++fx) {
669           PetscInt face    = firstXFace + (fz*numYEdges+fy)*numXVertices + fx;
670           PetscInt edgeL   = firstZEdge + (  fy*                 numXVertices+fx)*numZEdges + fz;
671           PetscInt edgeR   = firstZEdge + (((fy+1)%numYVertices)*numXVertices+fx)*numZEdges + fz;
672           PetscInt edgeB   = firstYEdge + (  fz*                 numXVertices+fx)*numYEdges + fy;
673           PetscInt edgeT   = firstYEdge + (((fz+1)%numZVertices)*numXVertices+fx)*numYEdges + fy;
674           PetscInt ornt[4] = {0, 0, -2, -2};
675           PetscInt cone[4];
676 
677           if (dim == 3) {
678             /* markers */
679             if (bdX != DM_BOUNDARY_PERIODIC) {
680               if (fx == numXVertices-1) {
681                 ierr = DMSetLabelValue(dm, "Face Sets", face, faceMarkerRight);CHKERRQ(ierr);
682                 ierr = DMSetLabelValue(dm, "marker", face, markerRight);CHKERRQ(ierr);
683               }
684               else if (fx == 0) {
685                 ierr = DMSetLabelValue(dm, "Face Sets", face, faceMarkerLeft);CHKERRQ(ierr);
686                 ierr = DMSetLabelValue(dm, "marker", face, markerLeft);CHKERRQ(ierr);
687               }
688             }
689           }
690           cone[0] = edgeB; cone[1] = edgeR; cone[2] = edgeT; cone[3] = edgeL;
691           ierr    = DMPlexSetCone(dm, face, cone);CHKERRQ(ierr);
692           ierr    = DMPlexSetConeOrientation(dm, face, ornt);CHKERRQ(ierr);
693         }
694       }
695     }
696     /* Build y faces */
697     for (fz = 0; fz < numZEdges; ++fz) {
698       for (fx = 0; fx < numXEdges; ++fx) {
699         for (fy = 0; fy < numYVertices; ++fy) {
700           PetscInt face    = firstYFace + (fz*numXEdges+fx)*numYVertices + fy;
701           PetscInt edgeL   = firstZEdge + (fy*numXVertices+  fx                 )*numZEdges + fz;
702           PetscInt edgeR   = firstZEdge + (fy*numXVertices+((fx+1)%numXVertices))*numZEdges + fz;
703           PetscInt edgeB   = firstXEdge + (  fz                 *numYVertices+fy)*numXEdges + fx;
704           PetscInt edgeT   = firstXEdge + (((fz+1)%numZVertices)*numYVertices+fy)*numXEdges + fx;
705           PetscInt ornt[4] = {0, 0, -2, -2};
706           PetscInt cone[4];
707 
708           if (dim == 3) {
709             /* markers */
710             if (bdY != DM_BOUNDARY_PERIODIC) {
711               if (fy == numYVertices-1) {
712                 ierr = DMSetLabelValue(dm, "Face Sets", face, faceMarkerBack);CHKERRQ(ierr);
713                 ierr = DMSetLabelValue(dm, "marker", face, markerBack);CHKERRQ(ierr);
714               }
715               else if (fy == 0) {
716                 ierr = DMSetLabelValue(dm, "Face Sets", face, faceMarkerFront);CHKERRQ(ierr);
717                 ierr = DMSetLabelValue(dm, "marker", face, markerFront);CHKERRQ(ierr);
718               }
719             }
720           }
721           cone[0] = edgeB; cone[1] = edgeR; cone[2] = edgeT; cone[3] = edgeL;
722           ierr    = DMPlexSetCone(dm, face, cone);CHKERRQ(ierr);
723           ierr    = DMPlexSetConeOrientation(dm, face, ornt);CHKERRQ(ierr);
724         }
725       }
726     }
727     /* Build z faces */
728     for (fy = 0; fy < numYEdges; ++fy) {
729       for (fx = 0; fx < numXEdges; ++fx) {
730         for (fz = 0; fz < numZVertices; fz++) {
731           PetscInt face    = firstZFace + (fy*numXEdges+fx)*numZVertices + fz;
732           PetscInt edgeL   = firstYEdge + (fz*numXVertices+  fx                 )*numYEdges + fy;
733           PetscInt edgeR   = firstYEdge + (fz*numXVertices+((fx+1)%numXVertices))*numYEdges + fy;
734           PetscInt edgeB   = firstXEdge + (fz*numYVertices+  fy                 )*numXEdges + fx;
735           PetscInt edgeT   = firstXEdge + (fz*numYVertices+((fy+1)%numYVertices))*numXEdges + fx;
736           PetscInt ornt[4] = {0, 0, -2, -2};
737           PetscInt cone[4];
738 
739           if (dim == 2) {
740             if (bdX == DM_BOUNDARY_TWIST && fx == numXEdges-1) {edgeR += numYEdges-1-2*fy; ornt[1] = -2;}
741             if (bdY == DM_BOUNDARY_TWIST && fy == numYEdges-1) {edgeT += numXEdges-1-2*fx; ornt[2] =  0;}
742             if (bdX != DM_BOUNDARY_NONE && fx == numXEdges-1 && cutLabel) {ierr = DMLabelSetValue(cutLabel, face, 2);CHKERRQ(ierr);}
743             if (bdY != DM_BOUNDARY_NONE && fy == numYEdges-1 && cutLabel) {ierr = DMLabelSetValue(cutLabel, face, 2);CHKERRQ(ierr);}
744           } else {
745             /* markers */
746             if (bdZ != DM_BOUNDARY_PERIODIC) {
747               if (fz == numZVertices-1) {
748                 ierr = DMSetLabelValue(dm, "Face Sets", face, faceMarkerTop);CHKERRQ(ierr);
749                 ierr = DMSetLabelValue(dm, "marker", face, markerTop);CHKERRQ(ierr);
750               }
751               else if (fz == 0) {
752                 ierr = DMSetLabelValue(dm, "Face Sets", face, faceMarkerBottom);CHKERRQ(ierr);
753                 ierr = DMSetLabelValue(dm, "marker", face, markerBottom);CHKERRQ(ierr);
754               }
755             }
756           }
757           cone[0] = edgeB; cone[1] = edgeR; cone[2] = edgeT; cone[3] = edgeL;
758           ierr    = DMPlexSetCone(dm, face, cone);CHKERRQ(ierr);
759           ierr    = DMPlexSetConeOrientation(dm, face, ornt);CHKERRQ(ierr);
760         }
761       }
762     }
763     /* Build Z edges*/
764     for (vy = 0; vy < numYVertices; vy++) {
765       for (vx = 0; vx < numXVertices; vx++) {
766         for (ez = 0; ez < numZEdges; ez++) {
767           const PetscInt edge    = firstZEdge  + (vy*numXVertices+vx)*numZEdges + ez;
768           const PetscInt vertexB = firstVertex + (  ez                 *numYVertices+vy)*numXVertices + vx;
769           const PetscInt vertexT = firstVertex + (((ez+1)%numZVertices)*numYVertices+vy)*numXVertices + vx;
770           PetscInt       cone[2];
771 
772           if (dim == 3) {
773             if (bdX != DM_BOUNDARY_PERIODIC) {
774               if (vx == numXVertices-1) {
775                 ierr = DMSetLabelValue(dm, "marker", edge, markerRight);CHKERRQ(ierr);
776               }
777               else if (vx == 0) {
778                 ierr = DMSetLabelValue(dm, "marker", edge, markerLeft);CHKERRQ(ierr);
779               }
780             }
781             if (bdY != DM_BOUNDARY_PERIODIC) {
782               if (vy == numYVertices-1) {
783                 ierr = DMSetLabelValue(dm, "marker", edge, markerBack);CHKERRQ(ierr);
784               }
785               else if (vy == 0) {
786                 ierr = DMSetLabelValue(dm, "marker", edge, markerFront);CHKERRQ(ierr);
787               }
788             }
789           }
790           cone[0] = vertexB; cone[1] = vertexT;
791           ierr = DMPlexSetCone(dm, edge, cone);CHKERRQ(ierr);
792         }
793       }
794     }
795     /* Build Y edges*/
796     for (vz = 0; vz < numZVertices; vz++) {
797       for (vx = 0; vx < numXVertices; vx++) {
798         for (ey = 0; ey < numYEdges; ey++) {
799           const PetscInt nextv   = (dim == 2 && bdY == DM_BOUNDARY_TWIST && ey == numYEdges-1) ? (numXVertices-vx-1) : (vz*numYVertices+((ey+1)%numYVertices))*numXVertices + vx;
800           const PetscInt edge    = firstYEdge  + (vz*numXVertices+vx)*numYEdges + ey;
801           const PetscInt vertexF = firstVertex + (vz*numYVertices+ey)*numXVertices + vx;
802           const PetscInt vertexK = firstVertex + nextv;
803           PetscInt       cone[2];
804 
805           cone[0] = vertexF; cone[1] = vertexK;
806           ierr = DMPlexSetCone(dm, edge, cone);CHKERRQ(ierr);
807           if (dim == 2) {
808             if ((bdX != DM_BOUNDARY_PERIODIC) && (bdX != DM_BOUNDARY_TWIST)) {
809               if (vx == numXVertices-1) {
810                 ierr = DMSetLabelValue(dm, "Face Sets", edge, faceMarkerRight);CHKERRQ(ierr);
811                 ierr = DMSetLabelValue(dm, "marker", edge,    markerRight);CHKERRQ(ierr);
812                 ierr = DMSetLabelValue(dm, "marker", cone[0], markerRight);CHKERRQ(ierr);
813                 if (ey == numYEdges-1) {
814                   ierr = DMSetLabelValue(dm, "marker", cone[1], markerRight);CHKERRQ(ierr);
815                 }
816               } else if (vx == 0) {
817                 ierr = DMSetLabelValue(dm, "Face Sets", edge, faceMarkerLeft);CHKERRQ(ierr);
818                 ierr = DMSetLabelValue(dm, "marker", edge,    markerLeft);CHKERRQ(ierr);
819                 ierr = DMSetLabelValue(dm, "marker", cone[0], markerLeft);CHKERRQ(ierr);
820                 if (ey == numYEdges-1) {
821                   ierr = DMSetLabelValue(dm, "marker", cone[1], markerLeft);CHKERRQ(ierr);
822                 }
823               }
824             } else {
825               if (vx == 0 && cutLabel) {
826                 ierr = DMLabelSetValue(cutLabel, edge,    1);CHKERRQ(ierr);
827                 ierr = DMLabelSetValue(cutLabel, cone[0], 1);CHKERRQ(ierr);
828                 if (ey == numYEdges-1) {
829                   ierr = DMLabelSetValue(cutLabel, cone[1], 1);CHKERRQ(ierr);
830                 }
831               }
832             }
833           } else {
834             if (bdX != DM_BOUNDARY_PERIODIC) {
835               if (vx == numXVertices-1) {
836                 ierr = DMSetLabelValue(dm, "marker", edge, markerRight);CHKERRQ(ierr);
837               } else if (vx == 0) {
838                 ierr = DMSetLabelValue(dm, "marker", edge, markerLeft);CHKERRQ(ierr);
839               }
840             }
841             if (bdZ != DM_BOUNDARY_PERIODIC) {
842               if (vz == numZVertices-1) {
843                 ierr = DMSetLabelValue(dm, "marker", edge, markerTop);CHKERRQ(ierr);
844               } else if (vz == 0) {
845                 ierr = DMSetLabelValue(dm, "marker", edge, markerBottom);CHKERRQ(ierr);
846               }
847             }
848           }
849         }
850       }
851     }
852     /* Build X edges*/
853     for (vz = 0; vz < numZVertices; vz++) {
854       for (vy = 0; vy < numYVertices; vy++) {
855         for (ex = 0; ex < numXEdges; ex++) {
856           const PetscInt nextv   = (dim == 2 && bdX == DM_BOUNDARY_TWIST && ex == numXEdges-1) ? (numYVertices-vy-1)*numXVertices : (vz*numYVertices+vy)*numXVertices + (ex+1)%numXVertices;
857           const PetscInt edge    = firstXEdge  + (vz*numYVertices+vy)*numXEdges + ex;
858           const PetscInt vertexL = firstVertex + (vz*numYVertices+vy)*numXVertices + ex;
859           const PetscInt vertexR = firstVertex + nextv;
860           PetscInt       cone[2];
861 
862           cone[0] = vertexL; cone[1] = vertexR;
863           ierr = DMPlexSetCone(dm, edge, cone);CHKERRQ(ierr);
864           if (dim == 2) {
865             if ((bdY != DM_BOUNDARY_PERIODIC) && (bdY != DM_BOUNDARY_TWIST)) {
866               if (vy == numYVertices-1) {
867                 ierr = DMSetLabelValue(dm, "Face Sets", edge, faceMarkerTop);CHKERRQ(ierr);
868                 ierr = DMSetLabelValue(dm, "marker", edge,    markerTop);CHKERRQ(ierr);
869                 ierr = DMSetLabelValue(dm, "marker", cone[0], markerTop);CHKERRQ(ierr);
870                 if (ex == numXEdges-1) {
871                   ierr = DMSetLabelValue(dm, "marker", cone[1], markerTop);CHKERRQ(ierr);
872                 }
873               } else if (vy == 0) {
874                 ierr = DMSetLabelValue(dm, "Face Sets", edge, faceMarkerBottom);CHKERRQ(ierr);
875                 ierr = DMSetLabelValue(dm, "marker", edge,    markerBottom);CHKERRQ(ierr);
876                 ierr = DMSetLabelValue(dm, "marker", cone[0], markerBottom);CHKERRQ(ierr);
877                 if (ex == numXEdges-1) {
878                   ierr = DMSetLabelValue(dm, "marker", cone[1], markerBottom);CHKERRQ(ierr);
879                 }
880               }
881             } else {
882               if (vy == 0 && cutLabel) {
883                 ierr = DMLabelSetValue(cutLabel, edge,    1);CHKERRQ(ierr);
884                 ierr = DMLabelSetValue(cutLabel, cone[0], 1);CHKERRQ(ierr);
885                 if (ex == numXEdges-1) {
886                   ierr = DMLabelSetValue(cutLabel, cone[1], 1);CHKERRQ(ierr);
887                 }
888               }
889             }
890           } else {
891             if (bdY != DM_BOUNDARY_PERIODIC) {
892               if (vy == numYVertices-1) {
893                 ierr = DMSetLabelValue(dm, "marker", edge, markerBack);CHKERRQ(ierr);
894               }
895               else if (vy == 0) {
896                 ierr = DMSetLabelValue(dm, "marker", edge, markerFront);CHKERRQ(ierr);
897               }
898             }
899             if (bdZ != DM_BOUNDARY_PERIODIC) {
900               if (vz == numZVertices-1) {
901                 ierr = DMSetLabelValue(dm, "marker", edge, markerTop);CHKERRQ(ierr);
902               }
903               else if (vz == 0) {
904                 ierr = DMSetLabelValue(dm, "marker", edge, markerBottom);CHKERRQ(ierr);
905               }
906             }
907           }
908         }
909       }
910     }
911     ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
912     ierr = DMPlexStratify(dm);CHKERRQ(ierr);
913     /* Build coordinates */
914     ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
915     ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
916     ierr = PetscSectionSetFieldComponents(coordSection, 0, dim);CHKERRQ(ierr);
917     ierr = PetscSectionSetChart(coordSection, firstVertex, firstVertex+numVertices);CHKERRQ(ierr);
918     for (v = firstVertex; v < firstVertex+numVertices; ++v) {
919       ierr = PetscSectionSetDof(coordSection, v, dim);CHKERRQ(ierr);
920       ierr = PetscSectionSetFieldDof(coordSection, v, 0, dim);CHKERRQ(ierr);
921     }
922     ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
923     ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
924     ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
925     ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
926     ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
927     ierr = VecSetBlockSize(coordinates, dim);CHKERRQ(ierr);
928     ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
929     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
930     for (vz = 0; vz < numZVertices; ++vz) {
931       for (vy = 0; vy < numYVertices; ++vy) {
932         for (vx = 0; vx < numXVertices; ++vx) {
933           coords[((vz*numYVertices+vy)*numXVertices+vx)*dim+0] = lower[0] + ((upper[0] - lower[0])/numXEdges)*vx;
934           coords[((vz*numYVertices+vy)*numXVertices+vx)*dim+1] = lower[1] + ((upper[1] - lower[1])/numYEdges)*vy;
935           if (dim == 3) {
936             coords[((vz*numYVertices+vy)*numXVertices+vx)*dim+2] = lower[2] + ((upper[2] - lower[2])/numZEdges)*vz;
937           }
938         }
939       }
940     }
941     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
942     ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
943     ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
944   }
945   PetscFunctionReturn(0);
946 }
947 
948 static PetscErrorCode DMPlexCreateBoxMesh_Tensor_Internal(MPI_Comm comm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate, DM *dm)
949 {
950   PetscInt       i;
951   PetscErrorCode ierr;
952 
953   PetscFunctionBegin;
954   PetscValidPointer(dm, 7);
955   ierr = DMCreate(comm, dm);CHKERRQ(ierr);
956   PetscValidLogicalCollectiveInt(*dm,dim,2);
957   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
958   ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
959   switch (dim) {
960   case 2: {ierr = DMPlexCreateCubeMesh_Internal(*dm, lower, upper, faces, periodicity[0], periodicity[1], DM_BOUNDARY_NONE);CHKERRQ(ierr);break;}
961   case 3: {ierr = DMPlexCreateCubeMesh_Internal(*dm, lower, upper, faces, periodicity[0], periodicity[1], periodicity[2]);CHKERRQ(ierr);break;}
962   default: SETERRQ1(comm, PETSC_ERR_SUP, "Dimension not supported: %d", dim);
963   }
964   if (periodicity[0] == DM_BOUNDARY_PERIODIC || periodicity[0] == DM_BOUNDARY_TWIST ||
965       periodicity[1] == DM_BOUNDARY_PERIODIC || periodicity[1] == DM_BOUNDARY_TWIST ||
966       (dim > 2 && (periodicity[2] == DM_BOUNDARY_PERIODIC || periodicity[2] == DM_BOUNDARY_TWIST))) {
967     PetscReal L[3];
968     PetscReal maxCell[3];
969 
970     for (i = 0; i < dim; i++) {
971       L[i]       = upper[i] - lower[i];
972       maxCell[i] = 1.1 * (L[i] / PetscMax(1,faces[i]));
973     }
974     ierr = DMSetPeriodicity(*dm,PETSC_TRUE,maxCell,L,periodicity);CHKERRQ(ierr);
975   }
976   if (!interpolate) {
977     DM udm;
978 
979     ierr = DMPlexUninterpolate(*dm, &udm);CHKERRQ(ierr);
980     ierr = DMPlexCopyCoordinates(*dm, udm);CHKERRQ(ierr);
981     ierr = DMDestroy(dm);CHKERRQ(ierr);
982     *dm  = udm;
983   }
984   PetscFunctionReturn(0);
985 }
986 
987 /*@C
988   DMPlexCreateBoxMesh - Creates a mesh on the tensor product of unit intervals (box) using simplices or tensor cells (hexahedra).
989 
990   Collective on MPI_Comm
991 
992   Input Parameters:
993 + comm        - The communicator for the DM object
994 . dim         - The spatial dimension
995 . simplex     - PETSC_TRUE for simplices, PETSC_FALSE for tensor cells
996 . faces       - Number of faces per dimension, or NULL for (1,) in 1D and (2, 2) in 2D and (1, 1, 1) in 3D
997 . lower       - The lower left corner, or NULL for (0, 0, 0)
998 . upper       - The upper right corner, or NULL for (1, 1, 1)
999 . periodicity - The boundary type for the X,Y,Z direction, or NULL for DM_BOUNDARY_NONE
1000 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
1001 
1002   Output Parameter:
1003 . dm  - The DM object
1004 
1005   Options Database Keys:
1006 + -dm_plex_box_lower <x,y,z> - Specify lower-left-bottom coordinates for the box
1007 - -dm_plex_box_upper <x,y,z> - Specify upper-right-top coordinates for the box
1008 
1009   Note: Here is the numbering returned for 2 faces in each direction for tensor cells:
1010 $ 10---17---11---18----12
1011 $  |         |         |
1012 $  |         |         |
1013 $ 20    2   22    3    24
1014 $  |         |         |
1015 $  |         |         |
1016 $  7---15----8---16----9
1017 $  |         |         |
1018 $  |         |         |
1019 $ 19    0   21    1   23
1020 $  |         |         |
1021 $  |         |         |
1022 $  4---13----5---14----6
1023 
1024 and for simplicial cells
1025 
1026 $ 14----8---15----9----16
1027 $  |\     5  |\      7 |
1028 $  | \       | \       |
1029 $ 13   2    14    3    15
1030 $  | 4   \   | 6   \   |
1031 $  |       \ |       \ |
1032 $ 11----6---12----7----13
1033 $  |\        |\        |
1034 $  | \    1  | \     3 |
1035 $ 10   0    11    1    12
1036 $  | 0   \   | 2   \   |
1037 $  |       \ |       \ |
1038 $  8----4----9----5----10
1039 
1040   Level: beginner
1041 
1042 .keywords: DM, create
1043 .seealso: DMPlexCreateFromFile(), DMPlexCreateHexCylinderMesh(), DMSetType(), DMCreate()
1044 @*/
1045 PetscErrorCode DMPlexCreateBoxMesh(MPI_Comm comm, PetscInt dim, PetscBool simplex, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate, DM *dm)
1046 {
1047   PetscInt       fac[3] = {0, 0, 0};
1048   PetscReal      low[3] = {0, 0, 0};
1049   PetscReal      upp[3] = {1, 1, 1};
1050   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1051   PetscInt       i, n;
1052   PetscBool      flg;
1053   PetscErrorCode ierr;
1054 
1055   PetscFunctionBegin;
1056   for (i = 0; i < dim; ++i) fac[i] = faces ? faces[i] : (dim == 1 ? 1 : 4-dim);
1057   if (lower) for (i = 0; i < dim; ++i) low[i] = lower[i];
1058   if (upper) for (i = 0; i < dim; ++i) upp[i] = upper[i];
1059   if (periodicity) for (i = 0; i < dim; ++i) bdt[i] = periodicity[i];
1060   /* Allow bounds to be specified from the command line */
1061   n    = 3;
1062   ierr = PetscOptionsGetRealArray(NULL, NULL, "-dm_plex_box_lower", low, &n, &flg);CHKERRQ(ierr);
1063   if (flg && (n != dim)) SETERRQ2(comm, PETSC_ERR_ARG_SIZ, "Lower box point had %D values, should have been %D", n, dim);
1064   n    = 3;
1065   ierr = PetscOptionsGetRealArray(NULL, NULL, "-dm_plex_box_upper", upp, &n, &flg);CHKERRQ(ierr);
1066   if (flg && (n != dim)) SETERRQ2(comm, PETSC_ERR_ARG_SIZ, "Upper box point had %D values, should have been %D", n, dim);
1067 
1068   if (dim == 1)      {ierr = DMPlexCreateLineMesh_Internal(comm, fac[0], low[0], upp[0], bdt[0], dm);CHKERRQ(ierr);}
1069   else if (simplex)  {ierr = DMPlexCreateBoxMesh_Simplex_Internal(comm, dim, fac, low, upp, bdt, interpolate, dm);CHKERRQ(ierr);}
1070   else               {ierr = DMPlexCreateBoxMesh_Tensor_Internal(comm, dim, fac, low, upp, bdt, interpolate, dm);CHKERRQ(ierr);}
1071   PetscFunctionReturn(0);
1072 }
1073 
1074 /*@
1075   DMPlexCreateWedgeBoxMesh - Creates a 3-D mesh tesselating the (x,y) plane and extruding in the third direction using wedge cells.
1076 
1077   Collective on MPI_Comm
1078 
1079   Input Parameters:
1080 + comm        - The communicator for the DM object
1081 . faces       - Number of faces per dimension, or NULL for (1, 1, 1)
1082 . lower       - The lower left corner, or NULL for (0, 0, 0)
1083 . upper       - The upper right corner, or NULL for (1, 1, 1)
1084 . periodicity - The boundary type for the X,Y,Z direction, or NULL for DM_BOUNDARY_NONE
1085 . ordExt      - If PETSC_TRUE, orders the extruded cells in the height first. Otherwise, orders the cell on the layers first
1086 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
1087 
1088   Output Parameter:
1089 . dm  - The DM object
1090 
1091   Level: beginner
1092 
1093 .keywords: DM, create
1094 .seealso: DMPlexCreateHexCylinderMesh(), DMPlexCreateWedgeCylinderMesh(), DMPlexExtrude(), DMPlexCreateBoxMesh(), DMSetType(), DMCreate()
1095 @*/
1096 PetscErrorCode DMPlexCreateWedgeBoxMesh(MPI_Comm comm, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool ordExt, PetscBool interpolate, DM *dm)
1097 {
1098   DM             bdm, botdm;
1099   PetscInt       i;
1100   PetscInt       fac[3] = {0, 0, 0};
1101   PetscReal      low[3] = {0, 0, 0};
1102   PetscReal      upp[3] = {1, 1, 1};
1103   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1104   PetscErrorCode ierr;
1105 
1106   PetscFunctionBegin;
1107   for (i = 0; i < 3; ++i) fac[i] = faces ? (faces[i] > 0 ? faces[i] : 1) : 1;
1108   if (lower) for (i = 0; i < 3; ++i) low[i] = lower[i];
1109   if (upper) for (i = 0; i < 3; ++i) upp[i] = upper[i];
1110   if (periodicity) for (i = 0; i < 3; ++i) bdt[i] = periodicity[i];
1111   for (i = 0; i < 3; ++i) if (bdt[i] != DM_BOUNDARY_NONE) SETERRQ(comm, PETSC_ERR_SUP, "Periodicity not yet supported");
1112 
1113   ierr = DMCreate(comm, &bdm);CHKERRQ(ierr);
1114   ierr = DMSetType(bdm, DMPLEX);CHKERRQ(ierr);
1115   ierr = DMSetDimension(bdm, 1);CHKERRQ(ierr);
1116   ierr = DMSetCoordinateDim(bdm, 2);CHKERRQ(ierr);
1117   ierr = DMPlexCreateSquareBoundary(bdm, low, upp, fac);CHKERRQ(ierr);
1118   ierr = DMPlexGenerate(bdm, NULL, PETSC_FALSE, &botdm);CHKERRQ(ierr);
1119   ierr = DMDestroy(&bdm);CHKERRQ(ierr);
1120   ierr = DMPlexExtrude(botdm, fac[2], upp[2] - low[2], ordExt, interpolate, dm);CHKERRQ(ierr);
1121   if (low[2] != 0.0) {
1122     Vec         v;
1123     PetscScalar *x;
1124     PetscInt    cDim, n;
1125 
1126     ierr = DMGetCoordinatesLocal(*dm, &v);CHKERRQ(ierr);
1127     ierr = VecGetBlockSize(v, &cDim);CHKERRQ(ierr);
1128     ierr = VecGetLocalSize(v, &n);CHKERRQ(ierr);
1129     ierr = VecGetArray(v, &x);CHKERRQ(ierr);
1130     x   += cDim;
1131     for (i=0; i<n; i+=cDim) x[i] += low[2];
1132     ierr = VecRestoreArray(v,&x);CHKERRQ(ierr);
1133     ierr = DMSetCoordinatesLocal(*dm, v);CHKERRQ(ierr);
1134   }
1135   ierr = DMDestroy(&botdm);CHKERRQ(ierr);
1136   PetscFunctionReturn(0);
1137 }
1138 
1139 /*@
1140   DMPlexExtrude - Creates a (d+1)-D mesh by extruding a d-D mesh in the normal direction using prismatic cells.
1141 
1142   Collective on idm
1143 
1144   Input Parameters:
1145 + idm - The mesh to be extruted
1146 . layers - The number of layers
1147 . height - The height of the extruded layer
1148 . ordExt - If PETSC_TRUE, orders the extruded cells in the height first. Otherwise, orders the cell on the layers first
1149 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
1150 
1151   Output Parameter:
1152 . dm  - The DM object
1153 
1154   Notes: The object created is an hybrid mesh, the vertex ordering in the cone of the cell is that of the prismatic cells
1155 
1156   Level: advanced
1157 
1158 .keywords: DM, create
1159 .seealso: DMPlexCreateWedgeCylinderMesh(), DMPlexCreateWedgeBoxMesh(), DMPlexSetHybridBounds(), DMSetType(), DMCreate()
1160 @*/
1161 PetscErrorCode DMPlexExtrude(DM idm, PetscInt layers, PetscReal height, PetscBool ordExt, PetscBool interpolate, DM* dm)
1162 {
1163   PetscScalar       *coordsB;
1164   const PetscScalar *coordsA;
1165   PetscReal         *normals = NULL;
1166   Vec               coordinatesA, coordinatesB;
1167   PetscSection      coordSectionA, coordSectionB;
1168   PetscInt          dim, cDim, cDimB, c, l, v, coordSize, *newCone;
1169   PetscInt          cStart, cEnd, vStart, vEnd, cellV, numCells, numVertices;
1170   PetscErrorCode    ierr;
1171 
1172   PetscFunctionBegin;
1173   PetscValidHeaderSpecific(idm, DM_CLASSID, 1);
1174   PetscValidLogicalCollectiveInt(idm, layers, 2);
1175   PetscValidLogicalCollectiveReal(idm, height, 3);
1176   PetscValidLogicalCollectiveBool(idm, interpolate, 4);
1177   ierr = DMGetDimension(idm, &dim);CHKERRQ(ierr);
1178   if (dim < 1 || dim > 3) SETERRQ1(PetscObjectComm((PetscObject)idm), PETSC_ERR_SUP, "Support for dimension %D not coded", dim);
1179 
1180   ierr = DMPlexGetHeightStratum(idm, 0, &cStart, &cEnd);CHKERRQ(ierr);
1181   ierr = DMPlexGetDepthStratum(idm, 0, &vStart, &vEnd);CHKERRQ(ierr);
1182   numCells = (cEnd - cStart)*layers;
1183   numVertices = (vEnd - vStart)*(layers+1);
1184   ierr = DMCreate(PetscObjectComm((PetscObject)idm), dm);CHKERRQ(ierr);
1185   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
1186   ierr = DMSetDimension(*dm, dim+1);CHKERRQ(ierr);
1187   ierr = DMPlexSetChart(*dm, 0, numCells+numVertices);CHKERRQ(ierr);
1188   for (c = cStart, cellV = 0; c < cEnd; ++c) {
1189     PetscInt *closure = NULL;
1190     PetscInt closureSize, numCorners = 0;
1191 
1192     ierr = DMPlexGetTransitiveClosure(idm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1193     for (v = 0; v < closureSize*2; v += 2) if ((closure[v] >= vStart) && (closure[v] < vEnd)) numCorners++;
1194     ierr = DMPlexRestoreTransitiveClosure(idm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1195     for (l = 0; l < layers; ++l) {
1196       ierr = DMPlexSetConeSize(*dm, ordExt ? layers*(c - cStart) + l : l*(cEnd - cStart) + c - cStart, 2*numCorners);CHKERRQ(ierr);
1197     }
1198     cellV = PetscMax(numCorners,cellV);
1199   }
1200   ierr = DMPlexSetHybridBounds(*dm, 0, PETSC_DETERMINE, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
1201   ierr = DMSetUp(*dm);CHKERRQ(ierr);
1202 
1203   ierr = DMGetCoordinateDim(idm, &cDim);CHKERRQ(ierr);
1204   if (dim != cDim) {
1205     ierr = PetscCalloc1(cDim*(vEnd - vStart), &normals);CHKERRQ(ierr);
1206   }
1207   ierr = PetscMalloc1(3*cellV,&newCone);CHKERRQ(ierr);
1208   for (c = cStart; c < cEnd; ++c) {
1209     PetscInt *closure = NULL;
1210     PetscInt closureSize, numCorners = 0, l;
1211     PetscReal normal[3] = {0, 0, 0};
1212 
1213     if (normals) {
1214       ierr = DMPlexComputeCellGeometryFVM(idm, c, NULL, NULL, normal);CHKERRQ(ierr);
1215     }
1216     ierr = DMPlexGetTransitiveClosure(idm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1217     for (v = 0; v < closureSize*2; v += 2) {
1218       if ((closure[v] >= vStart) && (closure[v] < vEnd)) {
1219         PetscInt d;
1220 
1221         newCone[numCorners++] = closure[v] - vStart;
1222         if (normals) { for (d = 0; d < cDim; ++d) normals[cDim*(closure[v]-vStart)+d] += normal[d]; }
1223       }
1224     }
1225     switch (numCorners) {
1226     case 4: /* do nothing */
1227     case 2: /* do nothing */
1228       break;
1229     case 3: /* from counter-clockwise to wedge ordering */
1230       l = newCone[1];
1231       newCone[1] = newCone[2];
1232       newCone[2] = l;
1233       break;
1234     default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_SUP, "Unsupported number of corners: %D", numCorners);
1235     }
1236     ierr = DMPlexRestoreTransitiveClosure(idm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1237     for (l = 0; l < layers; ++l) {
1238       PetscInt i;
1239 
1240       for (i = 0; i < numCorners; ++i) {
1241         newCone[  numCorners + i] = ordExt ? (layers+1)*newCone[i] + l     + numCells :     l*(vEnd - vStart) + newCone[i] + numCells;
1242         newCone[2*numCorners + i] = ordExt ? (layers+1)*newCone[i] + l + 1 + numCells : (l+1)*(vEnd - vStart) + newCone[i] + numCells;
1243       }
1244       ierr = DMPlexSetCone(*dm, ordExt ? layers*(c - cStart) + l : l*(cEnd - cStart) + c - cStart, newCone + numCorners);CHKERRQ(ierr);
1245     }
1246   }
1247   ierr = DMPlexSymmetrize(*dm);CHKERRQ(ierr);
1248   ierr = DMPlexStratify(*dm);CHKERRQ(ierr);
1249   ierr = PetscFree(newCone);CHKERRQ(ierr);
1250 
1251   cDimB = cDim == dim ? cDim+1 : cDim;
1252   ierr = DMGetCoordinateSection(*dm, &coordSectionB);CHKERRQ(ierr);
1253   ierr = PetscSectionSetNumFields(coordSectionB, 1);CHKERRQ(ierr);
1254   ierr = PetscSectionSetFieldComponents(coordSectionB, 0, cDimB);CHKERRQ(ierr);
1255   ierr = PetscSectionSetChart(coordSectionB, numCells, numCells+numVertices);CHKERRQ(ierr);
1256   for (v = numCells; v < numCells+numVertices; ++v) {
1257     ierr = PetscSectionSetDof(coordSectionB, v, cDimB);CHKERRQ(ierr);
1258     ierr = PetscSectionSetFieldDof(coordSectionB, v, 0, cDimB);CHKERRQ(ierr);
1259   }
1260   ierr = PetscSectionSetUp(coordSectionB);CHKERRQ(ierr);
1261   ierr = PetscSectionGetStorageSize(coordSectionB, &coordSize);CHKERRQ(ierr);
1262   ierr = VecCreate(PETSC_COMM_SELF, &coordinatesB);CHKERRQ(ierr);
1263   ierr = PetscObjectSetName((PetscObject) coordinatesB, "coordinates");CHKERRQ(ierr);
1264   ierr = VecSetSizes(coordinatesB, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
1265   ierr = VecSetBlockSize(coordinatesB, cDimB);CHKERRQ(ierr);
1266   ierr = VecSetType(coordinatesB,VECSTANDARD);CHKERRQ(ierr);
1267 
1268   ierr = DMGetCoordinateSection(idm, &coordSectionA);CHKERRQ(ierr);
1269   ierr = DMGetCoordinatesLocal(idm, &coordinatesA);CHKERRQ(ierr);
1270   ierr = VecGetArray(coordinatesB, &coordsB);CHKERRQ(ierr);
1271   ierr = VecGetArrayRead(coordinatesA, &coordsA);CHKERRQ(ierr);
1272   for (v = vStart; v < vEnd; ++v) {
1273     const PetscScalar *cptr;
1274     PetscReal         ones2[2] = { 0., 1.}, ones3[3] = { 0., 0., 1.};
1275     PetscReal         *normal, norm, h = height/layers;
1276     PetscInt          offA, d, cDimA = cDim;
1277 
1278     normal = normals ? normals + cDimB*(v - vStart) : (cDim > 1 ? ones3 : ones2);
1279     if (normals) {
1280       for (d = 0, norm = 0.0; d < cDimB; ++d) norm += normal[d]*normal[d];
1281       for (d = 0; d < cDimB; ++d) normal[d] *= 1./PetscSqrtReal(norm);
1282     }
1283 
1284     ierr = PetscSectionGetOffset(coordSectionA, v, &offA);CHKERRQ(ierr);
1285     cptr = coordsA + offA;
1286     for (l = 0; l < layers+1; ++l) {
1287       PetscInt offB, d, newV;
1288 
1289       newV = ordExt ? (layers+1)*(v -vStart) + l + numCells : (vEnd -vStart)*l + (v -vStart) + numCells;
1290       ierr = PetscSectionGetOffset(coordSectionB, newV, &offB);CHKERRQ(ierr);
1291       for (d = 0; d < cDimA; ++d) { coordsB[offB+d]  = cptr[d]; }
1292       for (d = 0; d < cDimB; ++d) { coordsB[offB+d] += l ? normal[d]*h : 0.0; }
1293       cptr    = coordsB + offB;
1294       cDimA   = cDimB;
1295     }
1296   }
1297   ierr = VecRestoreArrayRead(coordinatesA, &coordsA);CHKERRQ(ierr);
1298   ierr = VecRestoreArray(coordinatesB, &coordsB);CHKERRQ(ierr);
1299   ierr = DMSetCoordinatesLocal(*dm, coordinatesB);CHKERRQ(ierr);
1300   ierr = VecDestroy(&coordinatesB);CHKERRQ(ierr);
1301   ierr = PetscFree(normals);CHKERRQ(ierr);
1302   if (interpolate) {
1303     DM idm;
1304 
1305     ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
1306     ierr = DMPlexCopyCoordinates(*dm, idm);CHKERRQ(ierr);
1307     ierr = DMDestroy(dm);CHKERRQ(ierr);
1308     *dm  = idm;
1309   }
1310   PetscFunctionReturn(0);
1311 }
1312 
1313 /*@C
1314   DMPlexSetOptionsPrefix - Sets the prefix used for searching for all DM options in the database.
1315 
1316   Logically Collective on DM
1317 
1318   Input Parameters:
1319 + dm - the DM context
1320 - prefix - the prefix to prepend to all option names
1321 
1322   Notes:
1323   A hyphen (-) must NOT be given at the beginning of the prefix name.
1324   The first character of all runtime options is AUTOMATICALLY the hyphen.
1325 
1326   Level: advanced
1327 
1328 .seealso: SNESSetFromOptions()
1329 @*/
1330 PetscErrorCode DMPlexSetOptionsPrefix(DM dm, const char prefix[])
1331 {
1332   DM_Plex       *mesh = (DM_Plex *) dm->data;
1333   PetscErrorCode ierr;
1334 
1335   PetscFunctionBegin;
1336   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1337   ierr = PetscObjectSetOptionsPrefix((PetscObject) dm, prefix);CHKERRQ(ierr);
1338   ierr = PetscObjectSetOptionsPrefix((PetscObject) mesh->partitioner, prefix);CHKERRQ(ierr);
1339   PetscFunctionReturn(0);
1340 }
1341 
1342 /*@
1343   DMPlexCreateHexCylinderMesh - Creates a mesh on the tensor product of the unit interval with the circle (cylinder) using hexahedra.
1344 
1345   Collective on MPI_Comm
1346 
1347   Input Parameters:
1348 + comm      - The communicator for the DM object
1349 . numRefine - The number of regular refinements to the basic 5 cell structure
1350 - periodicZ - The boundary type for the Z direction
1351 
1352   Output Parameter:
1353 . dm  - The DM object
1354 
1355   Note: Here is the output numbering looking from the bottom of the cylinder:
1356 $       17-----14
1357 $        |     |
1358 $        |  2  |
1359 $        |     |
1360 $ 17-----8-----7-----14
1361 $  |     |     |     |
1362 $  |  3  |  0  |  1  |
1363 $  |     |     |     |
1364 $ 19-----5-----6-----13
1365 $        |     |
1366 $        |  4  |
1367 $        |     |
1368 $       19-----13
1369 $
1370 $ and up through the top
1371 $
1372 $       18-----16
1373 $        |     |
1374 $        |  2  |
1375 $        |     |
1376 $ 18----10----11-----16
1377 $  |     |     |     |
1378 $  |  3  |  0  |  1  |
1379 $  |     |     |     |
1380 $ 20-----9----12-----15
1381 $        |     |
1382 $        |  4  |
1383 $        |     |
1384 $       20-----15
1385 
1386   Level: beginner
1387 
1388 .keywords: DM, create
1389 .seealso: DMPlexCreateBoxMesh(), DMSetType(), DMCreate()
1390 @*/
1391 PetscErrorCode DMPlexCreateHexCylinderMesh(MPI_Comm comm, PetscInt numRefine, DMBoundaryType periodicZ, DM *dm)
1392 {
1393   const PetscInt dim = 3;
1394   PetscInt       numCells, numVertices, r;
1395   PetscMPIInt    rank;
1396   PetscErrorCode ierr;
1397 
1398   PetscFunctionBegin;
1399   PetscValidPointer(dm, 4);
1400   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
1401   if (numRefine < 0) SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Number of refinements %D cannot be negative", numRefine);
1402   ierr = DMCreate(comm, dm);CHKERRQ(ierr);
1403   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
1404   ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
1405   /* Create topology */
1406   {
1407     PetscInt cone[8], c;
1408 
1409     numCells    = !rank ?  5 : 0;
1410     numVertices = !rank ? 16 : 0;
1411     if (periodicZ == DM_BOUNDARY_PERIODIC) {
1412       numCells   *= 3;
1413       numVertices = !rank ? 24 : 0;
1414     }
1415     ierr = DMPlexSetChart(*dm, 0, numCells+numVertices);CHKERRQ(ierr);
1416     for (c = 0; c < numCells; c++) {ierr = DMPlexSetConeSize(*dm, c, 8);CHKERRQ(ierr);}
1417     ierr = DMSetUp(*dm);CHKERRQ(ierr);
1418     if (!rank) {
1419       if (periodicZ == DM_BOUNDARY_PERIODIC) {
1420         cone[0] = 15; cone[1] = 18; cone[2] = 17; cone[3] = 16;
1421         cone[4] = 31; cone[5] = 32; cone[6] = 33; cone[7] = 34;
1422         ierr = DMPlexSetCone(*dm, 0, cone);CHKERRQ(ierr);
1423         cone[0] = 16; cone[1] = 17; cone[2] = 24; cone[3] = 23;
1424         cone[4] = 32; cone[5] = 36; cone[6] = 37; cone[7] = 33; /* 22 25 26 21 */
1425         ierr = DMPlexSetCone(*dm, 1, cone);CHKERRQ(ierr);
1426         cone[0] = 18; cone[1] = 27; cone[2] = 24; cone[3] = 17;
1427         cone[4] = 34; cone[5] = 33; cone[6] = 37; cone[7] = 38;
1428         ierr = DMPlexSetCone(*dm, 2, cone);CHKERRQ(ierr);
1429         cone[0] = 29; cone[1] = 27; cone[2] = 18; cone[3] = 15;
1430         cone[4] = 35; cone[5] = 31; cone[6] = 34; cone[7] = 38;
1431         ierr = DMPlexSetCone(*dm, 3, cone);CHKERRQ(ierr);
1432         cone[0] = 29; cone[1] = 15; cone[2] = 16; cone[3] = 23;
1433         cone[4] = 35; cone[5] = 36; cone[6] = 32; cone[7] = 31;
1434         ierr = DMPlexSetCone(*dm, 4, cone);CHKERRQ(ierr);
1435 
1436         cone[0] = 31; cone[1] = 34; cone[2] = 33; cone[3] = 32;
1437         cone[4] = 19; cone[5] = 22; cone[6] = 21; cone[7] = 20;
1438         ierr = DMPlexSetCone(*dm, 5, cone);CHKERRQ(ierr);
1439         cone[0] = 32; cone[1] = 33; cone[2] = 37; cone[3] = 36;
1440         cone[4] = 22; cone[5] = 25; cone[6] = 26; cone[7] = 21;
1441         ierr = DMPlexSetCone(*dm, 6, cone);CHKERRQ(ierr);
1442         cone[0] = 34; cone[1] = 38; cone[2] = 37; cone[3] = 33;
1443         cone[4] = 20; cone[5] = 21; cone[6] = 26; cone[7] = 28;
1444         ierr = DMPlexSetCone(*dm, 7, cone);CHKERRQ(ierr);
1445         cone[0] = 35; cone[1] = 38; cone[2] = 34; cone[3] = 31;
1446         cone[4] = 30; cone[5] = 19; cone[6] = 20; cone[7] = 28;
1447         ierr = DMPlexSetCone(*dm, 8, cone);CHKERRQ(ierr);
1448         cone[0] = 35; cone[1] = 31; cone[2] = 32; cone[3] = 36;
1449         cone[4] = 30; cone[5] = 25; cone[6] = 22; cone[7] = 19;
1450         ierr = DMPlexSetCone(*dm, 9, cone);CHKERRQ(ierr);
1451 
1452         cone[0] = 19; cone[1] = 20; cone[2] = 21; cone[3] = 22;
1453         cone[4] = 15; cone[5] = 16; cone[6] = 17; cone[7] = 18;
1454         ierr = DMPlexSetCone(*dm, 10, cone);CHKERRQ(ierr);
1455         cone[0] = 22; cone[1] = 21; cone[2] = 26; cone[3] = 25;
1456         cone[4] = 16; cone[5] = 23; cone[6] = 24; cone[7] = 17;
1457         ierr = DMPlexSetCone(*dm, 11, cone);CHKERRQ(ierr);
1458         cone[0] = 20; cone[1] = 28; cone[2] = 26; cone[3] = 21;
1459         cone[4] = 18; cone[5] = 17; cone[6] = 24; cone[7] = 27;
1460         ierr = DMPlexSetCone(*dm, 12, cone);CHKERRQ(ierr);
1461         cone[0] = 30; cone[1] = 28; cone[2] = 20; cone[3] = 19;
1462         cone[4] = 29; cone[5] = 15; cone[6] = 18; cone[7] = 27;
1463         ierr = DMPlexSetCone(*dm, 13, cone);CHKERRQ(ierr);
1464         cone[0] = 30; cone[1] = 19; cone[2] = 22; cone[3] = 25;
1465         cone[4] = 29; cone[5] = 23; cone[6] = 16; cone[7] = 15;
1466         ierr = DMPlexSetCone(*dm, 14, cone);CHKERRQ(ierr);
1467       } else {
1468         cone[0] =  5; cone[1] =  8; cone[2] =  7; cone[3] =  6;
1469         cone[4] =  9; cone[5] = 12; cone[6] = 11; cone[7] = 10;
1470         ierr = DMPlexSetCone(*dm, 0, cone);CHKERRQ(ierr);
1471         cone[0] =  6; cone[1] =  7; cone[2] = 14; cone[3] = 13;
1472         cone[4] = 12; cone[5] = 15; cone[6] = 16; cone[7] = 11;
1473         ierr = DMPlexSetCone(*dm, 1, cone);CHKERRQ(ierr);
1474         cone[0] =  8; cone[1] = 17; cone[2] = 14; cone[3] =  7;
1475         cone[4] = 10; cone[5] = 11; cone[6] = 16; cone[7] = 18;
1476         ierr = DMPlexSetCone(*dm, 2, cone);CHKERRQ(ierr);
1477         cone[0] = 19; cone[1] = 17; cone[2] =  8; cone[3] =  5;
1478         cone[4] = 20; cone[5] =  9; cone[6] = 10; cone[7] = 18;
1479         ierr = DMPlexSetCone(*dm, 3, cone);CHKERRQ(ierr);
1480         cone[0] = 19; cone[1] =  5; cone[2] =  6; cone[3] = 13;
1481         cone[4] = 20; cone[5] = 15; cone[6] = 12; cone[7] =  9;
1482         ierr = DMPlexSetCone(*dm, 4, cone);CHKERRQ(ierr);
1483       }
1484     }
1485     ierr = DMPlexSymmetrize(*dm);CHKERRQ(ierr);
1486     ierr = DMPlexStratify(*dm);CHKERRQ(ierr);
1487   }
1488   /* Interpolate */
1489   {
1490     DM idm;
1491 
1492     ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
1493     ierr = DMDestroy(dm);CHKERRQ(ierr);
1494     *dm  = idm;
1495   }
1496   /* Create cube geometry */
1497   {
1498     Vec             coordinates;
1499     PetscSection    coordSection;
1500     PetscScalar    *coords;
1501     PetscInt        coordSize, v;
1502     const PetscReal dis = 1.0/PetscSqrtReal(2.0);
1503     const PetscReal ds2 = dis/2.0;
1504 
1505     /* Build coordinates */
1506     ierr = DMGetCoordinateSection(*dm, &coordSection);CHKERRQ(ierr);
1507     ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
1508     ierr = PetscSectionSetFieldComponents(coordSection, 0, dim);CHKERRQ(ierr);
1509     ierr = PetscSectionSetChart(coordSection, numCells, numCells+numVertices);CHKERRQ(ierr);
1510     for (v = numCells; v < numCells+numVertices; ++v) {
1511       ierr = PetscSectionSetDof(coordSection, v, dim);CHKERRQ(ierr);
1512       ierr = PetscSectionSetFieldDof(coordSection, v, 0, dim);CHKERRQ(ierr);
1513     }
1514     ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
1515     ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
1516     ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
1517     ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
1518     ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
1519     ierr = VecSetBlockSize(coordinates, dim);CHKERRQ(ierr);
1520     ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
1521     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
1522     if (!rank) {
1523       coords[0*dim+0] = -ds2; coords[0*dim+1] = -ds2; coords[0*dim+2] = 0.0;
1524       coords[1*dim+0] =  ds2; coords[1*dim+1] = -ds2; coords[1*dim+2] = 0.0;
1525       coords[2*dim+0] =  ds2; coords[2*dim+1] =  ds2; coords[2*dim+2] = 0.0;
1526       coords[3*dim+0] = -ds2; coords[3*dim+1] =  ds2; coords[3*dim+2] = 0.0;
1527       coords[4*dim+0] = -ds2; coords[4*dim+1] = -ds2; coords[4*dim+2] = 1.0;
1528       coords[5*dim+0] = -ds2; coords[5*dim+1] =  ds2; coords[5*dim+2] = 1.0;
1529       coords[6*dim+0] =  ds2; coords[6*dim+1] =  ds2; coords[6*dim+2] = 1.0;
1530       coords[7*dim+0] =  ds2; coords[7*dim+1] = -ds2; coords[7*dim+2] = 1.0;
1531       coords[ 8*dim+0] =  dis; coords[ 8*dim+1] = -dis; coords[ 8*dim+2] = 0.0;
1532       coords[ 9*dim+0] =  dis; coords[ 9*dim+1] =  dis; coords[ 9*dim+2] = 0.0;
1533       coords[10*dim+0] =  dis; coords[10*dim+1] = -dis; coords[10*dim+2] = 1.0;
1534       coords[11*dim+0] =  dis; coords[11*dim+1] =  dis; coords[11*dim+2] = 1.0;
1535       coords[12*dim+0] = -dis; coords[12*dim+1] =  dis; coords[12*dim+2] = 0.0;
1536       coords[13*dim+0] = -dis; coords[13*dim+1] =  dis; coords[13*dim+2] = 1.0;
1537       coords[14*dim+0] = -dis; coords[14*dim+1] = -dis; coords[14*dim+2] = 0.0;
1538       coords[15*dim+0] = -dis; coords[15*dim+1] = -dis; coords[15*dim+2] = 1.0;
1539       if (periodicZ == DM_BOUNDARY_PERIODIC) {
1540         /* 15 31 19 */ coords[16*dim+0] = -ds2; coords[16*dim+1] = -ds2; coords[16*dim+2] = 0.5;
1541         /* 16 32 22 */ coords[17*dim+0] =  ds2; coords[17*dim+1] = -ds2; coords[17*dim+2] = 0.5;
1542         /* 17 33 21 */ coords[18*dim+0] =  ds2; coords[18*dim+1] =  ds2; coords[18*dim+2] = 0.5;
1543         /* 18 34 20 */ coords[19*dim+0] = -ds2; coords[19*dim+1] =  ds2; coords[19*dim+2] = 0.5;
1544         /* 29 35 30 */ coords[20*dim+0] = -dis; coords[20*dim+1] = -dis; coords[20*dim+2] = 0.5;
1545         /* 23 36 25 */ coords[21*dim+0] =  dis; coords[21*dim+1] = -dis; coords[21*dim+2] = 0.5;
1546         /* 24 37 26 */ coords[22*dim+0] =  dis; coords[22*dim+1] =  dis; coords[22*dim+2] = 0.5;
1547         /* 27 38 28 */ coords[23*dim+0] = -dis; coords[23*dim+1] =  dis; coords[23*dim+2] = 0.5;
1548       }
1549     }
1550     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
1551     ierr = DMSetCoordinatesLocal(*dm, coordinates);CHKERRQ(ierr);
1552     ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
1553   }
1554   /* Create periodicity */
1555   if (periodicZ == DM_BOUNDARY_PERIODIC || periodicZ == DM_BOUNDARY_TWIST) {
1556     PetscReal      L[3];
1557     PetscReal      maxCell[3];
1558     DMBoundaryType bdType[3];
1559     PetscReal      lower[3] = {0.0, 0.0, 0.0};
1560     PetscReal      upper[3] = {1.0, 1.0, 1.5};
1561     PetscInt       i, numZCells = 3;
1562 
1563     bdType[0] = DM_BOUNDARY_NONE;
1564     bdType[1] = DM_BOUNDARY_NONE;
1565     bdType[2] = periodicZ;
1566     for (i = 0; i < dim; i++) {
1567       L[i]       = upper[i] - lower[i];
1568       maxCell[i] = 1.1 * (L[i] / numZCells);
1569     }
1570     ierr = DMSetPeriodicity(*dm, PETSC_TRUE, maxCell, L, bdType);CHKERRQ(ierr);
1571   }
1572   /* Refine topology */
1573   for (r = 0; r < numRefine; ++r) {
1574     DM rdm = NULL;
1575 
1576     ierr = DMRefine(*dm, comm, &rdm);CHKERRQ(ierr);
1577     ierr = DMDestroy(dm);CHKERRQ(ierr);
1578     *dm  = rdm;
1579   }
1580   /* Remap geometry to cylinder
1581        Interior square: Linear interpolation is correct
1582        The other cells all have vertices on rays from the origin. We want to uniformly expand the spacing
1583        such that the last vertex is on the unit circle. So the closest and farthest vertices are at distance
1584 
1585          phi     = arctan(y/x)
1586          d_close = sqrt(1/8 + 1/4 sin^2(phi))
1587          d_far   = sqrt(1/2 + sin^2(phi))
1588 
1589        so we remap them using
1590 
1591          x_new = x_close + (x - x_close) (1 - d_close) / (d_far - d_close)
1592          y_new = y_close + (y - y_close) (1 - d_close) / (d_far - d_close)
1593 
1594        If pi/4 < phi < 3pi/4 or -3pi/4 < phi < -pi/4, then we switch x and y.
1595   */
1596   {
1597     Vec           coordinates;
1598     PetscSection  coordSection;
1599     PetscScalar  *coords;
1600     PetscInt      vStart, vEnd, v;
1601     const PetscReal dis = 1.0/PetscSqrtReal(2.0);
1602     const PetscReal ds2 = 0.5*dis;
1603 
1604     ierr = DMPlexGetDepthStratum(*dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
1605     ierr = DMGetCoordinateSection(*dm, &coordSection);CHKERRQ(ierr);
1606     ierr = DMGetCoordinatesLocal(*dm, &coordinates);CHKERRQ(ierr);
1607     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
1608     for (v = vStart; v < vEnd; ++v) {
1609       PetscReal phi, sinp, cosp, dc, df, x, y, xc, yc;
1610       PetscInt  off;
1611 
1612       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
1613       if ((PetscAbsScalar(coords[off+0]) <= ds2) && (PetscAbsScalar(coords[off+1]) <= ds2)) continue;
1614       x    = PetscRealPart(coords[off]);
1615       y    = PetscRealPart(coords[off+1]);
1616       phi  = PetscAtan2Real(y, x);
1617       sinp = PetscSinReal(phi);
1618       cosp = PetscCosReal(phi);
1619       if ((PetscAbsReal(phi) > PETSC_PI/4.0) && (PetscAbsReal(phi) < 3.0*PETSC_PI/4.0)) {
1620         dc = PetscAbsReal(ds2/sinp);
1621         df = PetscAbsReal(dis/sinp);
1622         xc = ds2*x/PetscAbsReal(y);
1623         yc = ds2*PetscSignReal(y);
1624       } else {
1625         dc = PetscAbsReal(ds2/cosp);
1626         df = PetscAbsReal(dis/cosp);
1627         xc = ds2*PetscSignReal(x);
1628         yc = ds2*y/PetscAbsReal(x);
1629       }
1630       coords[off+0] = xc + (coords[off+0] - xc)*(1.0 - dc)/(df - dc);
1631       coords[off+1] = yc + (coords[off+1] - yc)*(1.0 - dc)/(df - dc);
1632     }
1633     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
1634     if (periodicZ == DM_BOUNDARY_PERIODIC || periodicZ == DM_BOUNDARY_TWIST) {
1635       ierr = DMLocalizeCoordinates(*dm);CHKERRQ(ierr);
1636     }
1637   }
1638   PetscFunctionReturn(0);
1639 }
1640 
1641 /*@
1642   DMPlexCreateWedgeCylinderMesh - Creates a mesh on the tensor product of the unit interval with the circle (cylinder) using wedges.
1643 
1644   Collective on MPI_Comm
1645 
1646   Input Parameters:
1647 + comm - The communicator for the DM object
1648 . n    - The number of wedges around the origin
1649 - interpolate - Create edges and faces
1650 
1651   Output Parameter:
1652 . dm  - The DM object
1653 
1654   Level: beginner
1655 
1656 .keywords: DM, create
1657 .seealso: DMPlexCreateHexCylinderMesh(), DMPlexCreateBoxMesh(), DMSetType(), DMCreate()
1658 @*/
1659 PetscErrorCode DMPlexCreateWedgeCylinderMesh(MPI_Comm comm, PetscInt n, PetscBool interpolate, DM *dm)
1660 {
1661   const PetscInt dim = 3;
1662   PetscInt       numCells, numVertices;
1663   PetscMPIInt    rank;
1664   PetscErrorCode ierr;
1665 
1666   PetscFunctionBegin;
1667   PetscValidPointer(dm, 3);
1668   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
1669   if (n < 0) SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Number of wedges %D cannot be negative", n);
1670   ierr = DMCreate(comm, dm);CHKERRQ(ierr);
1671   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
1672   ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
1673   /* Create topology */
1674   {
1675     PetscInt cone[6], c;
1676 
1677     numCells    = !rank ?        n : 0;
1678     numVertices = !rank ?  2*(n+1) : 0;
1679     ierr = DMPlexSetChart(*dm, 0, numCells+numVertices);CHKERRQ(ierr);
1680     ierr = DMPlexSetHybridBounds(*dm, 0, PETSC_DETERMINE, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
1681     for (c = 0; c < numCells; c++) {ierr = DMPlexSetConeSize(*dm, c, 6);CHKERRQ(ierr);}
1682     ierr = DMSetUp(*dm);CHKERRQ(ierr);
1683     for (c = 0; c < numCells; c++) {
1684       cone[0] =  c+n*1; cone[1] = (c+1)%n+n*1; cone[2] = 0+3*n;
1685       cone[3] =  c+n*2; cone[4] = (c+1)%n+n*2; cone[5] = 1+3*n;
1686       ierr = DMPlexSetCone(*dm, c, cone);CHKERRQ(ierr);
1687     }
1688     ierr = DMPlexSymmetrize(*dm);CHKERRQ(ierr);
1689     ierr = DMPlexStratify(*dm);CHKERRQ(ierr);
1690   }
1691   /* Interpolate */
1692   if (interpolate) {
1693     DM idm;
1694 
1695     ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
1696     ierr = DMDestroy(dm);CHKERRQ(ierr);
1697     *dm  = idm;
1698   }
1699   /* Create cylinder geometry */
1700   {
1701     Vec          coordinates;
1702     PetscSection coordSection;
1703     PetscScalar *coords;
1704     PetscInt     coordSize, v, c;
1705 
1706     /* Build coordinates */
1707     ierr = DMGetCoordinateSection(*dm, &coordSection);CHKERRQ(ierr);
1708     ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
1709     ierr = PetscSectionSetFieldComponents(coordSection, 0, dim);CHKERRQ(ierr);
1710     ierr = PetscSectionSetChart(coordSection, numCells, numCells+numVertices);CHKERRQ(ierr);
1711     for (v = numCells; v < numCells+numVertices; ++v) {
1712       ierr = PetscSectionSetDof(coordSection, v, dim);CHKERRQ(ierr);
1713       ierr = PetscSectionSetFieldDof(coordSection, v, 0, dim);CHKERRQ(ierr);
1714     }
1715     ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
1716     ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
1717     ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
1718     ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
1719     ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
1720     ierr = VecSetBlockSize(coordinates, dim);CHKERRQ(ierr);
1721     ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
1722     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
1723     for (c = 0; c < numCells; c++) {
1724       coords[(c+0*n)*dim+0] = PetscCosReal(2.0*c*PETSC_PI/n); coords[(c+0*n)*dim+1] = PetscSinReal(2.0*c*PETSC_PI/n); coords[(c+0*n)*dim+2] = 1.0;
1725       coords[(c+1*n)*dim+0] = PetscCosReal(2.0*c*PETSC_PI/n); coords[(c+1*n)*dim+1] = PetscSinReal(2.0*c*PETSC_PI/n); coords[(c+1*n)*dim+2] = 0.0;
1726     }
1727     if (!rank) {
1728       coords[(2*n+0)*dim+0] = 0.0; coords[(2*n+0)*dim+1] = 0.0; coords[(2*n+0)*dim+2] = 1.0;
1729       coords[(2*n+1)*dim+0] = 0.0; coords[(2*n+1)*dim+1] = 0.0; coords[(2*n+1)*dim+2] = 0.0;
1730     }
1731     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
1732     ierr = DMSetCoordinatesLocal(*dm, coordinates);CHKERRQ(ierr);
1733     ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
1734   }
1735   PetscFunctionReturn(0);
1736 }
1737 
1738 PETSC_STATIC_INLINE PetscReal DiffNormReal(PetscInt dim, const PetscReal x[], const PetscReal y[])
1739 {
1740   PetscReal prod = 0.0;
1741   PetscInt  i;
1742   for (i = 0; i < dim; ++i) prod += PetscSqr(x[i] - y[i]);
1743   return PetscSqrtReal(prod);
1744 }
1745 PETSC_STATIC_INLINE PetscReal DotReal(PetscInt dim, const PetscReal x[], const PetscReal y[])
1746 {
1747   PetscReal prod = 0.0;
1748   PetscInt  i;
1749   for (i = 0; i < dim; ++i) prod += x[i]*y[i];
1750   return prod;
1751 }
1752 
1753 /*@
1754   DMPlexCreateSphereMesh - Creates a mesh on the d-dimensional sphere, S^d.
1755 
1756   Collective on MPI_Comm
1757 
1758   Input Parameters:
1759 . comm  - The communicator for the DM object
1760 . dim   - The dimension
1761 - simplex - Use simplices, or tensor product cells
1762 
1763   Output Parameter:
1764 . dm  - The DM object
1765 
1766   Level: beginner
1767 
1768 .keywords: DM, create
1769 .seealso: DMPlexCreateBoxMesh(), DMSetType(), DMCreate()
1770 @*/
1771 PetscErrorCode DMPlexCreateSphereMesh(MPI_Comm comm, PetscInt dim, PetscBool simplex, DM *dm)
1772 {
1773   const PetscInt  embedDim = dim+1;
1774   PetscSection    coordSection;
1775   Vec             coordinates;
1776   PetscScalar    *coords;
1777   PetscReal      *coordsIn;
1778   PetscInt        numCells, numEdges, numVerts, firstVertex, v, firstEdge, coordSize, d, c, e;
1779   PetscMPIInt     rank;
1780   PetscErrorCode  ierr;
1781 
1782   PetscFunctionBegin;
1783   PetscValidPointer(dm, 4);
1784   ierr = DMCreate(comm, dm);CHKERRQ(ierr);
1785   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
1786   ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
1787   ierr = DMSetCoordinateDim(*dm, dim+1);CHKERRQ(ierr);
1788   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) *dm), &rank);CHKERRQ(ierr);
1789   switch (dim) {
1790   case 2:
1791     if (simplex) {
1792       DM              idm;
1793       const PetscReal edgeLen     = 2.0/(1.0 + PETSC_PHI);
1794       const PetscReal vertex[3]   = {0.0, 1.0/(1.0 + PETSC_PHI), PETSC_PHI/(1.0 + PETSC_PHI)};
1795       const PetscInt  degree      = 5;
1796       PetscInt        s[3]        = {1, 1, 1};
1797       PetscInt        cone[3];
1798       PetscInt       *graph, p, i, j, k;
1799 
1800       numCells    = !rank ? 20 : 0;
1801       numVerts    = !rank ? 12 : 0;
1802       firstVertex = numCells;
1803       /* Use icosahedron, which for a unit sphere has coordinates which are all cyclic permutations of
1804 
1805            (0, \pm 1/\phi+1, \pm \phi/\phi+1)
1806 
1807          where \phi^2 - \phi - 1 = 0, meaning \phi is the golden ratio \frac{1 + \sqrt{5}}{2}. The edge
1808          length is then given by 2/\phi = 2 * 2.73606 = 5.47214.
1809        */
1810       /* Construct vertices */
1811       ierr = PetscCalloc1(numVerts * embedDim, &coordsIn);CHKERRQ(ierr);
1812       for (p = 0, i = 0; p < embedDim; ++p) {
1813         for (s[1] = -1; s[1] < 2; s[1] += 2) {
1814           for (s[2] = -1; s[2] < 2; s[2] += 2) {
1815             for (d = 0; d < embedDim; ++d) coordsIn[i*embedDim+d] = s[(d+p)%embedDim]*vertex[(d+p)%embedDim];
1816             ++i;
1817           }
1818         }
1819       }
1820       /* Construct graph */
1821       ierr = PetscCalloc1(numVerts * numVerts, &graph);CHKERRQ(ierr);
1822       for (i = 0; i < numVerts; ++i) {
1823         for (j = 0, k = 0; j < numVerts; ++j) {
1824           if (PetscAbsReal(DiffNormReal(embedDim, &coordsIn[i*embedDim], &coordsIn[j*embedDim]) - edgeLen) < PETSC_SMALL) {graph[i*numVerts+j] = 1; ++k;}
1825         }
1826         if (k != degree) SETERRQ3(comm, PETSC_ERR_PLIB, "Invalid icosahedron, vertex %D degree %D != %D", i, k, degree);
1827       }
1828       /* Build Topology */
1829       ierr = DMPlexSetChart(*dm, 0, numCells+numVerts);CHKERRQ(ierr);
1830       for (c = 0; c < numCells; c++) {
1831         ierr = DMPlexSetConeSize(*dm, c, embedDim);CHKERRQ(ierr);
1832       }
1833       ierr = DMSetUp(*dm);CHKERRQ(ierr); /* Allocate space for cones */
1834       /* Cells */
1835       for (i = 0, c = 0; i < numVerts; ++i) {
1836         for (j = 0; j < i; ++j) {
1837           for (k = 0; k < j; ++k) {
1838             if (graph[i*numVerts+j] && graph[j*numVerts+k] && graph[k*numVerts+i]) {
1839               cone[0] = firstVertex+i; cone[1] = firstVertex+j; cone[2] = firstVertex+k;
1840               /* Check orientation */
1841               {
1842                 const PetscInt epsilon[3][3][3] = {{{0, 0, 0}, {0, 0, 1}, {0, -1, 0}}, {{0, 0, -1}, {0, 0, 0}, {1, 0, 0}}, {{0, 1, 0}, {-1, 0, 0}, {0, 0, 0}}};
1843                 PetscReal normal[3];
1844                 PetscInt  e, f;
1845 
1846                 for (d = 0; d < embedDim; ++d) {
1847                   normal[d] = 0.0;
1848                   for (e = 0; e < embedDim; ++e) {
1849                     for (f = 0; f < embedDim; ++f) {
1850                       normal[d] += epsilon[d][e][f]*(coordsIn[j*embedDim+e] - coordsIn[i*embedDim+e])*(coordsIn[k*embedDim+f] - coordsIn[i*embedDim+f]);
1851                     }
1852                   }
1853                 }
1854                 if (DotReal(embedDim, normal, &coordsIn[i*embedDim]) < 0) {PetscInt tmp = cone[1]; cone[1] = cone[2]; cone[2] = tmp;}
1855               }
1856               ierr = DMPlexSetCone(*dm, c++, cone);CHKERRQ(ierr);
1857             }
1858           }
1859         }
1860       }
1861       ierr = DMPlexSymmetrize(*dm);CHKERRQ(ierr);
1862       ierr = DMPlexStratify(*dm);CHKERRQ(ierr);
1863       ierr = PetscFree(graph);CHKERRQ(ierr);
1864       /* Interpolate mesh */
1865       ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
1866       ierr = DMDestroy(dm);CHKERRQ(ierr);
1867       *dm  = idm;
1868     } else {
1869       /*
1870         12-21--13
1871          |     |
1872         25  4  24
1873          |     |
1874   12-25--9-16--8-24--13
1875    |     |     |     |
1876   23  5 17  0 15  3  22
1877    |     |     |     |
1878   10-20--6-14--7-19--11
1879          |     |
1880         20  1  19
1881          |     |
1882         10-18--11
1883          |     |
1884         23  2  22
1885          |     |
1886         12-21--13
1887        */
1888       const PetscReal dist = 1.0/PetscSqrtReal(3.0);
1889       PetscInt        cone[4], ornt[4];
1890 
1891       numCells    = !rank ?  6 : 0;
1892       numEdges    = !rank ? 12 : 0;
1893       numVerts    = !rank ?  8 : 0;
1894       firstVertex = numCells;
1895       firstEdge   = numCells + numVerts;
1896       /* Build Topology */
1897       ierr = DMPlexSetChart(*dm, 0, numCells+numEdges+numVerts);CHKERRQ(ierr);
1898       for (c = 0; c < numCells; c++) {
1899         ierr = DMPlexSetConeSize(*dm, c, 4);CHKERRQ(ierr);
1900       }
1901       for (e = firstEdge; e < firstEdge+numEdges; ++e) {
1902         ierr = DMPlexSetConeSize(*dm, e, 2);CHKERRQ(ierr);
1903       }
1904       ierr = DMSetUp(*dm);CHKERRQ(ierr); /* Allocate space for cones */
1905       /* Cell 0 */
1906       cone[0] = 14; cone[1] = 15; cone[2] = 16; cone[3] = 17;
1907       ierr = DMPlexSetCone(*dm, 0, cone);CHKERRQ(ierr);
1908       ornt[0] = 0; ornt[1] = 0; ornt[2] = 0; ornt[3] = 0;
1909       ierr = DMPlexSetConeOrientation(*dm, 0, ornt);CHKERRQ(ierr);
1910       /* Cell 1 */
1911       cone[0] = 18; cone[1] = 19; cone[2] = 14; cone[3] = 20;
1912       ierr = DMPlexSetCone(*dm, 1, cone);CHKERRQ(ierr);
1913       ornt[0] = 0; ornt[1] = 0; ornt[2] = -2; ornt[3] = 0;
1914       ierr = DMPlexSetConeOrientation(*dm, 1, ornt);CHKERRQ(ierr);
1915       /* Cell 2 */
1916       cone[0] = 21; cone[1] = 22; cone[2] = 18; cone[3] = 23;
1917       ierr = DMPlexSetCone(*dm, 2, cone);CHKERRQ(ierr);
1918       ornt[0] = 0; ornt[1] = 0; ornt[2] = -2; ornt[3] = 0;
1919       ierr = DMPlexSetConeOrientation(*dm, 2, ornt);CHKERRQ(ierr);
1920       /* Cell 3 */
1921       cone[0] = 19; cone[1] = 22; cone[2] = 24; cone[3] = 15;
1922       ierr = DMPlexSetCone(*dm, 3, cone);CHKERRQ(ierr);
1923       ornt[0] = -2; ornt[1] = -2; ornt[2] = 0; ornt[3] = -2;
1924       ierr = DMPlexSetConeOrientation(*dm, 3, ornt);CHKERRQ(ierr);
1925       /* Cell 4 */
1926       cone[0] = 16; cone[1] = 24; cone[2] = 21; cone[3] = 25;
1927       ierr = DMPlexSetCone(*dm, 4, cone);CHKERRQ(ierr);
1928       ornt[0] = -2; ornt[1] = -2; ornt[2] = -2; ornt[3] = 0;
1929       ierr = DMPlexSetConeOrientation(*dm, 4, ornt);CHKERRQ(ierr);
1930       /* Cell 5 */
1931       cone[0] = 20; cone[1] = 17; cone[2] = 25; cone[3] = 23;
1932       ierr = DMPlexSetCone(*dm, 5, cone);CHKERRQ(ierr);
1933       ornt[0] = -2; ornt[1] = -2; ornt[2] = -2; ornt[3] = -2;
1934       ierr = DMPlexSetConeOrientation(*dm, 5, ornt);CHKERRQ(ierr);
1935       /* Edges */
1936       cone[0] =  6; cone[1] =  7;
1937       ierr = DMPlexSetCone(*dm, 14, cone);CHKERRQ(ierr);
1938       cone[0] =  7; cone[1] =  8;
1939       ierr = DMPlexSetCone(*dm, 15, cone);CHKERRQ(ierr);
1940       cone[0] =  8; cone[1] =  9;
1941       ierr = DMPlexSetCone(*dm, 16, cone);CHKERRQ(ierr);
1942       cone[0] =  9; cone[1] =  6;
1943       ierr = DMPlexSetCone(*dm, 17, cone);CHKERRQ(ierr);
1944       cone[0] = 10; cone[1] = 11;
1945       ierr = DMPlexSetCone(*dm, 18, cone);CHKERRQ(ierr);
1946       cone[0] = 11; cone[1] =  7;
1947       ierr = DMPlexSetCone(*dm, 19, cone);CHKERRQ(ierr);
1948       cone[0] =  6; cone[1] = 10;
1949       ierr = DMPlexSetCone(*dm, 20, cone);CHKERRQ(ierr);
1950       cone[0] = 12; cone[1] = 13;
1951       ierr = DMPlexSetCone(*dm, 21, cone);CHKERRQ(ierr);
1952       cone[0] = 13; cone[1] = 11;
1953       ierr = DMPlexSetCone(*dm, 22, cone);CHKERRQ(ierr);
1954       cone[0] = 10; cone[1] = 12;
1955       ierr = DMPlexSetCone(*dm, 23, cone);CHKERRQ(ierr);
1956       cone[0] = 13; cone[1] =  8;
1957       ierr = DMPlexSetCone(*dm, 24, cone);CHKERRQ(ierr);
1958       cone[0] = 12; cone[1] =  9;
1959       ierr = DMPlexSetCone(*dm, 25, cone);CHKERRQ(ierr);
1960       ierr = DMPlexSymmetrize(*dm);CHKERRQ(ierr);
1961       ierr = DMPlexStratify(*dm);CHKERRQ(ierr);
1962       /* Build coordinates */
1963       ierr = PetscCalloc1(numVerts * embedDim, &coordsIn);CHKERRQ(ierr);
1964       coordsIn[0*embedDim+0] = -dist; coordsIn[0*embedDim+1] =  dist; coordsIn[0*embedDim+2] = -dist;
1965       coordsIn[1*embedDim+0] =  dist; coordsIn[1*embedDim+1] =  dist; coordsIn[1*embedDim+2] = -dist;
1966       coordsIn[2*embedDim+0] =  dist; coordsIn[2*embedDim+1] = -dist; coordsIn[2*embedDim+2] = -dist;
1967       coordsIn[3*embedDim+0] = -dist; coordsIn[3*embedDim+1] = -dist; coordsIn[3*embedDim+2] = -dist;
1968       coordsIn[4*embedDim+0] = -dist; coordsIn[4*embedDim+1] =  dist; coordsIn[4*embedDim+2] =  dist;
1969       coordsIn[5*embedDim+0] =  dist; coordsIn[5*embedDim+1] =  dist; coordsIn[5*embedDim+2] =  dist;
1970       coordsIn[6*embedDim+0] = -dist; coordsIn[6*embedDim+1] = -dist; coordsIn[6*embedDim+2] =  dist;
1971       coordsIn[7*embedDim+0] =  dist; coordsIn[7*embedDim+1] = -dist; coordsIn[7*embedDim+2] =  dist;
1972     }
1973     break;
1974   case 3:
1975     if (simplex) {
1976       DM              idm;
1977       const PetscReal edgeLen         = 1.0/PETSC_PHI;
1978       const PetscReal vertexA[4]      = {0.5, 0.5, 0.5, 0.5};
1979       const PetscReal vertexB[4]      = {1.0, 0.0, 0.0, 0.0};
1980       const PetscReal vertexC[4]      = {0.5, 0.5*PETSC_PHI, 0.5/PETSC_PHI, 0.0};
1981       const PetscInt  degree          = 12;
1982       PetscInt        s[4]            = {1, 1, 1};
1983       PetscInt        evenPerm[12][4] = {{0, 1, 2, 3}, {0, 2, 3, 1}, {0, 3, 1, 2}, {1, 0, 3, 2}, {1, 2, 0, 3}, {1, 3, 2, 0},
1984                                          {2, 0, 1, 3}, {2, 1, 3, 0}, {2, 3, 0, 1}, {3, 0, 2, 1}, {3, 1, 0, 2}, {3, 2, 1, 0}};
1985       PetscInt        cone[4];
1986       PetscInt       *graph, p, i, j, k, l;
1987 
1988       numCells    = !rank ? 600 : 0;
1989       numVerts    = !rank ? 120 : 0;
1990       firstVertex = numCells;
1991       /* Use the 600-cell, which for a unit sphere has coordinates which are
1992 
1993            1/2 (\pm 1, \pm 1,    \pm 1, \pm 1)                          16
1994                (\pm 1,    0,       0,      0)  all cyclic permutations   8
1995            1/2 (\pm 1, \pm phi, \pm 1/phi, 0)  all even permutations    96
1996 
1997          where \phi^2 - \phi - 1 = 0, meaning \phi is the golden ratio \frac{1 + \sqrt{5}}{2}. The edge
1998          length is then given by 1/\phi = 2.73606.
1999 
2000          http://buzzard.pugetsound.edu/sage-practice/ch03s03.html
2001          http://mathworld.wolfram.com/600-Cell.html
2002        */
2003       /* Construct vertices */
2004       ierr = PetscCalloc1(numVerts * embedDim, &coordsIn);CHKERRQ(ierr);
2005       i    = 0;
2006       for (s[0] = -1; s[0] < 2; s[0] += 2) {
2007         for (s[1] = -1; s[1] < 2; s[1] += 2) {
2008           for (s[2] = -1; s[2] < 2; s[2] += 2) {
2009             for (s[3] = -1; s[3] < 2; s[3] += 2) {
2010               for (d = 0; d < embedDim; ++d) coordsIn[i*embedDim+d] = s[d]*vertexA[d];
2011               ++i;
2012             }
2013           }
2014         }
2015       }
2016       for (p = 0; p < embedDim; ++p) {
2017         s[1] = s[2] = s[3] = 1;
2018         for (s[0] = -1; s[0] < 2; s[0] += 2) {
2019           for (d = 0; d < embedDim; ++d) coordsIn[i*embedDim+d] = s[(d+p)%embedDim]*vertexB[(d+p)%embedDim];
2020           ++i;
2021         }
2022       }
2023       for (p = 0; p < 12; ++p) {
2024         s[3] = 1;
2025         for (s[0] = -1; s[0] < 2; s[0] += 2) {
2026           for (s[1] = -1; s[1] < 2; s[1] += 2) {
2027             for (s[2] = -1; s[2] < 2; s[2] += 2) {
2028               for (d = 0; d < embedDim; ++d) coordsIn[i*embedDim+d] = s[evenPerm[p][d]]*vertexC[evenPerm[p][d]];
2029               ++i;
2030             }
2031           }
2032         }
2033       }
2034       if (i != numVerts) SETERRQ2(comm, PETSC_ERR_PLIB, "Invalid 600-cell, vertices %D != %D", i, numVerts);
2035       /* Construct graph */
2036       ierr = PetscCalloc1(numVerts * numVerts, &graph);CHKERRQ(ierr);
2037       for (i = 0; i < numVerts; ++i) {
2038         for (j = 0, k = 0; j < numVerts; ++j) {
2039           if (PetscAbsReal(DiffNormReal(embedDim, &coordsIn[i*embedDim], &coordsIn[j*embedDim]) - edgeLen) < PETSC_SMALL) {graph[i*numVerts+j] = 1; ++k;}
2040         }
2041         if (k != degree) SETERRQ3(comm, PETSC_ERR_PLIB, "Invalid 600-cell, vertex %D degree %D != %D", i, k, degree);
2042       }
2043       /* Build Topology */
2044       ierr = DMPlexSetChart(*dm, 0, numCells+numVerts);CHKERRQ(ierr);
2045       for (c = 0; c < numCells; c++) {
2046         ierr = DMPlexSetConeSize(*dm, c, embedDim);CHKERRQ(ierr);
2047       }
2048       ierr = DMSetUp(*dm);CHKERRQ(ierr); /* Allocate space for cones */
2049       /* Cells */
2050       for (i = 0, c = 0; i < numVerts; ++i) {
2051         for (j = 0; j < i; ++j) {
2052           for (k = 0; k < j; ++k) {
2053             for (l = 0; l < k; ++l) {
2054               if (graph[i*numVerts+j] && graph[j*numVerts+k] && graph[k*numVerts+i] &&
2055                   graph[l*numVerts+i] && graph[l*numVerts+j] && graph[l*numVerts+k]) {
2056                 cone[0] = firstVertex+i; cone[1] = firstVertex+j; cone[2] = firstVertex+k; cone[3] = firstVertex+l;
2057                 /* Check orientation: https://ef.gy/linear-algebra:normal-vectors-in-higher-dimensional-spaces */
2058                 {
2059                   const PetscInt epsilon[4][4][4][4] = {{{{0,  0,  0,  0}, { 0, 0,  0,  0}, { 0,  0, 0,  0}, { 0,  0,  0, 0}},
2060                                                          {{0,  0,  0,  0}, { 0, 0,  0,  0}, { 0,  0, 0,  1}, { 0,  0, -1, 0}},
2061                                                          {{0,  0,  0,  0}, { 0, 0,  0, -1}, { 0,  0, 0,  0}, { 0,  1,  0, 0}},
2062                                                          {{0,  0,  0,  0}, { 0, 0,  1,  0}, { 0, -1, 0,  0}, { 0,  0,  0, 0}}},
2063 
2064                                                         {{{0,  0,  0,  0}, { 0, 0,  0,  0}, { 0,  0, 0, -1}, { 0,  0,  1, 0}},
2065                                                          {{0,  0,  0,  0}, { 0, 0,  0,  0}, { 0,  0, 0,  0}, { 0,  0,  0, 0}},
2066                                                          {{0,  0,  0,  1}, { 0, 0,  0,  0}, { 0,  0, 0,  0}, {-1,  0,  0, 0}},
2067                                                          {{0,  0, -1,  0}, { 0, 0,  0,  0}, { 1,  0, 0,  0}, { 0,  0,  0, 0}}},
2068 
2069                                                         {{{0,  0,  0,  0}, { 0, 0,  0,  1}, { 0,  0, 0,  0}, { 0, -1,  0, 0}},
2070                                                          {{0,  0,  0, -1}, { 0, 0,  0,  0}, { 0,  0, 0,  0}, { 1,  0,  0, 0}},
2071                                                          {{0,  0,  0,  0}, { 0, 0,  0,  0}, { 0,  0, 0,  0}, { 0,  0,  0, 0}},
2072                                                          {{0,  1,  0,  0}, {-1, 0,  0,  0}, { 0,  0, 0,  0}, { 0,  0,  0, 0}}},
2073 
2074                                                         {{{0,  0,  0,  0}, { 0, 0, -1,  0}, { 0,  1, 0,  0}, { 0,  0,  0, 0}},
2075                                                          {{0,  0,  1,  0}, { 0, 0,  0,  0}, {-1,  0, 0,  0}, { 0,  0,  0, 0}},
2076                                                          {{0, -1,  0,  0}, { 1, 0,  0,  0}, { 0,  0, 0,  0}, { 0,  0,  0, 0}},
2077                                                          {{0,  0,  0,  0}, { 0, 0,  0,  0}, { 0,  0, 0,  0}, { 0,  0,  0, 0}}}};
2078                   PetscReal normal[4];
2079                   PetscInt  e, f, g;
2080 
2081                   for (d = 0; d < embedDim; ++d) {
2082                     normal[d] = 0.0;
2083                     for (e = 0; e < embedDim; ++e) {
2084                       for (f = 0; f < embedDim; ++f) {
2085                         for (g = 0; g < embedDim; ++g) {
2086                           normal[d] += epsilon[d][e][f][g]*(coordsIn[j*embedDim+e] - coordsIn[i*embedDim+e])*(coordsIn[k*embedDim+f] - coordsIn[i*embedDim+f])*(coordsIn[l*embedDim+f] - coordsIn[i*embedDim+f]);
2087                         }
2088                       }
2089                     }
2090                   }
2091                   if (DotReal(embedDim, normal, &coordsIn[i*embedDim]) < 0) {PetscInt tmp = cone[1]; cone[1] = cone[2]; cone[2] = tmp;}
2092                 }
2093                 ierr = DMPlexSetCone(*dm, c++, cone);CHKERRQ(ierr);
2094               }
2095             }
2096           }
2097         }
2098       }
2099       ierr = DMPlexSymmetrize(*dm);CHKERRQ(ierr);
2100       ierr = DMPlexStratify(*dm);CHKERRQ(ierr);
2101       ierr = PetscFree(graph);CHKERRQ(ierr);
2102       /* Interpolate mesh */
2103       ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
2104       ierr = DMDestroy(dm);CHKERRQ(ierr);
2105       *dm  = idm;
2106       break;
2107     }
2108   default: SETERRQ1(comm, PETSC_ERR_SUP, "Unsupported dimension for sphere: %D", dim);
2109   }
2110   /* Create coordinates */
2111   ierr = DMGetCoordinateSection(*dm, &coordSection);CHKERRQ(ierr);
2112   ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
2113   ierr = PetscSectionSetFieldComponents(coordSection, 0, embedDim);CHKERRQ(ierr);
2114   ierr = PetscSectionSetChart(coordSection, firstVertex, firstVertex+numVerts);CHKERRQ(ierr);
2115   for (v = firstVertex; v < firstVertex+numVerts; ++v) {
2116     ierr = PetscSectionSetDof(coordSection, v, embedDim);CHKERRQ(ierr);
2117     ierr = PetscSectionSetFieldDof(coordSection, v, 0, embedDim);CHKERRQ(ierr);
2118   }
2119   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
2120   ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
2121   ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
2122   ierr = VecSetBlockSize(coordinates, embedDim);CHKERRQ(ierr);
2123   ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
2124   ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
2125   ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
2126   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
2127   for (v = 0; v < numVerts; ++v) for (d = 0; d < embedDim; ++d) {coords[v*embedDim+d] = coordsIn[v*embedDim+d];}
2128   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
2129   ierr = DMSetCoordinatesLocal(*dm, coordinates);CHKERRQ(ierr);
2130   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
2131   ierr = PetscFree(coordsIn);CHKERRQ(ierr);
2132   PetscFunctionReturn(0);
2133 }
2134 
2135 /* External function declarations here */
2136 extern PetscErrorCode DMCreateInterpolation_Plex(DM dmCoarse, DM dmFine, Mat *interpolation, Vec *scaling);
2137 extern PetscErrorCode DMCreateInjection_Plex(DM dmCoarse, DM dmFine, Mat *mat);
2138 extern PetscErrorCode DMCreateMassMatrix_Plex(DM dmCoarse, DM dmFine, Mat *mat);
2139 extern PetscErrorCode DMCreateDefaultSection_Plex(DM dm);
2140 extern PetscErrorCode DMCreateDefaultConstraints_Plex(DM dm);
2141 extern PetscErrorCode DMCreateMatrix_Plex(DM dm,  Mat *J);
2142 extern PetscErrorCode DMCreateCoordinateDM_Plex(DM dm, DM *cdm);
2143 extern PetscErrorCode DMCreateCoordinateField_Plex(DM dm, DMField *field);
2144 PETSC_INTERN PetscErrorCode DMClone_Plex(DM dm, DM *newdm);
2145 extern PetscErrorCode DMSetUp_Plex(DM dm);
2146 extern PetscErrorCode DMDestroy_Plex(DM dm);
2147 extern PetscErrorCode DMView_Plex(DM dm, PetscViewer viewer);
2148 extern PetscErrorCode DMLoad_Plex(DM dm, PetscViewer viewer);
2149 extern PetscErrorCode DMCreateSubDM_Plex(DM dm, PetscInt numFields, const PetscInt fields[], IS *is, DM *subdm);
2150 extern PetscErrorCode DMCreateSuperDM_Plex(DM dms[], PetscInt len, IS **is, DM *superdm);
2151 static PetscErrorCode DMInitialize_Plex(DM dm);
2152 
2153 /* Replace dm with the contents of dmNew
2154    - Share the DM_Plex structure
2155    - Share the coordinates
2156    - Share the SF
2157 */
2158 static PetscErrorCode DMPlexReplace_Static(DM dm, DM dmNew)
2159 {
2160   PetscSF               sf;
2161   DM                    coordDM, coarseDM;
2162   Vec                   coords;
2163   PetscBool             isper;
2164   const PetscReal      *maxCell, *L;
2165   const DMBoundaryType *bd;
2166   PetscErrorCode        ierr;
2167 
2168   PetscFunctionBegin;
2169   ierr = DMGetPointSF(dmNew, &sf);CHKERRQ(ierr);
2170   ierr = DMSetPointSF(dm, sf);CHKERRQ(ierr);
2171   ierr = DMGetCoordinateDM(dmNew, &coordDM);CHKERRQ(ierr);
2172   ierr = DMGetCoordinatesLocal(dmNew, &coords);CHKERRQ(ierr);
2173   ierr = DMSetCoordinateDM(dm, coordDM);CHKERRQ(ierr);
2174   ierr = DMSetCoordinatesLocal(dm, coords);CHKERRQ(ierr);
2175   ierr = DMGetPeriodicity(dm, &isper, &maxCell, &L, &bd);CHKERRQ(ierr);
2176   ierr = DMSetPeriodicity(dmNew, isper, maxCell, L, bd);CHKERRQ(ierr);
2177   ierr = DMDestroy_Plex(dm);CHKERRQ(ierr);
2178   ierr = DMInitialize_Plex(dm);CHKERRQ(ierr);
2179   dm->data = dmNew->data;
2180   ((DM_Plex *) dmNew->data)->refct++;
2181   dmNew->labels->refct++;
2182   if (!--(dm->labels->refct)) {
2183     DMLabelLink next = dm->labels->next;
2184 
2185     /* destroy the labels */
2186     while (next) {
2187       DMLabelLink tmp = next->next;
2188 
2189       ierr = DMLabelDestroy(&next->label);CHKERRQ(ierr);
2190       ierr = PetscFree(next);CHKERRQ(ierr);
2191       next = tmp;
2192     }
2193     ierr = PetscFree(dm->labels);CHKERRQ(ierr);
2194   }
2195   dm->labels = dmNew->labels;
2196   dm->depthLabel = dmNew->depthLabel;
2197   ierr = DMGetCoarseDM(dmNew,&coarseDM);CHKERRQ(ierr);
2198   ierr = DMSetCoarseDM(dm,coarseDM);CHKERRQ(ierr);
2199   PetscFunctionReturn(0);
2200 }
2201 
2202 /* Swap dm with the contents of dmNew
2203    - Swap the DM_Plex structure
2204    - Swap the coordinates
2205    - Swap the point PetscSF
2206 */
2207 static PetscErrorCode DMPlexSwap_Static(DM dmA, DM dmB)
2208 {
2209   DM              coordDMA, coordDMB;
2210   Vec             coordsA,  coordsB;
2211   PetscSF         sfA,      sfB;
2212   void            *tmp;
2213   DMLabelLinkList listTmp;
2214   DMLabel         depthTmp;
2215   PetscInt        tmpI;
2216   PetscErrorCode  ierr;
2217 
2218   PetscFunctionBegin;
2219   ierr = DMGetPointSF(dmA, &sfA);CHKERRQ(ierr);
2220   ierr = DMGetPointSF(dmB, &sfB);CHKERRQ(ierr);
2221   ierr = PetscObjectReference((PetscObject) sfA);CHKERRQ(ierr);
2222   ierr = DMSetPointSF(dmA, sfB);CHKERRQ(ierr);
2223   ierr = DMSetPointSF(dmB, sfA);CHKERRQ(ierr);
2224   ierr = PetscObjectDereference((PetscObject) sfA);CHKERRQ(ierr);
2225 
2226   ierr = DMGetCoordinateDM(dmA, &coordDMA);CHKERRQ(ierr);
2227   ierr = DMGetCoordinateDM(dmB, &coordDMB);CHKERRQ(ierr);
2228   ierr = PetscObjectReference((PetscObject) coordDMA);CHKERRQ(ierr);
2229   ierr = DMSetCoordinateDM(dmA, coordDMB);CHKERRQ(ierr);
2230   ierr = DMSetCoordinateDM(dmB, coordDMA);CHKERRQ(ierr);
2231   ierr = PetscObjectDereference((PetscObject) coordDMA);CHKERRQ(ierr);
2232 
2233   ierr = DMGetCoordinatesLocal(dmA, &coordsA);CHKERRQ(ierr);
2234   ierr = DMGetCoordinatesLocal(dmB, &coordsB);CHKERRQ(ierr);
2235   ierr = PetscObjectReference((PetscObject) coordsA);CHKERRQ(ierr);
2236   ierr = DMSetCoordinatesLocal(dmA, coordsB);CHKERRQ(ierr);
2237   ierr = DMSetCoordinatesLocal(dmB, coordsA);CHKERRQ(ierr);
2238   ierr = PetscObjectDereference((PetscObject) coordsA);CHKERRQ(ierr);
2239 
2240   tmp       = dmA->data;
2241   dmA->data = dmB->data;
2242   dmB->data = tmp;
2243   listTmp   = dmA->labels;
2244   dmA->labels = dmB->labels;
2245   dmB->labels = listTmp;
2246   depthTmp  = dmA->depthLabel;
2247   dmA->depthLabel = dmB->depthLabel;
2248   dmB->depthLabel = depthTmp;
2249   tmpI         = dmA->levelup;
2250   dmA->levelup = dmB->levelup;
2251   dmB->levelup = tmpI;
2252   PetscFunctionReturn(0);
2253 }
2254 
2255 PetscErrorCode DMSetFromOptions_NonRefinement_Plex(PetscOptionItems *PetscOptionsObject,DM dm)
2256 {
2257   DM_Plex       *mesh = (DM_Plex*) dm->data;
2258   PetscErrorCode ierr;
2259 
2260   PetscFunctionBegin;
2261   /* Handle viewing */
2262   ierr = PetscOptionsBool("-dm_plex_print_set_values", "Output all set values info", "DMPlexMatSetClosure", PETSC_FALSE, &mesh->printSetValues, NULL);CHKERRQ(ierr);
2263   ierr = PetscOptionsInt("-dm_plex_print_fem", "Debug output level all fem computations", "DMPlexSNESComputeResidualFEM", 0, &mesh->printFEM, NULL);CHKERRQ(ierr);
2264   ierr = PetscOptionsReal("-dm_plex_print_tol", "Tolerance for FEM output", "DMPlexSNESComputeResidualFEM", mesh->printTol, &mesh->printTol, NULL);CHKERRQ(ierr);
2265   ierr = PetscOptionsInt("-dm_plex_print_l2", "Debug output level all L2 diff computations", "DMComputeL2Diff", 0, &mesh->printL2, NULL);CHKERRQ(ierr);
2266   /* Point Location */
2267   ierr = PetscOptionsBool("-dm_plex_hash_location", "Use grid hashing for point location", "DMInterpolate", PETSC_FALSE, &mesh->useHashLocation, NULL);CHKERRQ(ierr);
2268   /* Partitioning and distribution */
2269   ierr = PetscOptionsBool("-dm_plex_partition_balance", "Attempt to evenly divide points on partition boundary between processes", "DMPlexSetPartitionBalance", PETSC_FALSE, &mesh->partitionBalance, NULL);CHKERRQ(ierr);
2270   /* Generation and remeshing */
2271   ierr = PetscOptionsBool("-dm_plex_remesh_bd", "Allow changes to the boundary on remeshing", "DMAdapt", PETSC_FALSE, &mesh->remeshBd, NULL);CHKERRQ(ierr);
2272   /* Projection behavior */
2273   ierr = PetscOptionsInt("-dm_plex_max_projection_height", "Maxmimum mesh point height used to project locally", "DMPlexSetMaxProjectionHeight", 0, &mesh->maxProjectionHeight, NULL);CHKERRQ(ierr);
2274   ierr = PetscOptionsBool("-dm_plex_regular_refinement", "Use special nested projection algorithm for regular refinement", "DMPlexSetRegularRefinement", mesh->regularRefinement, &mesh->regularRefinement, NULL);CHKERRQ(ierr);
2275 
2276   ierr = PetscPartitionerSetFromOptions(mesh->partitioner);CHKERRQ(ierr);
2277   PetscFunctionReturn(0);
2278 }
2279 
2280 static PetscErrorCode DMSetFromOptions_Plex(PetscOptionItems *PetscOptionsObject,DM dm)
2281 {
2282   PetscInt       refine = 0, coarsen = 0, r;
2283   PetscBool      isHierarchy;
2284   PetscErrorCode ierr;
2285 
2286   PetscFunctionBegin;
2287   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2288   ierr = PetscOptionsHead(PetscOptionsObject,"DMPlex Options");CHKERRQ(ierr);
2289   /* Handle DMPlex refinement */
2290   ierr = PetscOptionsInt("-dm_refine", "The number of uniform refinements", "DMCreate", refine, &refine, NULL);CHKERRQ(ierr);
2291   ierr = PetscOptionsInt("-dm_refine_hierarchy", "The number of uniform refinements", "DMCreate", refine, &refine, &isHierarchy);CHKERRQ(ierr);
2292   if (refine) {ierr = DMPlexSetRefinementUniform(dm, PETSC_TRUE);CHKERRQ(ierr);}
2293   if (refine && isHierarchy) {
2294     DM *dms, coarseDM;
2295 
2296     ierr = DMGetCoarseDM(dm, &coarseDM);CHKERRQ(ierr);
2297     ierr = PetscObjectReference((PetscObject)coarseDM);CHKERRQ(ierr);
2298     ierr = PetscMalloc1(refine,&dms);CHKERRQ(ierr);
2299     ierr = DMRefineHierarchy(dm, refine, dms);CHKERRQ(ierr);
2300     /* Total hack since we do not pass in a pointer */
2301     ierr = DMPlexSwap_Static(dm, dms[refine-1]);CHKERRQ(ierr);
2302     if (refine == 1) {
2303       ierr = DMSetCoarseDM(dm, dms[0]);CHKERRQ(ierr);
2304       ierr = DMPlexSetRegularRefinement(dm, PETSC_TRUE);CHKERRQ(ierr);
2305     } else {
2306       ierr = DMSetCoarseDM(dm, dms[refine-2]);CHKERRQ(ierr);
2307       ierr = DMPlexSetRegularRefinement(dm, PETSC_TRUE);CHKERRQ(ierr);
2308       ierr = DMSetCoarseDM(dms[0], dms[refine-1]);CHKERRQ(ierr);
2309       ierr = DMPlexSetRegularRefinement(dms[0], PETSC_TRUE);CHKERRQ(ierr);
2310     }
2311     ierr = DMSetCoarseDM(dms[refine-1], coarseDM);CHKERRQ(ierr);
2312     ierr = PetscObjectDereference((PetscObject)coarseDM);CHKERRQ(ierr);
2313     /* Free DMs */
2314     for (r = 0; r < refine; ++r) {
2315       ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dms[r]);CHKERRQ(ierr);
2316       ierr = DMDestroy(&dms[r]);CHKERRQ(ierr);
2317     }
2318     ierr = PetscFree(dms);CHKERRQ(ierr);
2319   } else {
2320     for (r = 0; r < refine; ++r) {
2321       DM refinedMesh;
2322 
2323       ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dm);CHKERRQ(ierr);
2324       ierr = DMRefine(dm, PetscObjectComm((PetscObject) dm), &refinedMesh);CHKERRQ(ierr);
2325       /* Total hack since we do not pass in a pointer */
2326       ierr = DMPlexReplace_Static(dm, refinedMesh);CHKERRQ(ierr);
2327       ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dm);CHKERRQ(ierr);
2328       ierr = DMDestroy(&refinedMesh);CHKERRQ(ierr);
2329     }
2330   }
2331   /* Handle DMPlex coarsening */
2332   ierr = PetscOptionsInt("-dm_coarsen", "Coarsen the mesh", "DMCreate", coarsen, &coarsen, NULL);CHKERRQ(ierr);
2333   ierr = PetscOptionsInt("-dm_coarsen_hierarchy", "The number of coarsenings", "DMCreate", coarsen, &coarsen, &isHierarchy);CHKERRQ(ierr);
2334   if (coarsen && isHierarchy) {
2335     DM *dms;
2336 
2337     ierr = PetscMalloc1(coarsen, &dms);CHKERRQ(ierr);
2338     ierr = DMCoarsenHierarchy(dm, coarsen, dms);CHKERRQ(ierr);
2339     /* Free DMs */
2340     for (r = 0; r < coarsen; ++r) {
2341       ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dms[r]);CHKERRQ(ierr);
2342       ierr = DMDestroy(&dms[r]);CHKERRQ(ierr);
2343     }
2344     ierr = PetscFree(dms);CHKERRQ(ierr);
2345   } else {
2346     for (r = 0; r < coarsen; ++r) {
2347       DM coarseMesh;
2348 
2349       ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dm);CHKERRQ(ierr);
2350       ierr = DMCoarsen(dm, PetscObjectComm((PetscObject) dm), &coarseMesh);CHKERRQ(ierr);
2351       /* Total hack since we do not pass in a pointer */
2352       ierr = DMPlexReplace_Static(dm, coarseMesh);CHKERRQ(ierr);
2353       ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dm);CHKERRQ(ierr);
2354       ierr = DMDestroy(&coarseMesh);CHKERRQ(ierr);
2355     }
2356   }
2357   /* Handle */
2358   ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dm);CHKERRQ(ierr);
2359   ierr = PetscOptionsTail();CHKERRQ(ierr);
2360   PetscFunctionReturn(0);
2361 }
2362 
2363 static PetscErrorCode DMCreateGlobalVector_Plex(DM dm,Vec *vec)
2364 {
2365   PetscErrorCode ierr;
2366 
2367   PetscFunctionBegin;
2368   ierr = DMCreateGlobalVector_Section_Private(dm,vec);CHKERRQ(ierr);
2369   /* ierr = VecSetOperation(*vec, VECOP_DUPLICATE, (void(*)(void)) VecDuplicate_MPI_DM);CHKERRQ(ierr); */
2370   ierr = VecSetOperation(*vec, VECOP_VIEW, (void (*)(void)) VecView_Plex);CHKERRQ(ierr);
2371   ierr = VecSetOperation(*vec, VECOP_VIEWNATIVE, (void (*)(void)) VecView_Plex_Native);CHKERRQ(ierr);
2372   ierr = VecSetOperation(*vec, VECOP_LOAD, (void (*)(void)) VecLoad_Plex);CHKERRQ(ierr);
2373   ierr = VecSetOperation(*vec, VECOP_LOADNATIVE, (void (*)(void)) VecLoad_Plex_Native);CHKERRQ(ierr);
2374   PetscFunctionReturn(0);
2375 }
2376 
2377 static PetscErrorCode DMCreateLocalVector_Plex(DM dm,Vec *vec)
2378 {
2379   PetscErrorCode ierr;
2380 
2381   PetscFunctionBegin;
2382   ierr = DMCreateLocalVector_Section_Private(dm,vec);CHKERRQ(ierr);
2383   ierr = VecSetOperation(*vec, VECOP_VIEW, (void (*)(void)) VecView_Plex_Local);CHKERRQ(ierr);
2384   ierr = VecSetOperation(*vec, VECOP_LOAD, (void (*)(void)) VecLoad_Plex_Local);CHKERRQ(ierr);
2385   PetscFunctionReturn(0);
2386 }
2387 
2388 static PetscErrorCode DMGetDimPoints_Plex(DM dm, PetscInt dim, PetscInt *pStart, PetscInt *pEnd)
2389 {
2390   PetscInt       depth, d;
2391   PetscErrorCode ierr;
2392 
2393   PetscFunctionBegin;
2394   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
2395   if (depth == 1) {
2396     ierr = DMGetDimension(dm, &d);CHKERRQ(ierr);
2397     if (dim == 0)      {ierr = DMPlexGetDepthStratum(dm, dim, pStart, pEnd);CHKERRQ(ierr);}
2398     else if (dim == d) {ierr = DMPlexGetDepthStratum(dm, 1, pStart, pEnd);CHKERRQ(ierr);}
2399     else               {*pStart = 0; *pEnd = 0;}
2400   } else {
2401     ierr = DMPlexGetDepthStratum(dm, dim, pStart, pEnd);CHKERRQ(ierr);
2402   }
2403   PetscFunctionReturn(0);
2404 }
2405 
2406 static PetscErrorCode DMGetNeighors_Plex(DM dm, PetscInt *nranks, const PetscMPIInt *ranks[])
2407 {
2408   PetscSF        sf;
2409   PetscErrorCode ierr;
2410 
2411   PetscFunctionBegin;
2412   ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr);
2413   ierr = PetscSFGetRanks(sf, nranks, ranks, NULL, NULL, NULL);CHKERRQ(ierr);
2414   PetscFunctionReturn(0);
2415 }
2416 
2417 static PetscErrorCode DMHasCreateInjection_Plex(DM dm, PetscBool *flg)
2418 {
2419   PetscFunctionBegin;
2420   PetscValidHeaderSpecific(dm,DM_CLASSID,1);
2421   PetscValidPointer(flg,2);
2422   *flg = PETSC_TRUE;
2423   PetscFunctionReturn(0);
2424 }
2425 
2426 static PetscErrorCode DMInitialize_Plex(DM dm)
2427 {
2428   PetscErrorCode ierr;
2429 
2430   PetscFunctionBegin;
2431   dm->ops->view                            = DMView_Plex;
2432   dm->ops->load                            = DMLoad_Plex;
2433   dm->ops->setfromoptions                  = DMSetFromOptions_Plex;
2434   dm->ops->clone                           = DMClone_Plex;
2435   dm->ops->setup                           = DMSetUp_Plex;
2436   dm->ops->createdefaultsection            = DMCreateDefaultSection_Plex;
2437   dm->ops->createdefaultconstraints        = DMCreateDefaultConstraints_Plex;
2438   dm->ops->createglobalvector              = DMCreateGlobalVector_Plex;
2439   dm->ops->createlocalvector               = DMCreateLocalVector_Plex;
2440   dm->ops->getlocaltoglobalmapping         = NULL;
2441   dm->ops->createfieldis                   = NULL;
2442   dm->ops->createcoordinatedm              = DMCreateCoordinateDM_Plex;
2443   dm->ops->createcoordinatefield           = DMCreateCoordinateField_Plex;
2444   dm->ops->getcoloring                     = NULL;
2445   dm->ops->creatematrix                    = DMCreateMatrix_Plex;
2446   dm->ops->createinterpolation             = DMCreateInterpolation_Plex;
2447   dm->ops->createmassmatrix                = DMCreateMassMatrix_Plex;
2448   dm->ops->getaggregates                   = NULL;
2449   dm->ops->getinjection                    = DMCreateInjection_Plex;
2450   dm->ops->hascreateinjection              = DMHasCreateInjection_Plex;
2451   dm->ops->refine                          = DMRefine_Plex;
2452   dm->ops->coarsen                         = DMCoarsen_Plex;
2453   dm->ops->refinehierarchy                 = DMRefineHierarchy_Plex;
2454   dm->ops->coarsenhierarchy                = DMCoarsenHierarchy_Plex;
2455   dm->ops->adaptlabel                      = DMAdaptLabel_Plex;
2456   dm->ops->adaptmetric                     = DMAdaptMetric_Plex;
2457   dm->ops->globaltolocalbegin              = NULL;
2458   dm->ops->globaltolocalend                = NULL;
2459   dm->ops->localtoglobalbegin              = NULL;
2460   dm->ops->localtoglobalend                = NULL;
2461   dm->ops->destroy                         = DMDestroy_Plex;
2462   dm->ops->createsubdm                     = DMCreateSubDM_Plex;
2463   dm->ops->createsuperdm                   = DMCreateSuperDM_Plex;
2464   dm->ops->getdimpoints                    = DMGetDimPoints_Plex;
2465   dm->ops->locatepoints                    = DMLocatePoints_Plex;
2466   dm->ops->projectfunctionlocal            = DMProjectFunctionLocal_Plex;
2467   dm->ops->projectfunctionlabellocal       = DMProjectFunctionLabelLocal_Plex;
2468   dm->ops->projectfieldlocal               = DMProjectFieldLocal_Plex;
2469   dm->ops->projectfieldlabellocal          = DMProjectFieldLabelLocal_Plex;
2470   dm->ops->computel2diff                   = DMComputeL2Diff_Plex;
2471   dm->ops->computel2gradientdiff           = DMComputeL2GradientDiff_Plex;
2472   dm->ops->computel2fielddiff              = DMComputeL2FieldDiff_Plex;
2473   dm->ops->getneighbors                    = DMGetNeighors_Plex;
2474   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMPlexInsertBoundaryValues_C",DMPlexInsertBoundaryValues_Plex);CHKERRQ(ierr);
2475   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMSetUpGLVisViewer_C",DMSetUpGLVisViewer_Plex);CHKERRQ(ierr);
2476   PetscFunctionReturn(0);
2477 }
2478 
2479 PETSC_INTERN PetscErrorCode DMClone_Plex(DM dm, DM *newdm)
2480 {
2481   DM_Plex        *mesh = (DM_Plex *) dm->data;
2482   PetscErrorCode ierr;
2483 
2484   PetscFunctionBegin;
2485   mesh->refct++;
2486   (*newdm)->data = mesh;
2487   ierr = PetscObjectChangeTypeName((PetscObject) *newdm, DMPLEX);CHKERRQ(ierr);
2488   ierr = DMInitialize_Plex(*newdm);CHKERRQ(ierr);
2489   PetscFunctionReturn(0);
2490 }
2491 
2492 /*MC
2493   DMPLEX = "plex" - A DM object that encapsulates an unstructured mesh, or CW Complex, which can be expressed using a Hasse Diagram.
2494                     In the local representation, Vecs contain all unknowns in the interior and shared boundary. This is
2495                     specified by a PetscSection object. Ownership in the global representation is determined by
2496                     ownership of the underlying DMPlex points. This is specified by another PetscSection object.
2497 
2498   Options Database Keys:
2499 + -dm_view :mesh.tex:ascii_latex - View the mesh in LaTeX/TikZ
2500 . -dm_plex_view_scale <num>      - Scale the TikZ
2501 - -dm_plex_print_fem <num>       - View FEM assembly information, such as element vectors and matrices
2502 
2503 
2504   Level: intermediate
2505 
2506 .seealso: DMType, DMPlexCreate(), DMCreate(), DMSetType()
2507 M*/
2508 
2509 PETSC_EXTERN PetscErrorCode DMCreate_Plex(DM dm)
2510 {
2511   DM_Plex        *mesh;
2512   PetscInt       unit, d;
2513   PetscErrorCode ierr;
2514 
2515   PetscFunctionBegin;
2516   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2517   ierr     = PetscNewLog(dm,&mesh);CHKERRQ(ierr);
2518   dm->dim  = 0;
2519   dm->data = mesh;
2520 
2521   mesh->refct             = 1;
2522   ierr                    = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &mesh->coneSection);CHKERRQ(ierr);
2523   mesh->maxConeSize       = 0;
2524   mesh->cones             = NULL;
2525   mesh->coneOrientations  = NULL;
2526   ierr                    = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &mesh->supportSection);CHKERRQ(ierr);
2527   mesh->maxSupportSize    = 0;
2528   mesh->supports          = NULL;
2529   mesh->refinementUniform = PETSC_TRUE;
2530   mesh->refinementLimit   = -1.0;
2531 
2532   mesh->facesTmp = NULL;
2533 
2534   mesh->tetgenOpts   = NULL;
2535   mesh->triangleOpts = NULL;
2536   ierr = PetscPartitionerCreate(PetscObjectComm((PetscObject)dm), &mesh->partitioner);CHKERRQ(ierr);
2537   mesh->remeshBd     = PETSC_FALSE;
2538 
2539   mesh->subpointMap = NULL;
2540 
2541   for (unit = 0; unit < NUM_PETSC_UNITS; ++unit) mesh->scale[unit] = 1.0;
2542 
2543   mesh->regularRefinement   = PETSC_FALSE;
2544   mesh->depthState          = -1;
2545   mesh->globalVertexNumbers = NULL;
2546   mesh->globalCellNumbers   = NULL;
2547   mesh->anchorSection       = NULL;
2548   mesh->anchorIS            = NULL;
2549   mesh->createanchors       = NULL;
2550   mesh->computeanchormatrix = NULL;
2551   mesh->parentSection       = NULL;
2552   mesh->parents             = NULL;
2553   mesh->childIDs            = NULL;
2554   mesh->childSection        = NULL;
2555   mesh->children            = NULL;
2556   mesh->referenceTree       = NULL;
2557   mesh->getchildsymmetry    = NULL;
2558   for (d = 0; d < 8; ++d) mesh->hybridPointMax[d] = PETSC_DETERMINE;
2559   mesh->vtkCellHeight       = 0;
2560   mesh->useAnchors          = PETSC_FALSE;
2561 
2562   mesh->maxProjectionHeight = 0;
2563 
2564   mesh->printSetValues = PETSC_FALSE;
2565   mesh->printFEM       = 0;
2566   mesh->printTol       = 1.0e-10;
2567 
2568   ierr = DMInitialize_Plex(dm);CHKERRQ(ierr);
2569   PetscFunctionReturn(0);
2570 }
2571 
2572 /*@
2573   DMPlexCreate - Creates a DMPlex object, which encapsulates an unstructured mesh, or CW complex, which can be expressed using a Hasse Diagram.
2574 
2575   Collective on MPI_Comm
2576 
2577   Input Parameter:
2578 . comm - The communicator for the DMPlex object
2579 
2580   Output Parameter:
2581 . mesh  - The DMPlex object
2582 
2583   Level: beginner
2584 
2585 .keywords: DMPlex, create
2586 @*/
2587 PetscErrorCode DMPlexCreate(MPI_Comm comm, DM *mesh)
2588 {
2589   PetscErrorCode ierr;
2590 
2591   PetscFunctionBegin;
2592   PetscValidPointer(mesh,2);
2593   ierr = DMCreate(comm, mesh);CHKERRQ(ierr);
2594   ierr = DMSetType(*mesh, DMPLEX);CHKERRQ(ierr);
2595   PetscFunctionReturn(0);
2596 }
2597 
2598 /*
2599   This takes as input the common mesh generator output, a list of the vertices for each cell, but vertex numbers are global and an SF is built for them
2600 */
2601 /* TODO: invertCells and spaceDim arguments could be added also to to DMPlexCreateFromCellListParallel(), DMPlexBuildFromCellList_Internal() and DMPlexCreateFromCellList() */
2602 PetscErrorCode DMPlexBuildFromCellList_Parallel_Internal(DM dm, PetscInt spaceDim, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, const int cells[], PetscBool invertCells, PetscSF *sfVert)
2603 {
2604   PetscSF         sfPoint;
2605   PetscLayout     vLayout;
2606   PetscHSetI      vhash;
2607   PetscSFNode    *remoteVerticesAdj, *vertexLocal, *vertexOwner, *remoteVertex;
2608   const PetscInt *vrange;
2609   PetscInt        numVerticesAdj, off = 0, *verticesAdj, numVerticesGhost = 0, *localVertex, *cone, c, p, v, g;
2610   PetscMPIInt     rank, size;
2611   PetscErrorCode  ierr;
2612 
2613   PetscFunctionBegin;
2614   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRQ(ierr);
2615   ierr = MPI_Comm_size(PetscObjectComm((PetscObject) dm), &size);CHKERRQ(ierr);
2616   /* Partition vertices */
2617   ierr = PetscLayoutCreate(PetscObjectComm((PetscObject) dm), &vLayout);CHKERRQ(ierr);
2618   ierr = PetscLayoutSetLocalSize(vLayout, numVertices);CHKERRQ(ierr);
2619   ierr = PetscLayoutSetBlockSize(vLayout, 1);CHKERRQ(ierr);
2620   ierr = PetscLayoutSetUp(vLayout);CHKERRQ(ierr);
2621   ierr = PetscLayoutGetRanges(vLayout, &vrange);CHKERRQ(ierr);
2622   /* Count vertices and map them to procs */
2623   ierr = PetscHSetICreate(&vhash);CHKERRQ(ierr);
2624   for (c = 0; c < numCells; ++c) {
2625     for (p = 0; p < numCorners; ++p) {
2626       ierr = PetscHSetIAdd(vhash, cells[c*numCorners+p]);CHKERRQ(ierr);
2627     }
2628   }
2629   ierr = PetscHSetIGetSize(vhash, &numVerticesAdj);CHKERRQ(ierr);
2630   ierr = PetscMalloc1(numVerticesAdj, &verticesAdj);CHKERRQ(ierr);
2631   ierr = PetscHSetIGetElems(vhash, &off, verticesAdj);CHKERRQ(ierr);
2632   ierr = PetscHSetIDestroy(&vhash);CHKERRQ(ierr);
2633   if (off != numVerticesAdj) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid number of local vertices %D should be %D", off, numVerticesAdj);
2634   ierr = PetscSortInt(numVerticesAdj, verticesAdj);CHKERRQ(ierr);
2635   ierr = PetscMalloc1(numVerticesAdj, &remoteVerticesAdj);CHKERRQ(ierr);
2636   for (v = 0; v < numVerticesAdj; ++v) {
2637     const PetscInt gv = verticesAdj[v];
2638     PetscInt       vrank;
2639 
2640     ierr = PetscFindInt(gv, size+1, vrange, &vrank);CHKERRQ(ierr);
2641     vrank = vrank < 0 ? -(vrank+2) : vrank;
2642     remoteVerticesAdj[v].index = gv - vrange[vrank];
2643     remoteVerticesAdj[v].rank  = vrank;
2644   }
2645   /* Create cones */
2646   ierr = DMPlexSetChart(dm, 0, numCells+numVerticesAdj);CHKERRQ(ierr);
2647   for (c = 0; c < numCells; ++c) {ierr = DMPlexSetConeSize(dm, c, numCorners);CHKERRQ(ierr);}
2648   ierr = DMSetUp(dm);CHKERRQ(ierr);
2649   ierr = DMGetWorkArray(dm, numCorners, MPIU_INT, &cone);CHKERRQ(ierr);
2650   for (c = 0; c < numCells; ++c) {
2651     for (p = 0; p < numCorners; ++p) {
2652       const PetscInt gv = cells[c*numCorners+p];
2653       PetscInt       lv;
2654 
2655       ierr = PetscFindInt(gv, numVerticesAdj, verticesAdj, &lv);CHKERRQ(ierr);
2656       if (lv < 0) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Could not find global vertex %D in local connectivity", gv);
2657       cone[p] = lv+numCells;
2658     }
2659     if (invertCells) { ierr = DMPlexInvertCell_Internal(spaceDim, numCorners, cone);CHKERRQ(ierr); }
2660     ierr = DMPlexSetCone(dm, c, cone);CHKERRQ(ierr);
2661   }
2662   ierr = DMRestoreWorkArray(dm, numCorners, MPIU_INT, &cone);CHKERRQ(ierr);
2663   /* Create SF for vertices */
2664   ierr = PetscSFCreate(PetscObjectComm((PetscObject)dm), sfVert);CHKERRQ(ierr);
2665   ierr = PetscObjectSetName((PetscObject) *sfVert, "Vertex Ownership SF");CHKERRQ(ierr);
2666   ierr = PetscSFSetFromOptions(*sfVert);CHKERRQ(ierr);
2667   ierr = PetscSFSetGraph(*sfVert, numVertices, numVerticesAdj, NULL, PETSC_OWN_POINTER, remoteVerticesAdj, PETSC_OWN_POINTER);CHKERRQ(ierr);
2668   ierr = PetscFree(verticesAdj);CHKERRQ(ierr);
2669   /* Build pointSF */
2670   ierr = PetscMalloc2(numVerticesAdj, &vertexLocal, numVertices, &vertexOwner);CHKERRQ(ierr);
2671   for (v = 0; v < numVerticesAdj; ++v) {vertexLocal[v].index = v+numCells; vertexLocal[v].rank = rank;}
2672   for (v = 0; v < numVertices;    ++v) {vertexOwner[v].index = -1;         vertexOwner[v].rank = -1;}
2673   ierr = PetscSFReduceBegin(*sfVert, MPIU_2INT, vertexLocal, vertexOwner, MPI_MAXLOC);CHKERRQ(ierr);
2674   ierr = PetscSFReduceEnd(*sfVert, MPIU_2INT, vertexLocal, vertexOwner, MPI_MAXLOC);CHKERRQ(ierr);
2675   for (v = 0; v < numVertices;    ++v) if (vertexOwner[v].rank < 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Global vertex %d on rank %d was unclaimed", v + vrange[rank], rank);
2676   ierr = PetscSFBcastBegin(*sfVert, MPIU_2INT, vertexOwner, vertexLocal);CHKERRQ(ierr);
2677   ierr = PetscSFBcastEnd(*sfVert, MPIU_2INT, vertexOwner, vertexLocal);CHKERRQ(ierr);
2678   for (v = 0; v < numVerticesAdj; ++v) if (vertexLocal[v].rank != rank) ++numVerticesGhost;
2679   ierr = PetscMalloc1(numVerticesGhost, &localVertex);CHKERRQ(ierr);
2680   ierr = PetscMalloc1(numVerticesGhost, &remoteVertex);CHKERRQ(ierr);
2681   for (v = 0, g = 0; v < numVerticesAdj; ++v) {
2682     if (vertexLocal[v].rank != rank) {
2683       localVertex[g]        = v+numCells;
2684       remoteVertex[g].index = vertexLocal[v].index;
2685       remoteVertex[g].rank  = vertexLocal[v].rank;
2686       ++g;
2687     }
2688   }
2689   ierr = PetscFree2(vertexLocal, vertexOwner);CHKERRQ(ierr);
2690   if (g != numVerticesGhost) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid number of vertex ghosts %D should be %D", g, numVerticesGhost);
2691   ierr = DMGetPointSF(dm, &sfPoint);CHKERRQ(ierr);
2692   ierr = PetscObjectSetName((PetscObject) sfPoint, "point SF");CHKERRQ(ierr);
2693   ierr = PetscSFSetGraph(sfPoint, numCells+numVerticesAdj, numVerticesGhost, localVertex, PETSC_OWN_POINTER, remoteVertex, PETSC_OWN_POINTER);CHKERRQ(ierr);
2694   ierr = PetscLayoutDestroy(&vLayout);CHKERRQ(ierr);
2695   /* Fill in the rest of the topology structure */
2696   ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
2697   ierr = DMPlexStratify(dm);CHKERRQ(ierr);
2698   PetscFunctionReturn(0);
2699 }
2700 
2701 /*
2702   This takes as input the coordinates for each owned vertex
2703 */
2704 PetscErrorCode DMPlexBuildCoordinates_Parallel_Internal(DM dm, PetscInt spaceDim, PetscInt numCells, PetscInt numV, PetscSF sfVert, const PetscReal vertexCoords[])
2705 {
2706   PetscSection   coordSection;
2707   Vec            coordinates;
2708   PetscScalar   *coords;
2709   PetscInt       numVertices, numVerticesAdj, coordSize, v;
2710   PetscErrorCode ierr;
2711 
2712   PetscFunctionBegin;
2713   ierr = DMSetCoordinateDim(dm, spaceDim);CHKERRQ(ierr);
2714   ierr = PetscSFGetGraph(sfVert, &numVertices, &numVerticesAdj, NULL, NULL);CHKERRQ(ierr);
2715   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
2716   ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
2717   ierr = PetscSectionSetFieldComponents(coordSection, 0, spaceDim);CHKERRQ(ierr);
2718   ierr = PetscSectionSetChart(coordSection, numCells, numCells + numVerticesAdj);CHKERRQ(ierr);
2719   for (v = numCells; v < numCells+numVerticesAdj; ++v) {
2720     ierr = PetscSectionSetDof(coordSection, v, spaceDim);CHKERRQ(ierr);
2721     ierr = PetscSectionSetFieldDof(coordSection, v, 0, spaceDim);CHKERRQ(ierr);
2722   }
2723   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
2724   ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
2725   ierr = VecCreate(PetscObjectComm((PetscObject)dm), &coordinates);CHKERRQ(ierr);
2726   ierr = VecSetBlockSize(coordinates, spaceDim);CHKERRQ(ierr);
2727   ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
2728   ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
2729   ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
2730   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
2731   {
2732     MPI_Datatype coordtype;
2733 
2734     /* Need a temp buffer for coords if we have complex/single */
2735     ierr = MPI_Type_contiguous(spaceDim, MPIU_SCALAR, &coordtype);CHKERRQ(ierr);
2736     ierr = MPI_Type_commit(&coordtype);CHKERRQ(ierr);
2737 #if defined(PETSC_USE_COMPLEX)
2738     {
2739     PetscScalar *svertexCoords;
2740     PetscInt    i;
2741     ierr = PetscMalloc1(numV*spaceDim,&svertexCoords);CHKERRQ(ierr);
2742     for (i=0; i<numV*spaceDim; i++) svertexCoords[i] = vertexCoords[i];
2743     ierr = PetscSFBcastBegin(sfVert, coordtype, svertexCoords, coords);CHKERRQ(ierr);
2744     ierr = PetscSFBcastEnd(sfVert, coordtype, svertexCoords, coords);CHKERRQ(ierr);
2745     ierr = PetscFree(svertexCoords);CHKERRQ(ierr);
2746     }
2747 #else
2748     ierr = PetscSFBcastBegin(sfVert, coordtype, vertexCoords, coords);CHKERRQ(ierr);
2749     ierr = PetscSFBcastEnd(sfVert, coordtype, vertexCoords, coords);CHKERRQ(ierr);
2750 #endif
2751     ierr = MPI_Type_free(&coordtype);CHKERRQ(ierr);
2752   }
2753   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
2754   ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
2755   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
2756   PetscFunctionReturn(0);
2757 }
2758 
2759 /*@C
2760   DMPlexCreateFromCellListParallel - This takes as input common mesh generator output, a list of the vertices for each cell, and produces a DM
2761 
2762   Input Parameters:
2763 + comm - The communicator
2764 . dim - The topological dimension of the mesh
2765 . numCells - The number of cells owned by this process
2766 . numVertices - The number of vertices owned by this process
2767 . numCorners - The number of vertices for each cell
2768 . interpolate - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
2769 . cells - An array of numCells*numCorners numbers, the global vertex numbers for each cell
2770 . spaceDim - The spatial dimension used for coordinates
2771 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
2772 
2773   Output Parameter:
2774 + dm - The DM
2775 - vertexSF - Optional, SF describing complete vertex ownership
2776 
2777   Note: Two triangles sharing a face
2778 $
2779 $        2
2780 $      / | \
2781 $     /  |  \
2782 $    /   |   \
2783 $   0  0 | 1  3
2784 $    \   |   /
2785 $     \  |  /
2786 $      \ | /
2787 $        1
2788 would have input
2789 $  numCells = 2, numVertices = 4
2790 $  cells = [0 1 2  1 3 2]
2791 $
2792 which would result in the DMPlex
2793 $
2794 $        4
2795 $      / | \
2796 $     /  |  \
2797 $    /   |   \
2798 $   2  0 | 1  5
2799 $    \   |   /
2800 $     \  |  /
2801 $      \ | /
2802 $        3
2803 
2804   Level: beginner
2805 
2806 .seealso: DMPlexCreateFromCellList(), DMPlexCreateFromDAG(), DMPlexCreate()
2807 @*/
2808 PetscErrorCode DMPlexCreateFromCellListParallel(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, PetscBool interpolate, const int cells[], PetscInt spaceDim, const PetscReal vertexCoords[], PetscSF *vertexSF, DM *dm)
2809 {
2810   PetscSF        sfVert;
2811   PetscErrorCode ierr;
2812 
2813   PetscFunctionBegin;
2814   ierr = DMCreate(comm, dm);CHKERRQ(ierr);
2815   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
2816   PetscValidLogicalCollectiveInt(*dm, dim, 2);
2817   PetscValidLogicalCollectiveInt(*dm, spaceDim, 8);
2818   ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
2819   ierr = DMPlexBuildFromCellList_Parallel_Internal(*dm, spaceDim, numCells, numVertices, numCorners, cells, PETSC_FALSE, &sfVert);CHKERRQ(ierr);
2820   if (interpolate) {
2821     DM idm;
2822 
2823     ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
2824     ierr = DMDestroy(dm);CHKERRQ(ierr);
2825     *dm  = idm;
2826   }
2827   ierr = DMPlexBuildCoordinates_Parallel_Internal(*dm, spaceDim, numCells, numVertices, sfVert, vertexCoords);CHKERRQ(ierr);
2828   if (vertexSF) *vertexSF = sfVert;
2829   else {ierr = PetscSFDestroy(&sfVert);CHKERRQ(ierr);}
2830   PetscFunctionReturn(0);
2831 }
2832 
2833 /*
2834   This takes as input the common mesh generator output, a list of the vertices for each cell
2835 */
2836 PetscErrorCode DMPlexBuildFromCellList_Internal(DM dm, PetscInt spaceDim, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, const int cells[], PetscBool invertCells)
2837 {
2838   PetscInt      *cone, c, p;
2839   PetscErrorCode ierr;
2840 
2841   PetscFunctionBegin;
2842   ierr = DMPlexSetChart(dm, 0, numCells+numVertices);CHKERRQ(ierr);
2843   for (c = 0; c < numCells; ++c) {
2844     ierr = DMPlexSetConeSize(dm, c, numCorners);CHKERRQ(ierr);
2845   }
2846   ierr = DMSetUp(dm);CHKERRQ(ierr);
2847   ierr = DMGetWorkArray(dm, numCorners, MPIU_INT, &cone);CHKERRQ(ierr);
2848   for (c = 0; c < numCells; ++c) {
2849     for (p = 0; p < numCorners; ++p) {
2850       cone[p] = cells[c*numCorners+p]+numCells;
2851     }
2852     if (invertCells) { ierr = DMPlexInvertCell_Internal(spaceDim, numCorners, cone);CHKERRQ(ierr); }
2853     ierr = DMPlexSetCone(dm, c, cone);CHKERRQ(ierr);
2854   }
2855   ierr = DMRestoreWorkArray(dm, numCorners, MPIU_INT, &cone);CHKERRQ(ierr);
2856   ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
2857   ierr = DMPlexStratify(dm);CHKERRQ(ierr);
2858   PetscFunctionReturn(0);
2859 }
2860 
2861 /*
2862   This takes as input the coordinates for each vertex
2863 */
2864 PetscErrorCode DMPlexBuildCoordinates_Internal(DM dm, PetscInt spaceDim, PetscInt numCells, PetscInt numVertices, const double vertexCoords[])
2865 {
2866   PetscSection   coordSection;
2867   Vec            coordinates;
2868   DM             cdm;
2869   PetscScalar   *coords;
2870   PetscInt       v, d;
2871   PetscErrorCode ierr;
2872 
2873   PetscFunctionBegin;
2874   ierr = DMSetCoordinateDim(dm, spaceDim);CHKERRQ(ierr);
2875   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
2876   ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
2877   ierr = PetscSectionSetFieldComponents(coordSection, 0, spaceDim);CHKERRQ(ierr);
2878   ierr = PetscSectionSetChart(coordSection, numCells, numCells + numVertices);CHKERRQ(ierr);
2879   for (v = numCells; v < numCells+numVertices; ++v) {
2880     ierr = PetscSectionSetDof(coordSection, v, spaceDim);CHKERRQ(ierr);
2881     ierr = PetscSectionSetFieldDof(coordSection, v, 0, spaceDim);CHKERRQ(ierr);
2882   }
2883   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
2884 
2885   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
2886   ierr = DMCreateLocalVector(cdm, &coordinates);CHKERRQ(ierr);
2887   ierr = VecSetBlockSize(coordinates, spaceDim);CHKERRQ(ierr);
2888   ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
2889   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
2890   for (v = 0; v < numVertices; ++v) {
2891     for (d = 0; d < spaceDim; ++d) {
2892       coords[v*spaceDim+d] = vertexCoords[v*spaceDim+d];
2893     }
2894   }
2895   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
2896   ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
2897   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
2898   PetscFunctionReturn(0);
2899 }
2900 
2901 /*@C
2902   DMPlexCreateFromCellList - This takes as input common mesh generator output, a list of the vertices for each cell, and produces a DM
2903 
2904   Input Parameters:
2905 + comm - The communicator
2906 . dim - The topological dimension of the mesh
2907 . numCells - The number of cells
2908 . numVertices - The number of vertices
2909 . numCorners - The number of vertices for each cell
2910 . interpolate - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
2911 . cells - An array of numCells*numCorners numbers, the vertices for each cell
2912 . spaceDim - The spatial dimension used for coordinates
2913 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
2914 
2915   Output Parameter:
2916 . dm - The DM
2917 
2918   Note: Two triangles sharing a face
2919 $
2920 $        2
2921 $      / | \
2922 $     /  |  \
2923 $    /   |   \
2924 $   0  0 | 1  3
2925 $    \   |   /
2926 $     \  |  /
2927 $      \ | /
2928 $        1
2929 would have input
2930 $  numCells = 2, numVertices = 4
2931 $  cells = [0 1 2  1 3 2]
2932 $
2933 which would result in the DMPlex
2934 $
2935 $        4
2936 $      / | \
2937 $     /  |  \
2938 $    /   |   \
2939 $   2  0 | 1  5
2940 $    \   |   /
2941 $     \  |  /
2942 $      \ | /
2943 $        3
2944 
2945   Level: beginner
2946 
2947 .seealso: DMPlexCreateFromDAG(), DMPlexCreate()
2948 @*/
2949 PetscErrorCode DMPlexCreateFromCellList(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, PetscBool interpolate, const int cells[], PetscInt spaceDim, const double vertexCoords[], DM *dm)
2950 {
2951   PetscErrorCode ierr;
2952 
2953   PetscFunctionBegin;
2954   ierr = DMCreate(comm, dm);CHKERRQ(ierr);
2955   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
2956   ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
2957   ierr = DMPlexBuildFromCellList_Internal(*dm, spaceDim, numCells, numVertices, numCorners, cells, PETSC_FALSE);CHKERRQ(ierr);
2958   if (interpolate) {
2959     DM idm;
2960 
2961     ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
2962     ierr = DMDestroy(dm);CHKERRQ(ierr);
2963     *dm  = idm;
2964   }
2965   ierr = DMPlexBuildCoordinates_Internal(*dm, spaceDim, numCells, numVertices, vertexCoords);CHKERRQ(ierr);
2966   PetscFunctionReturn(0);
2967 }
2968 
2969 /*@
2970   DMPlexCreateFromDAG - This takes as input the adjacency-list representation of the Directed Acyclic Graph (Hasse Diagram) encoding a mesh, and produces a DM
2971 
2972   Input Parameters:
2973 + dm - The empty DM object, usually from DMCreate() and DMSetDimension()
2974 . depth - The depth of the DAG
2975 . numPoints - Array of size depth + 1 containing the number of points at each depth
2976 . coneSize - The cone size of each point
2977 . cones - The concatenation of the cone points for each point, the cone list must be oriented correctly for each point
2978 . coneOrientations - The orientation of each cone point
2979 - vertexCoords - An array of numPoints[0]*spacedim numbers representing the coordinates of each vertex, with spacedim the value set via DMSetCoordinateDim()
2980 
2981   Output Parameter:
2982 . dm - The DM
2983 
2984   Note: Two triangles sharing a face would have input
2985 $  depth = 1, numPoints = [4 2], coneSize = [3 3 0 0 0 0]
2986 $  cones = [2 3 4  3 5 4], coneOrientations = [0 0 0  0 0 0]
2987 $ vertexCoords = [-1.0 0.0  0.0 -1.0  0.0 1.0  1.0 0.0]
2988 $
2989 which would result in the DMPlex
2990 $
2991 $        4
2992 $      / | \
2993 $     /  |  \
2994 $    /   |   \
2995 $   2  0 | 1  5
2996 $    \   |   /
2997 $     \  |  /
2998 $      \ | /
2999 $        3
3000 $
3001 $ Notice that all points are numbered consecutively, unlikely DMPlexCreateFromCellList()
3002 
3003   Level: advanced
3004 
3005 .seealso: DMPlexCreateFromCellList(), DMPlexCreate()
3006 @*/
3007 PetscErrorCode DMPlexCreateFromDAG(DM dm, PetscInt depth, const PetscInt numPoints[], const PetscInt coneSize[], const PetscInt cones[], const PetscInt coneOrientations[], const PetscScalar vertexCoords[])
3008 {
3009   Vec            coordinates;
3010   PetscSection   coordSection;
3011   PetscScalar    *coords;
3012   PetscInt       coordSize, firstVertex = -1, pStart = 0, pEnd = 0, p, v, dim, dimEmbed, d, off;
3013   PetscErrorCode ierr;
3014 
3015   PetscFunctionBegin;
3016   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
3017   ierr = DMGetCoordinateDim(dm, &dimEmbed);CHKERRQ(ierr);
3018   if (dimEmbed < dim) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Embedding dimension %d cannot be less than intrinsic dimension %d",dimEmbed,dim);
3019   for (d = 0; d <= depth; ++d) pEnd += numPoints[d];
3020   ierr = DMPlexSetChart(dm, pStart, pEnd);CHKERRQ(ierr);
3021   for (p = pStart; p < pEnd; ++p) {
3022     ierr = DMPlexSetConeSize(dm, p, coneSize[p-pStart]);CHKERRQ(ierr);
3023     if (firstVertex < 0 && !coneSize[p - pStart]) {
3024       firstVertex = p - pStart;
3025     }
3026   }
3027   if (firstVertex < 0 && numPoints[0]) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONG,"Expected %d vertices but could not find any", numPoints[0]);
3028   ierr = DMSetUp(dm);CHKERRQ(ierr); /* Allocate space for cones */
3029   for (p = pStart, off = 0; p < pEnd; off += coneSize[p-pStart], ++p) {
3030     ierr = DMPlexSetCone(dm, p, &cones[off]);CHKERRQ(ierr);
3031     ierr = DMPlexSetConeOrientation(dm, p, &coneOrientations[off]);CHKERRQ(ierr);
3032   }
3033   ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
3034   ierr = DMPlexStratify(dm);CHKERRQ(ierr);
3035   /* Build coordinates */
3036   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
3037   ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
3038   ierr = PetscSectionSetFieldComponents(coordSection, 0, dimEmbed);CHKERRQ(ierr);
3039   ierr = PetscSectionSetChart(coordSection, firstVertex, firstVertex+numPoints[0]);CHKERRQ(ierr);
3040   for (v = firstVertex; v < firstVertex+numPoints[0]; ++v) {
3041     ierr = PetscSectionSetDof(coordSection, v, dimEmbed);CHKERRQ(ierr);
3042     ierr = PetscSectionSetFieldDof(coordSection, v, 0, dimEmbed);CHKERRQ(ierr);
3043   }
3044   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
3045   ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
3046   ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
3047   ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
3048   ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
3049   ierr = VecSetBlockSize(coordinates, dimEmbed);CHKERRQ(ierr);
3050   ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
3051   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
3052   for (v = 0; v < numPoints[0]; ++v) {
3053     PetscInt off;
3054 
3055     ierr = PetscSectionGetOffset(coordSection, v+firstVertex, &off);CHKERRQ(ierr);
3056     for (d = 0; d < dimEmbed; ++d) {
3057       coords[off+d] = vertexCoords[v*dimEmbed+d];
3058     }
3059   }
3060   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
3061   ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
3062   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
3063   PetscFunctionReturn(0);
3064 }
3065 
3066 /*@C
3067   DMPlexCreateCellVertexFromFile - Create a DMPlex mesh from a simple cell-vertex file.
3068 
3069 + comm        - The MPI communicator
3070 . filename    - Name of the .dat file
3071 - interpolate - Create faces and edges in the mesh
3072 
3073   Output Parameter:
3074 . dm  - The DM object representing the mesh
3075 
3076   Note: The format is the simplest possible:
3077 $ Ne
3078 $ v0 v1 ... vk
3079 $ Nv
3080 $ x y z marker
3081 
3082   Level: beginner
3083 
3084 .seealso: DMPlexCreateFromFile(), DMPlexCreateMedFromFile(), DMPlexCreateGmsh(), DMPlexCreate()
3085 @*/
3086 PetscErrorCode DMPlexCreateCellVertexFromFile(MPI_Comm comm, const char filename[], PetscBool interpolate, DM *dm)
3087 {
3088   DMLabel         marker;
3089   PetscViewer     viewer;
3090   Vec             coordinates;
3091   PetscSection    coordSection;
3092   PetscScalar    *coords;
3093   char            line[PETSC_MAX_PATH_LEN];
3094   PetscInt        dim = 3, cdim = 3, coordSize, v, c, d;
3095   PetscMPIInt     rank;
3096   int             snum, Nv, Nc;
3097   PetscErrorCode  ierr;
3098 
3099   PetscFunctionBegin;
3100   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
3101   ierr = PetscViewerCreate(comm, &viewer);CHKERRQ(ierr);
3102   ierr = PetscViewerSetType(viewer, PETSCVIEWERASCII);CHKERRQ(ierr);
3103   ierr = PetscViewerFileSetMode(viewer, FILE_MODE_READ);CHKERRQ(ierr);
3104   ierr = PetscViewerFileSetName(viewer, filename);CHKERRQ(ierr);
3105   if (!rank) {
3106     ierr = PetscViewerRead(viewer, line, 2, NULL, PETSC_STRING);CHKERRQ(ierr);
3107     snum = sscanf(line, "%d %d", &Nc, &Nv);
3108     if (snum != 2) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
3109   } else {
3110     Nc = Nv = 0;
3111   }
3112   ierr = DMCreate(comm, dm);CHKERRQ(ierr);
3113   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
3114   ierr = DMPlexSetChart(*dm, 0, Nc+Nv);CHKERRQ(ierr);
3115   ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
3116   ierr = DMSetCoordinateDim(*dm, cdim);CHKERRQ(ierr);
3117   /* Read topology */
3118   if (!rank) {
3119     PetscInt cone[8], corners = 8;
3120     int      vbuf[8], v;
3121 
3122     for (c = 0; c < Nc; ++c) {ierr = DMPlexSetConeSize(*dm, c, corners);CHKERRQ(ierr);}
3123     ierr = DMSetUp(*dm);CHKERRQ(ierr);
3124     for (c = 0; c < Nc; ++c) {
3125       ierr = PetscViewerRead(viewer, line, corners, NULL, PETSC_STRING);CHKERRQ(ierr);
3126       snum = sscanf(line, "%d %d %d %d %d %d %d %d", &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3], &vbuf[4], &vbuf[5], &vbuf[6], &vbuf[7]);
3127       if (snum != corners) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
3128       for (v = 0; v < corners; ++v) cone[v] = vbuf[v] + Nc;
3129       /* Hexahedra are inverted */
3130       {
3131         PetscInt tmp = cone[1];
3132         cone[1] = cone[3];
3133         cone[3] = tmp;
3134       }
3135       ierr = DMPlexSetCone(*dm, c, cone);CHKERRQ(ierr);
3136     }
3137   }
3138   ierr = DMPlexSymmetrize(*dm);CHKERRQ(ierr);
3139   ierr = DMPlexStratify(*dm);CHKERRQ(ierr);
3140   /* Read coordinates */
3141   ierr = DMGetCoordinateSection(*dm, &coordSection);CHKERRQ(ierr);
3142   ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
3143   ierr = PetscSectionSetFieldComponents(coordSection, 0, cdim);CHKERRQ(ierr);
3144   ierr = PetscSectionSetChart(coordSection, Nc, Nc + Nv);CHKERRQ(ierr);
3145   for (v = Nc; v < Nc+Nv; ++v) {
3146     ierr = PetscSectionSetDof(coordSection, v, cdim);CHKERRQ(ierr);
3147     ierr = PetscSectionSetFieldDof(coordSection, v, 0, cdim);CHKERRQ(ierr);
3148   }
3149   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
3150   ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
3151   ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
3152   ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
3153   ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
3154   ierr = VecSetBlockSize(coordinates, cdim);CHKERRQ(ierr);
3155   ierr = VecSetType(coordinates, VECSTANDARD);CHKERRQ(ierr);
3156   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
3157   if (!rank) {
3158     double x[3];
3159     int    val;
3160 
3161     ierr = DMCreateLabel(*dm, "marker");CHKERRQ(ierr);
3162     ierr = DMGetLabel(*dm, "marker", &marker);CHKERRQ(ierr);
3163     for (v = 0; v < Nv; ++v) {
3164       ierr = PetscViewerRead(viewer, line, 4, NULL, PETSC_STRING);CHKERRQ(ierr);
3165       snum = sscanf(line, "%lg %lg %lg %d", &x[0], &x[1], &x[2], &val);
3166       if (snum != 4) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
3167       for (d = 0; d < cdim; ++d) coords[v*cdim+d] = x[d];
3168       if (val) {ierr = DMLabelSetValue(marker, v+Nc, val);CHKERRQ(ierr);}
3169     }
3170   }
3171   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
3172   ierr = DMSetCoordinatesLocal(*dm, coordinates);CHKERRQ(ierr);
3173   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
3174   ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
3175   if (interpolate) {
3176     DM      idm;
3177     DMLabel bdlabel;
3178 
3179     ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
3180     ierr = DMDestroy(dm);CHKERRQ(ierr);
3181     *dm  = idm;
3182 
3183     ierr = DMGetLabel(*dm, "marker", &bdlabel);CHKERRQ(ierr);
3184     ierr = DMPlexMarkBoundaryFaces(*dm, PETSC_DETERMINE, bdlabel);CHKERRQ(ierr);
3185     ierr = DMPlexLabelComplete(*dm, bdlabel);CHKERRQ(ierr);
3186   }
3187   PetscFunctionReturn(0);
3188 }
3189 
3190 /*@C
3191   DMPlexCreateFromFile - This takes a filename and produces a DM
3192 
3193   Input Parameters:
3194 + comm - The communicator
3195 . filename - A file name
3196 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
3197 
3198   Output Parameter:
3199 . dm - The DM
3200 
3201   Options Database Keys:
3202 . -dm_plex_create_from_hdf5_xdmf - use the PETSC_VIEWER_HDF5_XDMF format for reading HDF5
3203 
3204   Level: beginner
3205 
3206 .seealso: DMPlexCreateFromDAG(), DMPlexCreateFromCellList(), DMPlexCreate()
3207 @*/
3208 PetscErrorCode DMPlexCreateFromFile(MPI_Comm comm, const char filename[], PetscBool interpolate, DM *dm)
3209 {
3210   const char    *extGmsh    = ".msh";
3211   const char    *extCGNS    = ".cgns";
3212   const char    *extExodus  = ".exo";
3213   const char    *extGenesis = ".gen";
3214   const char    *extFluent  = ".cas";
3215   const char    *extHDF5    = ".h5";
3216   const char    *extMed     = ".med";
3217   const char    *extPLY     = ".ply";
3218   const char    *extCV      = ".dat";
3219   size_t         len;
3220   PetscBool      isGmsh, isCGNS, isExodus, isGenesis, isFluent, isHDF5, isMed, isPLY, isCV;
3221   PetscMPIInt    rank;
3222   PetscErrorCode ierr;
3223 
3224   PetscFunctionBegin;
3225   PetscValidPointer(filename, 2);
3226   PetscValidPointer(dm, 4);
3227   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
3228   ierr = PetscStrlen(filename, &len);CHKERRQ(ierr);
3229   if (!len) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Filename must be a valid path");
3230   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)], extGmsh,    4, &isGmsh);CHKERRQ(ierr);
3231   ierr = PetscStrncmp(&filename[PetscMax(0,len-5)], extCGNS,    5, &isCGNS);CHKERRQ(ierr);
3232   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)], extExodus,  4, &isExodus);CHKERRQ(ierr);
3233   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)], extGenesis, 4, &isGenesis);CHKERRQ(ierr);
3234   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)], extFluent,  4, &isFluent);CHKERRQ(ierr);
3235   ierr = PetscStrncmp(&filename[PetscMax(0,len-3)], extHDF5,    3, &isHDF5);CHKERRQ(ierr);
3236   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)], extMed,     4, &isMed);CHKERRQ(ierr);
3237   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)], extPLY,     4, &isPLY);CHKERRQ(ierr);
3238   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)], extCV,      4, &isCV);CHKERRQ(ierr);
3239   if (isGmsh) {
3240     ierr = DMPlexCreateGmshFromFile(comm, filename, interpolate, dm);CHKERRQ(ierr);
3241   } else if (isCGNS) {
3242     ierr = DMPlexCreateCGNSFromFile(comm, filename, interpolate, dm);CHKERRQ(ierr);
3243   } else if (isExodus || isGenesis) {
3244     ierr = DMPlexCreateExodusFromFile(comm, filename, interpolate, dm);CHKERRQ(ierr);
3245   } else if (isFluent) {
3246     ierr = DMPlexCreateFluentFromFile(comm, filename, interpolate, dm);CHKERRQ(ierr);
3247   } else if (isHDF5) {
3248     PetscBool      load_hdf5_xdmf = PETSC_FALSE;
3249     PetscViewer viewer;
3250 
3251     ierr = PetscOptionsGetBool(NULL, NULL, "-dm_plex_create_from_hdf5_xdmf", &load_hdf5_xdmf, NULL);CHKERRQ(ierr);
3252     ierr = PetscViewerCreate(comm, &viewer);CHKERRQ(ierr);
3253     ierr = PetscViewerSetType(viewer, PETSCVIEWERHDF5);CHKERRQ(ierr);
3254     ierr = PetscViewerFileSetMode(viewer, FILE_MODE_READ);CHKERRQ(ierr);
3255     ierr = PetscViewerFileSetName(viewer, filename);CHKERRQ(ierr);
3256     ierr = DMCreate(comm, dm);CHKERRQ(ierr);
3257     ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
3258     if (load_hdf5_xdmf) {ierr = PetscViewerPushFormat(viewer, PETSC_VIEWER_HDF5_XDMF);CHKERRQ(ierr);}
3259     ierr = DMLoad(*dm, viewer);CHKERRQ(ierr);
3260     if (load_hdf5_xdmf) {ierr = PetscViewerPopFormat(viewer);CHKERRQ(ierr);}
3261     ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
3262 
3263     if (interpolate) {
3264       DM idm;
3265 
3266       ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
3267       ierr = DMDestroy(dm);CHKERRQ(ierr);
3268       *dm  = idm;
3269     }
3270   } else if (isMed) {
3271     ierr = DMPlexCreateMedFromFile(comm, filename, interpolate, dm);CHKERRQ(ierr);
3272   } else if (isPLY) {
3273     ierr = DMPlexCreatePLYFromFile(comm, filename, interpolate, dm);CHKERRQ(ierr);
3274   } else if (isCV) {
3275     ierr = DMPlexCreateCellVertexFromFile(comm, filename, interpolate, dm);CHKERRQ(ierr);
3276   } else SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cannot load file %s: unrecognized extension", filename);
3277   PetscFunctionReturn(0);
3278 }
3279 
3280 /*@
3281   DMPlexCreateReferenceCell - Create a DMPLEX with the appropriate FEM reference cell
3282 
3283   Collective on comm
3284 
3285   Input Parameters:
3286 + comm    - The communicator
3287 . dim     - The spatial dimension
3288 - simplex - Flag for simplex, otherwise use a tensor-product cell
3289 
3290   Output Parameter:
3291 . refdm - The reference cell
3292 
3293   Level: intermediate
3294 
3295 .keywords: reference cell
3296 .seealso:
3297 @*/
3298 PetscErrorCode DMPlexCreateReferenceCell(MPI_Comm comm, PetscInt dim, PetscBool simplex, DM *refdm)
3299 {
3300   DM             rdm;
3301   Vec            coords;
3302   PetscErrorCode ierr;
3303 
3304   PetscFunctionBeginUser;
3305   ierr = DMCreate(comm, &rdm);CHKERRQ(ierr);
3306   ierr = DMSetType(rdm, DMPLEX);CHKERRQ(ierr);
3307   ierr = DMSetDimension(rdm, dim);CHKERRQ(ierr);
3308   switch (dim) {
3309   case 0:
3310   {
3311     PetscInt    numPoints[1]        = {1};
3312     PetscInt    coneSize[1]         = {0};
3313     PetscInt    cones[1]            = {0};
3314     PetscInt    coneOrientations[1] = {0};
3315     PetscScalar vertexCoords[1]     = {0.0};
3316 
3317     ierr = DMPlexCreateFromDAG(rdm, 0, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
3318   }
3319   break;
3320   case 1:
3321   {
3322     PetscInt    numPoints[2]        = {2, 1};
3323     PetscInt    coneSize[3]         = {2, 0, 0};
3324     PetscInt    cones[2]            = {1, 2};
3325     PetscInt    coneOrientations[2] = {0, 0};
3326     PetscScalar vertexCoords[2]     = {-1.0,  1.0};
3327 
3328     ierr = DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
3329   }
3330   break;
3331   case 2:
3332     if (simplex) {
3333       PetscInt    numPoints[2]        = {3, 1};
3334       PetscInt    coneSize[4]         = {3, 0, 0, 0};
3335       PetscInt    cones[3]            = {1, 2, 3};
3336       PetscInt    coneOrientations[3] = {0, 0, 0};
3337       PetscScalar vertexCoords[6]     = {-1.0, -1.0,  1.0, -1.0,  -1.0, 1.0};
3338 
3339       ierr = DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
3340     } else {
3341       PetscInt    numPoints[2]        = {4, 1};
3342       PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
3343       PetscInt    cones[4]            = {1, 2, 3, 4};
3344       PetscInt    coneOrientations[4] = {0, 0, 0, 0};
3345       PetscScalar vertexCoords[8]     = {-1.0, -1.0,  1.0, -1.0,  1.0, 1.0,  -1.0, 1.0};
3346 
3347       ierr = DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
3348     }
3349   break;
3350   case 3:
3351     if (simplex) {
3352       PetscInt    numPoints[2]        = {4, 1};
3353       PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
3354       PetscInt    cones[4]            = {1, 3, 2, 4};
3355       PetscInt    coneOrientations[4] = {0, 0, 0, 0};
3356       PetscScalar vertexCoords[12]    = {-1.0, -1.0, -1.0,  1.0, -1.0, -1.0,  -1.0, 1.0, -1.0,  -1.0, -1.0, 1.0};
3357 
3358       ierr = DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
3359     } else {
3360       PetscInt    numPoints[2]        = {8, 1};
3361       PetscInt    coneSize[9]         = {8, 0, 0, 0, 0, 0, 0, 0, 0};
3362       PetscInt    cones[8]            = {1, 4, 3, 2, 5, 6, 7, 8};
3363       PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
3364       PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0,  1.0, -1.0, -1.0,  1.0, 1.0, -1.0,  -1.0, 1.0, -1.0,
3365                                          -1.0, -1.0,  1.0,  1.0, -1.0,  1.0,  1.0, 1.0,  1.0,  -1.0, 1.0,  1.0};
3366 
3367       ierr = DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
3368     }
3369   break;
3370   default:
3371     SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Cannot create reference cell for dimension %d", dim);
3372   }
3373   ierr = DMPlexInterpolate(rdm, refdm);CHKERRQ(ierr);
3374   if (rdm->coordinateDM) {
3375     DM           ncdm;
3376     PetscSection cs;
3377     PetscInt     pEnd = -1;
3378 
3379     ierr = DMGetSection(rdm->coordinateDM, &cs);CHKERRQ(ierr);
3380     if (cs) {ierr = PetscSectionGetChart(cs, NULL, &pEnd);CHKERRQ(ierr);}
3381     if (pEnd >= 0) {
3382       ierr = DMClone(rdm->coordinateDM, &ncdm);CHKERRQ(ierr);
3383       ierr = DMSetSection(ncdm, cs);CHKERRQ(ierr);
3384       ierr = DMSetCoordinateDM(*refdm, ncdm);CHKERRQ(ierr);
3385       ierr = DMDestroy(&ncdm);CHKERRQ(ierr);
3386     }
3387   }
3388   ierr = DMGetCoordinatesLocal(rdm, &coords);CHKERRQ(ierr);
3389   if (coords) {
3390     ierr = DMSetCoordinatesLocal(*refdm, coords);CHKERRQ(ierr);
3391   } else {
3392     ierr = DMGetCoordinates(rdm, &coords);CHKERRQ(ierr);
3393     if (coords) {ierr = DMSetCoordinates(*refdm, coords);CHKERRQ(ierr);}
3394   }
3395   ierr = DMDestroy(&rdm);CHKERRQ(ierr);
3396   PetscFunctionReturn(0);
3397 }
3398