xref: /petsc/src/dm/impls/plex/plexcreate.c (revision 285fb4e2b69b3de46a0633bd0adc6a7f684caa1e)
1 #define PETSCDM_DLL
2 #include <petsc/private/dmpleximpl.h>    /*I   "petscdmplex.h"   I*/
3 #include <petsc/private/hashseti.h>          /*I   "petscdmplex.h"   I*/
4 #include <petscsf.h>
5 
6 /*@
7   DMPlexCreateDoublet - Creates a mesh of two cells of the specified type, optionally with later refinement.
8 
9   Collective on MPI_Comm
10 
11   Input Parameters:
12 + comm - The communicator for the DM object
13 . dim - The spatial dimension
14 . simplex - Flag for simplicial cells, otherwise they are tensor product cells
15 . interpolate - Flag to create intermediate mesh pieces (edges, faces)
16 . refinementUniform - Flag for uniform parallel refinement
17 - refinementLimit - A nonzero number indicates the largest admissible volume for a refined cell
18 
19   Output Parameter:
20 . dm - The DM object
21 
22   Level: beginner
23 
24 .keywords: DM, create
25 .seealso: DMSetType(), DMCreate()
26 @*/
27 PetscErrorCode DMPlexCreateDoublet(MPI_Comm comm, PetscInt dim, PetscBool simplex, PetscBool interpolate, PetscBool refinementUniform, PetscReal refinementLimit, DM *newdm)
28 {
29   DM             dm;
30   PetscInt       p;
31   PetscMPIInt    rank;
32   PetscErrorCode ierr;
33 
34   PetscFunctionBegin;
35   ierr = DMCreate(comm, &dm);CHKERRQ(ierr);
36   ierr = DMSetType(dm, DMPLEX);CHKERRQ(ierr);
37   ierr = DMSetDimension(dm, dim);CHKERRQ(ierr);
38   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
39   switch (dim) {
40   case 2:
41     if (simplex) {ierr = PetscObjectSetName((PetscObject) dm, "triangular");CHKERRQ(ierr);}
42     else         {ierr = PetscObjectSetName((PetscObject) dm, "quadrilateral");CHKERRQ(ierr);}
43     break;
44   case 3:
45     if (simplex) {ierr = PetscObjectSetName((PetscObject) dm, "tetrahedral");CHKERRQ(ierr);}
46     else         {ierr = PetscObjectSetName((PetscObject) dm, "hexahedral");CHKERRQ(ierr);}
47     break;
48   default:
49     SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make meshes for dimension %d", dim);
50   }
51   if (rank) {
52     PetscInt numPoints[2] = {0, 0};
53     ierr = DMPlexCreateFromDAG(dm, 1, numPoints, NULL, NULL, NULL, NULL);CHKERRQ(ierr);
54   } else {
55     switch (dim) {
56     case 2:
57       if (simplex) {
58         PetscInt    numPoints[2]        = {4, 2};
59         PetscInt    coneSize[6]         = {3, 3, 0, 0, 0, 0};
60         PetscInt    cones[6]            = {2, 3, 4,  5, 4, 3};
61         PetscInt    coneOrientations[6] = {0, 0, 0,  0, 0, 0};
62         PetscScalar vertexCoords[8]     = {-0.5, 0.5, 0.0, 0.0, 0.0, 1.0, 0.5, 0.5};
63         PetscInt    markerPoints[8]     = {2, 1, 3, 1, 4, 1, 5, 1};
64 
65         ierr = DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
66         for (p = 0; p < 4; ++p) {ierr = DMSetLabelValue(dm, "marker", markerPoints[p*2], markerPoints[p*2+1]);CHKERRQ(ierr);}
67       } else {
68         PetscInt    numPoints[2]        = {6, 2};
69         PetscInt    coneSize[8]         = {4, 4, 0, 0, 0, 0, 0, 0};
70         PetscInt    cones[8]            = {2, 3, 4, 5,  3, 6, 7, 4};
71         PetscInt    coneOrientations[8] = {0, 0, 0, 0,  0, 0, 0, 0};
72         PetscScalar vertexCoords[12]    = {-1.0, -0.5,  0.0, -0.5,  0.0, 0.5,  -1.0, 0.5,  1.0, -0.5,  1.0, 0.5};
73 
74         ierr = DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
75       }
76       break;
77     case 3:
78       if (simplex) {
79         PetscInt    numPoints[2]        = {5, 2};
80         PetscInt    coneSize[7]         = {4, 4, 0, 0, 0, 0, 0};
81         PetscInt    cones[8]            = {4, 3, 5, 2,  5, 3, 4, 6};
82         PetscInt    coneOrientations[8] = {0, 0, 0, 0,  0, 0, 0, 0};
83         PetscScalar vertexCoords[15]    = {-1.0, 0.0, 0.0,  0.0, -1.0, 0.0,  0.0, 0.0, 1.0,  0.0, 1.0, 0.0,  1.0, 0.0, 0.0};
84         PetscInt    markerPoints[10]    = {2, 1, 3, 1, 4, 1, 5, 1, 6, 1};
85 
86         ierr = DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
87         for (p = 0; p < 5; ++p) {ierr = DMSetLabelValue(dm, "marker", markerPoints[p*2], markerPoints[p*2+1]);CHKERRQ(ierr);}
88       } else {
89         PetscInt    numPoints[2]         = {12, 2};
90         PetscInt    coneSize[14]         = {8, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
91         PetscInt    cones[16]            = {2, 3, 4, 5, 6, 7, 8, 9,  5, 4, 10, 11, 7, 12, 13, 8};
92         PetscInt    coneOrientations[16] = {0, 0, 0, 0, 0, 0, 0, 0,  0, 0,  0,  0, 0,  0,  0, 0};
93         PetscScalar vertexCoords[36]     = {-1.0, -0.5, -0.5,  -1.0,  0.5, -0.5,  0.0,  0.5, -0.5,   0.0, -0.5, -0.5,
94                                             -1.0, -0.5,  0.5,   0.0, -0.5,  0.5,  0.0,  0.5,  0.5,  -1.0,  0.5,  0.5,
95                                              1.0,  0.5, -0.5,   1.0, -0.5, -0.5,  1.0, -0.5,  0.5,   1.0,  0.5,  0.5};
96 
97         ierr = DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
98       }
99       break;
100     default:
101       SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make meshes for dimension %d", dim);
102     }
103   }
104   *newdm = dm;
105   if (refinementLimit > 0.0) {
106     DM rdm;
107     const char *name;
108 
109     ierr = DMPlexSetRefinementUniform(*newdm, PETSC_FALSE);CHKERRQ(ierr);
110     ierr = DMPlexSetRefinementLimit(*newdm, refinementLimit);CHKERRQ(ierr);
111     ierr = DMRefine(*newdm, comm, &rdm);CHKERRQ(ierr);
112     ierr = PetscObjectGetName((PetscObject) *newdm, &name);CHKERRQ(ierr);
113     ierr = PetscObjectSetName((PetscObject)    rdm,  name);CHKERRQ(ierr);
114     ierr = DMDestroy(newdm);CHKERRQ(ierr);
115     *newdm = rdm;
116   }
117   if (interpolate) {
118     DM idm;
119 
120     ierr = DMPlexInterpolate(*newdm, &idm);CHKERRQ(ierr);
121     ierr = DMDestroy(newdm);CHKERRQ(ierr);
122     *newdm = idm;
123   }
124   {
125     DM refinedMesh     = NULL;
126     DM distributedMesh = NULL;
127 
128     /* Distribute mesh over processes */
129     ierr = DMPlexDistribute(*newdm, 0, NULL, &distributedMesh);CHKERRQ(ierr);
130     if (distributedMesh) {
131       ierr = DMDestroy(newdm);CHKERRQ(ierr);
132       *newdm = distributedMesh;
133     }
134     if (refinementUniform) {
135       ierr = DMPlexSetRefinementUniform(*newdm, refinementUniform);CHKERRQ(ierr);
136       ierr = DMRefine(*newdm, comm, &refinedMesh);CHKERRQ(ierr);
137       if (refinedMesh) {
138         ierr = DMDestroy(newdm);CHKERRQ(ierr);
139         *newdm = refinedMesh;
140       }
141     }
142   }
143   PetscFunctionReturn(0);
144 }
145 
146 /*@
147   DMPlexCreateSquareBoundary - Creates a 1D mesh the is the boundary of a square lattice.
148 
149   Collective on MPI_Comm
150 
151   Input Parameters:
152 + comm  - The communicator for the DM object
153 . lower - The lower left corner coordinates
154 . upper - The upper right corner coordinates
155 - edges - The number of cells in each direction
156 
157   Output Parameter:
158 . dm  - The DM object
159 
160   Note: Here is the numbering returned for 2 cells in each direction:
161 $ 18--5-17--4--16
162 $  |     |     |
163 $  6    10     3
164 $  |     |     |
165 $ 19-11-20--9--15
166 $  |     |     |
167 $  7     8     2
168 $  |     |     |
169 $ 12--0-13--1--14
170 
171   Level: beginner
172 
173 .keywords: DM, create
174 .seealso: DMPlexCreateBoxMesh(), DMPlexCreateCubeBoundary(), DMSetType(), DMCreate()
175 @*/
176 PetscErrorCode DMPlexCreateSquareBoundary(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[])
177 {
178   const PetscInt numVertices    = (edges[0]+1)*(edges[1]+1);
179   const PetscInt numEdges       = edges[0]*(edges[1]+1) + (edges[0]+1)*edges[1];
180   PetscInt       markerTop      = 1;
181   PetscInt       markerBottom   = 1;
182   PetscInt       markerRight    = 1;
183   PetscInt       markerLeft     = 1;
184   PetscBool      markerSeparate = PETSC_FALSE;
185   Vec            coordinates;
186   PetscSection   coordSection;
187   PetscScalar    *coords;
188   PetscInt       coordSize;
189   PetscMPIInt    rank;
190   PetscInt       v, vx, vy;
191   PetscErrorCode ierr;
192 
193   PetscFunctionBegin;
194   ierr = PetscOptionsGetBool(((PetscObject) dm)->options,((PetscObject) dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL);CHKERRQ(ierr);
195   if (markerSeparate) {
196     markerTop    = 3;
197     markerBottom = 1;
198     markerRight  = 2;
199     markerLeft   = 4;
200   }
201   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRQ(ierr);
202   if (!rank) {
203     PetscInt e, ex, ey;
204 
205     ierr = DMPlexSetChart(dm, 0, numEdges+numVertices);CHKERRQ(ierr);
206     for (e = 0; e < numEdges; ++e) {
207       ierr = DMPlexSetConeSize(dm, e, 2);CHKERRQ(ierr);
208     }
209     ierr = DMSetUp(dm);CHKERRQ(ierr); /* Allocate space for cones */
210     for (vx = 0; vx <= edges[0]; vx++) {
211       for (ey = 0; ey < edges[1]; ey++) {
212         PetscInt edge   = vx*edges[1] + ey + edges[0]*(edges[1]+1);
213         PetscInt vertex = ey*(edges[0]+1) + vx + numEdges;
214         PetscInt cone[2];
215 
216         cone[0] = vertex; cone[1] = vertex+edges[0]+1;
217         ierr    = DMPlexSetCone(dm, edge, cone);CHKERRQ(ierr);
218         if (vx == edges[0]) {
219           ierr = DMSetLabelValue(dm, "marker", edge,    markerRight);CHKERRQ(ierr);
220           ierr = DMSetLabelValue(dm, "marker", cone[0], markerRight);CHKERRQ(ierr);
221           if (ey == edges[1]-1) {
222             ierr = DMSetLabelValue(dm, "marker", cone[1], markerRight);CHKERRQ(ierr);
223             ierr = DMSetLabelValue(dm, "Face Sets", cone[1], markerRight);CHKERRQ(ierr);
224           }
225         } else if (vx == 0) {
226           ierr = DMSetLabelValue(dm, "marker", edge,    markerLeft);CHKERRQ(ierr);
227           ierr = DMSetLabelValue(dm, "marker", cone[0], markerLeft);CHKERRQ(ierr);
228           if (ey == edges[1]-1) {
229             ierr = DMSetLabelValue(dm, "marker", cone[1], markerLeft);CHKERRQ(ierr);
230             ierr = DMSetLabelValue(dm, "Face Sets", cone[1], markerLeft);CHKERRQ(ierr);
231           }
232         }
233       }
234     }
235     for (vy = 0; vy <= edges[1]; vy++) {
236       for (ex = 0; ex < edges[0]; ex++) {
237         PetscInt edge   = vy*edges[0]     + ex;
238         PetscInt vertex = vy*(edges[0]+1) + ex + numEdges;
239         PetscInt cone[2];
240 
241         cone[0] = vertex; cone[1] = vertex+1;
242         ierr    = DMPlexSetCone(dm, edge, cone);CHKERRQ(ierr);
243         if (vy == edges[1]) {
244           ierr = DMSetLabelValue(dm, "marker", edge,    markerTop);CHKERRQ(ierr);
245           ierr = DMSetLabelValue(dm, "marker", cone[0], markerTop);CHKERRQ(ierr);
246           if (ex == edges[0]-1) {
247             ierr = DMSetLabelValue(dm, "marker", cone[1], markerTop);CHKERRQ(ierr);
248             ierr = DMSetLabelValue(dm, "Face Sets", cone[1], markerTop);CHKERRQ(ierr);
249           }
250         } else if (vy == 0) {
251           ierr = DMSetLabelValue(dm, "marker", edge,    markerBottom);CHKERRQ(ierr);
252           ierr = DMSetLabelValue(dm, "marker", cone[0], markerBottom);CHKERRQ(ierr);
253           if (ex == edges[0]-1) {
254             ierr = DMSetLabelValue(dm, "marker", cone[1], markerBottom);CHKERRQ(ierr);
255             ierr = DMSetLabelValue(dm, "Face Sets", cone[1], markerBottom);CHKERRQ(ierr);
256           }
257         }
258       }
259     }
260   }
261   ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
262   ierr = DMPlexStratify(dm);CHKERRQ(ierr);
263   /* Build coordinates */
264   ierr = DMSetCoordinateDim(dm, 2);CHKERRQ(ierr);
265   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
266   ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
267   ierr = PetscSectionSetChart(coordSection, numEdges, numEdges + numVertices);CHKERRQ(ierr);
268   ierr = PetscSectionSetFieldComponents(coordSection, 0, 2);CHKERRQ(ierr);
269   for (v = numEdges; v < numEdges+numVertices; ++v) {
270     ierr = PetscSectionSetDof(coordSection, v, 2);CHKERRQ(ierr);
271     ierr = PetscSectionSetFieldDof(coordSection, v, 0, 2);CHKERRQ(ierr);
272   }
273   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
274   ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
275   ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
276   ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
277   ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
278   ierr = VecSetBlockSize(coordinates, 2);CHKERRQ(ierr);
279   ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
280   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
281   for (vy = 0; vy <= edges[1]; ++vy) {
282     for (vx = 0; vx <= edges[0]; ++vx) {
283       coords[(vy*(edges[0]+1)+vx)*2+0] = lower[0] + ((upper[0] - lower[0])/edges[0])*vx;
284       coords[(vy*(edges[0]+1)+vx)*2+1] = lower[1] + ((upper[1] - lower[1])/edges[1])*vy;
285     }
286   }
287   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
288   ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
289   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
290   PetscFunctionReturn(0);
291 }
292 
293 /*@
294   DMPlexCreateCubeBoundary - Creates a 2D mesh that is the boundary of a cubic lattice.
295 
296   Collective on MPI_Comm
297 
298   Input Parameters:
299 + comm  - The communicator for the DM object
300 . lower - The lower left front corner coordinates
301 . upper - The upper right back corner coordinates
302 - edges - The number of cells in each direction
303 
304   Output Parameter:
305 . dm  - The DM object
306 
307   Level: beginner
308 
309 .keywords: DM, create
310 .seealso: DMPlexCreateBoxMesh(), DMPlexCreateSquareBoundary(), DMSetType(), DMCreate()
311 @*/
312 PetscErrorCode DMPlexCreateCubeBoundary(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt faces[])
313 {
314   PetscInt       vertices[3], numVertices;
315   PetscInt       numFaces    = 2*faces[0]*faces[1] + 2*faces[1]*faces[2] + 2*faces[0]*faces[2];
316   Vec            coordinates;
317   PetscSection   coordSection;
318   PetscScalar    *coords;
319   PetscInt       coordSize;
320   PetscMPIInt    rank;
321   PetscInt       v, vx, vy, vz;
322   PetscInt       voffset, iface=0, cone[4];
323   PetscErrorCode ierr;
324 
325   PetscFunctionBegin;
326   if ((faces[0] < 1) || (faces[1] < 1) || (faces[2] < 1)) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Must have at least 1 face per side");
327   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRQ(ierr);
328   vertices[0] = faces[0]+1; vertices[1] = faces[1]+1; vertices[2] = faces[2]+1;
329   numVertices = vertices[0]*vertices[1]*vertices[2];
330   if (!rank) {
331     PetscInt f;
332 
333     ierr = DMPlexSetChart(dm, 0, numFaces+numVertices);CHKERRQ(ierr);
334     for (f = 0; f < numFaces; ++f) {
335       ierr = DMPlexSetConeSize(dm, f, 4);CHKERRQ(ierr);
336     }
337     ierr = DMSetUp(dm);CHKERRQ(ierr); /* Allocate space for cones */
338     for (v = 0; v < numFaces+numVertices; ++v) {
339       ierr = DMSetLabelValue(dm, "marker", v, 1);CHKERRQ(ierr);
340     }
341 
342     /* Side 0 (Top) */
343     for (vy = 0; vy < faces[1]; vy++) {
344       for (vx = 0; vx < faces[0]; vx++) {
345         voffset = numFaces + vertices[0]*vertices[1]*(vertices[2]-1) + vy*vertices[0] + vx;
346         cone[0] = voffset; cone[1] = voffset+1; cone[2] = voffset+vertices[0]+1; cone[3] = voffset+vertices[0];
347         ierr    = DMPlexSetCone(dm, iface, cone);CHKERRQ(ierr);
348         iface++;
349       }
350     }
351 
352     /* Side 1 (Bottom) */
353     for (vy = 0; vy < faces[1]; vy++) {
354       for (vx = 0; vx < faces[0]; vx++) {
355         voffset = numFaces + vy*(faces[0]+1) + vx;
356         cone[0] = voffset+1; cone[1] = voffset; cone[2] = voffset+vertices[0]; cone[3] = voffset+vertices[0]+1;
357         ierr    = DMPlexSetCone(dm, iface, cone);CHKERRQ(ierr);
358         iface++;
359       }
360     }
361 
362     /* Side 2 (Front) */
363     for (vz = 0; vz < faces[2]; vz++) {
364       for (vx = 0; vx < faces[0]; vx++) {
365         voffset = numFaces + vz*vertices[0]*vertices[1] + vx;
366         cone[0] = voffset; cone[1] = voffset+1; cone[2] = voffset+vertices[0]*vertices[1]+1; cone[3] = voffset+vertices[0]*vertices[1];
367         ierr    = DMPlexSetCone(dm, iface, cone);CHKERRQ(ierr);
368         iface++;
369       }
370     }
371 
372     /* Side 3 (Back) */
373     for (vz = 0; vz < faces[2]; vz++) {
374       for (vx = 0; vx < faces[0]; vx++) {
375         voffset = numFaces + vz*vertices[0]*vertices[1] + vertices[0]*(vertices[1]-1) + vx;
376         cone[0] = voffset+vertices[0]*vertices[1]; cone[1] = voffset+vertices[0]*vertices[1]+1;
377         cone[2] = voffset+1; cone[3] = voffset;
378         ierr    = DMPlexSetCone(dm, iface, cone);CHKERRQ(ierr);
379         iface++;
380       }
381     }
382 
383     /* Side 4 (Left) */
384     for (vz = 0; vz < faces[2]; vz++) {
385       for (vy = 0; vy < faces[1]; vy++) {
386         voffset = numFaces + vz*vertices[0]*vertices[1] + vy*vertices[0];
387         cone[0] = voffset; cone[1] = voffset+vertices[0]*vertices[1];
388         cone[2] = voffset+vertices[0]*vertices[1]+vertices[0]; cone[3] = voffset+vertices[0];
389         ierr    = DMPlexSetCone(dm, iface, cone);CHKERRQ(ierr);
390         iface++;
391       }
392     }
393 
394     /* Side 5 (Right) */
395     for (vz = 0; vz < faces[2]; vz++) {
396       for (vy = 0; vy < faces[1]; vy++) {
397         voffset = numFaces + vz*vertices[0]*vertices[1] + vy*vertices[0] + faces[0];
398         cone[0] = voffset+vertices[0]*vertices[1]; cone[1] = voffset;
399         cone[2] = voffset+vertices[0]; cone[3] = voffset+vertices[0]*vertices[1]+vertices[0];
400         ierr    = DMPlexSetCone(dm, iface, cone);CHKERRQ(ierr);
401         iface++;
402       }
403     }
404   }
405   ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
406   ierr = DMPlexStratify(dm);CHKERRQ(ierr);
407   /* Build coordinates */
408   ierr = DMSetCoordinateDim(dm, 3);CHKERRQ(ierr);
409   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
410   ierr = PetscSectionSetChart(coordSection, numFaces, numFaces + numVertices);CHKERRQ(ierr);
411   for (v = numFaces; v < numFaces+numVertices; ++v) {
412     ierr = PetscSectionSetDof(coordSection, v, 3);CHKERRQ(ierr);
413   }
414   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
415   ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
416   ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
417   ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
418   ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
419   ierr = VecSetBlockSize(coordinates, 3);CHKERRQ(ierr);
420   ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
421   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
422   for (vz = 0; vz <= faces[2]; ++vz) {
423     for (vy = 0; vy <= faces[1]; ++vy) {
424       for (vx = 0; vx <= faces[0]; ++vx) {
425         coords[((vz*(faces[1]+1)+vy)*(faces[0]+1)+vx)*3+0] = lower[0] + ((upper[0] - lower[0])/faces[0])*vx;
426         coords[((vz*(faces[1]+1)+vy)*(faces[0]+1)+vx)*3+1] = lower[1] + ((upper[1] - lower[1])/faces[1])*vy;
427         coords[((vz*(faces[1]+1)+vy)*(faces[0]+1)+vx)*3+2] = lower[2] + ((upper[2] - lower[2])/faces[2])*vz;
428       }
429     }
430   }
431   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
432   ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
433   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
434   PetscFunctionReturn(0);
435 }
436 
437 static PetscErrorCode DMPlexCreateLineMesh_Internal(MPI_Comm comm,PetscInt segments,PetscReal lower,PetscReal upper,DMBoundaryType bd,DM *dm)
438 {
439   PetscInt       i,fStart,fEnd,numCells = 0,numVerts = 0;
440   PetscInt       numPoints[2],*coneSize,*cones,*coneOrientations;
441   PetscScalar    *vertexCoords;
442   PetscReal      L,maxCell;
443   PetscBool      markerSeparate = PETSC_FALSE;
444   PetscInt       markerLeft  = 1, faceMarkerLeft  = 1;
445   PetscInt       markerRight = 1, faceMarkerRight = 2;
446   PetscBool      wrap = (bd == DM_BOUNDARY_PERIODIC || bd == DM_BOUNDARY_TWIST) ? PETSC_TRUE : PETSC_FALSE;
447   PetscMPIInt    rank;
448   PetscErrorCode ierr;
449 
450   PetscFunctionBegin;
451   PetscValidPointer(dm,4);
452 
453   ierr = DMCreate(comm,dm);CHKERRQ(ierr);
454   ierr = DMSetType(*dm,DMPLEX);CHKERRQ(ierr);
455   ierr = DMSetDimension(*dm,1);CHKERRQ(ierr);
456   ierr = DMCreateLabel(*dm,"marker");CHKERRQ(ierr);
457   ierr = DMCreateLabel(*dm,"Face Sets");CHKERRQ(ierr);
458 
459   ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr);
460   if (!rank) numCells = segments;
461   if (!rank) numVerts = segments + (wrap ? 0 : 1);
462 
463   numPoints[0] = numVerts ; numPoints[1] = numCells;
464   ierr = PetscMalloc4(numCells+numVerts,&coneSize,numCells*2,&cones,numCells+numVerts,&coneOrientations,numVerts,&vertexCoords);CHKERRQ(ierr);
465   ierr = PetscMemzero(coneOrientations,(numCells+numVerts)*sizeof(PetscInt));CHKERRQ(ierr);
466   for (i = 0; i < numCells; ++i) { coneSize[i] = 2; }
467   for (i = 0; i < numVerts; ++i) { coneSize[numCells+i] = 0; }
468   for (i = 0; i < numCells; ++i) { cones[2*i] = numCells + i%numVerts; cones[2*i+1] = numCells + (i+1)%numVerts; }
469   for (i = 0; i < numVerts; ++i) { vertexCoords[i] = lower + (upper-lower)*((PetscReal)i/(PetscReal)numCells); }
470   ierr = DMPlexCreateFromDAG(*dm,1,numPoints,coneSize,cones,coneOrientations,vertexCoords);CHKERRQ(ierr);
471   ierr = PetscFree4(coneSize,cones,coneOrientations,vertexCoords);CHKERRQ(ierr);
472 
473   ierr = PetscOptionsGetBool(((PetscObject)*dm)->options,((PetscObject)*dm)->prefix,"-dm_plex_separate_marker",&markerSeparate,NULL);CHKERRQ(ierr);
474   if (markerSeparate) { markerLeft = faceMarkerLeft; markerRight = faceMarkerRight;}
475   if (!wrap && !rank) {
476     ierr = DMPlexGetHeightStratum(*dm,1,&fStart,&fEnd);CHKERRQ(ierr);
477     ierr = DMSetLabelValue(*dm,"marker",fStart,markerLeft);CHKERRQ(ierr);
478     ierr = DMSetLabelValue(*dm,"marker",fEnd-1,markerRight);CHKERRQ(ierr);
479     ierr = DMSetLabelValue(*dm,"Face Sets",fStart,faceMarkerLeft);CHKERRQ(ierr);
480     ierr = DMSetLabelValue(*dm,"Face Sets",fEnd-1,faceMarkerRight);CHKERRQ(ierr);
481   }
482   if (wrap) {
483     L       = upper - lower;
484     maxCell = (PetscReal)1.1*(L/(PetscReal)PetscMax(1,segments));
485     ierr = DMSetPeriodicity(*dm,PETSC_TRUE,&maxCell,&L,&bd);CHKERRQ(ierr);
486   }
487   PetscFunctionReturn(0);
488 }
489 
490 static PetscErrorCode DMPlexCreateBoxMesh_Simplex_Internal(MPI_Comm comm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate, DM *dm)
491 {
492   DM             boundary;
493   PetscInt       i;
494   PetscErrorCode ierr;
495 
496   PetscFunctionBegin;
497   PetscValidPointer(dm, 4);
498   for (i = 0; i < dim; ++i) if (periodicity[i] != DM_BOUNDARY_NONE) SETERRQ(comm, PETSC_ERR_SUP, "Periodicity is not supported for simplex meshes");
499   ierr = DMCreate(comm, &boundary);CHKERRQ(ierr);
500   PetscValidLogicalCollectiveInt(boundary,dim,2);
501   ierr = DMSetType(boundary, DMPLEX);CHKERRQ(ierr);
502   ierr = DMSetDimension(boundary, dim-1);CHKERRQ(ierr);
503   ierr = DMSetCoordinateDim(boundary, dim);CHKERRQ(ierr);
504   switch (dim) {
505   case 2: ierr = DMPlexCreateSquareBoundary(boundary, lower, upper, faces);CHKERRQ(ierr);break;
506   case 3: ierr = DMPlexCreateCubeBoundary(boundary, lower, upper, faces);CHKERRQ(ierr);break;
507   default: SETERRQ1(comm, PETSC_ERR_SUP, "Dimension not supported: %d", dim);
508   }
509   ierr = DMPlexGenerate(boundary, NULL, interpolate, dm);CHKERRQ(ierr);
510   ierr = DMDestroy(&boundary);CHKERRQ(ierr);
511   PetscFunctionReturn(0);
512 }
513 
514 static PetscErrorCode DMPlexCreateCubeMesh_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[], DMBoundaryType bdX, DMBoundaryType bdY, DMBoundaryType bdZ)
515 {
516   DMLabel        cutLabel = NULL;
517   PetscInt       markerTop      = 1, faceMarkerTop      = 1;
518   PetscInt       markerBottom   = 1, faceMarkerBottom   = 1;
519   PetscInt       markerFront    = 1, faceMarkerFront    = 1;
520   PetscInt       markerBack     = 1, faceMarkerBack     = 1;
521   PetscInt       markerRight    = 1, faceMarkerRight    = 1;
522   PetscInt       markerLeft     = 1, faceMarkerLeft     = 1;
523   PetscInt       dim;
524   PetscBool      markerSeparate = PETSC_FALSE, cutMarker = PETSC_FALSE;
525   PetscMPIInt    rank;
526   PetscErrorCode ierr;
527 
528   PetscFunctionBegin;
529   ierr = DMGetDimension(dm,&dim);CHKERRQ(ierr);
530   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRQ(ierr);
531   ierr = DMCreateLabel(dm,"marker");CHKERRQ(ierr);
532   ierr = DMCreateLabel(dm,"Face Sets");CHKERRQ(ierr);
533   ierr = PetscOptionsGetBool(((PetscObject) dm)->options,((PetscObject) dm)->prefix, "-dm_plex_periodic_cut", &cutMarker, NULL);CHKERRQ(ierr);
534   if (bdX == DM_BOUNDARY_PERIODIC || bdX == DM_BOUNDARY_TWIST ||
535       bdY == DM_BOUNDARY_PERIODIC || bdY == DM_BOUNDARY_TWIST ||
536       bdZ == DM_BOUNDARY_PERIODIC || bdZ == DM_BOUNDARY_TWIST) {
537 
538     if (cutMarker) {ierr = DMCreateLabel(dm, "periodic_cut");CHKERRQ(ierr); ierr = DMGetLabel(dm, "periodic_cut", &cutLabel);CHKERRQ(ierr);}
539   }
540   switch (dim) {
541   case 2:
542     faceMarkerTop    = 3;
543     faceMarkerBottom = 1;
544     faceMarkerRight  = 2;
545     faceMarkerLeft   = 4;
546     break;
547   case 3:
548     faceMarkerBottom = 1;
549     faceMarkerTop    = 2;
550     faceMarkerFront  = 3;
551     faceMarkerBack   = 4;
552     faceMarkerRight  = 5;
553     faceMarkerLeft   = 6;
554     break;
555   default:
556     SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SUP,"Dimension %d not supported",dim);
557     break;
558   }
559   ierr = PetscOptionsGetBool(((PetscObject) dm)->options,((PetscObject) dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL);CHKERRQ(ierr);
560   if (markerSeparate) {
561     markerBottom = faceMarkerBottom;
562     markerTop    = faceMarkerTop;
563     markerFront  = faceMarkerFront;
564     markerBack   = faceMarkerBack;
565     markerRight  = faceMarkerRight;
566     markerLeft   = faceMarkerLeft;
567   }
568   {
569     const PetscInt numXEdges    = !rank ? edges[0] : 0;
570     const PetscInt numYEdges    = !rank ? edges[1] : 0;
571     const PetscInt numZEdges    = !rank ? edges[2] : 0;
572     const PetscInt numXVertices = !rank ? (bdX == DM_BOUNDARY_PERIODIC || bdX == DM_BOUNDARY_TWIST ? edges[0] : edges[0]+1) : 0;
573     const PetscInt numYVertices = !rank ? (bdY == DM_BOUNDARY_PERIODIC || bdY == DM_BOUNDARY_TWIST ? edges[1] : edges[1]+1) : 0;
574     const PetscInt numZVertices = !rank ? (bdZ == DM_BOUNDARY_PERIODIC || bdZ == DM_BOUNDARY_TWIST ? edges[2] : edges[2]+1) : 0;
575     const PetscInt numCells     = numXEdges*numYEdges*numZEdges;
576     const PetscInt numXFaces    = numYEdges*numZEdges;
577     const PetscInt numYFaces    = numXEdges*numZEdges;
578     const PetscInt numZFaces    = numXEdges*numYEdges;
579     const PetscInt numTotXFaces = numXVertices*numXFaces;
580     const PetscInt numTotYFaces = numYVertices*numYFaces;
581     const PetscInt numTotZFaces = numZVertices*numZFaces;
582     const PetscInt numFaces     = numTotXFaces + numTotYFaces + numTotZFaces;
583     const PetscInt numTotXEdges = numXEdges*numYVertices*numZVertices;
584     const PetscInt numTotYEdges = numYEdges*numXVertices*numZVertices;
585     const PetscInt numTotZEdges = numZEdges*numXVertices*numYVertices;
586     const PetscInt numVertices  = numXVertices*numYVertices*numZVertices;
587     const PetscInt numEdges     = numTotXEdges + numTotYEdges + numTotZEdges;
588     const PetscInt firstVertex  = (dim == 2) ? numFaces : numCells;
589     const PetscInt firstXFace   = (dim == 2) ? 0 : numCells + numVertices;
590     const PetscInt firstYFace   = firstXFace + numTotXFaces;
591     const PetscInt firstZFace   = firstYFace + numTotYFaces;
592     const PetscInt firstXEdge   = numCells + numFaces + numVertices;
593     const PetscInt firstYEdge   = firstXEdge + numTotXEdges;
594     const PetscInt firstZEdge   = firstYEdge + numTotYEdges;
595     Vec            coordinates;
596     PetscSection   coordSection;
597     PetscScalar   *coords;
598     PetscInt       coordSize;
599     PetscInt       v, vx, vy, vz;
600     PetscInt       c, f, fx, fy, fz, e, ex, ey, ez;
601 
602     ierr = DMPlexSetChart(dm, 0, numCells+numFaces+numEdges+numVertices);CHKERRQ(ierr);
603     for (c = 0; c < numCells; c++) {
604       ierr = DMPlexSetConeSize(dm, c, 6);CHKERRQ(ierr);
605     }
606     for (f = firstXFace; f < firstXFace+numFaces; ++f) {
607       ierr = DMPlexSetConeSize(dm, f, 4);CHKERRQ(ierr);
608     }
609     for (e = firstXEdge; e < firstXEdge+numEdges; ++e) {
610       ierr = DMPlexSetConeSize(dm, e, 2);CHKERRQ(ierr);
611     }
612     ierr = DMSetUp(dm);CHKERRQ(ierr); /* Allocate space for cones */
613     /* Build cells */
614     for (fz = 0; fz < numZEdges; ++fz) {
615       for (fy = 0; fy < numYEdges; ++fy) {
616         for (fx = 0; fx < numXEdges; ++fx) {
617           PetscInt cell    = (fz*numYEdges + fy)*numXEdges + fx;
618           PetscInt faceB   = firstZFace + (fy*numXEdges+fx)*numZVertices +   fz;
619           PetscInt faceT   = firstZFace + (fy*numXEdges+fx)*numZVertices + ((fz+1)%numZVertices);
620           PetscInt faceF   = firstYFace + (fz*numXEdges+fx)*numYVertices +   fy;
621           PetscInt faceK   = firstYFace + (fz*numXEdges+fx)*numYVertices + ((fy+1)%numYVertices);
622           PetscInt faceL   = firstXFace + (fz*numYEdges+fy)*numXVertices +   fx;
623           PetscInt faceR   = firstXFace + (fz*numYEdges+fy)*numXVertices + ((fx+1)%numXVertices);
624                             /* B,  T,  F,  K,  R,  L */
625           PetscInt ornt[6] = {-4,  0,  0, -1,  0, -4}; /* ??? */
626           PetscInt cone[6];
627 
628           /* no boundary twisting in 3D */
629           cone[0] = faceB; cone[1] = faceT; cone[2] = faceF; cone[3] = faceK; cone[4] = faceR; cone[5] = faceL;
630           ierr    = DMPlexSetCone(dm, cell, cone);CHKERRQ(ierr);
631           ierr    = DMPlexSetConeOrientation(dm, cell, ornt);CHKERRQ(ierr);
632           if (bdX != DM_BOUNDARY_NONE && fx == numXEdges-1 && cutLabel) {ierr = DMLabelSetValue(cutLabel, cell, 2);CHKERRQ(ierr);}
633           if (bdY != DM_BOUNDARY_NONE && fy == numYEdges-1 && cutLabel) {ierr = DMLabelSetValue(cutLabel, cell, 2);CHKERRQ(ierr);}
634           if (bdZ != DM_BOUNDARY_NONE && fz == numZEdges-1 && cutLabel) {ierr = DMLabelSetValue(cutLabel, cell, 2);CHKERRQ(ierr);}
635         }
636       }
637     }
638     /* Build x faces */
639     for (fz = 0; fz < numZEdges; ++fz) {
640       for (fy = 0; fy < numYEdges; ++fy) {
641         for (fx = 0; fx < numXVertices; ++fx) {
642           PetscInt face    = firstXFace + (fz*numYEdges+fy)*numXVertices + fx;
643           PetscInt edgeL   = firstZEdge + (  fy*                 numXVertices+fx)*numZEdges + fz;
644           PetscInt edgeR   = firstZEdge + (((fy+1)%numYVertices)*numXVertices+fx)*numZEdges + fz;
645           PetscInt edgeB   = firstYEdge + (  fz*                 numXVertices+fx)*numYEdges + fy;
646           PetscInt edgeT   = firstYEdge + (((fz+1)%numZVertices)*numXVertices+fx)*numYEdges + fy;
647           PetscInt ornt[4] = {0, 0, -2, -2};
648           PetscInt cone[4];
649 
650           if (dim == 3) {
651             /* markers */
652             if (bdX != DM_BOUNDARY_PERIODIC) {
653               if (fx == numXVertices-1) {
654                 ierr = DMSetLabelValue(dm, "Face Sets", face, faceMarkerRight);CHKERRQ(ierr);
655                 ierr = DMSetLabelValue(dm, "marker", face, markerRight);CHKERRQ(ierr);
656               }
657               else if (fx == 0) {
658                 ierr = DMSetLabelValue(dm, "Face Sets", face, faceMarkerLeft);CHKERRQ(ierr);
659                 ierr = DMSetLabelValue(dm, "marker", face, markerLeft);CHKERRQ(ierr);
660               }
661             }
662           }
663           cone[0] = edgeB; cone[1] = edgeR; cone[2] = edgeT; cone[3] = edgeL;
664           ierr    = DMPlexSetCone(dm, face, cone);CHKERRQ(ierr);
665           ierr    = DMPlexSetConeOrientation(dm, face, ornt);CHKERRQ(ierr);
666         }
667       }
668     }
669     /* Build y faces */
670     for (fz = 0; fz < numZEdges; ++fz) {
671       for (fx = 0; fx < numXEdges; ++fx) {
672         for (fy = 0; fy < numYVertices; ++fy) {
673           PetscInt face    = firstYFace + (fz*numXEdges+fx)*numYVertices + fy;
674           PetscInt edgeL   = firstZEdge + (fy*numXVertices+  fx                 )*numZEdges + fz;
675           PetscInt edgeR   = firstZEdge + (fy*numXVertices+((fx+1)%numXVertices))*numZEdges + fz;
676           PetscInt edgeB   = firstXEdge + (  fz                 *numYVertices+fy)*numXEdges + fx;
677           PetscInt edgeT   = firstXEdge + (((fz+1)%numZVertices)*numYVertices+fy)*numXEdges + fx;
678           PetscInt ornt[4] = {0, 0, -2, -2};
679           PetscInt cone[4];
680 
681           if (dim == 3) {
682             /* markers */
683             if (bdY != DM_BOUNDARY_PERIODIC) {
684               if (fy == numYVertices-1) {
685                 ierr = DMSetLabelValue(dm, "Face Sets", face, faceMarkerBack);CHKERRQ(ierr);
686                 ierr = DMSetLabelValue(dm, "marker", face, markerBack);CHKERRQ(ierr);
687               }
688               else if (fy == 0) {
689                 ierr = DMSetLabelValue(dm, "Face Sets", face, faceMarkerFront);CHKERRQ(ierr);
690                 ierr = DMSetLabelValue(dm, "marker", face, markerFront);CHKERRQ(ierr);
691               }
692             }
693           }
694           cone[0] = edgeB; cone[1] = edgeR; cone[2] = edgeT; cone[3] = edgeL;
695           ierr    = DMPlexSetCone(dm, face, cone);CHKERRQ(ierr);
696           ierr    = DMPlexSetConeOrientation(dm, face, ornt);CHKERRQ(ierr);
697         }
698       }
699     }
700     /* Build z faces */
701     for (fy = 0; fy < numYEdges; ++fy) {
702       for (fx = 0; fx < numXEdges; ++fx) {
703         for (fz = 0; fz < numZVertices; fz++) {
704           PetscInt face    = firstZFace + (fy*numXEdges+fx)*numZVertices + fz;
705           PetscInt edgeL   = firstYEdge + (fz*numXVertices+  fx                 )*numYEdges + fy;
706           PetscInt edgeR   = firstYEdge + (fz*numXVertices+((fx+1)%numXVertices))*numYEdges + fy;
707           PetscInt edgeB   = firstXEdge + (fz*numYVertices+  fy                 )*numXEdges + fx;
708           PetscInt edgeT   = firstXEdge + (fz*numYVertices+((fy+1)%numYVertices))*numXEdges + fx;
709           PetscInt ornt[4] = {0, 0, -2, -2};
710           PetscInt cone[4];
711 
712           if (dim == 2) {
713             if (bdX == DM_BOUNDARY_TWIST && fx == numXEdges-1) {edgeR += numYEdges-1-2*fy; ornt[1] = -2;}
714             if (bdY == DM_BOUNDARY_TWIST && fy == numYEdges-1) {edgeT += numXEdges-1-2*fx; ornt[2] =  0;}
715             if (bdX != DM_BOUNDARY_NONE && fx == numXEdges-1 && cutLabel) {ierr = DMLabelSetValue(cutLabel, face, 2);CHKERRQ(ierr);}
716             if (bdY != DM_BOUNDARY_NONE && fy == numYEdges-1 && cutLabel) {ierr = DMLabelSetValue(cutLabel, face, 2);CHKERRQ(ierr);}
717           } else {
718             /* markers */
719             if (bdZ != DM_BOUNDARY_PERIODIC) {
720               if (fz == numZVertices-1) {
721                 ierr = DMSetLabelValue(dm, "Face Sets", face, faceMarkerTop);CHKERRQ(ierr);
722                 ierr = DMSetLabelValue(dm, "marker", face, markerTop);CHKERRQ(ierr);
723               }
724               else if (fz == 0) {
725                 ierr = DMSetLabelValue(dm, "Face Sets", face, faceMarkerBottom);CHKERRQ(ierr);
726                 ierr = DMSetLabelValue(dm, "marker", face, markerBottom);CHKERRQ(ierr);
727               }
728             }
729           }
730           cone[0] = edgeB; cone[1] = edgeR; cone[2] = edgeT; cone[3] = edgeL;
731           ierr    = DMPlexSetCone(dm, face, cone);CHKERRQ(ierr);
732           ierr    = DMPlexSetConeOrientation(dm, face, ornt);CHKERRQ(ierr);
733         }
734       }
735     }
736     /* Build Z edges*/
737     for (vy = 0; vy < numYVertices; vy++) {
738       for (vx = 0; vx < numXVertices; vx++) {
739         for (ez = 0; ez < numZEdges; ez++) {
740           const PetscInt edge    = firstZEdge  + (vy*numXVertices+vx)*numZEdges + ez;
741           const PetscInt vertexB = firstVertex + (  ez                 *numYVertices+vy)*numXVertices + vx;
742           const PetscInt vertexT = firstVertex + (((ez+1)%numZVertices)*numYVertices+vy)*numXVertices + vx;
743           PetscInt       cone[2];
744 
745           if (dim == 3) {
746             if (bdX != DM_BOUNDARY_PERIODIC) {
747               if (vx == numXVertices-1) {
748                 ierr = DMSetLabelValue(dm, "marker", edge, markerRight);CHKERRQ(ierr);
749               }
750               else if (vx == 0) {
751                 ierr = DMSetLabelValue(dm, "marker", edge, markerLeft);CHKERRQ(ierr);
752               }
753             }
754             if (bdY != DM_BOUNDARY_PERIODIC) {
755               if (vy == numYVertices-1) {
756                 ierr = DMSetLabelValue(dm, "marker", edge, markerBack);CHKERRQ(ierr);
757               }
758               else if (vy == 0) {
759                 ierr = DMSetLabelValue(dm, "marker", edge, markerFront);CHKERRQ(ierr);
760               }
761             }
762           }
763           cone[0] = vertexB; cone[1] = vertexT;
764           ierr = DMPlexSetCone(dm, edge, cone);CHKERRQ(ierr);
765         }
766       }
767     }
768     /* Build Y edges*/
769     for (vz = 0; vz < numZVertices; vz++) {
770       for (vx = 0; vx < numXVertices; vx++) {
771         for (ey = 0; ey < numYEdges; ey++) {
772           const PetscInt nextv   = (dim == 2 && bdY == DM_BOUNDARY_TWIST && ey == numYEdges-1) ? (numXVertices-vx-1) : (vz*numYVertices+((ey+1)%numYVertices))*numXVertices + vx;
773           const PetscInt edge    = firstYEdge  + (vz*numXVertices+vx)*numYEdges + ey;
774           const PetscInt vertexF = firstVertex + (vz*numYVertices+ey)*numXVertices + vx;
775           const PetscInt vertexK = firstVertex + nextv;
776           PetscInt       cone[2];
777 
778           cone[0] = vertexF; cone[1] = vertexK;
779           ierr = DMPlexSetCone(dm, edge, cone);CHKERRQ(ierr);
780           if (dim == 2) {
781             if ((bdX != DM_BOUNDARY_PERIODIC) && (bdX != DM_BOUNDARY_TWIST)) {
782               if (vx == numXVertices-1) {
783                 ierr = DMSetLabelValue(dm, "Face Sets", edge, faceMarkerRight);CHKERRQ(ierr);
784                 ierr = DMSetLabelValue(dm, "marker", edge,    markerRight);CHKERRQ(ierr);
785                 ierr = DMSetLabelValue(dm, "marker", cone[0], markerRight);CHKERRQ(ierr);
786                 if (ey == numYEdges-1) {
787                   ierr = DMSetLabelValue(dm, "marker", cone[1], markerRight);CHKERRQ(ierr);
788                 }
789               } else if (vx == 0) {
790                 ierr = DMSetLabelValue(dm, "Face Sets", edge, faceMarkerLeft);CHKERRQ(ierr);
791                 ierr = DMSetLabelValue(dm, "marker", edge,    markerLeft);CHKERRQ(ierr);
792                 ierr = DMSetLabelValue(dm, "marker", cone[0], markerLeft);CHKERRQ(ierr);
793                 if (ey == numYEdges-1) {
794                   ierr = DMSetLabelValue(dm, "marker", cone[1], markerLeft);CHKERRQ(ierr);
795                 }
796               }
797             } else {
798               if (vx == 0 && cutLabel) {
799                 ierr = DMLabelSetValue(cutLabel, edge,    1);CHKERRQ(ierr);
800                 ierr = DMLabelSetValue(cutLabel, cone[0], 1);CHKERRQ(ierr);
801                 if (ey == numYEdges-1) {
802                   ierr = DMLabelSetValue(cutLabel, cone[1], 1);CHKERRQ(ierr);
803                 }
804               }
805             }
806           } else {
807             if (bdX != DM_BOUNDARY_PERIODIC) {
808               if (vx == numXVertices-1) {
809                 ierr = DMSetLabelValue(dm, "marker", edge, markerRight);CHKERRQ(ierr);
810               } else if (vx == 0) {
811                 ierr = DMSetLabelValue(dm, "marker", edge, markerLeft);CHKERRQ(ierr);
812               }
813             }
814             if (bdZ != DM_BOUNDARY_PERIODIC) {
815               if (vz == numZVertices-1) {
816                 ierr = DMSetLabelValue(dm, "marker", edge, markerTop);CHKERRQ(ierr);
817               } else if (vz == 0) {
818                 ierr = DMSetLabelValue(dm, "marker", edge, markerBottom);CHKERRQ(ierr);
819               }
820             }
821           }
822         }
823       }
824     }
825     /* Build X edges*/
826     for (vz = 0; vz < numZVertices; vz++) {
827       for (vy = 0; vy < numYVertices; vy++) {
828         for (ex = 0; ex < numXEdges; ex++) {
829           const PetscInt nextv   = (dim == 2 && bdX == DM_BOUNDARY_TWIST && ex == numXEdges-1) ? (numYVertices-vy-1)*numXVertices : (vz*numYVertices+vy)*numXVertices + (ex+1)%numXVertices;
830           const PetscInt edge    = firstXEdge  + (vz*numYVertices+vy)*numXEdges + ex;
831           const PetscInt vertexL = firstVertex + (vz*numYVertices+vy)*numXVertices + ex;
832           const PetscInt vertexR = firstVertex + nextv;
833           PetscInt       cone[2];
834 
835           cone[0] = vertexL; cone[1] = vertexR;
836           ierr = DMPlexSetCone(dm, edge, cone);CHKERRQ(ierr);
837           if (dim == 2) {
838             if ((bdY != DM_BOUNDARY_PERIODIC) && (bdY != DM_BOUNDARY_TWIST)) {
839               if (vy == numYVertices-1) {
840                 ierr = DMSetLabelValue(dm, "Face Sets", edge, faceMarkerTop);CHKERRQ(ierr);
841                 ierr = DMSetLabelValue(dm, "marker", edge,    markerTop);CHKERRQ(ierr);
842                 ierr = DMSetLabelValue(dm, "marker", cone[0], markerTop);CHKERRQ(ierr);
843                 if (ex == numXEdges-1) {
844                   ierr = DMSetLabelValue(dm, "marker", cone[1], markerTop);CHKERRQ(ierr);
845                 }
846               } else if (vy == 0) {
847                 ierr = DMSetLabelValue(dm, "Face Sets", edge, faceMarkerBottom);CHKERRQ(ierr);
848                 ierr = DMSetLabelValue(dm, "marker", edge,    markerBottom);CHKERRQ(ierr);
849                 ierr = DMSetLabelValue(dm, "marker", cone[0], markerBottom);CHKERRQ(ierr);
850                 if (ex == numXEdges-1) {
851                   ierr = DMSetLabelValue(dm, "marker", cone[1], markerBottom);CHKERRQ(ierr);
852                 }
853               }
854             } else {
855               if (vy == 0 && cutLabel) {
856                 ierr = DMLabelSetValue(cutLabel, edge,    1);CHKERRQ(ierr);
857                 ierr = DMLabelSetValue(cutLabel, cone[0], 1);CHKERRQ(ierr);
858                 if (ex == numXEdges-1) {
859                   ierr = DMLabelSetValue(cutLabel, cone[1], 1);CHKERRQ(ierr);
860                 }
861               }
862             }
863           } else {
864             if (bdY != DM_BOUNDARY_PERIODIC) {
865               if (vy == numYVertices-1) {
866                 ierr = DMSetLabelValue(dm, "marker", edge, markerBack);CHKERRQ(ierr);
867               }
868               else if (vy == 0) {
869                 ierr = DMSetLabelValue(dm, "marker", edge, markerFront);CHKERRQ(ierr);
870               }
871             }
872             if (bdZ != DM_BOUNDARY_PERIODIC) {
873               if (vz == numZVertices-1) {
874                 ierr = DMSetLabelValue(dm, "marker", edge, markerTop);CHKERRQ(ierr);
875               }
876               else if (vz == 0) {
877                 ierr = DMSetLabelValue(dm, "marker", edge, markerBottom);CHKERRQ(ierr);
878               }
879             }
880           }
881         }
882       }
883     }
884     ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
885     ierr = DMPlexStratify(dm);CHKERRQ(ierr);
886     /* Build coordinates */
887     ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
888     ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
889     ierr = PetscSectionSetFieldComponents(coordSection, 0, dim);CHKERRQ(ierr);
890     ierr = PetscSectionSetChart(coordSection, firstVertex, firstVertex+numVertices);CHKERRQ(ierr);
891     for (v = firstVertex; v < firstVertex+numVertices; ++v) {
892       ierr = PetscSectionSetDof(coordSection, v, dim);CHKERRQ(ierr);
893       ierr = PetscSectionSetFieldDof(coordSection, v, 0, dim);CHKERRQ(ierr);
894     }
895     ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
896     ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
897     ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
898     ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
899     ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
900     ierr = VecSetBlockSize(coordinates, dim);CHKERRQ(ierr);
901     ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
902     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
903     for (vz = 0; vz < numZVertices; ++vz) {
904       for (vy = 0; vy < numYVertices; ++vy) {
905         for (vx = 0; vx < numXVertices; ++vx) {
906           coords[((vz*numYVertices+vy)*numXVertices+vx)*dim+0] = lower[0] + ((upper[0] - lower[0])/numXEdges)*vx;
907           coords[((vz*numYVertices+vy)*numXVertices+vx)*dim+1] = lower[1] + ((upper[1] - lower[1])/numYEdges)*vy;
908           if (dim == 3) {
909             coords[((vz*numYVertices+vy)*numXVertices+vx)*dim+2] = lower[2] + ((upper[2] - lower[2])/numZEdges)*vz;
910           }
911         }
912       }
913     }
914     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
915     ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
916     ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
917   }
918   PetscFunctionReturn(0);
919 }
920 
921 static PetscErrorCode DMPlexCreateBoxMesh_Tensor_Internal(MPI_Comm comm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate, DM *dm)
922 {
923   PetscInt       i;
924   PetscErrorCode ierr;
925 
926   PetscFunctionBegin;
927   PetscValidPointer(dm, 7);
928   ierr = DMCreate(comm, dm);CHKERRQ(ierr);
929   PetscValidLogicalCollectiveInt(*dm,dim,2);
930   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
931   ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
932   switch (dim) {
933   case 2: {ierr = DMPlexCreateCubeMesh_Internal(*dm, lower, upper, faces, periodicity[0], periodicity[1], DM_BOUNDARY_NONE);CHKERRQ(ierr);break;}
934   case 3: {ierr = DMPlexCreateCubeMesh_Internal(*dm, lower, upper, faces, periodicity[0], periodicity[1], periodicity[2]);CHKERRQ(ierr);break;}
935   default: SETERRQ1(comm, PETSC_ERR_SUP, "Dimension not supported: %d", dim);
936   }
937   if (periodicity[0] == DM_BOUNDARY_PERIODIC || periodicity[0] == DM_BOUNDARY_TWIST ||
938       periodicity[1] == DM_BOUNDARY_PERIODIC || periodicity[1] == DM_BOUNDARY_TWIST ||
939       (dim > 2 && (periodicity[2] == DM_BOUNDARY_PERIODIC || periodicity[2] == DM_BOUNDARY_TWIST))) {
940     PetscReal L[3];
941     PetscReal maxCell[3];
942 
943     for (i = 0; i < dim; i++) {
944       L[i]       = upper[i] - lower[i];
945       maxCell[i] = 1.1 * (L[i] / PetscMax(1,faces[i]));
946     }
947     ierr = DMSetPeriodicity(*dm,PETSC_TRUE,maxCell,L,periodicity);CHKERRQ(ierr);
948   }
949   if (!interpolate) {
950     DM udm;
951 
952     ierr = DMPlexUninterpolate(*dm, &udm);CHKERRQ(ierr);
953     ierr = DMPlexCopyCoordinates(*dm, udm);CHKERRQ(ierr);
954     ierr = DMDestroy(dm);CHKERRQ(ierr);
955     *dm  = udm;
956   }
957   PetscFunctionReturn(0);
958 }
959 
960 /*@C
961   DMPlexCreateBoxMesh - Creates a mesh on the tensor product of unit intervals (box) using simplices or tensor cells (hexahedra).
962 
963   Collective on MPI_Comm
964 
965   Input Parameters:
966 + comm        - The communicator for the DM object
967 . dim         - The spatial dimension
968 . simplex     - PETSC_TRUE for simplices, PETSC_FALSE for tensor cells
969 . faces       - Number of faces per dimension, or NULL for (1,) in 1D and (2, 2) in 2D and (1, 1, 1) in 3D
970 . lower       - The lower left corner, or NULL for (0, 0, 0)
971 . upper       - The upper right corner, or NULL for (1, 1, 1)
972 . periodicity - The boundary type for the X,Y,Z direction, or NULL for DM_BOUNDARY_NONE
973 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
974 
975   Output Parameter:
976 . dm  - The DM object
977 
978   Note: Here is the numbering returned for 2 faces in each direction for tensor cells:
979 $ 10---17---11---18----12
980 $  |         |         |
981 $  |         |         |
982 $ 20    2   22    3    24
983 $  |         |         |
984 $  |         |         |
985 $  7---15----8---16----9
986 $  |         |         |
987 $  |         |         |
988 $ 19    0   21    1   23
989 $  |         |         |
990 $  |         |         |
991 $  4---13----5---14----6
992 
993 and for simplicial cells
994 
995 $ 14----8---15----9----16
996 $  |\     5  |\      7 |
997 $  | \       | \       |
998 $ 13   2    14    3    15
999 $  | 4   \   | 6   \   |
1000 $  |       \ |       \ |
1001 $ 11----6---12----7----13
1002 $  |\        |\        |
1003 $  | \    1  | \     3 |
1004 $ 10   0    11    1    12
1005 $  | 0   \   | 2   \   |
1006 $  |       \ |       \ |
1007 $  8----4----9----5----10
1008 
1009   Level: beginner
1010 
1011 .keywords: DM, create
1012 .seealso: DMPlexCreateFromFile(), DMPlexCreateHexCylinderMesh(), DMSetType(), DMCreate()
1013 @*/
1014 PetscErrorCode DMPlexCreateBoxMesh(MPI_Comm comm, PetscInt dim, PetscBool simplex, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate, DM *dm)
1015 {
1016   PetscInt       i;
1017   PetscInt       fac[3] = {0, 0, 0};
1018   PetscReal      low[3] = {0, 0, 0};
1019   PetscReal      upp[3] = {1, 1, 1};
1020   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1021   PetscErrorCode ierr;
1022 
1023   PetscFunctionBegin;
1024   for (i = 0; i < dim; ++i) fac[i] = faces ? faces[i] : (dim == 1 ? 1 : 4-dim);
1025   if (lower) for (i = 0; i < dim; ++i) low[i] = lower[i];
1026   if (upper) for (i = 0; i < dim; ++i) upp[i] = upper[i];
1027   if (periodicity) for (i = 0; i < dim; ++i) bdt[i] = periodicity[i];
1028 
1029   if (dim == 1)      {ierr = DMPlexCreateLineMesh_Internal(comm, fac[0], low[0], upp[0], bdt[0], dm);CHKERRQ(ierr);}
1030   else if (simplex)  {ierr = DMPlexCreateBoxMesh_Simplex_Internal(comm, dim, fac, low, upp, bdt, interpolate, dm);CHKERRQ(ierr);}
1031   else               {ierr = DMPlexCreateBoxMesh_Tensor_Internal(comm, dim, fac, low, upp, bdt, interpolate, dm);CHKERRQ(ierr);}
1032   PetscFunctionReturn(0);
1033 }
1034 
1035 /*@
1036   DMPlexCreateWedgeBoxMesh - Creates a 3-D mesh tesselating the (x,y) plane and extruding in the third direction using wedge cells.
1037 
1038   Collective on MPI_Comm
1039 
1040   Input Parameters:
1041 + comm        - The communicator for the DM object
1042 . faces       - Number of faces per dimension, or NULL for (1, 1, 1)
1043 . lower       - The lower left corner, or NULL for (0, 0, 0)
1044 . upper       - The upper right corner, or NULL for (1, 1, 1)
1045 . periodicity - The boundary type for the X,Y,Z direction, or NULL for DM_BOUNDARY_NONE
1046 . ordExt      - If PETSC_TRUE, orders the extruded cells in the height first. Otherwise, orders the cell on the layers first
1047 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
1048 
1049   Output Parameter:
1050 . dm  - The DM object
1051 
1052   Level: beginner
1053 
1054 .keywords: DM, create
1055 .seealso: DMPlexCreateHexCylinderMesh(), DMPlexCreateWedgeCylinderMesh(), DMPlexExtrude(), DMPlexCreateBoxMesh(), DMSetType(), DMCreate()
1056 @*/
1057 PetscErrorCode DMPlexCreateWedgeBoxMesh(MPI_Comm comm, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool ordExt, PetscBool interpolate, DM *dm)
1058 {
1059   DM             bdm, botdm;
1060   PetscInt       i;
1061   PetscInt       fac[3] = {0, 0, 0};
1062   PetscReal      low[3] = {0, 0, 0};
1063   PetscReal      upp[3] = {1, 1, 1};
1064   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1065   PetscErrorCode ierr;
1066 
1067   PetscFunctionBegin;
1068   for (i = 0; i < 3; ++i) fac[i] = faces ? (faces[i] > 0 ? faces[i] : 1) : 1;
1069   if (lower) for (i = 0; i < 3; ++i) low[i] = lower[i];
1070   if (upper) for (i = 0; i < 3; ++i) upp[i] = upper[i];
1071   if (periodicity) for (i = 0; i < 3; ++i) bdt[i] = periodicity[i];
1072   for (i = 0; i < 3; ++i) if (bdt[i] != DM_BOUNDARY_NONE) SETERRQ(comm, PETSC_ERR_SUP, "Periodicity not yet supported");
1073 
1074   ierr = DMCreate(comm, &bdm);CHKERRQ(ierr);
1075   ierr = DMSetType(bdm, DMPLEX);CHKERRQ(ierr);
1076   ierr = DMSetDimension(bdm, 1);CHKERRQ(ierr);
1077   ierr = DMSetCoordinateDim(bdm, 2);CHKERRQ(ierr);
1078   ierr = DMPlexCreateSquareBoundary(bdm, low, upp, fac);CHKERRQ(ierr);
1079   ierr = DMPlexGenerate(bdm, NULL, PETSC_FALSE, &botdm);CHKERRQ(ierr);
1080   ierr = DMDestroy(&bdm);CHKERRQ(ierr);
1081   ierr = DMPlexExtrude(botdm, fac[2], upp[2] - low[2], ordExt, interpolate, dm);CHKERRQ(ierr);
1082   if (low[2] != 0.0) {
1083     Vec         v;
1084     PetscScalar *x;
1085     PetscInt    cDim, n;
1086 
1087     ierr = DMGetCoordinatesLocal(*dm, &v);CHKERRQ(ierr);
1088     ierr = VecGetBlockSize(v, &cDim);CHKERRQ(ierr);
1089     ierr = VecGetLocalSize(v, &n);CHKERRQ(ierr);
1090     ierr = VecGetArray(v, &x);CHKERRQ(ierr);
1091     x   += cDim;
1092     for (i=0; i<n; i+=cDim) x[i] += low[2];
1093     ierr = VecRestoreArray(v,&x);CHKERRQ(ierr);
1094     ierr = DMSetCoordinatesLocal(*dm, v);CHKERRQ(ierr);
1095   }
1096   ierr = DMDestroy(&botdm);CHKERRQ(ierr);
1097   PetscFunctionReturn(0);
1098 }
1099 
1100 /*@
1101   DMPlexExtrude - Creates a (d+1)-D mesh by extruding a d-D mesh in the normal direction using prismatic cells.
1102 
1103   Collective on idm
1104 
1105   Input Parameters:
1106 + idm - The mesh to be extruted
1107 . layers - The number of layers
1108 . height - The height of the extruded layer
1109 . ordExt - If PETSC_TRUE, orders the extruded cells in the height first. Otherwise, orders the cell on the layers first
1110 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
1111 
1112   Output Parameter:
1113 . dm  - The DM object
1114 
1115   Notes: The object created is an hybrid mesh, the vertex ordering in the cone of the cell is that of the prismatic cells
1116 
1117   Level: advanced
1118 
1119 .keywords: DM, create
1120 .seealso: DMPlexCreateWedgeCylinderMesh(), DMPlexCreateWedgeBoxMesh(), DMPlexSetHybridBounds(), DMSetType(), DMCreate()
1121 @*/
1122 PetscErrorCode DMPlexExtrude(DM idm, PetscInt layers, PetscReal height, PetscBool ordExt, PetscBool interpolate, DM* dm)
1123 {
1124   PetscScalar       *coordsB;
1125   const PetscScalar *coordsA;
1126   PetscReal         *normals = NULL;
1127   Vec               coordinatesA, coordinatesB;
1128   PetscSection      coordSectionA, coordSectionB;
1129   PetscInt          dim, cDim, cDimB, c, l, v, coordSize, *newCone;
1130   PetscInt          cStart, cEnd, vStart, vEnd, cellV, numCells, numVertices;
1131   PetscErrorCode    ierr;
1132 
1133   PetscFunctionBegin;
1134   PetscValidHeaderSpecific(idm, DM_CLASSID, 1);
1135   PetscValidLogicalCollectiveInt(idm, layers, 2);
1136   PetscValidLogicalCollectiveReal(idm, height, 3);
1137   PetscValidLogicalCollectiveBool(idm, interpolate, 4);
1138   ierr = DMGetDimension(idm, &dim);CHKERRQ(ierr);
1139   if (dim < 1 || dim > 3) SETERRQ1(PetscObjectComm((PetscObject)idm), PETSC_ERR_SUP, "Support for dimension %D not coded", dim);
1140 
1141   ierr = DMPlexGetHeightStratum(idm, 0, &cStart, &cEnd);CHKERRQ(ierr);
1142   ierr = DMPlexGetDepthStratum(idm, 0, &vStart, &vEnd);CHKERRQ(ierr);
1143   numCells = (cEnd - cStart)*layers;
1144   numVertices = (vEnd - vStart)*(layers+1);
1145   ierr = DMCreate(PetscObjectComm((PetscObject)idm), dm);CHKERRQ(ierr);
1146   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
1147   ierr = DMSetDimension(*dm, dim+1);CHKERRQ(ierr);
1148   ierr = DMPlexSetChart(*dm, 0, numCells+numVertices);CHKERRQ(ierr);
1149   for (c = cStart, cellV = 0; c < cEnd; ++c) {
1150     PetscInt *closure = NULL;
1151     PetscInt closureSize, numCorners = 0;
1152 
1153     ierr = DMPlexGetTransitiveClosure(idm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1154     for (v = 0; v < closureSize*2; v += 2) if ((closure[v] >= vStart) && (closure[v] < vEnd)) numCorners++;
1155     ierr = DMPlexRestoreTransitiveClosure(idm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1156     for (l = 0; l < layers; ++l) {
1157       ierr = DMPlexSetConeSize(*dm, ordExt ? layers*(c - cStart) + l : l*(cEnd - cStart) + c - cStart, 2*numCorners);CHKERRQ(ierr);
1158     }
1159     cellV = PetscMax(numCorners,cellV);
1160   }
1161   ierr = DMPlexSetHybridBounds(*dm, 0, PETSC_DETERMINE, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
1162   ierr = DMSetUp(*dm);CHKERRQ(ierr);
1163 
1164   ierr = DMGetCoordinateDim(idm, &cDim);CHKERRQ(ierr);
1165   if (dim != cDim) {
1166     ierr = PetscCalloc1(cDim*(vEnd - vStart), &normals);CHKERRQ(ierr);
1167   }
1168   ierr = PetscMalloc1(3*cellV,&newCone);CHKERRQ(ierr);
1169   for (c = cStart; c < cEnd; ++c) {
1170     PetscInt *closure = NULL;
1171     PetscInt closureSize, numCorners = 0, l;
1172     PetscReal normal[3] = {0, 0, 0};
1173 
1174     if (normals) {
1175       ierr = DMPlexComputeCellGeometryFVM(idm, c, NULL, NULL, normal);CHKERRQ(ierr);
1176     }
1177     ierr = DMPlexGetTransitiveClosure(idm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1178     for (v = 0; v < closureSize*2; v += 2) {
1179       if ((closure[v] >= vStart) && (closure[v] < vEnd)) {
1180         PetscInt d;
1181 
1182         newCone[numCorners++] = closure[v] - vStart;
1183         if (normals) { for (d = 0; d < cDim; ++d) normals[cDim*(closure[v]-vStart)+d] += normal[d]; }
1184       }
1185     }
1186     switch (numCorners) {
1187     case 4: /* do nothing */
1188     case 2: /* do nothing */
1189       break;
1190     case 3: /* from counter-clockwise to wedge ordering */
1191       l = newCone[1];
1192       newCone[1] = newCone[2];
1193       newCone[2] = l;
1194       break;
1195     default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_SUP, "Unsupported number of corners: %D", numCorners);
1196     }
1197     ierr = DMPlexRestoreTransitiveClosure(idm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1198     for (l = 0; l < layers; ++l) {
1199       PetscInt i;
1200 
1201       for (i = 0; i < numCorners; ++i) {
1202         newCone[  numCorners + i] = ordExt ? (layers+1)*newCone[i] + l     + numCells :     l*(vEnd - vStart) + newCone[i] + numCells;
1203         newCone[2*numCorners + i] = ordExt ? (layers+1)*newCone[i] + l + 1 + numCells : (l+1)*(vEnd - vStart) + newCone[i] + numCells;
1204       }
1205       ierr = DMPlexSetCone(*dm, ordExt ? layers*(c - cStart) + l : l*(cEnd - cStart) + c - cStart, newCone + numCorners);CHKERRQ(ierr);
1206     }
1207   }
1208   ierr = DMPlexSymmetrize(*dm);CHKERRQ(ierr);
1209   ierr = DMPlexStratify(*dm);CHKERRQ(ierr);
1210   ierr = PetscFree(newCone);CHKERRQ(ierr);
1211 
1212   cDimB = cDim == dim ? cDim+1 : cDim;
1213   ierr = DMGetCoordinateSection(*dm, &coordSectionB);CHKERRQ(ierr);
1214   ierr = PetscSectionSetNumFields(coordSectionB, 1);CHKERRQ(ierr);
1215   ierr = PetscSectionSetFieldComponents(coordSectionB, 0, cDimB);CHKERRQ(ierr);
1216   ierr = PetscSectionSetChart(coordSectionB, numCells, numCells+numVertices);CHKERRQ(ierr);
1217   for (v = numCells; v < numCells+numVertices; ++v) {
1218     ierr = PetscSectionSetDof(coordSectionB, v, cDimB);CHKERRQ(ierr);
1219     ierr = PetscSectionSetFieldDof(coordSectionB, v, 0, cDimB);CHKERRQ(ierr);
1220   }
1221   ierr = PetscSectionSetUp(coordSectionB);CHKERRQ(ierr);
1222   ierr = PetscSectionGetStorageSize(coordSectionB, &coordSize);CHKERRQ(ierr);
1223   ierr = VecCreate(PETSC_COMM_SELF, &coordinatesB);CHKERRQ(ierr);
1224   ierr = PetscObjectSetName((PetscObject) coordinatesB, "coordinates");CHKERRQ(ierr);
1225   ierr = VecSetSizes(coordinatesB, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
1226   ierr = VecSetBlockSize(coordinatesB, cDimB);CHKERRQ(ierr);
1227   ierr = VecSetType(coordinatesB,VECSTANDARD);CHKERRQ(ierr);
1228 
1229   ierr = DMGetCoordinateSection(idm, &coordSectionA);CHKERRQ(ierr);
1230   ierr = DMGetCoordinatesLocal(idm, &coordinatesA);CHKERRQ(ierr);
1231   ierr = VecGetArray(coordinatesB, &coordsB);CHKERRQ(ierr);
1232   ierr = VecGetArrayRead(coordinatesA, &coordsA);CHKERRQ(ierr);
1233   for (v = vStart; v < vEnd; ++v) {
1234     const PetscScalar *cptr;
1235     PetscReal         ones2[2] = { 0., 1.}, ones3[3] = { 0., 0., 1.};
1236     PetscReal         *normal, norm, h = height/layers;
1237     PetscInt          offA, d, cDimA = cDim;
1238 
1239     normal = normals ? normals + cDimB*(v - vStart) : (cDim > 1 ? ones3 : ones2);
1240     if (normals) {
1241       for (d = 0, norm = 0.0; d < cDimB; ++d) norm += normal[d]*normal[d];
1242       for (d = 0; d < cDimB; ++d) normal[d] *= 1./PetscSqrtReal(norm);
1243     }
1244 
1245     ierr = PetscSectionGetOffset(coordSectionA, v, &offA);CHKERRQ(ierr);
1246     cptr = coordsA + offA;
1247     for (l = 0; l < layers+1; ++l) {
1248       PetscInt offB, d, newV;
1249 
1250       newV = ordExt ? (layers+1)*(v -vStart) + l + numCells : (vEnd -vStart)*l + (v -vStart) + numCells;
1251       ierr = PetscSectionGetOffset(coordSectionB, newV, &offB);CHKERRQ(ierr);
1252       for (d = 0; d < cDimA; ++d) { coordsB[offB+d]  = cptr[d]; }
1253       for (d = 0; d < cDimB; ++d) { coordsB[offB+d] += l ? normal[d]*h : 0.0; }
1254       cptr    = coordsB + offB;
1255       cDimA   = cDimB;
1256     }
1257   }
1258   ierr = VecRestoreArrayRead(coordinatesA, &coordsA);CHKERRQ(ierr);
1259   ierr = VecRestoreArray(coordinatesB, &coordsB);CHKERRQ(ierr);
1260   ierr = DMSetCoordinatesLocal(*dm, coordinatesB);CHKERRQ(ierr);
1261   ierr = VecDestroy(&coordinatesB);CHKERRQ(ierr);
1262   ierr = PetscFree(normals);CHKERRQ(ierr);
1263   if (interpolate) {
1264     DM idm;
1265 
1266     ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
1267     ierr = DMPlexCopyCoordinates(*dm, idm);CHKERRQ(ierr);
1268     ierr = DMDestroy(dm);CHKERRQ(ierr);
1269     *dm  = idm;
1270   }
1271   PetscFunctionReturn(0);
1272 }
1273 
1274 /*@C
1275   DMPlexSetOptionsPrefix - Sets the prefix used for searching for all DM options in the database.
1276 
1277   Logically Collective on DM
1278 
1279   Input Parameters:
1280 + dm - the DM context
1281 - prefix - the prefix to prepend to all option names
1282 
1283   Notes:
1284   A hyphen (-) must NOT be given at the beginning of the prefix name.
1285   The first character of all runtime options is AUTOMATICALLY the hyphen.
1286 
1287   Level: advanced
1288 
1289 .seealso: SNESSetFromOptions()
1290 @*/
1291 PetscErrorCode DMPlexSetOptionsPrefix(DM dm, const char prefix[])
1292 {
1293   DM_Plex       *mesh = (DM_Plex *) dm->data;
1294   PetscErrorCode ierr;
1295 
1296   PetscFunctionBegin;
1297   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1298   ierr = PetscObjectSetOptionsPrefix((PetscObject) dm, prefix);CHKERRQ(ierr);
1299   ierr = PetscObjectSetOptionsPrefix((PetscObject) mesh->partitioner, prefix);CHKERRQ(ierr);
1300   PetscFunctionReturn(0);
1301 }
1302 
1303 /*@
1304   DMPlexCreateHexCylinderMesh - Creates a mesh on the tensor product of the unit interval with the circle (cylinder) using hexahedra.
1305 
1306   Collective on MPI_Comm
1307 
1308   Input Parameters:
1309 + comm      - The communicator for the DM object
1310 . numRefine - The number of regular refinements to the basic 5 cell structure
1311 - periodicZ - The boundary type for the Z direction
1312 
1313   Output Parameter:
1314 . dm  - The DM object
1315 
1316   Note: Here is the output numbering looking from the bottom of the cylinder:
1317 $       17-----14
1318 $        |     |
1319 $        |  2  |
1320 $        |     |
1321 $ 17-----8-----7-----14
1322 $  |     |     |     |
1323 $  |  3  |  0  |  1  |
1324 $  |     |     |     |
1325 $ 19-----5-----6-----13
1326 $        |     |
1327 $        |  4  |
1328 $        |     |
1329 $       19-----13
1330 $
1331 $ and up through the top
1332 $
1333 $       18-----16
1334 $        |     |
1335 $        |  2  |
1336 $        |     |
1337 $ 18----10----11-----16
1338 $  |     |     |     |
1339 $  |  3  |  0  |  1  |
1340 $  |     |     |     |
1341 $ 20-----9----12-----15
1342 $        |     |
1343 $        |  4  |
1344 $        |     |
1345 $       20-----15
1346 
1347   Level: beginner
1348 
1349 .keywords: DM, create
1350 .seealso: DMPlexCreateBoxMesh(), DMSetType(), DMCreate()
1351 @*/
1352 PetscErrorCode DMPlexCreateHexCylinderMesh(MPI_Comm comm, PetscInt numRefine, DMBoundaryType periodicZ, DM *dm)
1353 {
1354   const PetscInt dim = 3;
1355   PetscInt       numCells, numVertices, r;
1356   PetscMPIInt    rank;
1357   PetscErrorCode ierr;
1358 
1359   PetscFunctionBegin;
1360   PetscValidPointer(dm, 4);
1361   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
1362   if (numRefine < 0) SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Number of refinements %D cannot be negative", numRefine);
1363   ierr = DMCreate(comm, dm);CHKERRQ(ierr);
1364   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
1365   ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
1366   /* Create topology */
1367   {
1368     PetscInt cone[8], c;
1369 
1370     numCells    = !rank ?  5 : 0;
1371     numVertices = !rank ? 16 : 0;
1372     if (periodicZ == DM_BOUNDARY_PERIODIC) {
1373       numCells   *= 3;
1374       numVertices = !rank ? 24 : 0;
1375     }
1376     ierr = DMPlexSetChart(*dm, 0, numCells+numVertices);CHKERRQ(ierr);
1377     for (c = 0; c < numCells; c++) {ierr = DMPlexSetConeSize(*dm, c, 8);CHKERRQ(ierr);}
1378     ierr = DMSetUp(*dm);CHKERRQ(ierr);
1379     if (!rank) {
1380       if (periodicZ == DM_BOUNDARY_PERIODIC) {
1381         cone[0] = 15; cone[1] = 18; cone[2] = 17; cone[3] = 16;
1382         cone[4] = 31; cone[5] = 32; cone[6] = 33; cone[7] = 34;
1383         ierr = DMPlexSetCone(*dm, 0, cone);CHKERRQ(ierr);
1384         cone[0] = 16; cone[1] = 17; cone[2] = 24; cone[3] = 23;
1385         cone[4] = 32; cone[5] = 36; cone[6] = 37; cone[7] = 33; /* 22 25 26 21 */
1386         ierr = DMPlexSetCone(*dm, 1, cone);CHKERRQ(ierr);
1387         cone[0] = 18; cone[1] = 27; cone[2] = 24; cone[3] = 17;
1388         cone[4] = 34; cone[5] = 33; cone[6] = 37; cone[7] = 38;
1389         ierr = DMPlexSetCone(*dm, 2, cone);CHKERRQ(ierr);
1390         cone[0] = 29; cone[1] = 27; cone[2] = 18; cone[3] = 15;
1391         cone[4] = 35; cone[5] = 31; cone[6] = 34; cone[7] = 38;
1392         ierr = DMPlexSetCone(*dm, 3, cone);CHKERRQ(ierr);
1393         cone[0] = 29; cone[1] = 15; cone[2] = 16; cone[3] = 23;
1394         cone[4] = 35; cone[5] = 36; cone[6] = 32; cone[7] = 31;
1395         ierr = DMPlexSetCone(*dm, 4, cone);CHKERRQ(ierr);
1396 
1397         cone[0] = 31; cone[1] = 34; cone[2] = 33; cone[3] = 32;
1398         cone[4] = 19; cone[5] = 22; cone[6] = 21; cone[7] = 20;
1399         ierr = DMPlexSetCone(*dm, 5, cone);CHKERRQ(ierr);
1400         cone[0] = 32; cone[1] = 33; cone[2] = 37; cone[3] = 36;
1401         cone[4] = 22; cone[5] = 25; cone[6] = 26; cone[7] = 21;
1402         ierr = DMPlexSetCone(*dm, 6, cone);CHKERRQ(ierr);
1403         cone[0] = 34; cone[1] = 38; cone[2] = 37; cone[3] = 33;
1404         cone[4] = 20; cone[5] = 21; cone[6] = 26; cone[7] = 28;
1405         ierr = DMPlexSetCone(*dm, 7, cone);CHKERRQ(ierr);
1406         cone[0] = 35; cone[1] = 38; cone[2] = 34; cone[3] = 31;
1407         cone[4] = 30; cone[5] = 19; cone[6] = 20; cone[7] = 28;
1408         ierr = DMPlexSetCone(*dm, 8, cone);CHKERRQ(ierr);
1409         cone[0] = 35; cone[1] = 31; cone[2] = 32; cone[3] = 36;
1410         cone[4] = 30; cone[5] = 25; cone[6] = 22; cone[7] = 19;
1411         ierr = DMPlexSetCone(*dm, 9, cone);CHKERRQ(ierr);
1412 
1413         cone[0] = 19; cone[1] = 20; cone[2] = 21; cone[3] = 22;
1414         cone[4] = 15; cone[5] = 16; cone[6] = 17; cone[7] = 18;
1415         ierr = DMPlexSetCone(*dm, 10, cone);CHKERRQ(ierr);
1416         cone[0] = 22; cone[1] = 21; cone[2] = 26; cone[3] = 25;
1417         cone[4] = 16; cone[5] = 23; cone[6] = 24; cone[7] = 17;
1418         ierr = DMPlexSetCone(*dm, 11, cone);CHKERRQ(ierr);
1419         cone[0] = 20; cone[1] = 28; cone[2] = 26; cone[3] = 21;
1420         cone[4] = 18; cone[5] = 17; cone[6] = 24; cone[7] = 27;
1421         ierr = DMPlexSetCone(*dm, 12, cone);CHKERRQ(ierr);
1422         cone[0] = 30; cone[1] = 28; cone[2] = 20; cone[3] = 19;
1423         cone[4] = 29; cone[5] = 15; cone[6] = 18; cone[7] = 27;
1424         ierr = DMPlexSetCone(*dm, 13, cone);CHKERRQ(ierr);
1425         cone[0] = 30; cone[1] = 19; cone[2] = 22; cone[3] = 25;
1426         cone[4] = 29; cone[5] = 23; cone[6] = 16; cone[7] = 15;
1427         ierr = DMPlexSetCone(*dm, 14, cone);CHKERRQ(ierr);
1428       } else {
1429         cone[0] =  5; cone[1] =  8; cone[2] =  7; cone[3] =  6;
1430         cone[4] =  9; cone[5] = 12; cone[6] = 11; cone[7] = 10;
1431         ierr = DMPlexSetCone(*dm, 0, cone);CHKERRQ(ierr);
1432         cone[0] =  6; cone[1] =  7; cone[2] = 14; cone[3] = 13;
1433         cone[4] = 12; cone[5] = 15; cone[6] = 16; cone[7] = 11;
1434         ierr = DMPlexSetCone(*dm, 1, cone);CHKERRQ(ierr);
1435         cone[0] =  8; cone[1] = 17; cone[2] = 14; cone[3] =  7;
1436         cone[4] = 10; cone[5] = 11; cone[6] = 16; cone[7] = 18;
1437         ierr = DMPlexSetCone(*dm, 2, cone);CHKERRQ(ierr);
1438         cone[0] = 19; cone[1] = 17; cone[2] =  8; cone[3] =  5;
1439         cone[4] = 20; cone[5] =  9; cone[6] = 10; cone[7] = 18;
1440         ierr = DMPlexSetCone(*dm, 3, cone);CHKERRQ(ierr);
1441         cone[0] = 19; cone[1] =  5; cone[2] =  6; cone[3] = 13;
1442         cone[4] = 20; cone[5] = 15; cone[6] = 12; cone[7] =  9;
1443         ierr = DMPlexSetCone(*dm, 4, cone);CHKERRQ(ierr);
1444       }
1445     }
1446     ierr = DMPlexSymmetrize(*dm);CHKERRQ(ierr);
1447     ierr = DMPlexStratify(*dm);CHKERRQ(ierr);
1448   }
1449   /* Interpolate */
1450   {
1451     DM idm;
1452 
1453     ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
1454     ierr = DMDestroy(dm);CHKERRQ(ierr);
1455     *dm  = idm;
1456   }
1457   /* Create cube geometry */
1458   {
1459     Vec             coordinates;
1460     PetscSection    coordSection;
1461     PetscScalar    *coords;
1462     PetscInt        coordSize, v;
1463     const PetscReal dis = 1.0/PetscSqrtReal(2.0);
1464     const PetscReal ds2 = dis/2.0;
1465 
1466     /* Build coordinates */
1467     ierr = DMGetCoordinateSection(*dm, &coordSection);CHKERRQ(ierr);
1468     ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
1469     ierr = PetscSectionSetFieldComponents(coordSection, 0, dim);CHKERRQ(ierr);
1470     ierr = PetscSectionSetChart(coordSection, numCells, numCells+numVertices);CHKERRQ(ierr);
1471     for (v = numCells; v < numCells+numVertices; ++v) {
1472       ierr = PetscSectionSetDof(coordSection, v, dim);CHKERRQ(ierr);
1473       ierr = PetscSectionSetFieldDof(coordSection, v, 0, dim);CHKERRQ(ierr);
1474     }
1475     ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
1476     ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
1477     ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
1478     ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
1479     ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
1480     ierr = VecSetBlockSize(coordinates, dim);CHKERRQ(ierr);
1481     ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
1482     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
1483     if (!rank) {
1484       coords[0*dim+0] = -ds2; coords[0*dim+1] = -ds2; coords[0*dim+2] = 0.0;
1485       coords[1*dim+0] =  ds2; coords[1*dim+1] = -ds2; coords[1*dim+2] = 0.0;
1486       coords[2*dim+0] =  ds2; coords[2*dim+1] =  ds2; coords[2*dim+2] = 0.0;
1487       coords[3*dim+0] = -ds2; coords[3*dim+1] =  ds2; coords[3*dim+2] = 0.0;
1488       coords[4*dim+0] = -ds2; coords[4*dim+1] = -ds2; coords[4*dim+2] = 1.0;
1489       coords[5*dim+0] = -ds2; coords[5*dim+1] =  ds2; coords[5*dim+2] = 1.0;
1490       coords[6*dim+0] =  ds2; coords[6*dim+1] =  ds2; coords[6*dim+2] = 1.0;
1491       coords[7*dim+0] =  ds2; coords[7*dim+1] = -ds2; coords[7*dim+2] = 1.0;
1492       coords[ 8*dim+0] =  dis; coords[ 8*dim+1] = -dis; coords[ 8*dim+2] = 0.0;
1493       coords[ 9*dim+0] =  dis; coords[ 9*dim+1] =  dis; coords[ 9*dim+2] = 0.0;
1494       coords[10*dim+0] =  dis; coords[10*dim+1] = -dis; coords[10*dim+2] = 1.0;
1495       coords[11*dim+0] =  dis; coords[11*dim+1] =  dis; coords[11*dim+2] = 1.0;
1496       coords[12*dim+0] = -dis; coords[12*dim+1] =  dis; coords[12*dim+2] = 0.0;
1497       coords[13*dim+0] = -dis; coords[13*dim+1] =  dis; coords[13*dim+2] = 1.0;
1498       coords[14*dim+0] = -dis; coords[14*dim+1] = -dis; coords[14*dim+2] = 0.0;
1499       coords[15*dim+0] = -dis; coords[15*dim+1] = -dis; coords[15*dim+2] = 1.0;
1500       if (periodicZ == DM_BOUNDARY_PERIODIC) {
1501         /* 15 31 19 */ coords[16*dim+0] = -ds2; coords[16*dim+1] = -ds2; coords[16*dim+2] = 0.5;
1502         /* 16 32 22 */ coords[17*dim+0] =  ds2; coords[17*dim+1] = -ds2; coords[17*dim+2] = 0.5;
1503         /* 17 33 21 */ coords[18*dim+0] =  ds2; coords[18*dim+1] =  ds2; coords[18*dim+2] = 0.5;
1504         /* 18 34 20 */ coords[19*dim+0] = -ds2; coords[19*dim+1] =  ds2; coords[19*dim+2] = 0.5;
1505         /* 29 35 30 */ coords[20*dim+0] = -dis; coords[20*dim+1] = -dis; coords[20*dim+2] = 0.5;
1506         /* 23 36 25 */ coords[21*dim+0] =  dis; coords[21*dim+1] = -dis; coords[21*dim+2] = 0.5;
1507         /* 24 37 26 */ coords[22*dim+0] =  dis; coords[22*dim+1] =  dis; coords[22*dim+2] = 0.5;
1508         /* 27 38 28 */ coords[23*dim+0] = -dis; coords[23*dim+1] =  dis; coords[23*dim+2] = 0.5;
1509       }
1510     }
1511     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
1512     ierr = DMSetCoordinatesLocal(*dm, coordinates);CHKERRQ(ierr);
1513     ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
1514   }
1515   /* Create periodicity */
1516   if (periodicZ == DM_BOUNDARY_PERIODIC || periodicZ == DM_BOUNDARY_TWIST) {
1517     PetscReal      L[3];
1518     PetscReal      maxCell[3];
1519     DMBoundaryType bdType[3];
1520     PetscReal      lower[3] = {0.0, 0.0, 0.0};
1521     PetscReal      upper[3] = {1.0, 1.0, 1.5};
1522     PetscInt       i, numZCells = 3;
1523 
1524     bdType[0] = DM_BOUNDARY_NONE;
1525     bdType[1] = DM_BOUNDARY_NONE;
1526     bdType[2] = periodicZ;
1527     for (i = 0; i < dim; i++) {
1528       L[i]       = upper[i] - lower[i];
1529       maxCell[i] = 1.1 * (L[i] / numZCells);
1530     }
1531     ierr = DMSetPeriodicity(*dm, PETSC_TRUE, maxCell, L, bdType);CHKERRQ(ierr);
1532   }
1533   /* Refine topology */
1534   for (r = 0; r < numRefine; ++r) {
1535     DM rdm = NULL;
1536 
1537     ierr = DMRefine(*dm, comm, &rdm);CHKERRQ(ierr);
1538     ierr = DMDestroy(dm);CHKERRQ(ierr);
1539     *dm  = rdm;
1540   }
1541   /* Remap geometry to cylinder
1542        Interior square: Linear interpolation is correct
1543        The other cells all have vertices on rays from the origin. We want to uniformly expand the spacing
1544        such that the last vertex is on the unit circle. So the closest and farthest vertices are at distance
1545 
1546          phi     = arctan(y/x)
1547          d_close = sqrt(1/8 + 1/4 sin^2(phi))
1548          d_far   = sqrt(1/2 + sin^2(phi))
1549 
1550        so we remap them using
1551 
1552          x_new = x_close + (x - x_close) (1 - d_close) / (d_far - d_close)
1553          y_new = y_close + (y - y_close) (1 - d_close) / (d_far - d_close)
1554 
1555        If pi/4 < phi < 3pi/4 or -3pi/4 < phi < -pi/4, then we switch x and y.
1556   */
1557   {
1558     Vec           coordinates;
1559     PetscSection  coordSection;
1560     PetscScalar  *coords;
1561     PetscInt      vStart, vEnd, v;
1562     const PetscReal dis = 1.0/PetscSqrtReal(2.0);
1563     const PetscReal ds2 = 0.5*dis;
1564 
1565     ierr = DMPlexGetDepthStratum(*dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
1566     ierr = DMGetCoordinateSection(*dm, &coordSection);CHKERRQ(ierr);
1567     ierr = DMGetCoordinatesLocal(*dm, &coordinates);CHKERRQ(ierr);
1568     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
1569     for (v = vStart; v < vEnd; ++v) {
1570       PetscReal phi, sinp, cosp, dc, df, x, y, xc, yc;
1571       PetscInt  off;
1572 
1573       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
1574       if ((PetscAbsScalar(coords[off+0]) <= ds2) && (PetscAbsScalar(coords[off+1]) <= ds2)) continue;
1575       x    = PetscRealPart(coords[off]);
1576       y    = PetscRealPart(coords[off+1]);
1577       phi  = PetscAtan2Real(y, x);
1578       sinp = PetscSinReal(phi);
1579       cosp = PetscCosReal(phi);
1580       if ((PetscAbsReal(phi) > PETSC_PI/4.0) && (PetscAbsReal(phi) < 3.0*PETSC_PI/4.0)) {
1581         dc = PetscAbsReal(ds2/sinp);
1582         df = PetscAbsReal(dis/sinp);
1583         xc = ds2*x/PetscAbsReal(y);
1584         yc = ds2*PetscSignReal(y);
1585       } else {
1586         dc = PetscAbsReal(ds2/cosp);
1587         df = PetscAbsReal(dis/cosp);
1588         xc = ds2*PetscSignReal(x);
1589         yc = ds2*y/PetscAbsReal(x);
1590       }
1591       coords[off+0] = xc + (coords[off+0] - xc)*(1.0 - dc)/(df - dc);
1592       coords[off+1] = yc + (coords[off+1] - yc)*(1.0 - dc)/(df - dc);
1593     }
1594     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
1595     if (periodicZ == DM_BOUNDARY_PERIODIC || periodicZ == DM_BOUNDARY_TWIST) {
1596       ierr = DMLocalizeCoordinates(*dm);CHKERRQ(ierr);
1597     }
1598   }
1599   PetscFunctionReturn(0);
1600 }
1601 
1602 /*@
1603   DMPlexCreateWedgeCylinderMesh - Creates a mesh on the tensor product of the unit interval with the circle (cylinder) using wedges.
1604 
1605   Collective on MPI_Comm
1606 
1607   Input Parameters:
1608 + comm - The communicator for the DM object
1609 . n    - The number of wedges around the origin
1610 - interpolate - Create edges and faces
1611 
1612   Output Parameter:
1613 . dm  - The DM object
1614 
1615   Level: beginner
1616 
1617 .keywords: DM, create
1618 .seealso: DMPlexCreateHexCylinderMesh(), DMPlexCreateBoxMesh(), DMSetType(), DMCreate()
1619 @*/
1620 PetscErrorCode DMPlexCreateWedgeCylinderMesh(MPI_Comm comm, PetscInt n, PetscBool interpolate, DM *dm)
1621 {
1622   const PetscInt dim = 3;
1623   PetscInt       numCells, numVertices;
1624   PetscMPIInt    rank;
1625   PetscErrorCode ierr;
1626 
1627   PetscFunctionBegin;
1628   PetscValidPointer(dm, 3);
1629   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
1630   if (n < 0) SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Number of wedges %D cannot be negative", n);
1631   ierr = DMCreate(comm, dm);CHKERRQ(ierr);
1632   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
1633   ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
1634   /* Create topology */
1635   {
1636     PetscInt cone[6], c;
1637 
1638     numCells    = !rank ?        n : 0;
1639     numVertices = !rank ?  2*(n+1) : 0;
1640     ierr = DMPlexSetChart(*dm, 0, numCells+numVertices);CHKERRQ(ierr);
1641     ierr = DMPlexSetHybridBounds(*dm, 0, PETSC_DETERMINE, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
1642     for (c = 0; c < numCells; c++) {ierr = DMPlexSetConeSize(*dm, c, 6);CHKERRQ(ierr);}
1643     ierr = DMSetUp(*dm);CHKERRQ(ierr);
1644     for (c = 0; c < numCells; c++) {
1645       cone[0] =  c+n*1; cone[1] = (c+1)%n+n*1; cone[2] = 0+3*n;
1646       cone[3] =  c+n*2; cone[4] = (c+1)%n+n*2; cone[5] = 1+3*n;
1647       ierr = DMPlexSetCone(*dm, c, cone);CHKERRQ(ierr);
1648     }
1649     ierr = DMPlexSymmetrize(*dm);CHKERRQ(ierr);
1650     ierr = DMPlexStratify(*dm);CHKERRQ(ierr);
1651   }
1652   /* Interpolate */
1653   if (interpolate) {
1654     DM idm;
1655 
1656     ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
1657     ierr = DMDestroy(dm);CHKERRQ(ierr);
1658     *dm  = idm;
1659   }
1660   /* Create cylinder geometry */
1661   {
1662     Vec          coordinates;
1663     PetscSection coordSection;
1664     PetscScalar *coords;
1665     PetscInt     coordSize, v, c;
1666 
1667     /* Build coordinates */
1668     ierr = DMGetCoordinateSection(*dm, &coordSection);CHKERRQ(ierr);
1669     ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
1670     ierr = PetscSectionSetFieldComponents(coordSection, 0, dim);CHKERRQ(ierr);
1671     ierr = PetscSectionSetChart(coordSection, numCells, numCells+numVertices);CHKERRQ(ierr);
1672     for (v = numCells; v < numCells+numVertices; ++v) {
1673       ierr = PetscSectionSetDof(coordSection, v, dim);CHKERRQ(ierr);
1674       ierr = PetscSectionSetFieldDof(coordSection, v, 0, dim);CHKERRQ(ierr);
1675     }
1676     ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
1677     ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
1678     ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
1679     ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
1680     ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
1681     ierr = VecSetBlockSize(coordinates, dim);CHKERRQ(ierr);
1682     ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
1683     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
1684     for (c = 0; c < numCells; c++) {
1685       coords[(c+0*n)*dim+0] = PetscCosReal(2.0*c*PETSC_PI/n); coords[(c+0*n)*dim+1] = PetscSinReal(2.0*c*PETSC_PI/n); coords[(c+0*n)*dim+2] = 1.0;
1686       coords[(c+1*n)*dim+0] = PetscCosReal(2.0*c*PETSC_PI/n); coords[(c+1*n)*dim+1] = PetscSinReal(2.0*c*PETSC_PI/n); coords[(c+1*n)*dim+2] = 0.0;
1687     }
1688     if (!rank) {
1689       coords[(2*n+0)*dim+0] = 0.0; coords[(2*n+0)*dim+1] = 0.0; coords[(2*n+0)*dim+2] = 1.0;
1690       coords[(2*n+1)*dim+0] = 0.0; coords[(2*n+1)*dim+1] = 0.0; coords[(2*n+1)*dim+2] = 0.0;
1691     }
1692     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
1693     ierr = DMSetCoordinatesLocal(*dm, coordinates);CHKERRQ(ierr);
1694     ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
1695   }
1696   PetscFunctionReturn(0);
1697 }
1698 
1699 PETSC_STATIC_INLINE PetscReal DiffNormReal(PetscInt dim, const PetscReal x[], const PetscReal y[])
1700 {
1701   PetscReal prod = 0.0;
1702   PetscInt  i;
1703   for (i = 0; i < dim; ++i) prod += PetscSqr(x[i] - y[i]);
1704   return PetscSqrtReal(prod);
1705 }
1706 PETSC_STATIC_INLINE PetscReal DotReal(PetscInt dim, const PetscReal x[], const PetscReal y[])
1707 {
1708   PetscReal prod = 0.0;
1709   PetscInt  i;
1710   for (i = 0; i < dim; ++i) prod += x[i]*y[i];
1711   return prod;
1712 }
1713 
1714 /*@
1715   DMPlexCreateSphereMesh - Creates a mesh on the d-dimensional sphere, S^d.
1716 
1717   Collective on MPI_Comm
1718 
1719   Input Parameters:
1720 . comm  - The communicator for the DM object
1721 . dim   - The dimension
1722 - simplex - Use simplices, or tensor product cells
1723 
1724   Output Parameter:
1725 . dm  - The DM object
1726 
1727   Level: beginner
1728 
1729 .keywords: DM, create
1730 .seealso: DMPlexCreateBoxMesh(), DMSetType(), DMCreate()
1731 @*/
1732 PetscErrorCode DMPlexCreateSphereMesh(MPI_Comm comm, PetscInt dim, PetscBool simplex, DM *dm)
1733 {
1734   const PetscInt  embedDim = dim+1;
1735   PetscSection    coordSection;
1736   Vec             coordinates;
1737   PetscScalar    *coords;
1738   PetscReal      *coordsIn;
1739   PetscInt        numCells, numEdges, numVerts, firstVertex, v, firstEdge, coordSize, d, c, e;
1740   PetscMPIInt     rank;
1741   PetscErrorCode  ierr;
1742 
1743   PetscFunctionBegin;
1744   PetscValidPointer(dm, 4);
1745   ierr = DMCreate(comm, dm);CHKERRQ(ierr);
1746   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
1747   ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
1748   ierr = DMSetCoordinateDim(*dm, dim+1);CHKERRQ(ierr);
1749   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) *dm), &rank);CHKERRQ(ierr);
1750   switch (dim) {
1751   case 2:
1752     if (simplex) {
1753       DM              idm;
1754       const PetscReal edgeLen     = 2.0/(1.0 + PETSC_PHI);
1755       const PetscReal vertex[3]   = {0.0, 1.0/(1.0 + PETSC_PHI), PETSC_PHI/(1.0 + PETSC_PHI)};
1756       const PetscInt  degree      = 5;
1757       PetscInt        s[3]        = {1, 1, 1};
1758       PetscInt        cone[3];
1759       PetscInt       *graph, p, i, j, k;
1760 
1761       numCells    = !rank ? 20 : 0;
1762       numVerts    = !rank ? 12 : 0;
1763       firstVertex = numCells;
1764       /* Use icosahedron, which for a unit sphere has coordinates which are all cyclic permutations of
1765 
1766            (0, \pm 1/\phi+1, \pm \phi/\phi+1)
1767 
1768          where \phi^2 - \phi - 1 = 0, meaning \phi is the golden ratio \frac{1 + \sqrt{5}}{2}. The edge
1769          length is then given by 2/\phi = 2 * 2.73606 = 5.47214.
1770        */
1771       /* Construct vertices */
1772       ierr = PetscCalloc1(numVerts * embedDim, &coordsIn);CHKERRQ(ierr);
1773       for (p = 0, i = 0; p < embedDim; ++p) {
1774         for (s[1] = -1; s[1] < 2; s[1] += 2) {
1775           for (s[2] = -1; s[2] < 2; s[2] += 2) {
1776             for (d = 0; d < embedDim; ++d) coordsIn[i*embedDim+d] = s[(d+p)%embedDim]*vertex[(d+p)%embedDim];
1777             ++i;
1778           }
1779         }
1780       }
1781       /* Construct graph */
1782       ierr = PetscCalloc1(numVerts * numVerts, &graph);CHKERRQ(ierr);
1783       for (i = 0; i < numVerts; ++i) {
1784         for (j = 0, k = 0; j < numVerts; ++j) {
1785           if (PetscAbsReal(DiffNormReal(embedDim, &coordsIn[i*embedDim], &coordsIn[j*embedDim]) - edgeLen) < PETSC_SMALL) {graph[i*numVerts+j] = 1; ++k;}
1786         }
1787         if (k != degree) SETERRQ3(comm, PETSC_ERR_PLIB, "Invalid icosahedron, vertex %D degree %D != %D", i, k, degree);
1788       }
1789       /* Build Topology */
1790       ierr = DMPlexSetChart(*dm, 0, numCells+numVerts);CHKERRQ(ierr);
1791       for (c = 0; c < numCells; c++) {
1792         ierr = DMPlexSetConeSize(*dm, c, embedDim);CHKERRQ(ierr);
1793       }
1794       ierr = DMSetUp(*dm);CHKERRQ(ierr); /* Allocate space for cones */
1795       /* Cells */
1796       for (i = 0, c = 0; i < numVerts; ++i) {
1797         for (j = 0; j < i; ++j) {
1798           for (k = 0; k < j; ++k) {
1799             if (graph[i*numVerts+j] && graph[j*numVerts+k] && graph[k*numVerts+i]) {
1800               cone[0] = firstVertex+i; cone[1] = firstVertex+j; cone[2] = firstVertex+k;
1801               /* Check orientation */
1802               {
1803                 const PetscInt epsilon[3][3][3] = {{{0, 0, 0}, {0, 0, 1}, {0, -1, 0}}, {{0, 0, -1}, {0, 0, 0}, {1, 0, 0}}, {{0, 1, 0}, {-1, 0, 0}, {0, 0, 0}}};
1804                 PetscReal normal[3];
1805                 PetscInt  e, f;
1806 
1807                 for (d = 0; d < embedDim; ++d) {
1808                   normal[d] = 0.0;
1809                   for (e = 0; e < embedDim; ++e) {
1810                     for (f = 0; f < embedDim; ++f) {
1811                       normal[d] += epsilon[d][e][f]*(coordsIn[j*embedDim+e] - coordsIn[i*embedDim+e])*(coordsIn[k*embedDim+f] - coordsIn[i*embedDim+f]);
1812                     }
1813                   }
1814                 }
1815                 if (DotReal(embedDim, normal, &coordsIn[i*embedDim]) < 0) {PetscInt tmp = cone[1]; cone[1] = cone[2]; cone[2] = tmp;}
1816               }
1817               ierr = DMPlexSetCone(*dm, c++, cone);CHKERRQ(ierr);
1818             }
1819           }
1820         }
1821       }
1822       ierr = DMPlexSymmetrize(*dm);CHKERRQ(ierr);
1823       ierr = DMPlexStratify(*dm);CHKERRQ(ierr);
1824       ierr = PetscFree(graph);CHKERRQ(ierr);
1825       /* Interpolate mesh */
1826       ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
1827       ierr = DMDestroy(dm);CHKERRQ(ierr);
1828       *dm  = idm;
1829     } else {
1830       /*
1831         12-21--13
1832          |     |
1833         25  4  24
1834          |     |
1835   12-25--9-16--8-24--13
1836    |     |     |     |
1837   23  5 17  0 15  3  22
1838    |     |     |     |
1839   10-20--6-14--7-19--11
1840          |     |
1841         20  1  19
1842          |     |
1843         10-18--11
1844          |     |
1845         23  2  22
1846          |     |
1847         12-21--13
1848        */
1849       const PetscReal dist = 1.0/PetscSqrtReal(3.0);
1850       PetscInt        cone[4], ornt[4];
1851 
1852       numCells    = !rank ?  6 : 0;
1853       numEdges    = !rank ? 12 : 0;
1854       numVerts    = !rank ?  8 : 0;
1855       firstVertex = numCells;
1856       firstEdge   = numCells + numVerts;
1857       /* Build Topology */
1858       ierr = DMPlexSetChart(*dm, 0, numCells+numEdges+numVerts);CHKERRQ(ierr);
1859       for (c = 0; c < numCells; c++) {
1860         ierr = DMPlexSetConeSize(*dm, c, 4);CHKERRQ(ierr);
1861       }
1862       for (e = firstEdge; e < firstEdge+numEdges; ++e) {
1863         ierr = DMPlexSetConeSize(*dm, e, 2);CHKERRQ(ierr);
1864       }
1865       ierr = DMSetUp(*dm);CHKERRQ(ierr); /* Allocate space for cones */
1866       /* Cell 0 */
1867       cone[0] = 14; cone[1] = 15; cone[2] = 16; cone[3] = 17;
1868       ierr = DMPlexSetCone(*dm, 0, cone);CHKERRQ(ierr);
1869       ornt[0] = 0; ornt[1] = 0; ornt[2] = 0; ornt[3] = 0;
1870       ierr = DMPlexSetConeOrientation(*dm, 0, ornt);CHKERRQ(ierr);
1871       /* Cell 1 */
1872       cone[0] = 18; cone[1] = 19; cone[2] = 14; cone[3] = 20;
1873       ierr = DMPlexSetCone(*dm, 1, cone);CHKERRQ(ierr);
1874       ornt[0] = 0; ornt[1] = 0; ornt[2] = -2; ornt[3] = 0;
1875       ierr = DMPlexSetConeOrientation(*dm, 1, ornt);CHKERRQ(ierr);
1876       /* Cell 2 */
1877       cone[0] = 21; cone[1] = 22; cone[2] = 18; cone[3] = 23;
1878       ierr = DMPlexSetCone(*dm, 2, cone);CHKERRQ(ierr);
1879       ornt[0] = 0; ornt[1] = 0; ornt[2] = -2; ornt[3] = 0;
1880       ierr = DMPlexSetConeOrientation(*dm, 2, ornt);CHKERRQ(ierr);
1881       /* Cell 3 */
1882       cone[0] = 19; cone[1] = 22; cone[2] = 24; cone[3] = 15;
1883       ierr = DMPlexSetCone(*dm, 3, cone);CHKERRQ(ierr);
1884       ornt[0] = -2; ornt[1] = -2; ornt[2] = 0; ornt[3] = -2;
1885       ierr = DMPlexSetConeOrientation(*dm, 3, ornt);CHKERRQ(ierr);
1886       /* Cell 4 */
1887       cone[0] = 16; cone[1] = 24; cone[2] = 21; cone[3] = 25;
1888       ierr = DMPlexSetCone(*dm, 4, cone);CHKERRQ(ierr);
1889       ornt[0] = -2; ornt[1] = -2; ornt[2] = -2; ornt[3] = 0;
1890       ierr = DMPlexSetConeOrientation(*dm, 4, ornt);CHKERRQ(ierr);
1891       /* Cell 5 */
1892       cone[0] = 20; cone[1] = 17; cone[2] = 25; cone[3] = 23;
1893       ierr = DMPlexSetCone(*dm, 5, cone);CHKERRQ(ierr);
1894       ornt[0] = -2; ornt[1] = -2; ornt[2] = -2; ornt[3] = -2;
1895       ierr = DMPlexSetConeOrientation(*dm, 5, ornt);CHKERRQ(ierr);
1896       /* Edges */
1897       cone[0] =  6; cone[1] =  7;
1898       ierr = DMPlexSetCone(*dm, 14, cone);CHKERRQ(ierr);
1899       cone[0] =  7; cone[1] =  8;
1900       ierr = DMPlexSetCone(*dm, 15, cone);CHKERRQ(ierr);
1901       cone[0] =  8; cone[1] =  9;
1902       ierr = DMPlexSetCone(*dm, 16, cone);CHKERRQ(ierr);
1903       cone[0] =  9; cone[1] =  6;
1904       ierr = DMPlexSetCone(*dm, 17, cone);CHKERRQ(ierr);
1905       cone[0] = 10; cone[1] = 11;
1906       ierr = DMPlexSetCone(*dm, 18, cone);CHKERRQ(ierr);
1907       cone[0] = 11; cone[1] =  7;
1908       ierr = DMPlexSetCone(*dm, 19, cone);CHKERRQ(ierr);
1909       cone[0] =  6; cone[1] = 10;
1910       ierr = DMPlexSetCone(*dm, 20, cone);CHKERRQ(ierr);
1911       cone[0] = 12; cone[1] = 13;
1912       ierr = DMPlexSetCone(*dm, 21, cone);CHKERRQ(ierr);
1913       cone[0] = 13; cone[1] = 11;
1914       ierr = DMPlexSetCone(*dm, 22, cone);CHKERRQ(ierr);
1915       cone[0] = 10; cone[1] = 12;
1916       ierr = DMPlexSetCone(*dm, 23, cone);CHKERRQ(ierr);
1917       cone[0] = 13; cone[1] =  8;
1918       ierr = DMPlexSetCone(*dm, 24, cone);CHKERRQ(ierr);
1919       cone[0] = 12; cone[1] =  9;
1920       ierr = DMPlexSetCone(*dm, 25, cone);CHKERRQ(ierr);
1921       ierr = DMPlexSymmetrize(*dm);CHKERRQ(ierr);
1922       ierr = DMPlexStratify(*dm);CHKERRQ(ierr);
1923       /* Build coordinates */
1924       ierr = PetscCalloc1(numVerts * embedDim, &coordsIn);CHKERRQ(ierr);
1925       coordsIn[0*embedDim+0] = -dist; coordsIn[0*embedDim+1] =  dist; coordsIn[0*embedDim+2] = -dist;
1926       coordsIn[1*embedDim+0] =  dist; coordsIn[1*embedDim+1] =  dist; coordsIn[1*embedDim+2] = -dist;
1927       coordsIn[2*embedDim+0] =  dist; coordsIn[2*embedDim+1] = -dist; coordsIn[2*embedDim+2] = -dist;
1928       coordsIn[3*embedDim+0] = -dist; coordsIn[3*embedDim+1] = -dist; coordsIn[3*embedDim+2] = -dist;
1929       coordsIn[4*embedDim+0] = -dist; coordsIn[4*embedDim+1] =  dist; coordsIn[4*embedDim+2] =  dist;
1930       coordsIn[5*embedDim+0] =  dist; coordsIn[5*embedDim+1] =  dist; coordsIn[5*embedDim+2] =  dist;
1931       coordsIn[6*embedDim+0] = -dist; coordsIn[6*embedDim+1] = -dist; coordsIn[6*embedDim+2] =  dist;
1932       coordsIn[7*embedDim+0] =  dist; coordsIn[7*embedDim+1] = -dist; coordsIn[7*embedDim+2] =  dist;
1933     }
1934     break;
1935   case 3:
1936     if (simplex) {
1937       DM              idm;
1938       const PetscReal edgeLen         = 1.0/PETSC_PHI;
1939       const PetscReal vertexA[4]      = {0.5, 0.5, 0.5, 0.5};
1940       const PetscReal vertexB[4]      = {1.0, 0.0, 0.0, 0.0};
1941       const PetscReal vertexC[4]      = {0.5, 0.5*PETSC_PHI, 0.5/PETSC_PHI, 0.0};
1942       const PetscInt  degree          = 12;
1943       PetscInt        s[4]            = {1, 1, 1};
1944       PetscInt        evenPerm[12][4] = {{0, 1, 2, 3}, {0, 2, 3, 1}, {0, 3, 1, 2}, {1, 0, 3, 2}, {1, 2, 0, 3}, {1, 3, 2, 0},
1945                                          {2, 0, 1, 3}, {2, 1, 3, 0}, {2, 3, 0, 1}, {3, 0, 2, 1}, {3, 1, 0, 2}, {3, 2, 1, 0}};
1946       PetscInt        cone[4];
1947       PetscInt       *graph, p, i, j, k, l;
1948 
1949       numCells    = !rank ? 600 : 0;
1950       numVerts    = !rank ? 120 : 0;
1951       firstVertex = numCells;
1952       /* Use the 600-cell, which for a unit sphere has coordinates which are
1953 
1954            1/2 (\pm 1, \pm 1,    \pm 1, \pm 1)                          16
1955                (\pm 1,    0,       0,      0)  all cyclic permutations   8
1956            1/2 (\pm 1, \pm phi, \pm 1/phi, 0)  all even permutations    96
1957 
1958          where \phi^2 - \phi - 1 = 0, meaning \phi is the golden ratio \frac{1 + \sqrt{5}}{2}. The edge
1959          length is then given by 1/\phi = 2.73606.
1960 
1961          http://buzzard.pugetsound.edu/sage-practice/ch03s03.html
1962          http://mathworld.wolfram.com/600-Cell.html
1963        */
1964       /* Construct vertices */
1965       ierr = PetscCalloc1(numVerts * embedDim, &coordsIn);CHKERRQ(ierr);
1966       i    = 0;
1967       for (s[0] = -1; s[0] < 2; s[0] += 2) {
1968         for (s[1] = -1; s[1] < 2; s[1] += 2) {
1969           for (s[2] = -1; s[2] < 2; s[2] += 2) {
1970             for (s[3] = -1; s[3] < 2; s[3] += 2) {
1971               for (d = 0; d < embedDim; ++d) coordsIn[i*embedDim+d] = s[d]*vertexA[d];
1972               ++i;
1973             }
1974           }
1975         }
1976       }
1977       for (p = 0; p < embedDim; ++p) {
1978         s[1] = s[2] = s[3] = 1;
1979         for (s[0] = -1; s[0] < 2; s[0] += 2) {
1980           for (d = 0; d < embedDim; ++d) coordsIn[i*embedDim+d] = s[(d+p)%embedDim]*vertexB[(d+p)%embedDim];
1981           ++i;
1982         }
1983       }
1984       for (p = 0; p < 12; ++p) {
1985         s[3] = 1;
1986         for (s[0] = -1; s[0] < 2; s[0] += 2) {
1987           for (s[1] = -1; s[1] < 2; s[1] += 2) {
1988             for (s[2] = -1; s[2] < 2; s[2] += 2) {
1989               for (d = 0; d < embedDim; ++d) coordsIn[i*embedDim+d] = s[evenPerm[p][d]]*vertexC[evenPerm[p][d]];
1990               ++i;
1991             }
1992           }
1993         }
1994       }
1995       if (i != numVerts) SETERRQ2(comm, PETSC_ERR_PLIB, "Invalid 600-cell, vertices %D != %D", i, numVerts);
1996       /* Construct graph */
1997       ierr = PetscCalloc1(numVerts * numVerts, &graph);CHKERRQ(ierr);
1998       for (i = 0; i < numVerts; ++i) {
1999         for (j = 0, k = 0; j < numVerts; ++j) {
2000           if (PetscAbsReal(DiffNormReal(embedDim, &coordsIn[i*embedDim], &coordsIn[j*embedDim]) - edgeLen) < PETSC_SMALL) {graph[i*numVerts+j] = 1; ++k;}
2001         }
2002         if (k != degree) SETERRQ3(comm, PETSC_ERR_PLIB, "Invalid 600-cell, vertex %D degree %D != %D", i, k, degree);
2003       }
2004       /* Build Topology */
2005       ierr = DMPlexSetChart(*dm, 0, numCells+numVerts);CHKERRQ(ierr);
2006       for (c = 0; c < numCells; c++) {
2007         ierr = DMPlexSetConeSize(*dm, c, embedDim);CHKERRQ(ierr);
2008       }
2009       ierr = DMSetUp(*dm);CHKERRQ(ierr); /* Allocate space for cones */
2010       /* Cells */
2011       for (i = 0, c = 0; i < numVerts; ++i) {
2012         for (j = 0; j < i; ++j) {
2013           for (k = 0; k < j; ++k) {
2014             for (l = 0; l < k; ++l) {
2015               if (graph[i*numVerts+j] && graph[j*numVerts+k] && graph[k*numVerts+i] &&
2016                   graph[l*numVerts+i] && graph[l*numVerts+j] && graph[l*numVerts+k]) {
2017                 cone[0] = firstVertex+i; cone[1] = firstVertex+j; cone[2] = firstVertex+k; cone[3] = firstVertex+l;
2018                 /* Check orientation: https://ef.gy/linear-algebra:normal-vectors-in-higher-dimensional-spaces */
2019                 {
2020                   const PetscInt epsilon[4][4][4][4] = {{{{0,  0,  0,  0}, { 0, 0,  0,  0}, { 0,  0, 0,  0}, { 0,  0,  0, 0}},
2021                                                          {{0,  0,  0,  0}, { 0, 0,  0,  0}, { 0,  0, 0,  1}, { 0,  0, -1, 0}},
2022                                                          {{0,  0,  0,  0}, { 0, 0,  0, -1}, { 0,  0, 0,  0}, { 0,  1,  0, 0}},
2023                                                          {{0,  0,  0,  0}, { 0, 0,  1,  0}, { 0, -1, 0,  0}, { 0,  0,  0, 0}}},
2024 
2025                                                         {{{0,  0,  0,  0}, { 0, 0,  0,  0}, { 0,  0, 0, -1}, { 0,  0,  1, 0}},
2026                                                          {{0,  0,  0,  0}, { 0, 0,  0,  0}, { 0,  0, 0,  0}, { 0,  0,  0, 0}},
2027                                                          {{0,  0,  0,  1}, { 0, 0,  0,  0}, { 0,  0, 0,  0}, {-1,  0,  0, 0}},
2028                                                          {{0,  0, -1,  0}, { 0, 0,  0,  0}, { 1,  0, 0,  0}, { 0,  0,  0, 0}}},
2029 
2030                                                         {{{0,  0,  0,  0}, { 0, 0,  0,  1}, { 0,  0, 0,  0}, { 0, -1,  0, 0}},
2031                                                          {{0,  0,  0, -1}, { 0, 0,  0,  0}, { 0,  0, 0,  0}, { 1,  0,  0, 0}},
2032                                                          {{0,  0,  0,  0}, { 0, 0,  0,  0}, { 0,  0, 0,  0}, { 0,  0,  0, 0}},
2033                                                          {{0,  1,  0,  0}, {-1, 0,  0,  0}, { 0,  0, 0,  0}, { 0,  0,  0, 0}}},
2034 
2035                                                         {{{0,  0,  0,  0}, { 0, 0, -1,  0}, { 0,  1, 0,  0}, { 0,  0,  0, 0}},
2036                                                          {{0,  0,  1,  0}, { 0, 0,  0,  0}, {-1,  0, 0,  0}, { 0,  0,  0, 0}},
2037                                                          {{0, -1,  0,  0}, { 1, 0,  0,  0}, { 0,  0, 0,  0}, { 0,  0,  0, 0}},
2038                                                          {{0,  0,  0,  0}, { 0, 0,  0,  0}, { 0,  0, 0,  0}, { 0,  0,  0, 0}}}};
2039                   PetscReal normal[4];
2040                   PetscInt  e, f, g;
2041 
2042                   for (d = 0; d < embedDim; ++d) {
2043                     normal[d] = 0.0;
2044                     for (e = 0; e < embedDim; ++e) {
2045                       for (f = 0; f < embedDim; ++f) {
2046                         for (g = 0; g < embedDim; ++g) {
2047                           normal[d] += epsilon[d][e][f][g]*(coordsIn[j*embedDim+e] - coordsIn[i*embedDim+e])*(coordsIn[k*embedDim+f] - coordsIn[i*embedDim+f])*(coordsIn[l*embedDim+f] - coordsIn[i*embedDim+f]);
2048                         }
2049                       }
2050                     }
2051                   }
2052                   if (DotReal(embedDim, normal, &coordsIn[i*embedDim]) < 0) {PetscInt tmp = cone[1]; cone[1] = cone[2]; cone[2] = tmp;}
2053                 }
2054                 ierr = DMPlexSetCone(*dm, c++, cone);CHKERRQ(ierr);
2055               }
2056             }
2057           }
2058         }
2059       }
2060       ierr = DMPlexSymmetrize(*dm);CHKERRQ(ierr);
2061       ierr = DMPlexStratify(*dm);CHKERRQ(ierr);
2062       ierr = PetscFree(graph);CHKERRQ(ierr);
2063       /* Interpolate mesh */
2064       ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
2065       ierr = DMDestroy(dm);CHKERRQ(ierr);
2066       *dm  = idm;
2067       break;
2068     }
2069   default: SETERRQ1(comm, PETSC_ERR_SUP, "Unsupported dimension for sphere: %D", dim);
2070   }
2071   /* Create coordinates */
2072   ierr = DMGetCoordinateSection(*dm, &coordSection);CHKERRQ(ierr);
2073   ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
2074   ierr = PetscSectionSetFieldComponents(coordSection, 0, embedDim);CHKERRQ(ierr);
2075   ierr = PetscSectionSetChart(coordSection, firstVertex, firstVertex+numVerts);CHKERRQ(ierr);
2076   for (v = firstVertex; v < firstVertex+numVerts; ++v) {
2077     ierr = PetscSectionSetDof(coordSection, v, embedDim);CHKERRQ(ierr);
2078     ierr = PetscSectionSetFieldDof(coordSection, v, 0, embedDim);CHKERRQ(ierr);
2079   }
2080   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
2081   ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
2082   ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
2083   ierr = VecSetBlockSize(coordinates, embedDim);CHKERRQ(ierr);
2084   ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
2085   ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
2086   ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
2087   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
2088   for (v = 0; v < numVerts; ++v) for (d = 0; d < embedDim; ++d) {coords[v*embedDim+d] = coordsIn[v*embedDim+d];}
2089   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
2090   ierr = DMSetCoordinatesLocal(*dm, coordinates);CHKERRQ(ierr);
2091   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
2092   ierr = PetscFree(coordsIn);CHKERRQ(ierr);
2093   PetscFunctionReturn(0);
2094 }
2095 
2096 /* External function declarations here */
2097 extern PetscErrorCode DMCreateInterpolation_Plex(DM dmCoarse, DM dmFine, Mat *interpolation, Vec *scaling);
2098 extern PetscErrorCode DMCreateInjection_Plex(DM dmCoarse, DM dmFine, Mat *mat);
2099 extern PetscErrorCode DMCreateMassMatrix_Plex(DM dmCoarse, DM dmFine, Mat *mat);
2100 extern PetscErrorCode DMCreateDefaultSection_Plex(DM dm);
2101 extern PetscErrorCode DMCreateDefaultConstraints_Plex(DM dm);
2102 extern PetscErrorCode DMCreateMatrix_Plex(DM dm,  Mat *J);
2103 extern PetscErrorCode DMCreateCoordinateDM_Plex(DM dm, DM *cdm);
2104 extern PetscErrorCode DMCreateCoordinateField_Plex(DM dm, DMField *field);
2105 PETSC_INTERN PetscErrorCode DMClone_Plex(DM dm, DM *newdm);
2106 extern PetscErrorCode DMSetUp_Plex(DM dm);
2107 extern PetscErrorCode DMDestroy_Plex(DM dm);
2108 extern PetscErrorCode DMView_Plex(DM dm, PetscViewer viewer);
2109 extern PetscErrorCode DMLoad_Plex(DM dm, PetscViewer viewer);
2110 extern PetscErrorCode DMCreateSubDM_Plex(DM dm, PetscInt numFields, const PetscInt fields[], IS *is, DM *subdm);
2111 extern PetscErrorCode DMCreateSuperDM_Plex(DM dms[], PetscInt len, IS **is, DM *superdm);
2112 static PetscErrorCode DMInitialize_Plex(DM dm);
2113 
2114 /* Replace dm with the contents of dmNew
2115    - Share the DM_Plex structure
2116    - Share the coordinates
2117    - Share the SF
2118 */
2119 static PetscErrorCode DMPlexReplace_Static(DM dm, DM dmNew)
2120 {
2121   PetscSF               sf;
2122   DM                    coordDM, coarseDM;
2123   Vec                   coords;
2124   PetscBool             isper;
2125   const PetscReal      *maxCell, *L;
2126   const DMBoundaryType *bd;
2127   PetscErrorCode        ierr;
2128 
2129   PetscFunctionBegin;
2130   ierr = DMGetPointSF(dmNew, &sf);CHKERRQ(ierr);
2131   ierr = DMSetPointSF(dm, sf);CHKERRQ(ierr);
2132   ierr = DMGetCoordinateDM(dmNew, &coordDM);CHKERRQ(ierr);
2133   ierr = DMGetCoordinatesLocal(dmNew, &coords);CHKERRQ(ierr);
2134   ierr = DMSetCoordinateDM(dm, coordDM);CHKERRQ(ierr);
2135   ierr = DMSetCoordinatesLocal(dm, coords);CHKERRQ(ierr);
2136   ierr = DMGetPeriodicity(dm, &isper, &maxCell, &L, &bd);CHKERRQ(ierr);
2137   ierr = DMSetPeriodicity(dmNew, isper, maxCell, L, bd);CHKERRQ(ierr);
2138   ierr = DMDestroy_Plex(dm);CHKERRQ(ierr);
2139   ierr = DMInitialize_Plex(dm);CHKERRQ(ierr);
2140   dm->data = dmNew->data;
2141   ((DM_Plex *) dmNew->data)->refct++;
2142   dmNew->labels->refct++;
2143   if (!--(dm->labels->refct)) {
2144     DMLabelLink next = dm->labels->next;
2145 
2146     /* destroy the labels */
2147     while (next) {
2148       DMLabelLink tmp = next->next;
2149 
2150       ierr = DMLabelDestroy(&next->label);CHKERRQ(ierr);
2151       ierr = PetscFree(next);CHKERRQ(ierr);
2152       next = tmp;
2153     }
2154     ierr = PetscFree(dm->labels);CHKERRQ(ierr);
2155   }
2156   dm->labels = dmNew->labels;
2157   dm->depthLabel = dmNew->depthLabel;
2158   ierr = DMGetCoarseDM(dmNew,&coarseDM);CHKERRQ(ierr);
2159   ierr = DMSetCoarseDM(dm,coarseDM);CHKERRQ(ierr);
2160   PetscFunctionReturn(0);
2161 }
2162 
2163 /* Swap dm with the contents of dmNew
2164    - Swap the DM_Plex structure
2165    - Swap the coordinates
2166    - Swap the point PetscSF
2167 */
2168 static PetscErrorCode DMPlexSwap_Static(DM dmA, DM dmB)
2169 {
2170   DM              coordDMA, coordDMB;
2171   Vec             coordsA,  coordsB;
2172   PetscSF         sfA,      sfB;
2173   void            *tmp;
2174   DMLabelLinkList listTmp;
2175   DMLabel         depthTmp;
2176   PetscErrorCode  ierr;
2177 
2178   PetscFunctionBegin;
2179   ierr = DMGetPointSF(dmA, &sfA);CHKERRQ(ierr);
2180   ierr = DMGetPointSF(dmB, &sfB);CHKERRQ(ierr);
2181   ierr = PetscObjectReference((PetscObject) sfA);CHKERRQ(ierr);
2182   ierr = DMSetPointSF(dmA, sfB);CHKERRQ(ierr);
2183   ierr = DMSetPointSF(dmB, sfA);CHKERRQ(ierr);
2184   ierr = PetscObjectDereference((PetscObject) sfA);CHKERRQ(ierr);
2185 
2186   ierr = DMGetCoordinateDM(dmA, &coordDMA);CHKERRQ(ierr);
2187   ierr = DMGetCoordinateDM(dmB, &coordDMB);CHKERRQ(ierr);
2188   ierr = PetscObjectReference((PetscObject) coordDMA);CHKERRQ(ierr);
2189   ierr = DMSetCoordinateDM(dmA, coordDMB);CHKERRQ(ierr);
2190   ierr = DMSetCoordinateDM(dmB, coordDMA);CHKERRQ(ierr);
2191   ierr = PetscObjectDereference((PetscObject) coordDMA);CHKERRQ(ierr);
2192 
2193   ierr = DMGetCoordinatesLocal(dmA, &coordsA);CHKERRQ(ierr);
2194   ierr = DMGetCoordinatesLocal(dmB, &coordsB);CHKERRQ(ierr);
2195   ierr = PetscObjectReference((PetscObject) coordsA);CHKERRQ(ierr);
2196   ierr = DMSetCoordinatesLocal(dmA, coordsB);CHKERRQ(ierr);
2197   ierr = DMSetCoordinatesLocal(dmB, coordsA);CHKERRQ(ierr);
2198   ierr = PetscObjectDereference((PetscObject) coordsA);CHKERRQ(ierr);
2199 
2200   tmp       = dmA->data;
2201   dmA->data = dmB->data;
2202   dmB->data = tmp;
2203   listTmp   = dmA->labels;
2204   dmA->labels = dmB->labels;
2205   dmB->labels = listTmp;
2206   depthTmp  = dmA->depthLabel;
2207   dmA->depthLabel = dmB->depthLabel;
2208   dmB->depthLabel = depthTmp;
2209   PetscFunctionReturn(0);
2210 }
2211 
2212 PetscErrorCode DMSetFromOptions_NonRefinement_Plex(PetscOptionItems *PetscOptionsObject,DM dm)
2213 {
2214   DM_Plex       *mesh = (DM_Plex*) dm->data;
2215   PetscErrorCode ierr;
2216 
2217   PetscFunctionBegin;
2218   /* Handle viewing */
2219   ierr = PetscOptionsBool("-dm_plex_print_set_values", "Output all set values info", "DMView", PETSC_FALSE, &mesh->printSetValues, NULL);CHKERRQ(ierr);
2220   ierr = PetscOptionsInt("-dm_plex_print_fem", "Debug output level all fem computations", "DMView", 0, &mesh->printFEM, NULL);CHKERRQ(ierr);
2221   ierr = PetscOptionsReal("-dm_plex_print_tol", "Tolerance for FEM output", "DMView", mesh->printTol, &mesh->printTol, NULL);CHKERRQ(ierr);
2222   /* Point Location */
2223   ierr = PetscOptionsBool("-dm_plex_hash_location", "Use grid hashing for point location", "DMView", PETSC_FALSE, &mesh->useHashLocation, NULL);CHKERRQ(ierr);
2224   /* Generation and remeshing */
2225   ierr = PetscOptionsBool("-dm_plex_remesh_bd", "Allow changes to the boundary on remeshing", "DMView", PETSC_FALSE, &mesh->remeshBd, NULL);CHKERRQ(ierr);
2226   /* Projection behavior */
2227   ierr = PetscOptionsInt("-dm_plex_max_projection_height", "Maxmimum mesh point height used to project locally", "DMPlexSetMaxProjectionHeight", 0, &mesh->maxProjectionHeight, NULL);CHKERRQ(ierr);
2228   ierr = PetscOptionsBool("-dm_plex_regular_refinement", "Use special nested projection algorithm for regular refinement", "DMPlexSetRegularRefinement", mesh->regularRefinement, &mesh->regularRefinement, NULL);CHKERRQ(ierr);
2229 
2230   ierr = PetscPartitionerSetFromOptions(mesh->partitioner);CHKERRQ(ierr);
2231   PetscFunctionReturn(0);
2232 }
2233 
2234 static PetscErrorCode DMSetFromOptions_Plex(PetscOptionItems *PetscOptionsObject,DM dm)
2235 {
2236   PetscInt       refine = 0, coarsen = 0, r;
2237   PetscBool      isHierarchy;
2238   PetscErrorCode ierr;
2239 
2240   PetscFunctionBegin;
2241   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2242   ierr = PetscOptionsHead(PetscOptionsObject,"DMPlex Options");CHKERRQ(ierr);
2243   /* Handle DMPlex refinement */
2244   ierr = PetscOptionsInt("-dm_refine", "The number of uniform refinements", "DMCreate", refine, &refine, NULL);CHKERRQ(ierr);
2245   ierr = PetscOptionsInt("-dm_refine_hierarchy", "The number of uniform refinements", "DMCreate", refine, &refine, &isHierarchy);CHKERRQ(ierr);
2246   if (refine) {ierr = DMPlexSetRefinementUniform(dm, PETSC_TRUE);CHKERRQ(ierr);}
2247   if (refine && isHierarchy) {
2248     DM *dms, coarseDM;
2249 
2250     ierr = DMGetCoarseDM(dm, &coarseDM);CHKERRQ(ierr);
2251     ierr = PetscObjectReference((PetscObject)coarseDM);CHKERRQ(ierr);
2252     ierr = PetscMalloc1(refine,&dms);CHKERRQ(ierr);
2253     ierr = DMRefineHierarchy(dm, refine, dms);CHKERRQ(ierr);
2254     /* Total hack since we do not pass in a pointer */
2255     ierr = DMPlexSwap_Static(dm, dms[refine-1]);CHKERRQ(ierr);
2256     if (refine == 1) {
2257       ierr = DMSetCoarseDM(dm, dms[0]);CHKERRQ(ierr);
2258       ierr = DMPlexSetRegularRefinement(dm, PETSC_TRUE);CHKERRQ(ierr);
2259     } else {
2260       ierr = DMSetCoarseDM(dm, dms[refine-2]);CHKERRQ(ierr);
2261       ierr = DMPlexSetRegularRefinement(dm, PETSC_TRUE);CHKERRQ(ierr);
2262       ierr = DMSetCoarseDM(dms[0], dms[refine-1]);CHKERRQ(ierr);
2263       ierr = DMPlexSetRegularRefinement(dms[0], PETSC_TRUE);CHKERRQ(ierr);
2264     }
2265     ierr = DMSetCoarseDM(dms[refine-1], coarseDM);CHKERRQ(ierr);
2266     ierr = PetscObjectDereference((PetscObject)coarseDM);CHKERRQ(ierr);
2267     /* Free DMs */
2268     for (r = 0; r < refine; ++r) {
2269       ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dms[r]);CHKERRQ(ierr);
2270       ierr = DMDestroy(&dms[r]);CHKERRQ(ierr);
2271     }
2272     ierr = PetscFree(dms);CHKERRQ(ierr);
2273   } else {
2274     for (r = 0; r < refine; ++r) {
2275       DM refinedMesh;
2276 
2277       ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dm);CHKERRQ(ierr);
2278       ierr = DMRefine(dm, PetscObjectComm((PetscObject) dm), &refinedMesh);CHKERRQ(ierr);
2279       /* Total hack since we do not pass in a pointer */
2280       ierr = DMPlexReplace_Static(dm, refinedMesh);CHKERRQ(ierr);
2281       ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dm);CHKERRQ(ierr);
2282       ierr = DMDestroy(&refinedMesh);CHKERRQ(ierr);
2283     }
2284   }
2285   /* Handle DMPlex coarsening */
2286   ierr = PetscOptionsInt("-dm_coarsen", "Coarsen the mesh", "DMCreate", coarsen, &coarsen, NULL);CHKERRQ(ierr);
2287   ierr = PetscOptionsInt("-dm_coarsen_hierarchy", "The number of coarsenings", "DMCreate", coarsen, &coarsen, &isHierarchy);CHKERRQ(ierr);
2288   if (coarsen && isHierarchy) {
2289     DM *dms;
2290 
2291     ierr = PetscMalloc1(coarsen, &dms);CHKERRQ(ierr);
2292     ierr = DMCoarsenHierarchy(dm, coarsen, dms);CHKERRQ(ierr);
2293     /* Free DMs */
2294     for (r = 0; r < coarsen; ++r) {
2295       ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dms[r]);CHKERRQ(ierr);
2296       ierr = DMDestroy(&dms[r]);CHKERRQ(ierr);
2297     }
2298     ierr = PetscFree(dms);CHKERRQ(ierr);
2299   } else {
2300     for (r = 0; r < coarsen; ++r) {
2301       DM coarseMesh;
2302 
2303       ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dm);CHKERRQ(ierr);
2304       ierr = DMCoarsen(dm, PetscObjectComm((PetscObject) dm), &coarseMesh);CHKERRQ(ierr);
2305       /* Total hack since we do not pass in a pointer */
2306       ierr = DMPlexReplace_Static(dm, coarseMesh);CHKERRQ(ierr);
2307       ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dm);CHKERRQ(ierr);
2308       ierr = DMDestroy(&coarseMesh);CHKERRQ(ierr);
2309     }
2310   }
2311   /* Handle */
2312   ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dm);CHKERRQ(ierr);
2313   ierr = PetscOptionsTail();CHKERRQ(ierr);
2314   PetscFunctionReturn(0);
2315 }
2316 
2317 static PetscErrorCode DMCreateGlobalVector_Plex(DM dm,Vec *vec)
2318 {
2319   PetscErrorCode ierr;
2320 
2321   PetscFunctionBegin;
2322   ierr = DMCreateGlobalVector_Section_Private(dm,vec);CHKERRQ(ierr);
2323   /* ierr = VecSetOperation(*vec, VECOP_DUPLICATE, (void(*)(void)) VecDuplicate_MPI_DM);CHKERRQ(ierr); */
2324   ierr = VecSetOperation(*vec, VECOP_VIEW, (void (*)(void)) VecView_Plex);CHKERRQ(ierr);
2325   ierr = VecSetOperation(*vec, VECOP_VIEWNATIVE, (void (*)(void)) VecView_Plex_Native);CHKERRQ(ierr);
2326   ierr = VecSetOperation(*vec, VECOP_LOAD, (void (*)(void)) VecLoad_Plex);CHKERRQ(ierr);
2327   ierr = VecSetOperation(*vec, VECOP_LOADNATIVE, (void (*)(void)) VecLoad_Plex_Native);CHKERRQ(ierr);
2328   PetscFunctionReturn(0);
2329 }
2330 
2331 static PetscErrorCode DMCreateLocalVector_Plex(DM dm,Vec *vec)
2332 {
2333   PetscErrorCode ierr;
2334 
2335   PetscFunctionBegin;
2336   ierr = DMCreateLocalVector_Section_Private(dm,vec);CHKERRQ(ierr);
2337   ierr = VecSetOperation(*vec, VECOP_VIEW, (void (*)(void)) VecView_Plex_Local);CHKERRQ(ierr);
2338   ierr = VecSetOperation(*vec, VECOP_LOAD, (void (*)(void)) VecLoad_Plex_Local);CHKERRQ(ierr);
2339   PetscFunctionReturn(0);
2340 }
2341 
2342 static PetscErrorCode DMGetDimPoints_Plex(DM dm, PetscInt dim, PetscInt *pStart, PetscInt *pEnd)
2343 {
2344   PetscInt       depth, d;
2345   PetscErrorCode ierr;
2346 
2347   PetscFunctionBegin;
2348   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
2349   if (depth == 1) {
2350     ierr = DMGetDimension(dm, &d);CHKERRQ(ierr);
2351     if (dim == 0)      {ierr = DMPlexGetDepthStratum(dm, dim, pStart, pEnd);CHKERRQ(ierr);}
2352     else if (dim == d) {ierr = DMPlexGetDepthStratum(dm, 1, pStart, pEnd);CHKERRQ(ierr);}
2353     else               {*pStart = 0; *pEnd = 0;}
2354   } else {
2355     ierr = DMPlexGetDepthStratum(dm, dim, pStart, pEnd);CHKERRQ(ierr);
2356   }
2357   PetscFunctionReturn(0);
2358 }
2359 
2360 static PetscErrorCode DMGetNeighors_Plex(DM dm, PetscInt *nranks, const PetscMPIInt *ranks[])
2361 {
2362   PetscSF        sf;
2363   PetscErrorCode ierr;
2364 
2365   PetscFunctionBegin;
2366   ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr);
2367   ierr = PetscSFGetRanks(sf, nranks, ranks, NULL, NULL, NULL);CHKERRQ(ierr);
2368   PetscFunctionReturn(0);
2369 }
2370 
2371 static PetscErrorCode DMHasCreateInjection_Plex(DM dm, PetscBool *flg)
2372 {
2373   PetscFunctionBegin;
2374   PetscValidHeaderSpecific(dm,DM_CLASSID,1);
2375   PetscValidPointer(flg,2);
2376   *flg = PETSC_TRUE;
2377   PetscFunctionReturn(0);
2378 }
2379 
2380 static PetscErrorCode DMInitialize_Plex(DM dm)
2381 {
2382   PetscErrorCode ierr;
2383 
2384   PetscFunctionBegin;
2385   dm->ops->view                            = DMView_Plex;
2386   dm->ops->load                            = DMLoad_Plex;
2387   dm->ops->setfromoptions                  = DMSetFromOptions_Plex;
2388   dm->ops->clone                           = DMClone_Plex;
2389   dm->ops->setup                           = DMSetUp_Plex;
2390   dm->ops->createdefaultsection            = DMCreateDefaultSection_Plex;
2391   dm->ops->createdefaultconstraints        = DMCreateDefaultConstraints_Plex;
2392   dm->ops->createglobalvector              = DMCreateGlobalVector_Plex;
2393   dm->ops->createlocalvector               = DMCreateLocalVector_Plex;
2394   dm->ops->getlocaltoglobalmapping         = NULL;
2395   dm->ops->createfieldis                   = NULL;
2396   dm->ops->createcoordinatedm              = DMCreateCoordinateDM_Plex;
2397   dm->ops->createcoordinatefield           = DMCreateCoordinateField_Plex;
2398   dm->ops->getcoloring                     = NULL;
2399   dm->ops->creatematrix                    = DMCreateMatrix_Plex;
2400   dm->ops->createinterpolation             = DMCreateInterpolation_Plex;
2401   dm->ops->createmassmatrix                = DMCreateMassMatrix_Plex;
2402   dm->ops->getaggregates                   = NULL;
2403   dm->ops->getinjection                    = DMCreateInjection_Plex;
2404   dm->ops->hascreateinjection              = DMHasCreateInjection_Plex;
2405   dm->ops->refine                          = DMRefine_Plex;
2406   dm->ops->coarsen                         = DMCoarsen_Plex;
2407   dm->ops->refinehierarchy                 = DMRefineHierarchy_Plex;
2408   dm->ops->coarsenhierarchy                = DMCoarsenHierarchy_Plex;
2409   dm->ops->adaptlabel                      = DMAdaptLabel_Plex;
2410   dm->ops->adaptmetric                     = DMAdaptMetric_Plex;
2411   dm->ops->globaltolocalbegin              = NULL;
2412   dm->ops->globaltolocalend                = NULL;
2413   dm->ops->localtoglobalbegin              = NULL;
2414   dm->ops->localtoglobalend                = NULL;
2415   dm->ops->destroy                         = DMDestroy_Plex;
2416   dm->ops->createsubdm                     = DMCreateSubDM_Plex;
2417   dm->ops->createsuperdm                   = DMCreateSuperDM_Plex;
2418   dm->ops->getdimpoints                    = DMGetDimPoints_Plex;
2419   dm->ops->locatepoints                    = DMLocatePoints_Plex;
2420   dm->ops->projectfunctionlocal            = DMProjectFunctionLocal_Plex;
2421   dm->ops->projectfunctionlabellocal       = DMProjectFunctionLabelLocal_Plex;
2422   dm->ops->projectfieldlocal               = DMProjectFieldLocal_Plex;
2423   dm->ops->projectfieldlabellocal          = DMProjectFieldLabelLocal_Plex;
2424   dm->ops->computel2diff                   = DMComputeL2Diff_Plex;
2425   dm->ops->computel2gradientdiff           = DMComputeL2GradientDiff_Plex;
2426   dm->ops->computel2fielddiff              = DMComputeL2FieldDiff_Plex;
2427   dm->ops->getneighbors                    = DMGetNeighors_Plex;
2428   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMPlexInsertBoundaryValues_C",DMPlexInsertBoundaryValues_Plex);CHKERRQ(ierr);
2429   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMSetUpGLVisViewer_C",DMSetUpGLVisViewer_Plex);CHKERRQ(ierr);
2430   PetscFunctionReturn(0);
2431 }
2432 
2433 PETSC_INTERN PetscErrorCode DMClone_Plex(DM dm, DM *newdm)
2434 {
2435   DM_Plex        *mesh = (DM_Plex *) dm->data;
2436   PetscErrorCode ierr;
2437 
2438   PetscFunctionBegin;
2439   mesh->refct++;
2440   (*newdm)->data = mesh;
2441   ierr = PetscObjectChangeTypeName((PetscObject) *newdm, DMPLEX);CHKERRQ(ierr);
2442   ierr = DMInitialize_Plex(*newdm);CHKERRQ(ierr);
2443   PetscFunctionReturn(0);
2444 }
2445 
2446 /*MC
2447   DMPLEX = "plex" - A DM object that encapsulates an unstructured mesh, or CW Complex, which can be expressed using a Hasse Diagram.
2448                     In the local representation, Vecs contain all unknowns in the interior and shared boundary. This is
2449                     specified by a PetscSection object. Ownership in the global representation is determined by
2450                     ownership of the underlying DMPlex points. This is specified by another PetscSection object.
2451 
2452   Level: intermediate
2453 
2454 .seealso: DMType, DMPlexCreate(), DMCreate(), DMSetType()
2455 M*/
2456 
2457 PETSC_EXTERN PetscErrorCode DMCreate_Plex(DM dm)
2458 {
2459   DM_Plex        *mesh;
2460   PetscInt       unit, d;
2461   PetscErrorCode ierr;
2462 
2463   PetscFunctionBegin;
2464   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2465   ierr     = PetscNewLog(dm,&mesh);CHKERRQ(ierr);
2466   dm->dim  = 0;
2467   dm->data = mesh;
2468 
2469   mesh->refct             = 1;
2470   ierr                    = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &mesh->coneSection);CHKERRQ(ierr);
2471   mesh->maxConeSize       = 0;
2472   mesh->cones             = NULL;
2473   mesh->coneOrientations  = NULL;
2474   ierr                    = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &mesh->supportSection);CHKERRQ(ierr);
2475   mesh->maxSupportSize    = 0;
2476   mesh->supports          = NULL;
2477   mesh->refinementUniform = PETSC_TRUE;
2478   mesh->refinementLimit   = -1.0;
2479 
2480   mesh->facesTmp = NULL;
2481 
2482   mesh->tetgenOpts   = NULL;
2483   mesh->triangleOpts = NULL;
2484   ierr = PetscPartitionerCreate(PetscObjectComm((PetscObject)dm), &mesh->partitioner);CHKERRQ(ierr);
2485   mesh->remeshBd     = PETSC_FALSE;
2486 
2487   mesh->subpointMap = NULL;
2488 
2489   for (unit = 0; unit < NUM_PETSC_UNITS; ++unit) mesh->scale[unit] = 1.0;
2490 
2491   mesh->regularRefinement   = PETSC_FALSE;
2492   mesh->depthState          = -1;
2493   mesh->globalVertexNumbers = NULL;
2494   mesh->globalCellNumbers   = NULL;
2495   mesh->anchorSection       = NULL;
2496   mesh->anchorIS            = NULL;
2497   mesh->createanchors       = NULL;
2498   mesh->computeanchormatrix = NULL;
2499   mesh->parentSection       = NULL;
2500   mesh->parents             = NULL;
2501   mesh->childIDs            = NULL;
2502   mesh->childSection        = NULL;
2503   mesh->children            = NULL;
2504   mesh->referenceTree       = NULL;
2505   mesh->getchildsymmetry    = NULL;
2506   for (d = 0; d < 8; ++d) mesh->hybridPointMax[d] = PETSC_DETERMINE;
2507   mesh->vtkCellHeight       = 0;
2508   mesh->useAnchors          = PETSC_FALSE;
2509 
2510   mesh->maxProjectionHeight = 0;
2511 
2512   mesh->printSetValues = PETSC_FALSE;
2513   mesh->printFEM       = 0;
2514   mesh->printTol       = 1.0e-10;
2515 
2516   ierr = DMInitialize_Plex(dm);CHKERRQ(ierr);
2517   PetscFunctionReturn(0);
2518 }
2519 
2520 /*@
2521   DMPlexCreate - Creates a DMPlex object, which encapsulates an unstructured mesh, or CW complex, which can be expressed using a Hasse Diagram.
2522 
2523   Collective on MPI_Comm
2524 
2525   Input Parameter:
2526 . comm - The communicator for the DMPlex object
2527 
2528   Output Parameter:
2529 . mesh  - The DMPlex object
2530 
2531   Level: beginner
2532 
2533 .keywords: DMPlex, create
2534 @*/
2535 PetscErrorCode DMPlexCreate(MPI_Comm comm, DM *mesh)
2536 {
2537   PetscErrorCode ierr;
2538 
2539   PetscFunctionBegin;
2540   PetscValidPointer(mesh,2);
2541   ierr = DMCreate(comm, mesh);CHKERRQ(ierr);
2542   ierr = DMSetType(*mesh, DMPLEX);CHKERRQ(ierr);
2543   PetscFunctionReturn(0);
2544 }
2545 
2546 /*
2547   This takes as input the common mesh generator output, a list of the vertices for each cell, but vertex numbers are global and an SF is built for them
2548 */
2549 /* TODO: invertCells and spaceDim arguments could be added also to to DMPlexCreateFromCellListParallel(), DMPlexBuildFromCellList_Internal() and DMPlexCreateFromCellList() */
2550 PetscErrorCode DMPlexBuildFromCellList_Parallel_Internal(DM dm, PetscInt spaceDim, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, const int cells[], PetscBool invertCells, PetscSF *sfVert)
2551 {
2552   PetscSF         sfPoint;
2553   PetscLayout     vLayout;
2554   PetscHSetI      vhash;
2555   PetscSFNode    *remoteVerticesAdj, *vertexLocal, *vertexOwner, *remoteVertex;
2556   const PetscInt *vrange;
2557   PetscInt        numVerticesAdj, off = 0, *verticesAdj, numVerticesGhost = 0, *localVertex, *cone, c, p, v, g;
2558   PetscMPIInt     rank, size;
2559   PetscErrorCode  ierr;
2560 
2561   PetscFunctionBegin;
2562   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRQ(ierr);
2563   ierr = MPI_Comm_size(PetscObjectComm((PetscObject) dm), &size);CHKERRQ(ierr);
2564   /* Partition vertices */
2565   ierr = PetscLayoutCreate(PetscObjectComm((PetscObject) dm), &vLayout);CHKERRQ(ierr);
2566   ierr = PetscLayoutSetLocalSize(vLayout, numVertices);CHKERRQ(ierr);
2567   ierr = PetscLayoutSetBlockSize(vLayout, 1);CHKERRQ(ierr);
2568   ierr = PetscLayoutSetUp(vLayout);CHKERRQ(ierr);
2569   ierr = PetscLayoutGetRanges(vLayout, &vrange);CHKERRQ(ierr);
2570   /* Count vertices and map them to procs */
2571   ierr = PetscHSetICreate(&vhash);CHKERRQ(ierr);
2572   for (c = 0; c < numCells; ++c) {
2573     for (p = 0; p < numCorners; ++p) {
2574       ierr = PetscHSetIAdd(vhash, cells[c*numCorners+p]);CHKERRQ(ierr);
2575     }
2576   }
2577   ierr = PetscHSetIGetSize(vhash, &numVerticesAdj);CHKERRQ(ierr);
2578   ierr = PetscMalloc1(numVerticesAdj, &verticesAdj);CHKERRQ(ierr);
2579   ierr = PetscHSetIGetElems(vhash, &off, verticesAdj);CHKERRQ(ierr);
2580   ierr = PetscHSetIDestroy(&vhash);CHKERRQ(ierr);
2581   if (off != numVerticesAdj) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid number of local vertices %D should be %D", off, numVerticesAdj);
2582   ierr = PetscSortInt(numVerticesAdj, verticesAdj);CHKERRQ(ierr);
2583   ierr = PetscMalloc1(numVerticesAdj, &remoteVerticesAdj);CHKERRQ(ierr);
2584   for (v = 0; v < numVerticesAdj; ++v) {
2585     const PetscInt gv = verticesAdj[v];
2586     PetscInt       vrank;
2587 
2588     ierr = PetscFindInt(gv, size+1, vrange, &vrank);CHKERRQ(ierr);
2589     vrank = vrank < 0 ? -(vrank+2) : vrank;
2590     remoteVerticesAdj[v].index = gv - vrange[vrank];
2591     remoteVerticesAdj[v].rank  = vrank;
2592   }
2593   /* Create cones */
2594   ierr = DMPlexSetChart(dm, 0, numCells+numVerticesAdj);CHKERRQ(ierr);
2595   for (c = 0; c < numCells; ++c) {ierr = DMPlexSetConeSize(dm, c, numCorners);CHKERRQ(ierr);}
2596   ierr = DMSetUp(dm);CHKERRQ(ierr);
2597   ierr = DMGetWorkArray(dm, numCorners, MPIU_INT, &cone);CHKERRQ(ierr);
2598   for (c = 0; c < numCells; ++c) {
2599     for (p = 0; p < numCorners; ++p) {
2600       const PetscInt gv = cells[c*numCorners+p];
2601       PetscInt       lv;
2602 
2603       ierr = PetscFindInt(gv, numVerticesAdj, verticesAdj, &lv);CHKERRQ(ierr);
2604       if (lv < 0) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Could not find global vertex %D in local connectivity", gv);
2605       cone[p] = lv+numCells;
2606     }
2607     if (invertCells) { ierr = DMPlexInvertCell_Internal(spaceDim, numCorners, cone);CHKERRQ(ierr); }
2608     ierr = DMPlexSetCone(dm, c, cone);CHKERRQ(ierr);
2609   }
2610   ierr = DMRestoreWorkArray(dm, numCorners, MPIU_INT, &cone);CHKERRQ(ierr);
2611   /* Create SF for vertices */
2612   ierr = PetscSFCreate(PetscObjectComm((PetscObject)dm), sfVert);CHKERRQ(ierr);
2613   ierr = PetscObjectSetName((PetscObject) *sfVert, "Vertex Ownership SF");CHKERRQ(ierr);
2614   ierr = PetscSFSetFromOptions(*sfVert);CHKERRQ(ierr);
2615   ierr = PetscSFSetGraph(*sfVert, numVertices, numVerticesAdj, NULL, PETSC_OWN_POINTER, remoteVerticesAdj, PETSC_OWN_POINTER);CHKERRQ(ierr);
2616   ierr = PetscFree(verticesAdj);CHKERRQ(ierr);
2617   /* Build pointSF */
2618   ierr = PetscMalloc2(numVerticesAdj, &vertexLocal, numVertices, &vertexOwner);CHKERRQ(ierr);
2619   for (v = 0; v < numVerticesAdj; ++v) {vertexLocal[v].index = v+numCells; vertexLocal[v].rank = rank;}
2620   for (v = 0; v < numVertices;    ++v) {vertexOwner[v].index = -1;         vertexOwner[v].rank = -1;}
2621   ierr = PetscSFReduceBegin(*sfVert, MPIU_2INT, vertexLocal, vertexOwner, MPI_MAXLOC);CHKERRQ(ierr);
2622   ierr = PetscSFReduceEnd(*sfVert, MPIU_2INT, vertexLocal, vertexOwner, MPI_MAXLOC);CHKERRQ(ierr);
2623   for (v = 0; v < numVertices;    ++v) if (vertexOwner[v].rank < 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Global vertex %d on rank %d was unclaimed", v + vrange[rank], rank);
2624   ierr = PetscSFBcastBegin(*sfVert, MPIU_2INT, vertexOwner, vertexLocal);CHKERRQ(ierr);
2625   ierr = PetscSFBcastEnd(*sfVert, MPIU_2INT, vertexOwner, vertexLocal);CHKERRQ(ierr);
2626   for (v = 0; v < numVerticesAdj; ++v) if (vertexLocal[v].rank != rank) ++numVerticesGhost;
2627   ierr = PetscMalloc1(numVerticesGhost, &localVertex);CHKERRQ(ierr);
2628   ierr = PetscMalloc1(numVerticesGhost, &remoteVertex);CHKERRQ(ierr);
2629   for (v = 0, g = 0; v < numVerticesAdj; ++v) {
2630     if (vertexLocal[v].rank != rank) {
2631       localVertex[g]        = v+numCells;
2632       remoteVertex[g].index = vertexLocal[v].index;
2633       remoteVertex[g].rank  = vertexLocal[v].rank;
2634       ++g;
2635     }
2636   }
2637   ierr = PetscFree2(vertexLocal, vertexOwner);CHKERRQ(ierr);
2638   if (g != numVerticesGhost) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid number of vertex ghosts %D should be %D", g, numVerticesGhost);
2639   ierr = DMGetPointSF(dm, &sfPoint);CHKERRQ(ierr);
2640   ierr = PetscObjectSetName((PetscObject) sfPoint, "point SF");CHKERRQ(ierr);
2641   ierr = PetscSFSetGraph(sfPoint, numCells+numVerticesAdj, numVerticesGhost, localVertex, PETSC_OWN_POINTER, remoteVertex, PETSC_OWN_POINTER);CHKERRQ(ierr);
2642   ierr = PetscLayoutDestroy(&vLayout);CHKERRQ(ierr);
2643   /* Fill in the rest of the topology structure */
2644   ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
2645   ierr = DMPlexStratify(dm);CHKERRQ(ierr);
2646   PetscFunctionReturn(0);
2647 }
2648 
2649 /*
2650   This takes as input the coordinates for each owned vertex
2651 */
2652 PetscErrorCode DMPlexBuildCoordinates_Parallel_Internal(DM dm, PetscInt spaceDim, PetscInt numCells, PetscInt numV, PetscSF sfVert, const PetscReal vertexCoords[])
2653 {
2654   PetscSection   coordSection;
2655   Vec            coordinates;
2656   PetscScalar   *coords;
2657   PetscInt       numVertices, numVerticesAdj, coordSize, v;
2658   PetscErrorCode ierr;
2659 
2660   PetscFunctionBegin;
2661   ierr = DMSetCoordinateDim(dm, spaceDim);CHKERRQ(ierr);
2662   ierr = PetscSFGetGraph(sfVert, &numVertices, &numVerticesAdj, NULL, NULL);CHKERRQ(ierr);
2663   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
2664   ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
2665   ierr = PetscSectionSetFieldComponents(coordSection, 0, spaceDim);CHKERRQ(ierr);
2666   ierr = PetscSectionSetChart(coordSection, numCells, numCells + numVerticesAdj);CHKERRQ(ierr);
2667   for (v = numCells; v < numCells+numVerticesAdj; ++v) {
2668     ierr = PetscSectionSetDof(coordSection, v, spaceDim);CHKERRQ(ierr);
2669     ierr = PetscSectionSetFieldDof(coordSection, v, 0, spaceDim);CHKERRQ(ierr);
2670   }
2671   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
2672   ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
2673   ierr = VecCreate(PetscObjectComm((PetscObject)dm), &coordinates);CHKERRQ(ierr);
2674   ierr = VecSetBlockSize(coordinates, spaceDim);CHKERRQ(ierr);
2675   ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
2676   ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
2677   ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
2678   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
2679   {
2680     MPI_Datatype coordtype;
2681 
2682     /* Need a temp buffer for coords if we have complex/single */
2683     ierr = MPI_Type_contiguous(spaceDim, MPIU_SCALAR, &coordtype);CHKERRQ(ierr);
2684     ierr = MPI_Type_commit(&coordtype);CHKERRQ(ierr);
2685 #if defined(PETSC_USE_COMPLEX)
2686     {
2687     PetscScalar *svertexCoords;
2688     PetscInt    i;
2689     ierr = PetscMalloc1(numV*spaceDim,&svertexCoords);CHKERRQ(ierr);
2690     for (i=0; i<numV*spaceDim; i++) svertexCoords[i] = vertexCoords[i];
2691     ierr = PetscSFBcastBegin(sfVert, coordtype, svertexCoords, coords);CHKERRQ(ierr);
2692     ierr = PetscSFBcastEnd(sfVert, coordtype, svertexCoords, coords);CHKERRQ(ierr);
2693     ierr = PetscFree(svertexCoords);CHKERRQ(ierr);
2694     }
2695 #else
2696     ierr = PetscSFBcastBegin(sfVert, coordtype, vertexCoords, coords);CHKERRQ(ierr);
2697     ierr = PetscSFBcastEnd(sfVert, coordtype, vertexCoords, coords);CHKERRQ(ierr);
2698 #endif
2699     ierr = MPI_Type_free(&coordtype);CHKERRQ(ierr);
2700   }
2701   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
2702   ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
2703   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
2704   PetscFunctionReturn(0);
2705 }
2706 
2707 /*@C
2708   DMPlexCreateFromCellListParallel - This takes as input common mesh generator output, a list of the vertices for each cell, and produces a DM
2709 
2710   Input Parameters:
2711 + comm - The communicator
2712 . dim - The topological dimension of the mesh
2713 . numCells - The number of cells owned by this process
2714 . numVertices - The number of vertices owned by this process
2715 . numCorners - The number of vertices for each cell
2716 . interpolate - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
2717 . cells - An array of numCells*numCorners numbers, the global vertex numbers for each cell
2718 . spaceDim - The spatial dimension used for coordinates
2719 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
2720 
2721   Output Parameter:
2722 + dm - The DM
2723 - vertexSF - Optional, SF describing complete vertex ownership
2724 
2725   Note: Two triangles sharing a face
2726 $
2727 $        2
2728 $      / | \
2729 $     /  |  \
2730 $    /   |   \
2731 $   0  0 | 1  3
2732 $    \   |   /
2733 $     \  |  /
2734 $      \ | /
2735 $        1
2736 would have input
2737 $  numCells = 2, numVertices = 4
2738 $  cells = [0 1 2  1 3 2]
2739 $
2740 which would result in the DMPlex
2741 $
2742 $        4
2743 $      / | \
2744 $     /  |  \
2745 $    /   |   \
2746 $   2  0 | 1  5
2747 $    \   |   /
2748 $     \  |  /
2749 $      \ | /
2750 $        3
2751 
2752   Level: beginner
2753 
2754 .seealso: DMPlexCreateFromCellList(), DMPlexCreateFromDAG(), DMPlexCreate()
2755 @*/
2756 PetscErrorCode DMPlexCreateFromCellListParallel(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, PetscBool interpolate, const int cells[], PetscInt spaceDim, const PetscReal vertexCoords[], PetscSF *vertexSF, DM *dm)
2757 {
2758   PetscSF        sfVert;
2759   PetscErrorCode ierr;
2760 
2761   PetscFunctionBegin;
2762   ierr = DMCreate(comm, dm);CHKERRQ(ierr);
2763   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
2764   PetscValidLogicalCollectiveInt(*dm, dim, 2);
2765   PetscValidLogicalCollectiveInt(*dm, spaceDim, 8);
2766   ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
2767   ierr = DMPlexBuildFromCellList_Parallel_Internal(*dm, spaceDim, numCells, numVertices, numCorners, cells, PETSC_FALSE, &sfVert);CHKERRQ(ierr);
2768   if (interpolate) {
2769     DM idm;
2770 
2771     ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
2772     ierr = DMDestroy(dm);CHKERRQ(ierr);
2773     *dm  = idm;
2774   }
2775   ierr = DMPlexBuildCoordinates_Parallel_Internal(*dm, spaceDim, numCells, numVertices, sfVert, vertexCoords);CHKERRQ(ierr);
2776   if (vertexSF) *vertexSF = sfVert;
2777   else {ierr = PetscSFDestroy(&sfVert);CHKERRQ(ierr);}
2778   PetscFunctionReturn(0);
2779 }
2780 
2781 /*
2782   This takes as input the common mesh generator output, a list of the vertices for each cell
2783 */
2784 PetscErrorCode DMPlexBuildFromCellList_Internal(DM dm, PetscInt spaceDim, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, const int cells[], PetscBool invertCells)
2785 {
2786   PetscInt      *cone, c, p;
2787   PetscErrorCode ierr;
2788 
2789   PetscFunctionBegin;
2790   ierr = DMPlexSetChart(dm, 0, numCells+numVertices);CHKERRQ(ierr);
2791   for (c = 0; c < numCells; ++c) {
2792     ierr = DMPlexSetConeSize(dm, c, numCorners);CHKERRQ(ierr);
2793   }
2794   ierr = DMSetUp(dm);CHKERRQ(ierr);
2795   ierr = DMGetWorkArray(dm, numCorners, MPIU_INT, &cone);CHKERRQ(ierr);
2796   for (c = 0; c < numCells; ++c) {
2797     for (p = 0; p < numCorners; ++p) {
2798       cone[p] = cells[c*numCorners+p]+numCells;
2799     }
2800     if (invertCells) { ierr = DMPlexInvertCell_Internal(spaceDim, numCorners, cone);CHKERRQ(ierr); }
2801     ierr = DMPlexSetCone(dm, c, cone);CHKERRQ(ierr);
2802   }
2803   ierr = DMRestoreWorkArray(dm, numCorners, MPIU_INT, &cone);CHKERRQ(ierr);
2804   ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
2805   ierr = DMPlexStratify(dm);CHKERRQ(ierr);
2806   PetscFunctionReturn(0);
2807 }
2808 
2809 /*
2810   This takes as input the coordinates for each vertex
2811 */
2812 PetscErrorCode DMPlexBuildCoordinates_Internal(DM dm, PetscInt spaceDim, PetscInt numCells, PetscInt numVertices, const double vertexCoords[])
2813 {
2814   PetscSection   coordSection;
2815   Vec            coordinates;
2816   DM             cdm;
2817   PetscScalar   *coords;
2818   PetscInt       v, d;
2819   PetscErrorCode ierr;
2820 
2821   PetscFunctionBegin;
2822   ierr = DMSetCoordinateDim(dm, spaceDim);CHKERRQ(ierr);
2823   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
2824   ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
2825   ierr = PetscSectionSetFieldComponents(coordSection, 0, spaceDim);CHKERRQ(ierr);
2826   ierr = PetscSectionSetChart(coordSection, numCells, numCells + numVertices);CHKERRQ(ierr);
2827   for (v = numCells; v < numCells+numVertices; ++v) {
2828     ierr = PetscSectionSetDof(coordSection, v, spaceDim);CHKERRQ(ierr);
2829     ierr = PetscSectionSetFieldDof(coordSection, v, 0, spaceDim);CHKERRQ(ierr);
2830   }
2831   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
2832 
2833   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
2834   ierr = DMCreateLocalVector(cdm, &coordinates);CHKERRQ(ierr);
2835   ierr = VecSetBlockSize(coordinates, spaceDim);CHKERRQ(ierr);
2836   ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
2837   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
2838   for (v = 0; v < numVertices; ++v) {
2839     for (d = 0; d < spaceDim; ++d) {
2840       coords[v*spaceDim+d] = vertexCoords[v*spaceDim+d];
2841     }
2842   }
2843   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
2844   ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
2845   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
2846   PetscFunctionReturn(0);
2847 }
2848 
2849 /*@C
2850   DMPlexCreateFromCellList - This takes as input common mesh generator output, a list of the vertices for each cell, and produces a DM
2851 
2852   Input Parameters:
2853 + comm - The communicator
2854 . dim - The topological dimension of the mesh
2855 . numCells - The number of cells
2856 . numVertices - The number of vertices
2857 . numCorners - The number of vertices for each cell
2858 . interpolate - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
2859 . cells - An array of numCells*numCorners numbers, the vertices for each cell
2860 . spaceDim - The spatial dimension used for coordinates
2861 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
2862 
2863   Output Parameter:
2864 . dm - The DM
2865 
2866   Note: Two triangles sharing a face
2867 $
2868 $        2
2869 $      / | \
2870 $     /  |  \
2871 $    /   |   \
2872 $   0  0 | 1  3
2873 $    \   |   /
2874 $     \  |  /
2875 $      \ | /
2876 $        1
2877 would have input
2878 $  numCells = 2, numVertices = 4
2879 $  cells = [0 1 2  1 3 2]
2880 $
2881 which would result in the DMPlex
2882 $
2883 $        4
2884 $      / | \
2885 $     /  |  \
2886 $    /   |   \
2887 $   2  0 | 1  5
2888 $    \   |   /
2889 $     \  |  /
2890 $      \ | /
2891 $        3
2892 
2893   Level: beginner
2894 
2895 .seealso: DMPlexCreateFromDAG(), DMPlexCreate()
2896 @*/
2897 PetscErrorCode DMPlexCreateFromCellList(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, PetscBool interpolate, const int cells[], PetscInt spaceDim, const double vertexCoords[], DM *dm)
2898 {
2899   PetscErrorCode ierr;
2900 
2901   PetscFunctionBegin;
2902   ierr = DMCreate(comm, dm);CHKERRQ(ierr);
2903   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
2904   ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
2905   ierr = DMPlexBuildFromCellList_Internal(*dm, spaceDim, numCells, numVertices, numCorners, cells, PETSC_FALSE);CHKERRQ(ierr);
2906   if (interpolate) {
2907     DM idm;
2908 
2909     ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
2910     ierr = DMDestroy(dm);CHKERRQ(ierr);
2911     *dm  = idm;
2912   }
2913   ierr = DMPlexBuildCoordinates_Internal(*dm, spaceDim, numCells, numVertices, vertexCoords);CHKERRQ(ierr);
2914   PetscFunctionReturn(0);
2915 }
2916 
2917 /*@
2918   DMPlexCreateFromDAG - This takes as input the adjacency-list representation of the Directed Acyclic Graph (Hasse Diagram) encoding a mesh, and produces a DM
2919 
2920   Input Parameters:
2921 + dm - The empty DM object, usually from DMCreate() and DMSetDimension()
2922 . depth - The depth of the DAG
2923 . numPoints - The number of points at each depth
2924 . coneSize - The cone size of each point
2925 . cones - The concatenation of the cone points for each point, the cone list must be oriented correctly for each point
2926 . coneOrientations - The orientation of each cone point
2927 - vertexCoords - An array of numVertices*dim numbers, the coordinates of each vertex
2928 
2929   Output Parameter:
2930 . dm - The DM
2931 
2932   Note: Two triangles sharing a face would have input
2933 $  depth = 1, numPoints = [4 2], coneSize = [3 3 0 0 0 0]
2934 $  cones = [2 3 4  3 5 4], coneOrientations = [0 0 0  0 0 0]
2935 $ vertexCoords = [-1.0 0.0  0.0 -1.0  0.0 1.0  1.0 0.0]
2936 $
2937 which would result in the DMPlex
2938 $
2939 $        4
2940 $      / | \
2941 $     /  |  \
2942 $    /   |   \
2943 $   2  0 | 1  5
2944 $    \   |   /
2945 $     \  |  /
2946 $      \ | /
2947 $        3
2948 $
2949 $ Notice that all points are numbered consecutively, unlikely DMPlexCreateFromCellList()
2950 
2951   Level: advanced
2952 
2953 .seealso: DMPlexCreateFromCellList(), DMPlexCreate()
2954 @*/
2955 PetscErrorCode DMPlexCreateFromDAG(DM dm, PetscInt depth, const PetscInt numPoints[], const PetscInt coneSize[], const PetscInt cones[], const PetscInt coneOrientations[], const PetscScalar vertexCoords[])
2956 {
2957   Vec            coordinates;
2958   PetscSection   coordSection;
2959   PetscScalar    *coords;
2960   PetscInt       coordSize, firstVertex = -1, pStart = 0, pEnd = 0, p, v, dim, dimEmbed, d, off;
2961   PetscErrorCode ierr;
2962 
2963   PetscFunctionBegin;
2964   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
2965   ierr = DMGetCoordinateDim(dm, &dimEmbed);CHKERRQ(ierr);
2966   if (dimEmbed < dim) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Embedding dimension %d cannot be less than intrinsic dimension %d",dimEmbed,dim);
2967   for (d = 0; d <= depth; ++d) pEnd += numPoints[d];
2968   ierr = DMPlexSetChart(dm, pStart, pEnd);CHKERRQ(ierr);
2969   for (p = pStart; p < pEnd; ++p) {
2970     ierr = DMPlexSetConeSize(dm, p, coneSize[p-pStart]);CHKERRQ(ierr);
2971     if (firstVertex < 0 && !coneSize[p - pStart]) {
2972       firstVertex = p - pStart;
2973     }
2974   }
2975   if (firstVertex < 0 && numPoints[0]) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONG,"Expected %d vertices but could not find any", numPoints[0]);
2976   ierr = DMSetUp(dm);CHKERRQ(ierr); /* Allocate space for cones */
2977   for (p = pStart, off = 0; p < pEnd; off += coneSize[p-pStart], ++p) {
2978     ierr = DMPlexSetCone(dm, p, &cones[off]);CHKERRQ(ierr);
2979     ierr = DMPlexSetConeOrientation(dm, p, &coneOrientations[off]);CHKERRQ(ierr);
2980   }
2981   ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
2982   ierr = DMPlexStratify(dm);CHKERRQ(ierr);
2983   /* Build coordinates */
2984   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
2985   ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
2986   ierr = PetscSectionSetFieldComponents(coordSection, 0, dimEmbed);CHKERRQ(ierr);
2987   ierr = PetscSectionSetChart(coordSection, firstVertex, firstVertex+numPoints[0]);CHKERRQ(ierr);
2988   for (v = firstVertex; v < firstVertex+numPoints[0]; ++v) {
2989     ierr = PetscSectionSetDof(coordSection, v, dimEmbed);CHKERRQ(ierr);
2990     ierr = PetscSectionSetFieldDof(coordSection, v, 0, dimEmbed);CHKERRQ(ierr);
2991   }
2992   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
2993   ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
2994   ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
2995   ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
2996   ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
2997   ierr = VecSetBlockSize(coordinates, dimEmbed);CHKERRQ(ierr);
2998   ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
2999   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
3000   for (v = 0; v < numPoints[0]; ++v) {
3001     PetscInt off;
3002 
3003     ierr = PetscSectionGetOffset(coordSection, v+firstVertex, &off);CHKERRQ(ierr);
3004     for (d = 0; d < dimEmbed; ++d) {
3005       coords[off+d] = vertexCoords[v*dimEmbed+d];
3006     }
3007   }
3008   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
3009   ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
3010   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
3011   PetscFunctionReturn(0);
3012 }
3013 
3014 /*@C
3015   DMPlexCreateCellVertexFromFile - Create a DMPlex mesh from a simple cell-vertex file.
3016 
3017 + comm        - The MPI communicator
3018 . filename    - Name of the .dat file
3019 - interpolate - Create faces and edges in the mesh
3020 
3021   Output Parameter:
3022 . dm  - The DM object representing the mesh
3023 
3024   Note: The format is the simplest possible:
3025 $ Ne
3026 $ v0 v1 ... vk
3027 $ Nv
3028 $ x y z marker
3029 
3030   Level: beginner
3031 
3032 .seealso: DMPlexCreateFromFile(), DMPlexCreateMedFromFile(), DMPlexCreateGmsh(), DMPlexCreate()
3033 @*/
3034 PetscErrorCode DMPlexCreateCellVertexFromFile(MPI_Comm comm, const char filename[], PetscBool interpolate, DM *dm)
3035 {
3036   DMLabel         marker;
3037   PetscViewer     viewer;
3038   Vec             coordinates;
3039   PetscSection    coordSection;
3040   PetscScalar    *coords;
3041   char            line[PETSC_MAX_PATH_LEN];
3042   PetscInt        dim = 3, cdim = 3, coordSize, v, c, d;
3043   PetscMPIInt     rank;
3044   int             snum, Nv, Nc;
3045   PetscErrorCode  ierr;
3046 
3047   PetscFunctionBegin;
3048   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
3049   ierr = PetscViewerCreate(comm, &viewer);CHKERRQ(ierr);
3050   ierr = PetscViewerSetType(viewer, PETSCVIEWERASCII);CHKERRQ(ierr);
3051   ierr = PetscViewerFileSetMode(viewer, FILE_MODE_READ);CHKERRQ(ierr);
3052   ierr = PetscViewerFileSetName(viewer, filename);CHKERRQ(ierr);
3053   if (!rank) {
3054     ierr = PetscViewerRead(viewer, line, 2, NULL, PETSC_STRING);CHKERRQ(ierr);
3055     snum = sscanf(line, "%d %d", &Nc, &Nv);
3056     if (snum != 2) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
3057   } else {
3058     Nc = Nv = 0;
3059   }
3060   ierr = DMCreate(comm, dm);CHKERRQ(ierr);
3061   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
3062   ierr = DMPlexSetChart(*dm, 0, Nc+Nv);CHKERRQ(ierr);
3063   ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
3064   ierr = DMSetCoordinateDim(*dm, cdim);CHKERRQ(ierr);
3065   /* Read topology */
3066   if (!rank) {
3067     PetscInt cone[8], corners = 8;
3068     int      vbuf[8], v;
3069 
3070     for (c = 0; c < Nc; ++c) {ierr = DMPlexSetConeSize(*dm, c, corners);CHKERRQ(ierr);}
3071     ierr = DMSetUp(*dm);CHKERRQ(ierr);
3072     for (c = 0; c < Nc; ++c) {
3073       ierr = PetscViewerRead(viewer, line, corners, NULL, PETSC_STRING);CHKERRQ(ierr);
3074       snum = sscanf(line, "%d %d %d %d %d %d %d %d", &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3], &vbuf[4], &vbuf[5], &vbuf[6], &vbuf[7]);
3075       if (snum != corners) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
3076       for (v = 0; v < corners; ++v) cone[v] = vbuf[v] + Nc;
3077       /* Hexahedra are inverted */
3078       {
3079         PetscInt tmp = cone[1];
3080         cone[1] = cone[3];
3081         cone[3] = tmp;
3082       }
3083       ierr = DMPlexSetCone(*dm, c, cone);CHKERRQ(ierr);
3084     }
3085   }
3086   ierr = DMPlexSymmetrize(*dm);CHKERRQ(ierr);
3087   ierr = DMPlexStratify(*dm);CHKERRQ(ierr);
3088   /* Read coordinates */
3089   ierr = DMGetCoordinateSection(*dm, &coordSection);CHKERRQ(ierr);
3090   ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
3091   ierr = PetscSectionSetFieldComponents(coordSection, 0, cdim);CHKERRQ(ierr);
3092   ierr = PetscSectionSetChart(coordSection, Nc, Nc + Nv);CHKERRQ(ierr);
3093   for (v = Nc; v < Nc+Nv; ++v) {
3094     ierr = PetscSectionSetDof(coordSection, v, cdim);CHKERRQ(ierr);
3095     ierr = PetscSectionSetFieldDof(coordSection, v, 0, cdim);CHKERRQ(ierr);
3096   }
3097   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
3098   ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
3099   ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
3100   ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
3101   ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
3102   ierr = VecSetBlockSize(coordinates, cdim);CHKERRQ(ierr);
3103   ierr = VecSetType(coordinates, VECSTANDARD);CHKERRQ(ierr);
3104   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
3105   if (!rank) {
3106     double x[3];
3107     int    val;
3108 
3109     ierr = DMCreateLabel(*dm, "marker");CHKERRQ(ierr);
3110     ierr = DMGetLabel(*dm, "marker", &marker);CHKERRQ(ierr);
3111     for (v = 0; v < Nv; ++v) {
3112       ierr = PetscViewerRead(viewer, line, 4, NULL, PETSC_STRING);CHKERRQ(ierr);
3113       snum = sscanf(line, "%lg %lg %lg %d", &x[0], &x[1], &x[2], &val);
3114       if (snum != 4) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
3115       for (d = 0; d < cdim; ++d) coords[v*cdim+d] = x[d];
3116       if (val) {ierr = DMLabelSetValue(marker, v+Nc, val);CHKERRQ(ierr);}
3117     }
3118   }
3119   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
3120   ierr = DMSetCoordinatesLocal(*dm, coordinates);CHKERRQ(ierr);
3121   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
3122   ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
3123   if (interpolate) {
3124     DM      idm;
3125     DMLabel bdlabel;
3126 
3127     ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
3128     ierr = DMDestroy(dm);CHKERRQ(ierr);
3129     *dm  = idm;
3130 
3131     ierr = DMGetLabel(*dm, "marker", &bdlabel);CHKERRQ(ierr);
3132     ierr = DMPlexMarkBoundaryFaces(*dm, PETSC_DETERMINE, bdlabel);CHKERRQ(ierr);
3133     ierr = DMPlexLabelComplete(*dm, bdlabel);CHKERRQ(ierr);
3134   }
3135   PetscFunctionReturn(0);
3136 }
3137 
3138 /*@C
3139   DMPlexCreateFromFile - This takes a filename and produces a DM
3140 
3141   Input Parameters:
3142 + comm - The communicator
3143 . filename - A file name
3144 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
3145 
3146   Output Parameter:
3147 . dm - The DM
3148 
3149   Options Database Keys:
3150 . -dm_plex_create_from_hdf5_xdmf - use the PETSC_VIEWER_HDF5_XDMF format for reading HDF5
3151 
3152   Level: beginner
3153 
3154 .seealso: DMPlexCreateFromDAG(), DMPlexCreateFromCellList(), DMPlexCreate()
3155 @*/
3156 PetscErrorCode DMPlexCreateFromFile(MPI_Comm comm, const char filename[], PetscBool interpolate, DM *dm)
3157 {
3158   const char    *extGmsh    = ".msh";
3159   const char    *extCGNS    = ".cgns";
3160   const char    *extExodus  = ".exo";
3161   const char    *extGenesis = ".gen";
3162   const char    *extFluent  = ".cas";
3163   const char    *extHDF5    = ".h5";
3164   const char    *extMed     = ".med";
3165   const char    *extPLY     = ".ply";
3166   const char    *extCV      = ".dat";
3167   size_t         len;
3168   PetscBool      isGmsh, isCGNS, isExodus, isGenesis, isFluent, isHDF5, isMed, isPLY, isCV;
3169   PetscMPIInt    rank;
3170   PetscErrorCode ierr;
3171 
3172   PetscFunctionBegin;
3173   PetscValidPointer(filename, 2);
3174   PetscValidPointer(dm, 4);
3175   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
3176   ierr = PetscStrlen(filename, &len);CHKERRQ(ierr);
3177   if (!len) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Filename must be a valid path");
3178   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)], extGmsh,    4, &isGmsh);CHKERRQ(ierr);
3179   ierr = PetscStrncmp(&filename[PetscMax(0,len-5)], extCGNS,    5, &isCGNS);CHKERRQ(ierr);
3180   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)], extExodus,  4, &isExodus);CHKERRQ(ierr);
3181   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)], extGenesis, 4, &isGenesis);CHKERRQ(ierr);
3182   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)], extFluent,  4, &isFluent);CHKERRQ(ierr);
3183   ierr = PetscStrncmp(&filename[PetscMax(0,len-3)], extHDF5,    3, &isHDF5);CHKERRQ(ierr);
3184   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)], extMed,     4, &isMed);CHKERRQ(ierr);
3185   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)], extPLY,     4, &isPLY);CHKERRQ(ierr);
3186   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)], extCV,      4, &isCV);CHKERRQ(ierr);
3187   if (isGmsh) {
3188     ierr = DMPlexCreateGmshFromFile(comm, filename, interpolate, dm);CHKERRQ(ierr);
3189   } else if (isCGNS) {
3190     ierr = DMPlexCreateCGNSFromFile(comm, filename, interpolate, dm);CHKERRQ(ierr);
3191   } else if (isExodus || isGenesis) {
3192     ierr = DMPlexCreateExodusFromFile(comm, filename, interpolate, dm);CHKERRQ(ierr);
3193   } else if (isFluent) {
3194     ierr = DMPlexCreateFluentFromFile(comm, filename, interpolate, dm);CHKERRQ(ierr);
3195   } else if (isHDF5) {
3196     PetscBool      load_hdf5_xdmf = PETSC_FALSE;
3197     PetscViewer viewer;
3198 
3199     ierr = PetscOptionsGetBool(NULL, NULL, "-dm_plex_create_from_hdf5_xdmf", &load_hdf5_xdmf, NULL);CHKERRQ(ierr);
3200     ierr = PetscViewerCreate(comm, &viewer);CHKERRQ(ierr);
3201     ierr = PetscViewerSetType(viewer, PETSCVIEWERHDF5);CHKERRQ(ierr);
3202     ierr = PetscViewerFileSetMode(viewer, FILE_MODE_READ);CHKERRQ(ierr);
3203     ierr = PetscViewerFileSetName(viewer, filename);CHKERRQ(ierr);
3204     ierr = DMCreate(comm, dm);CHKERRQ(ierr);
3205     ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
3206     if (load_hdf5_xdmf) {ierr = PetscViewerPushFormat(viewer, PETSC_VIEWER_HDF5_XDMF);CHKERRQ(ierr);}
3207     ierr = DMLoad(*dm, viewer);CHKERRQ(ierr);
3208     if (load_hdf5_xdmf) {ierr = PetscViewerPopFormat(viewer);CHKERRQ(ierr);}
3209     ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
3210 
3211     if (interpolate) {
3212       DM idm;
3213 
3214       ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
3215       ierr = DMDestroy(dm);CHKERRQ(ierr);
3216       *dm  = idm;
3217     }
3218   } else if (isMed) {
3219     ierr = DMPlexCreateMedFromFile(comm, filename, interpolate, dm);CHKERRQ(ierr);
3220   } else if (isPLY) {
3221     ierr = DMPlexCreatePLYFromFile(comm, filename, interpolate, dm);CHKERRQ(ierr);
3222   } else if (isCV) {
3223     ierr = DMPlexCreateCellVertexFromFile(comm, filename, interpolate, dm);CHKERRQ(ierr);
3224   } else SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cannot load file %s: unrecognized extension", filename);
3225   PetscFunctionReturn(0);
3226 }
3227 
3228 /*@
3229   DMPlexCreateReferenceCell - Create a DMPLEX with the appropriate FEM reference cell
3230 
3231   Collective on comm
3232 
3233   Input Parameters:
3234 + comm    - The communicator
3235 . dim     - The spatial dimension
3236 - simplex - Flag for simplex, otherwise use a tensor-product cell
3237 
3238   Output Parameter:
3239 . refdm - The reference cell
3240 
3241   Level: intermediate
3242 
3243 .keywords: reference cell
3244 .seealso:
3245 @*/
3246 PetscErrorCode DMPlexCreateReferenceCell(MPI_Comm comm, PetscInt dim, PetscBool simplex, DM *refdm)
3247 {
3248   DM             rdm;
3249   Vec            coords;
3250   PetscErrorCode ierr;
3251 
3252   PetscFunctionBeginUser;
3253   ierr = DMCreate(comm, &rdm);CHKERRQ(ierr);
3254   ierr = DMSetType(rdm, DMPLEX);CHKERRQ(ierr);
3255   ierr = DMSetDimension(rdm, dim);CHKERRQ(ierr);
3256   switch (dim) {
3257   case 0:
3258   {
3259     PetscInt    numPoints[1]        = {1};
3260     PetscInt    coneSize[1]         = {0};
3261     PetscInt    cones[1]            = {0};
3262     PetscInt    coneOrientations[1] = {0};
3263     PetscScalar vertexCoords[1]     = {0.0};
3264 
3265     ierr = DMPlexCreateFromDAG(rdm, 0, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
3266   }
3267   break;
3268   case 1:
3269   {
3270     PetscInt    numPoints[2]        = {2, 1};
3271     PetscInt    coneSize[3]         = {2, 0, 0};
3272     PetscInt    cones[2]            = {1, 2};
3273     PetscInt    coneOrientations[2] = {0, 0};
3274     PetscScalar vertexCoords[2]     = {-1.0,  1.0};
3275 
3276     ierr = DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
3277   }
3278   break;
3279   case 2:
3280     if (simplex) {
3281       PetscInt    numPoints[2]        = {3, 1};
3282       PetscInt    coneSize[4]         = {3, 0, 0, 0};
3283       PetscInt    cones[3]            = {1, 2, 3};
3284       PetscInt    coneOrientations[3] = {0, 0, 0};
3285       PetscScalar vertexCoords[6]     = {-1.0, -1.0,  1.0, -1.0,  -1.0, 1.0};
3286 
3287       ierr = DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
3288     } else {
3289       PetscInt    numPoints[2]        = {4, 1};
3290       PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
3291       PetscInt    cones[4]            = {1, 2, 3, 4};
3292       PetscInt    coneOrientations[4] = {0, 0, 0, 0};
3293       PetscScalar vertexCoords[8]     = {-1.0, -1.0,  1.0, -1.0,  1.0, 1.0,  -1.0, 1.0};
3294 
3295       ierr = DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
3296     }
3297   break;
3298   case 3:
3299     if (simplex) {
3300       PetscInt    numPoints[2]        = {4, 1};
3301       PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
3302       PetscInt    cones[4]            = {1, 3, 2, 4};
3303       PetscInt    coneOrientations[4] = {0, 0, 0, 0};
3304       PetscScalar vertexCoords[12]    = {-1.0, -1.0, -1.0,  1.0, -1.0, -1.0,  -1.0, 1.0, -1.0,  -1.0, -1.0, 1.0};
3305 
3306       ierr = DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
3307     } else {
3308       PetscInt    numPoints[2]        = {8, 1};
3309       PetscInt    coneSize[9]         = {8, 0, 0, 0, 0, 0, 0, 0, 0};
3310       PetscInt    cones[8]            = {1, 4, 3, 2, 5, 6, 7, 8};
3311       PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
3312       PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0,  1.0, -1.0, -1.0,  1.0, 1.0, -1.0,  -1.0, 1.0, -1.0,
3313                                          -1.0, -1.0,  1.0,  1.0, -1.0,  1.0,  1.0, 1.0,  1.0,  -1.0, 1.0,  1.0};
3314 
3315       ierr = DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
3316     }
3317   break;
3318   default:
3319     SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Cannot create reference cell for dimension %d", dim);
3320   }
3321   ierr = DMPlexInterpolate(rdm, refdm);CHKERRQ(ierr);
3322   if (rdm->coordinateDM) {
3323     DM           ncdm;
3324     PetscSection cs;
3325     PetscInt     pEnd = -1;
3326 
3327     ierr = DMGetDefaultSection(rdm->coordinateDM, &cs);CHKERRQ(ierr);
3328     if (cs) {ierr = PetscSectionGetChart(cs, NULL, &pEnd);CHKERRQ(ierr);}
3329     if (pEnd >= 0) {
3330       ierr = DMClone(rdm->coordinateDM, &ncdm);CHKERRQ(ierr);
3331       ierr = DMSetDefaultSection(ncdm, cs);CHKERRQ(ierr);
3332       ierr = DMSetCoordinateDM(*refdm, ncdm);CHKERRQ(ierr);
3333       ierr = DMDestroy(&ncdm);CHKERRQ(ierr);
3334     }
3335   }
3336   ierr = DMGetCoordinatesLocal(rdm, &coords);CHKERRQ(ierr);
3337   if (coords) {
3338     ierr = DMSetCoordinatesLocal(*refdm, coords);CHKERRQ(ierr);
3339   } else {
3340     ierr = DMGetCoordinates(rdm, &coords);CHKERRQ(ierr);
3341     if (coords) {ierr = DMSetCoordinates(*refdm, coords);CHKERRQ(ierr);}
3342   }
3343   ierr = DMDestroy(&rdm);CHKERRQ(ierr);
3344   PetscFunctionReturn(0);
3345 }
3346