xref: /petsc/src/dm/impls/plex/plexcreate.c (revision feff33ee0b5b037fa8f9f294dede656a2f85cc47)
1 #define PETSCDM_DLL
2 #include <petsc/private/dmpleximpl.h>    /*I   "petscdmplex.h"   I*/
3 #include <petscdmda.h>
4 #include <petscsf.h>
5 
6 /*@
7   DMPlexCreateDoublet - Creates a mesh of two cells of the specified type, optionally with later refinement.
8 
9   Collective on MPI_Comm
10 
11   Input Parameters:
12 + comm - The communicator for the DM object
13 . dim - The spatial dimension
14 . simplex - Flag for simplicial cells, otherwise they are tensor product cells
15 . interpolate - Flag to create intermediate mesh pieces (edges, faces)
16 . refinementUniform - Flag for uniform parallel refinement
17 - refinementLimit - A nonzero number indicates the largest admissible volume for a refined cell
18 
19   Output Parameter:
20 . dm - The DM object
21 
22   Level: beginner
23 
24 .keywords: DM, create
25 .seealso: DMSetType(), DMCreate()
26 @*/
27 PetscErrorCode DMPlexCreateDoublet(MPI_Comm comm, PetscInt dim, PetscBool simplex, PetscBool interpolate, PetscBool refinementUniform, PetscReal refinementLimit, DM *newdm)
28 {
29   DM             dm;
30   PetscInt       p;
31   PetscMPIInt    rank;
32   PetscErrorCode ierr;
33 
34   PetscFunctionBegin;
35   ierr = DMCreate(comm, &dm);CHKERRQ(ierr);
36   ierr = DMSetType(dm, DMPLEX);CHKERRQ(ierr);
37   ierr = DMSetDimension(dm, dim);CHKERRQ(ierr);
38   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
39   switch (dim) {
40   case 2:
41     if (simplex) {ierr = PetscObjectSetName((PetscObject) dm, "triangular");CHKERRQ(ierr);}
42     else         {ierr = PetscObjectSetName((PetscObject) dm, "quadrilateral");CHKERRQ(ierr);}
43     break;
44   case 3:
45     if (simplex) {ierr = PetscObjectSetName((PetscObject) dm, "tetrahedral");CHKERRQ(ierr);}
46     else         {ierr = PetscObjectSetName((PetscObject) dm, "hexahedral");CHKERRQ(ierr);}
47     break;
48   default:
49     SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make meshes for dimension %d", dim);
50   }
51   if (rank) {
52     PetscInt numPoints[2] = {0, 0};
53     ierr = DMPlexCreateFromDAG(dm, 1, numPoints, NULL, NULL, NULL, NULL);CHKERRQ(ierr);
54   } else {
55     switch (dim) {
56     case 2:
57       if (simplex) {
58         PetscInt    numPoints[2]        = {4, 2};
59         PetscInt    coneSize[6]         = {3, 3, 0, 0, 0, 0};
60         PetscInt    cones[6]            = {2, 3, 4,  5, 4, 3};
61         PetscInt    coneOrientations[6] = {0, 0, 0,  0, 0, 0};
62         PetscScalar vertexCoords[8]     = {-0.5, 0.5, 0.0, 0.0, 0.0, 1.0, 0.5, 0.5};
63         PetscInt    markerPoints[8]     = {2, 1, 3, 1, 4, 1, 5, 1};
64 
65         ierr = DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
66         for (p = 0; p < 4; ++p) {ierr = DMSetLabelValue(dm, "marker", markerPoints[p*2], markerPoints[p*2+1]);CHKERRQ(ierr);}
67       } else {
68         PetscInt    numPoints[2]        = {6, 2};
69         PetscInt    coneSize[8]         = {4, 4, 0, 0, 0, 0, 0, 0};
70         PetscInt    cones[8]            = {2, 3, 4, 5,  3, 6, 7, 4};
71         PetscInt    coneOrientations[8] = {0, 0, 0, 0,  0, 0, 0, 0};
72         PetscScalar vertexCoords[12]    = {-1.0, -0.5,  0.0, -0.5,  0.0, 0.5,  -1.0, 0.5,  1.0, -0.5,  1.0, 0.5};
73 
74         ierr = DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
75       }
76       break;
77     case 3:
78       if (simplex) {
79         PetscInt    numPoints[2]        = {5, 2};
80         PetscInt    coneSize[7]         = {4, 4, 0, 0, 0, 0, 0};
81         PetscInt    cones[8]            = {4, 3, 5, 2,  5, 3, 4, 6};
82         PetscInt    coneOrientations[8] = {0, 0, 0, 0,  0, 0, 0, 0};
83         PetscScalar vertexCoords[15]    = {-1.0, 0.0, 0.0,  0.0, -1.0, 0.0,  0.0, 0.0, 1.0,  0.0, 1.0, 0.0,  1.0, 0.0, 0.0};
84         PetscInt    markerPoints[10]    = {2, 1, 3, 1, 4, 1, 5, 1, 6, 1};
85 
86         ierr = DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
87         for (p = 0; p < 5; ++p) {ierr = DMSetLabelValue(dm, "marker", markerPoints[p*2], markerPoints[p*2+1]);CHKERRQ(ierr);}
88       } else {
89         PetscInt    numPoints[2]         = {12, 2};
90         PetscInt    coneSize[14]         = {8, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
91         PetscInt    cones[16]            = {2, 3, 4, 5, 6, 7, 8, 9,  5, 4, 10, 11, 7, 12, 13, 8};
92         PetscInt    coneOrientations[16] = {0, 0, 0, 0, 0, 0, 0, 0,  0, 0,  0,  0, 0,  0,  0, 0};
93         PetscScalar vertexCoords[36]     = {-1.0, -0.5, -0.5,  -1.0,  0.5, -0.5,  0.0,  0.5, -0.5,   0.0, -0.5, -0.5,
94                                             -1.0, -0.5,  0.5,   0.0, -0.5,  0.5,  0.0,  0.5,  0.5,  -1.0,  0.5,  0.5,
95                                              1.0,  0.5, -0.5,   1.0, -0.5, -0.5,  1.0, -0.5,  0.5,   1.0,  0.5,  0.5};
96 
97         ierr = DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
98       }
99       break;
100     default:
101       SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make meshes for dimension %d", dim);
102     }
103   }
104   *newdm = dm;
105   if (refinementLimit > 0.0) {
106     DM rdm;
107     const char *name;
108 
109     ierr = DMPlexSetRefinementUniform(*newdm, PETSC_FALSE);CHKERRQ(ierr);
110     ierr = DMPlexSetRefinementLimit(*newdm, refinementLimit);CHKERRQ(ierr);
111     ierr = DMRefine(*newdm, comm, &rdm);CHKERRQ(ierr);
112     ierr = PetscObjectGetName((PetscObject) *newdm, &name);CHKERRQ(ierr);
113     ierr = PetscObjectSetName((PetscObject)    rdm,  name);CHKERRQ(ierr);
114     ierr = DMDestroy(newdm);CHKERRQ(ierr);
115     *newdm = rdm;
116   }
117   if (interpolate) {
118     DM idm;
119 
120     ierr = DMPlexInterpolate(*newdm, &idm);CHKERRQ(ierr);
121     ierr = DMDestroy(newdm);CHKERRQ(ierr);
122     *newdm = idm;
123   }
124   {
125     DM refinedMesh     = NULL;
126     DM distributedMesh = NULL;
127 
128     /* Distribute mesh over processes */
129     ierr = DMPlexDistribute(*newdm, 0, NULL, &distributedMesh);CHKERRQ(ierr);
130     if (distributedMesh) {
131       ierr = DMDestroy(newdm);CHKERRQ(ierr);
132       *newdm = distributedMesh;
133     }
134     if (refinementUniform) {
135       ierr = DMPlexSetRefinementUniform(*newdm, refinementUniform);CHKERRQ(ierr);
136       ierr = DMRefine(*newdm, comm, &refinedMesh);CHKERRQ(ierr);
137       if (refinedMesh) {
138         ierr = DMDestroy(newdm);CHKERRQ(ierr);
139         *newdm = refinedMesh;
140       }
141     }
142   }
143   PetscFunctionReturn(0);
144 }
145 
146 /*@
147   DMPlexCreateSquareBoundary - Creates a 1D mesh the is the boundary of a square lattice.
148 
149   Collective on MPI_Comm
150 
151   Input Parameters:
152 + comm  - The communicator for the DM object
153 . lower - The lower left corner coordinates
154 . upper - The upper right corner coordinates
155 - edges - The number of cells in each direction
156 
157   Output Parameter:
158 . dm  - The DM object
159 
160   Note: Here is the numbering returned for 2 cells in each direction:
161 $ 18--5-17--4--16
162 $  |     |     |
163 $  6    10     3
164 $  |     |     |
165 $ 19-11-20--9--15
166 $  |     |     |
167 $  7     8     2
168 $  |     |     |
169 $ 12--0-13--1--14
170 
171   Level: beginner
172 
173 .keywords: DM, create
174 .seealso: DMPlexCreateBoxMesh(), DMPlexCreateCubeBoundary(), DMSetType(), DMCreate()
175 @*/
176 PetscErrorCode DMPlexCreateSquareBoundary(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[])
177 {
178   const PetscInt numVertices    = (edges[0]+1)*(edges[1]+1);
179   const PetscInt numEdges       = edges[0]*(edges[1]+1) + (edges[0]+1)*edges[1];
180   PetscInt       markerTop      = 1;
181   PetscInt       markerBottom   = 1;
182   PetscInt       markerRight    = 1;
183   PetscInt       markerLeft     = 1;
184   PetscBool      markerSeparate = PETSC_FALSE;
185   Vec            coordinates;
186   PetscSection   coordSection;
187   PetscScalar    *coords;
188   PetscInt       coordSize;
189   PetscMPIInt    rank;
190   PetscInt       v, vx, vy;
191   PetscErrorCode ierr;
192 
193   PetscFunctionBegin;
194   ierr = PetscOptionsGetBool(((PetscObject) dm)->options,((PetscObject) dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL);CHKERRQ(ierr);
195   if (markerSeparate) {
196     markerTop    = 3;
197     markerBottom = 1;
198     markerRight  = 2;
199     markerLeft   = 4;
200   }
201   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRQ(ierr);
202   if (!rank) {
203     PetscInt e, ex, ey;
204 
205     ierr = DMPlexSetChart(dm, 0, numEdges+numVertices);CHKERRQ(ierr);
206     for (e = 0; e < numEdges; ++e) {
207       ierr = DMPlexSetConeSize(dm, e, 2);CHKERRQ(ierr);
208     }
209     ierr = DMSetUp(dm);CHKERRQ(ierr); /* Allocate space for cones */
210     for (vx = 0; vx <= edges[0]; vx++) {
211       for (ey = 0; ey < edges[1]; ey++) {
212         PetscInt edge   = vx*edges[1] + ey + edges[0]*(edges[1]+1);
213         PetscInt vertex = ey*(edges[0]+1) + vx + numEdges;
214         PetscInt cone[2];
215 
216         cone[0] = vertex; cone[1] = vertex+edges[0]+1;
217         ierr    = DMPlexSetCone(dm, edge, cone);CHKERRQ(ierr);
218         if (vx == edges[0]) {
219           ierr = DMSetLabelValue(dm, "marker", edge,    markerRight);CHKERRQ(ierr);
220           ierr = DMSetLabelValue(dm, "marker", cone[0], markerRight);CHKERRQ(ierr);
221           if (ey == edges[1]-1) {
222             ierr = DMSetLabelValue(dm, "marker", cone[1], markerRight);CHKERRQ(ierr);
223             ierr = DMSetLabelValue(dm, "Face Sets", cone[1], markerRight);CHKERRQ(ierr);
224           }
225         } else if (vx == 0) {
226           ierr = DMSetLabelValue(dm, "marker", edge,    markerLeft);CHKERRQ(ierr);
227           ierr = DMSetLabelValue(dm, "marker", cone[0], markerLeft);CHKERRQ(ierr);
228           if (ey == edges[1]-1) {
229             ierr = DMSetLabelValue(dm, "marker", cone[1], markerLeft);CHKERRQ(ierr);
230             ierr = DMSetLabelValue(dm, "Face Sets", cone[1], markerLeft);CHKERRQ(ierr);
231           }
232         }
233       }
234     }
235     for (vy = 0; vy <= edges[1]; vy++) {
236       for (ex = 0; ex < edges[0]; ex++) {
237         PetscInt edge   = vy*edges[0]     + ex;
238         PetscInt vertex = vy*(edges[0]+1) + ex + numEdges;
239         PetscInt cone[2];
240 
241         cone[0] = vertex; cone[1] = vertex+1;
242         ierr    = DMPlexSetCone(dm, edge, cone);CHKERRQ(ierr);
243         if (vy == edges[1]) {
244           ierr = DMSetLabelValue(dm, "marker", edge,    markerTop);CHKERRQ(ierr);
245           ierr = DMSetLabelValue(dm, "marker", cone[0], markerTop);CHKERRQ(ierr);
246           if (ex == edges[0]-1) {
247             ierr = DMSetLabelValue(dm, "marker", cone[1], markerTop);CHKERRQ(ierr);
248             ierr = DMSetLabelValue(dm, "Face Sets", cone[1], markerTop);CHKERRQ(ierr);
249           }
250         } else if (vy == 0) {
251           ierr = DMSetLabelValue(dm, "marker", edge,    markerBottom);CHKERRQ(ierr);
252           ierr = DMSetLabelValue(dm, "marker", cone[0], markerBottom);CHKERRQ(ierr);
253           if (ex == edges[0]-1) {
254             ierr = DMSetLabelValue(dm, "marker", cone[1], markerBottom);CHKERRQ(ierr);
255             ierr = DMSetLabelValue(dm, "Face Sets", cone[1], markerBottom);CHKERRQ(ierr);
256           }
257         }
258       }
259     }
260   }
261   ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
262   ierr = DMPlexStratify(dm);CHKERRQ(ierr);
263   /* Build coordinates */
264   ierr = DMSetCoordinateDim(dm, 2);CHKERRQ(ierr);
265   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
266   ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
267   ierr = PetscSectionSetChart(coordSection, numEdges, numEdges + numVertices);CHKERRQ(ierr);
268   ierr = PetscSectionSetFieldComponents(coordSection, 0, 2);CHKERRQ(ierr);
269   for (v = numEdges; v < numEdges+numVertices; ++v) {
270     ierr = PetscSectionSetDof(coordSection, v, 2);CHKERRQ(ierr);
271     ierr = PetscSectionSetFieldDof(coordSection, v, 0, 2);CHKERRQ(ierr);
272   }
273   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
274   ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
275   ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
276   ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
277   ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
278   ierr = VecSetBlockSize(coordinates, 2);CHKERRQ(ierr);
279   ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
280   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
281   for (vy = 0; vy <= edges[1]; ++vy) {
282     for (vx = 0; vx <= edges[0]; ++vx) {
283       coords[(vy*(edges[0]+1)+vx)*2+0] = lower[0] + ((upper[0] - lower[0])/edges[0])*vx;
284       coords[(vy*(edges[0]+1)+vx)*2+1] = lower[1] + ((upper[1] - lower[1])/edges[1])*vy;
285     }
286   }
287   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
288   ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
289   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
290   PetscFunctionReturn(0);
291 }
292 
293 /*@
294   DMPlexCreateCubeBoundary - Creates a 2D mesh that is the boundary of a cubic lattice.
295 
296   Collective on MPI_Comm
297 
298   Input Parameters:
299 + comm  - The communicator for the DM object
300 . lower - The lower left front corner coordinates
301 . upper - The upper right back corner coordinates
302 - edges - The number of cells in each direction
303 
304   Output Parameter:
305 . dm  - The DM object
306 
307   Level: beginner
308 
309 .keywords: DM, create
310 .seealso: DMPlexCreateBoxMesh(), DMPlexCreateSquareBoundary(), DMSetType(), DMCreate()
311 @*/
312 PetscErrorCode DMPlexCreateCubeBoundary(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt faces[])
313 {
314   PetscInt       vertices[3], numVertices;
315   PetscInt       numFaces    = 2*faces[0]*faces[1] + 2*faces[1]*faces[2] + 2*faces[0]*faces[2];
316   Vec            coordinates;
317   PetscSection   coordSection;
318   PetscScalar    *coords;
319   PetscInt       coordSize;
320   PetscMPIInt    rank;
321   PetscInt       v, vx, vy, vz;
322   PetscInt       voffset, iface=0, cone[4];
323   PetscErrorCode ierr;
324 
325   PetscFunctionBegin;
326   if ((faces[0] < 1) || (faces[1] < 1) || (faces[2] < 1)) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Must have at least 1 face per side");
327   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRQ(ierr);
328   vertices[0] = faces[0]+1; vertices[1] = faces[1]+1; vertices[2] = faces[2]+1;
329   numVertices = vertices[0]*vertices[1]*vertices[2];
330   if (!rank) {
331     PetscInt f;
332 
333     ierr = DMPlexSetChart(dm, 0, numFaces+numVertices);CHKERRQ(ierr);
334     for (f = 0; f < numFaces; ++f) {
335       ierr = DMPlexSetConeSize(dm, f, 4);CHKERRQ(ierr);
336     }
337     ierr = DMSetUp(dm);CHKERRQ(ierr); /* Allocate space for cones */
338     for (v = 0; v < numFaces+numVertices; ++v) {
339       ierr = DMSetLabelValue(dm, "marker", v, 1);CHKERRQ(ierr);
340     }
341 
342     /* Side 0 (Top) */
343     for (vy = 0; vy < faces[1]; vy++) {
344       for (vx = 0; vx < faces[0]; vx++) {
345         voffset = numFaces + vertices[0]*vertices[1]*(vertices[2]-1) + vy*vertices[0] + vx;
346         cone[0] = voffset; cone[1] = voffset+1; cone[2] = voffset+vertices[0]+1; cone[3] = voffset+vertices[0];
347         ierr    = DMPlexSetCone(dm, iface, cone);CHKERRQ(ierr);
348         iface++;
349       }
350     }
351 
352     /* Side 1 (Bottom) */
353     for (vy = 0; vy < faces[1]; vy++) {
354       for (vx = 0; vx < faces[0]; vx++) {
355         voffset = numFaces + vy*(faces[0]+1) + vx;
356         cone[0] = voffset+1; cone[1] = voffset; cone[2] = voffset+vertices[0]; cone[3] = voffset+vertices[0]+1;
357         ierr    = DMPlexSetCone(dm, iface, cone);CHKERRQ(ierr);
358         iface++;
359       }
360     }
361 
362     /* Side 2 (Front) */
363     for (vz = 0; vz < faces[2]; vz++) {
364       for (vx = 0; vx < faces[0]; vx++) {
365         voffset = numFaces + vz*vertices[0]*vertices[1] + vx;
366         cone[0] = voffset; cone[1] = voffset+1; cone[2] = voffset+vertices[0]*vertices[1]+1; cone[3] = voffset+vertices[0]*vertices[1];
367         ierr    = DMPlexSetCone(dm, iface, cone);CHKERRQ(ierr);
368         iface++;
369       }
370     }
371 
372     /* Side 3 (Back) */
373     for (vz = 0; vz < faces[2]; vz++) {
374       for (vx = 0; vx < faces[0]; vx++) {
375         voffset = numFaces + vz*vertices[0]*vertices[1] + vertices[0]*(vertices[1]-1) + vx;
376         cone[0] = voffset+vertices[0]*vertices[1]; cone[1] = voffset+vertices[0]*vertices[1]+1;
377         cone[2] = voffset+1; cone[3] = voffset;
378         ierr    = DMPlexSetCone(dm, iface, cone);CHKERRQ(ierr);
379         iface++;
380       }
381     }
382 
383     /* Side 4 (Left) */
384     for (vz = 0; vz < faces[2]; vz++) {
385       for (vy = 0; vy < faces[1]; vy++) {
386         voffset = numFaces + vz*vertices[0]*vertices[1] + vy*vertices[0];
387         cone[0] = voffset; cone[1] = voffset+vertices[0]*vertices[1];
388         cone[2] = voffset+vertices[0]*vertices[1]+vertices[0]; cone[3] = voffset+vertices[0];
389         ierr    = DMPlexSetCone(dm, iface, cone);CHKERRQ(ierr);
390         iface++;
391       }
392     }
393 
394     /* Side 5 (Right) */
395     for (vz = 0; vz < faces[2]; vz++) {
396       for (vy = 0; vy < faces[1]; vy++) {
397         voffset = numFaces + vz*vertices[0]*vertices[1] + vy*vertices[0] + faces[0];
398         cone[0] = voffset+vertices[0]*vertices[1]; cone[1] = voffset;
399         cone[2] = voffset+vertices[0]; cone[3] = voffset+vertices[0]*vertices[1]+vertices[0];
400         ierr    = DMPlexSetCone(dm, iface, cone);CHKERRQ(ierr);
401         iface++;
402       }
403     }
404   }
405   ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
406   ierr = DMPlexStratify(dm);CHKERRQ(ierr);
407   /* Build coordinates */
408   ierr = DMSetCoordinateDim(dm, 3);CHKERRQ(ierr);
409   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
410   ierr = PetscSectionSetChart(coordSection, numFaces, numFaces + numVertices);CHKERRQ(ierr);
411   for (v = numFaces; v < numFaces+numVertices; ++v) {
412     ierr = PetscSectionSetDof(coordSection, v, 3);CHKERRQ(ierr);
413   }
414   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
415   ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
416   ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
417   ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
418   ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
419   ierr = VecSetBlockSize(coordinates, 3);CHKERRQ(ierr);
420   ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
421   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
422   for (vz = 0; vz <= faces[2]; ++vz) {
423     for (vy = 0; vy <= faces[1]; ++vy) {
424       for (vx = 0; vx <= faces[0]; ++vx) {
425         coords[((vz*(faces[1]+1)+vy)*(faces[0]+1)+vx)*3+0] = lower[0] + ((upper[0] - lower[0])/faces[0])*vx;
426         coords[((vz*(faces[1]+1)+vy)*(faces[0]+1)+vx)*3+1] = lower[1] + ((upper[1] - lower[1])/faces[1])*vy;
427         coords[((vz*(faces[1]+1)+vy)*(faces[0]+1)+vx)*3+2] = lower[2] + ((upper[2] - lower[2])/faces[2])*vz;
428       }
429     }
430   }
431   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
432   ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
433   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
434   PetscFunctionReturn(0);
435 }
436 
437 static PetscErrorCode DMPlexCreateLineMesh_Internal(MPI_Comm comm,PetscInt segments,PetscReal lower,PetscReal upper,DMBoundaryType bd,DM *dm)
438 {
439   PetscInt       i,fStart,fEnd,numCells = 0,numVerts = 0;
440   PetscInt       numPoints[2],*coneSize,*cones,*coneOrientations;
441   PetscScalar    *vertexCoords;
442   PetscReal      L,maxCell;
443   PetscBool      markerSeparate = PETSC_FALSE;
444   PetscInt       markerLeft  = 1, faceMarkerLeft  = 1;
445   PetscInt       markerRight = 1, faceMarkerRight = 2;
446   PetscBool      wrap = (bd == DM_BOUNDARY_PERIODIC || bd == DM_BOUNDARY_TWIST) ? PETSC_TRUE : PETSC_FALSE;
447   PetscMPIInt    rank;
448   PetscErrorCode ierr;
449 
450   PetscFunctionBegin;
451   PetscValidPointer(dm,4);
452 
453   ierr = DMCreate(comm,dm);CHKERRQ(ierr);
454   ierr = DMSetType(*dm,DMPLEX);CHKERRQ(ierr);
455   ierr = DMSetDimension(*dm,1);CHKERRQ(ierr);
456   ierr = DMCreateLabel(*dm,"marker");CHKERRQ(ierr);
457   ierr = DMCreateLabel(*dm,"Face Sets");CHKERRQ(ierr);
458 
459   ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr);
460   if (!rank) numCells = segments;
461   if (!rank) numVerts = segments + (wrap ? 0 : 1);
462 
463   numPoints[0] = numVerts ; numPoints[1] = numCells;
464   ierr = PetscMalloc4(numCells+numVerts,&coneSize,numCells*2,&cones,numCells+numVerts,&coneOrientations,numVerts,&vertexCoords);CHKERRQ(ierr);
465   ierr = PetscMemzero(coneOrientations,(numCells+numVerts)*sizeof(PetscInt));CHKERRQ(ierr);
466   for (i = 0; i < numCells; ++i) { coneSize[i] = 2; }
467   for (i = 0; i < numVerts; ++i) { coneSize[numCells+i] = 0; }
468   for (i = 0; i < numCells; ++i) { cones[2*i] = numCells + i%numVerts; cones[2*i+1] = numCells + (i+1)%numVerts; }
469   for (i = 0; i < numVerts; ++i) { vertexCoords[i] = lower + (upper-lower)*((PetscReal)i/(PetscReal)numCells); }
470   ierr = DMPlexCreateFromDAG(*dm,1,numPoints,coneSize,cones,coneOrientations,vertexCoords);CHKERRQ(ierr);
471   ierr = PetscFree4(coneSize,cones,coneOrientations,vertexCoords);CHKERRQ(ierr);
472 
473   ierr = PetscOptionsGetBool(((PetscObject)*dm)->options,((PetscObject)*dm)->prefix,"-dm_plex_separate_marker",&markerSeparate,NULL);CHKERRQ(ierr);
474   if (markerSeparate) { markerLeft = faceMarkerLeft; markerRight = faceMarkerRight;}
475   if (!wrap && !rank) {
476     ierr = DMPlexGetHeightStratum(*dm,1,&fStart,&fEnd);CHKERRQ(ierr);
477     ierr = DMSetLabelValue(*dm,"marker",fStart,markerLeft);CHKERRQ(ierr);
478     ierr = DMSetLabelValue(*dm,"marker",fEnd-1,markerRight);CHKERRQ(ierr);
479     ierr = DMSetLabelValue(*dm,"Face Sets",fStart,faceMarkerLeft);CHKERRQ(ierr);
480     ierr = DMSetLabelValue(*dm,"Face Sets",fEnd-1,faceMarkerRight);CHKERRQ(ierr);
481   }
482   if (wrap) {
483     L       = upper - lower;
484     maxCell = (PetscReal)1.1*(L/(PetscReal)PetscMax(1,segments));
485     ierr = DMSetPeriodicity(*dm,PETSC_TRUE,&maxCell,&L,&bd);CHKERRQ(ierr);
486   }
487   PetscFunctionReturn(0);
488 }
489 
490 static PetscErrorCode DMPlexCreateBoxMesh_Simplex_Internal(MPI_Comm comm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate, DM *dm)
491 {
492   DM             boundary;
493   PetscInt       i;
494   PetscErrorCode ierr;
495 
496   PetscFunctionBegin;
497   PetscValidPointer(dm, 4);
498   for (i = 0; i < dim; ++i) if (periodicity[i] != DM_BOUNDARY_NONE) SETERRQ(comm, PETSC_ERR_SUP, "Periodicity is not supported for simplex meshes");
499   ierr = DMCreate(comm, &boundary);CHKERRQ(ierr);
500   PetscValidLogicalCollectiveInt(boundary,dim,2);
501   ierr = DMSetType(boundary, DMPLEX);CHKERRQ(ierr);
502   ierr = DMSetDimension(boundary, dim-1);CHKERRQ(ierr);
503   ierr = DMSetCoordinateDim(boundary, dim);CHKERRQ(ierr);
504   switch (dim) {
505   case 2: ierr = DMPlexCreateSquareBoundary(boundary, lower, upper, faces);CHKERRQ(ierr);break;
506   case 3: ierr = DMPlexCreateCubeBoundary(boundary, lower, upper, faces);CHKERRQ(ierr);break;
507   default: SETERRQ1(comm, PETSC_ERR_SUP, "Dimension not supported: %d", dim);
508   }
509   ierr = DMPlexGenerate(boundary, NULL, interpolate, dm);CHKERRQ(ierr);
510   ierr = DMDestroy(&boundary);CHKERRQ(ierr);
511   PetscFunctionReturn(0);
512 }
513 
514 static PetscErrorCode DMPlexCreateCubeMesh_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[], DMBoundaryType bdX, DMBoundaryType bdY, DMBoundaryType bdZ)
515 {
516   DMLabel        cutLabel = NULL;
517   PetscInt       markerTop      = 1, faceMarkerTop      = 1;
518   PetscInt       markerBottom   = 1, faceMarkerBottom   = 1;
519   PetscInt       markerFront    = 1, faceMarkerFront    = 1;
520   PetscInt       markerBack     = 1, faceMarkerBack     = 1;
521   PetscInt       markerRight    = 1, faceMarkerRight    = 1;
522   PetscInt       markerLeft     = 1, faceMarkerLeft     = 1;
523   PetscInt       dim;
524   PetscBool      markerSeparate = PETSC_FALSE, cutMarker = PETSC_FALSE;
525   PetscMPIInt    rank;
526   PetscErrorCode ierr;
527 
528   PetscFunctionBegin;
529   ierr = DMGetDimension(dm,&dim);CHKERRQ(ierr);
530   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRQ(ierr);
531   ierr = DMCreateLabel(dm,"marker");CHKERRQ(ierr);
532   ierr = DMCreateLabel(dm,"Face Sets");CHKERRQ(ierr);
533   ierr = PetscOptionsGetBool(((PetscObject) dm)->options,((PetscObject) dm)->prefix, "-dm_plex_periodic_cut", &cutMarker, NULL);CHKERRQ(ierr);
534   if (bdX == DM_BOUNDARY_PERIODIC || bdX == DM_BOUNDARY_TWIST ||
535       bdY == DM_BOUNDARY_PERIODIC || bdY == DM_BOUNDARY_TWIST ||
536       bdZ == DM_BOUNDARY_PERIODIC || bdZ == DM_BOUNDARY_TWIST) {
537 
538     if (cutMarker) {ierr = DMCreateLabel(dm, "periodic_cut");CHKERRQ(ierr); ierr = DMGetLabel(dm, "periodic_cut", &cutLabel);CHKERRQ(ierr);}
539   }
540   switch (dim) {
541   case 2:
542     faceMarkerTop    = 3;
543     faceMarkerBottom = 1;
544     faceMarkerRight  = 2;
545     faceMarkerLeft   = 4;
546     break;
547   case 3:
548     faceMarkerBottom = 1;
549     faceMarkerTop    = 2;
550     faceMarkerFront  = 3;
551     faceMarkerBack   = 4;
552     faceMarkerRight  = 5;
553     faceMarkerLeft   = 6;
554     break;
555   default:
556     SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SUP,"Dimension %d not supported",dim);
557     break;
558   }
559   ierr = PetscOptionsGetBool(((PetscObject) dm)->options,((PetscObject) dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL);CHKERRQ(ierr);
560   if (markerSeparate) {
561     markerBottom = faceMarkerBottom;
562     markerTop    = faceMarkerTop;
563     markerFront  = faceMarkerFront;
564     markerBack   = faceMarkerBack;
565     markerRight  = faceMarkerRight;
566     markerLeft   = faceMarkerLeft;
567   }
568   {
569     const PetscInt numXEdges    = !rank ? edges[0] : 0;
570     const PetscInt numYEdges    = !rank ? edges[1] : 0;
571     const PetscInt numZEdges    = !rank ? edges[2] : 0;
572     const PetscInt numXVertices = !rank ? (bdX == DM_BOUNDARY_PERIODIC || bdX == DM_BOUNDARY_TWIST ? edges[0] : edges[0]+1) : 0;
573     const PetscInt numYVertices = !rank ? (bdY == DM_BOUNDARY_PERIODIC || bdY == DM_BOUNDARY_TWIST ? edges[1] : edges[1]+1) : 0;
574     const PetscInt numZVertices = !rank ? (bdZ == DM_BOUNDARY_PERIODIC || bdZ == DM_BOUNDARY_TWIST ? edges[2] : edges[2]+1) : 0;
575     const PetscInt numCells     = numXEdges*numYEdges*numZEdges;
576     const PetscInt numXFaces    = numYEdges*numZEdges;
577     const PetscInt numYFaces    = numXEdges*numZEdges;
578     const PetscInt numZFaces    = numXEdges*numYEdges;
579     const PetscInt numTotXFaces = numXVertices*numXFaces;
580     const PetscInt numTotYFaces = numYVertices*numYFaces;
581     const PetscInt numTotZFaces = numZVertices*numZFaces;
582     const PetscInt numFaces     = numTotXFaces + numTotYFaces + numTotZFaces;
583     const PetscInt numTotXEdges = numXEdges*numYVertices*numZVertices;
584     const PetscInt numTotYEdges = numYEdges*numXVertices*numZVertices;
585     const PetscInt numTotZEdges = numZEdges*numXVertices*numYVertices;
586     const PetscInt numVertices  = numXVertices*numYVertices*numZVertices;
587     const PetscInt numEdges     = numTotXEdges + numTotYEdges + numTotZEdges;
588     const PetscInt firstVertex  = (dim == 2) ? numFaces : numCells;
589     const PetscInt firstXFace   = (dim == 2) ? 0 : numCells + numVertices;
590     const PetscInt firstYFace   = firstXFace + numTotXFaces;
591     const PetscInt firstZFace   = firstYFace + numTotYFaces;
592     const PetscInt firstXEdge   = numCells + numFaces + numVertices;
593     const PetscInt firstYEdge   = firstXEdge + numTotXEdges;
594     const PetscInt firstZEdge   = firstYEdge + numTotYEdges;
595     Vec            coordinates;
596     PetscSection   coordSection;
597     PetscScalar   *coords;
598     PetscInt       coordSize;
599     PetscInt       v, vx, vy, vz;
600     PetscInt       c, f, fx, fy, fz, e, ex, ey, ez;
601 
602     ierr = DMPlexSetChart(dm, 0, numCells+numFaces+numEdges+numVertices);CHKERRQ(ierr);
603     for (c = 0; c < numCells; c++) {
604       ierr = DMPlexSetConeSize(dm, c, 6);CHKERRQ(ierr);
605     }
606     for (f = firstXFace; f < firstXFace+numFaces; ++f) {
607       ierr = DMPlexSetConeSize(dm, f, 4);CHKERRQ(ierr);
608     }
609     for (e = firstXEdge; e < firstXEdge+numEdges; ++e) {
610       ierr = DMPlexSetConeSize(dm, e, 2);CHKERRQ(ierr);
611     }
612     ierr = DMSetUp(dm);CHKERRQ(ierr); /* Allocate space for cones */
613     /* Build cells */
614     for (fz = 0; fz < numZEdges; ++fz) {
615       for (fy = 0; fy < numYEdges; ++fy) {
616         for (fx = 0; fx < numXEdges; ++fx) {
617           PetscInt cell    = (fz*numYEdges + fy)*numXEdges + fx;
618           PetscInt faceB   = firstZFace + (fy*numXEdges+fx)*numZVertices +   fz;
619           PetscInt faceT   = firstZFace + (fy*numXEdges+fx)*numZVertices + ((fz+1)%numZVertices);
620           PetscInt faceF   = firstYFace + (fz*numXEdges+fx)*numYVertices +   fy;
621           PetscInt faceK   = firstYFace + (fz*numXEdges+fx)*numYVertices + ((fy+1)%numYVertices);
622           PetscInt faceL   = firstXFace + (fz*numYEdges+fy)*numXVertices +   fx;
623           PetscInt faceR   = firstXFace + (fz*numYEdges+fy)*numXVertices + ((fx+1)%numXVertices);
624                             /* B,  T,  F,  K,  R,  L */
625           PetscInt ornt[6] = {-4,  0,  0, -1,  0, -4}; /* ??? */
626           PetscInt cone[6];
627 
628           /* no boundary twisting in 3D */
629           cone[0] = faceB; cone[1] = faceT; cone[2] = faceF; cone[3] = faceK; cone[4] = faceR; cone[5] = faceL;
630           ierr    = DMPlexSetCone(dm, cell, cone);CHKERRQ(ierr);
631           ierr    = DMPlexSetConeOrientation(dm, cell, ornt);CHKERRQ(ierr);
632           if (bdX != DM_BOUNDARY_NONE && fx == numXEdges-1 && cutLabel) {ierr = DMLabelSetValue(cutLabel, cell, 2);CHKERRQ(ierr);}
633           if (bdY != DM_BOUNDARY_NONE && fy == numYEdges-1 && cutLabel) {ierr = DMLabelSetValue(cutLabel, cell, 2);CHKERRQ(ierr);}
634           if (bdZ != DM_BOUNDARY_NONE && fz == numZEdges-1 && cutLabel) {ierr = DMLabelSetValue(cutLabel, cell, 2);CHKERRQ(ierr);}
635         }
636       }
637     }
638     /* Build x faces */
639     for (fz = 0; fz < numZEdges; ++fz) {
640       for (fy = 0; fy < numYEdges; ++fy) {
641         for (fx = 0; fx < numXVertices; ++fx) {
642           PetscInt face    = firstXFace + (fz*numYEdges+fy)*numXVertices + fx;
643           PetscInt edgeL   = firstZEdge + (  fy*                 numXVertices+fx)*numZEdges + fz;
644           PetscInt edgeR   = firstZEdge + (((fy+1)%numYVertices)*numXVertices+fx)*numZEdges + fz;
645           PetscInt edgeB   = firstYEdge + (  fz*                 numXVertices+fx)*numYEdges + fy;
646           PetscInt edgeT   = firstYEdge + (((fz+1)%numZVertices)*numXVertices+fx)*numYEdges + fy;
647           PetscInt ornt[4] = {0, 0, -2, -2};
648           PetscInt cone[4];
649 
650           if (dim == 3) {
651             /* markers */
652             if (bdX != DM_BOUNDARY_PERIODIC) {
653               if (fx == numXVertices-1) {
654                 ierr = DMSetLabelValue(dm, "Face Sets", face, faceMarkerRight);CHKERRQ(ierr);
655                 ierr = DMSetLabelValue(dm, "marker", face, markerRight);CHKERRQ(ierr);
656               }
657               else if (fx == 0) {
658                 ierr = DMSetLabelValue(dm, "Face Sets", face, faceMarkerLeft);CHKERRQ(ierr);
659                 ierr = DMSetLabelValue(dm, "marker", face, markerLeft);CHKERRQ(ierr);
660               }
661             }
662           }
663           cone[0] = edgeB; cone[1] = edgeR; cone[2] = edgeT; cone[3] = edgeL;
664           ierr    = DMPlexSetCone(dm, face, cone);CHKERRQ(ierr);
665           ierr    = DMPlexSetConeOrientation(dm, face, ornt);CHKERRQ(ierr);
666         }
667       }
668     }
669     /* Build y faces */
670     for (fz = 0; fz < numZEdges; ++fz) {
671       for (fx = 0; fx < numXEdges; ++fx) {
672         for (fy = 0; fy < numYVertices; ++fy) {
673           PetscInt face    = firstYFace + (fz*numXEdges+fx)*numYVertices + fy;
674           PetscInt edgeL   = firstZEdge + (fy*numXVertices+  fx                 )*numZEdges + fz;
675           PetscInt edgeR   = firstZEdge + (fy*numXVertices+((fx+1)%numXVertices))*numZEdges + fz;
676           PetscInt edgeB   = firstXEdge + (  fz                 *numYVertices+fy)*numXEdges + fx;
677           PetscInt edgeT   = firstXEdge + (((fz+1)%numZVertices)*numYVertices+fy)*numXEdges + fx;
678           PetscInt ornt[4] = {0, 0, -2, -2};
679           PetscInt cone[4];
680 
681           if (dim == 3) {
682             /* markers */
683             if (bdY != DM_BOUNDARY_PERIODIC) {
684               if (fy == numYVertices-1) {
685                 ierr = DMSetLabelValue(dm, "Face Sets", face, faceMarkerBack);CHKERRQ(ierr);
686                 ierr = DMSetLabelValue(dm, "marker", face, markerBack);CHKERRQ(ierr);
687               }
688               else if (fy == 0) {
689                 ierr = DMSetLabelValue(dm, "Face Sets", face, faceMarkerFront);CHKERRQ(ierr);
690                 ierr = DMSetLabelValue(dm, "marker", face, markerFront);CHKERRQ(ierr);
691               }
692             }
693           }
694           cone[0] = edgeB; cone[1] = edgeR; cone[2] = edgeT; cone[3] = edgeL;
695           ierr    = DMPlexSetCone(dm, face, cone);CHKERRQ(ierr);
696           ierr    = DMPlexSetConeOrientation(dm, face, ornt);CHKERRQ(ierr);
697         }
698       }
699     }
700     /* Build z faces */
701     for (fy = 0; fy < numYEdges; ++fy) {
702       for (fx = 0; fx < numXEdges; ++fx) {
703         for (fz = 0; fz < numZVertices; fz++) {
704           PetscInt face    = firstZFace + (fy*numXEdges+fx)*numZVertices + fz;
705           PetscInt edgeL   = firstYEdge + (fz*numXVertices+  fx                 )*numYEdges + fy;
706           PetscInt edgeR   = firstYEdge + (fz*numXVertices+((fx+1)%numXVertices))*numYEdges + fy;
707           PetscInt edgeB   = firstXEdge + (fz*numYVertices+  fy                 )*numXEdges + fx;
708           PetscInt edgeT   = firstXEdge + (fz*numYVertices+((fy+1)%numYVertices))*numXEdges + fx;
709           PetscInt ornt[4] = {0, 0, -2, -2};
710           PetscInt cone[4];
711 
712           if (dim == 2) {
713             if (bdX == DM_BOUNDARY_TWIST && fx == numXEdges-1) {edgeR += numYEdges-1-2*fy; ornt[1] = -2;}
714             if (bdY == DM_BOUNDARY_TWIST && fy == numYEdges-1) {edgeT += numXEdges-1-2*fx; ornt[2] =  0;}
715             if (bdX != DM_BOUNDARY_NONE && fx == numXEdges-1 && cutLabel) {ierr = DMLabelSetValue(cutLabel, face, 2);CHKERRQ(ierr);}
716             if (bdY != DM_BOUNDARY_NONE && fy == numYEdges-1 && cutLabel) {ierr = DMLabelSetValue(cutLabel, face, 2);CHKERRQ(ierr);}
717           } else {
718             /* markers */
719             if (bdZ != DM_BOUNDARY_PERIODIC) {
720               if (fz == numZVertices-1) {
721                 ierr = DMSetLabelValue(dm, "Face Sets", face, faceMarkerTop);CHKERRQ(ierr);
722                 ierr = DMSetLabelValue(dm, "marker", face, markerTop);CHKERRQ(ierr);
723               }
724               else if (fz == 0) {
725                 ierr = DMSetLabelValue(dm, "Face Sets", face, faceMarkerBottom);CHKERRQ(ierr);
726                 ierr = DMSetLabelValue(dm, "marker", face, markerBottom);CHKERRQ(ierr);
727               }
728             }
729           }
730           cone[0] = edgeB; cone[1] = edgeR; cone[2] = edgeT; cone[3] = edgeL;
731           ierr    = DMPlexSetCone(dm, face, cone);CHKERRQ(ierr);
732           ierr    = DMPlexSetConeOrientation(dm, face, ornt);CHKERRQ(ierr);
733         }
734       }
735     }
736     /* Build Z edges*/
737     for (vy = 0; vy < numYVertices; vy++) {
738       for (vx = 0; vx < numXVertices; vx++) {
739         for (ez = 0; ez < numZEdges; ez++) {
740           const PetscInt edge    = firstZEdge  + (vy*numXVertices+vx)*numZEdges + ez;
741           const PetscInt vertexB = firstVertex + (  ez                 *numYVertices+vy)*numXVertices + vx;
742           const PetscInt vertexT = firstVertex + (((ez+1)%numZVertices)*numYVertices+vy)*numXVertices + vx;
743           PetscInt       cone[2];
744 
745           if (dim == 3) {
746             if (bdX != DM_BOUNDARY_PERIODIC) {
747               if (vx == numXVertices-1) {
748                 ierr = DMSetLabelValue(dm, "marker", edge, markerRight);CHKERRQ(ierr);
749               }
750               else if (vx == 0) {
751                 ierr = DMSetLabelValue(dm, "marker", edge, markerLeft);CHKERRQ(ierr);
752               }
753             }
754             if (bdY != DM_BOUNDARY_PERIODIC) {
755               if (vy == numYVertices-1) {
756                 ierr = DMSetLabelValue(dm, "marker", edge, markerBack);CHKERRQ(ierr);
757               }
758               else if (vy == 0) {
759                 ierr = DMSetLabelValue(dm, "marker", edge, markerFront);CHKERRQ(ierr);
760               }
761             }
762           }
763           cone[0] = vertexB; cone[1] = vertexT;
764           ierr = DMPlexSetCone(dm, edge, cone);CHKERRQ(ierr);
765         }
766       }
767     }
768     /* Build Y edges*/
769     for (vz = 0; vz < numZVertices; vz++) {
770       for (vx = 0; vx < numXVertices; vx++) {
771         for (ey = 0; ey < numYEdges; ey++) {
772           const PetscInt nextv   = (dim == 2 && bdY == DM_BOUNDARY_TWIST && ey == numYEdges-1) ? (numXVertices-vx-1) : (vz*numYVertices+((ey+1)%numYVertices))*numXVertices + vx;
773           const PetscInt edge    = firstYEdge  + (vz*numXVertices+vx)*numYEdges + ey;
774           const PetscInt vertexF = firstVertex + (vz*numYVertices+ey)*numXVertices + vx;
775           const PetscInt vertexK = firstVertex + nextv;
776           PetscInt       cone[2];
777 
778           cone[0] = vertexF; cone[1] = vertexK;
779           ierr = DMPlexSetCone(dm, edge, cone);CHKERRQ(ierr);
780           if (dim == 2) {
781             if ((bdX != DM_BOUNDARY_PERIODIC) && (bdX != DM_BOUNDARY_TWIST)) {
782               if (vx == numXVertices-1) {
783                 ierr = DMSetLabelValue(dm, "Face Sets", edge, faceMarkerRight);CHKERRQ(ierr);
784                 ierr = DMSetLabelValue(dm, "marker", edge,    markerRight);CHKERRQ(ierr);
785                 ierr = DMSetLabelValue(dm, "marker", cone[0], markerRight);CHKERRQ(ierr);
786                 if (ey == numYEdges-1) {
787                   ierr = DMSetLabelValue(dm, "marker", cone[1], markerRight);CHKERRQ(ierr);
788                 }
789               } else if (vx == 0) {
790                 ierr = DMSetLabelValue(dm, "Face Sets", edge, faceMarkerLeft);CHKERRQ(ierr);
791                 ierr = DMSetLabelValue(dm, "marker", edge,    markerLeft);CHKERRQ(ierr);
792                 ierr = DMSetLabelValue(dm, "marker", cone[0], markerLeft);CHKERRQ(ierr);
793                 if (ey == numYEdges-1) {
794                   ierr = DMSetLabelValue(dm, "marker", cone[1], markerLeft);CHKERRQ(ierr);
795                 }
796               }
797             } else {
798               if (vx == 0 && cutLabel) {
799                 ierr = DMLabelSetValue(cutLabel, edge,    1);CHKERRQ(ierr);
800                 ierr = DMLabelSetValue(cutLabel, cone[0], 1);CHKERRQ(ierr);
801                 if (ey == numYEdges-1) {
802                   ierr = DMLabelSetValue(cutLabel, cone[1], 1);CHKERRQ(ierr);
803                 }
804               }
805             }
806           } else {
807             if (bdX != DM_BOUNDARY_PERIODIC) {
808               if (vx == numXVertices-1) {
809                 ierr = DMSetLabelValue(dm, "marker", edge, markerRight);CHKERRQ(ierr);
810               } else if (vx == 0) {
811                 ierr = DMSetLabelValue(dm, "marker", edge, markerLeft);CHKERRQ(ierr);
812               }
813             }
814             if (bdZ != DM_BOUNDARY_PERIODIC) {
815               if (vz == numZVertices-1) {
816                 ierr = DMSetLabelValue(dm, "marker", edge, markerTop);CHKERRQ(ierr);
817               } else if (vz == 0) {
818                 ierr = DMSetLabelValue(dm, "marker", edge, markerBottom);CHKERRQ(ierr);
819               }
820             }
821           }
822         }
823       }
824     }
825     /* Build X edges*/
826     for (vz = 0; vz < numZVertices; vz++) {
827       for (vy = 0; vy < numYVertices; vy++) {
828         for (ex = 0; ex < numXEdges; ex++) {
829           const PetscInt nextv   = (dim == 2 && bdX == DM_BOUNDARY_TWIST && ex == numXEdges-1) ? (numYVertices-vy-1)*numXVertices : (vz*numYVertices+vy)*numXVertices + (ex+1)%numXVertices;
830           const PetscInt edge    = firstXEdge  + (vz*numYVertices+vy)*numXEdges + ex;
831           const PetscInt vertexL = firstVertex + (vz*numYVertices+vy)*numXVertices + ex;
832           const PetscInt vertexR = firstVertex + nextv;
833           PetscInt       cone[2];
834 
835           cone[0] = vertexL; cone[1] = vertexR;
836           ierr = DMPlexSetCone(dm, edge, cone);CHKERRQ(ierr);
837           if (dim == 2) {
838             if ((bdY != DM_BOUNDARY_PERIODIC) && (bdY != DM_BOUNDARY_TWIST)) {
839               if (vy == numYVertices-1) {
840                 ierr = DMSetLabelValue(dm, "Face Sets", edge, faceMarkerTop);CHKERRQ(ierr);
841                 ierr = DMSetLabelValue(dm, "marker", edge,    markerTop);CHKERRQ(ierr);
842                 ierr = DMSetLabelValue(dm, "marker", cone[0], markerTop);CHKERRQ(ierr);
843                 if (ex == numXEdges-1) {
844                   ierr = DMSetLabelValue(dm, "marker", cone[1], markerTop);CHKERRQ(ierr);
845                 }
846               } else if (vy == 0) {
847                 ierr = DMSetLabelValue(dm, "Face Sets", edge, faceMarkerBottom);CHKERRQ(ierr);
848                 ierr = DMSetLabelValue(dm, "marker", edge,    markerBottom);CHKERRQ(ierr);
849                 ierr = DMSetLabelValue(dm, "marker", cone[0], markerBottom);CHKERRQ(ierr);
850                 if (ex == numXEdges-1) {
851                   ierr = DMSetLabelValue(dm, "marker", cone[1], markerBottom);CHKERRQ(ierr);
852                 }
853               }
854             } else {
855               if (vy == 0 && cutLabel) {
856                 ierr = DMLabelSetValue(cutLabel, edge,    1);CHKERRQ(ierr);
857                 ierr = DMLabelSetValue(cutLabel, cone[0], 1);CHKERRQ(ierr);
858                 if (ex == numXEdges-1) {
859                   ierr = DMLabelSetValue(cutLabel, cone[1], 1);CHKERRQ(ierr);
860                 }
861               }
862             }
863           } else {
864             if (bdY != DM_BOUNDARY_PERIODIC) {
865               if (vy == numYVertices-1) {
866                 ierr = DMSetLabelValue(dm, "marker", edge, markerBack);CHKERRQ(ierr);
867               }
868               else if (vy == 0) {
869                 ierr = DMSetLabelValue(dm, "marker", edge, markerFront);CHKERRQ(ierr);
870               }
871             }
872             if (bdZ != DM_BOUNDARY_PERIODIC) {
873               if (vz == numZVertices-1) {
874                 ierr = DMSetLabelValue(dm, "marker", edge, markerTop);CHKERRQ(ierr);
875               }
876               else if (vz == 0) {
877                 ierr = DMSetLabelValue(dm, "marker", edge, markerBottom);CHKERRQ(ierr);
878               }
879             }
880           }
881         }
882       }
883     }
884     ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
885     ierr = DMPlexStratify(dm);CHKERRQ(ierr);
886     /* Build coordinates */
887     ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
888     ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
889     ierr = PetscSectionSetFieldComponents(coordSection, 0, dim);CHKERRQ(ierr);
890     ierr = PetscSectionSetChart(coordSection, firstVertex, firstVertex+numVertices);CHKERRQ(ierr);
891     for (v = firstVertex; v < firstVertex+numVertices; ++v) {
892       ierr = PetscSectionSetDof(coordSection, v, dim);CHKERRQ(ierr);
893       ierr = PetscSectionSetFieldDof(coordSection, v, 0, dim);CHKERRQ(ierr);
894     }
895     ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
896     ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
897     ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
898     ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
899     ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
900     ierr = VecSetBlockSize(coordinates, dim);CHKERRQ(ierr);
901     ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
902     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
903     for (vz = 0; vz < numZVertices; ++vz) {
904       for (vy = 0; vy < numYVertices; ++vy) {
905         for (vx = 0; vx < numXVertices; ++vx) {
906           coords[((vz*numYVertices+vy)*numXVertices+vx)*dim+0] = lower[0] + ((upper[0] - lower[0])/numXEdges)*vx;
907           coords[((vz*numYVertices+vy)*numXVertices+vx)*dim+1] = lower[1] + ((upper[1] - lower[1])/numYEdges)*vy;
908           if (dim == 3) {
909             coords[((vz*numYVertices+vy)*numXVertices+vx)*dim+2] = lower[2] + ((upper[2] - lower[2])/numZEdges)*vz;
910           }
911         }
912       }
913     }
914     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
915     ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
916     ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
917   }
918   PetscFunctionReturn(0);
919 }
920 
921 static PetscErrorCode DMPlexCreateBoxMesh_Tensor_Internal(MPI_Comm comm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate, DM *dm)
922 {
923   PetscInt       i;
924   PetscErrorCode ierr;
925 
926   PetscFunctionBegin;
927   PetscValidPointer(dm, 7);
928   ierr = DMCreate(comm, dm);CHKERRQ(ierr);
929   PetscValidLogicalCollectiveInt(*dm,dim,2);
930   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
931   ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
932   switch (dim) {
933   case 2: {ierr = DMPlexCreateCubeMesh_Internal(*dm, lower, upper, faces, periodicity[0], periodicity[1], DM_BOUNDARY_NONE);CHKERRQ(ierr);break;}
934   case 3: {ierr = DMPlexCreateCubeMesh_Internal(*dm, lower, upper, faces, periodicity[0], periodicity[1], periodicity[2]);CHKERRQ(ierr);break;}
935   default: SETERRQ1(comm, PETSC_ERR_SUP, "Dimension not supported: %d", dim);
936   }
937   if (periodicity[0] == DM_BOUNDARY_PERIODIC || periodicity[0] == DM_BOUNDARY_TWIST ||
938       periodicity[1] == DM_BOUNDARY_PERIODIC || periodicity[1] == DM_BOUNDARY_TWIST ||
939       (dim > 2 && (periodicity[2] == DM_BOUNDARY_PERIODIC || periodicity[2] == DM_BOUNDARY_TWIST))) {
940     PetscReal L[3];
941     PetscReal maxCell[3];
942 
943     for (i = 0; i < dim; i++) {
944       L[i]       = upper[i] - lower[i];
945       maxCell[i] = 1.1 * (L[i] / PetscMax(1,faces[i]));
946     }
947     ierr = DMSetPeriodicity(*dm,PETSC_TRUE,maxCell,L,periodicity);CHKERRQ(ierr);
948   }
949   if (!interpolate) {
950     DM udm;
951 
952     ierr = DMPlexUninterpolate(*dm, &udm);CHKERRQ(ierr);
953     ierr = DMPlexCopyCoordinates(*dm, udm);CHKERRQ(ierr);
954     ierr = DMDestroy(dm);CHKERRQ(ierr);
955     *dm  = udm;
956   }
957   PetscFunctionReturn(0);
958 }
959 
960 /*@C
961   DMPlexCreateBoxMesh - Creates a mesh on the tensor product of unit intervals (box) using simplices or tensor cells (hexahedra).
962 
963   Collective on MPI_Comm
964 
965   Input Parameters:
966 + comm        - The communicator for the DM object
967 . dim         - The spatial dimension
968 . simplex     - PETSC_TRUE for simplices, PETSC_FALSE for tensor cells
969 . faces       - Number of faces per dimension, or NULL for (1,) in 1D and (2, 2) in 2D and (1, 1, 1) in 3D
970 . lower       - The lower left corner, or NULL for (0, 0, 0)
971 . upper       - The upper right corner, or NULL for (1, 1, 1)
972 . periodicity - The boundary type for the X,Y,Z direction, or NULL for DM_BOUNDARY_NONE
973 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
974 
975   Output Parameter:
976 . dm  - The DM object
977 
978   Note: Here is the numbering returned for 2 faces in each direction for tensor cells:
979 $ 10---17---11---18----12
980 $  |         |         |
981 $  |         |         |
982 $ 20    2   22    3    24
983 $  |         |         |
984 $  |         |         |
985 $  7---15----8---16----9
986 $  |         |         |
987 $  |         |         |
988 $ 19    0   21    1   23
989 $  |         |         |
990 $  |         |         |
991 $  4---13----5---14----6
992 
993 and for simplicial cells
994 
995 $ 14----8---15----9----16
996 $  |\     5  |\      7 |
997 $  | \       | \       |
998 $ 13   2    14    3    15
999 $  | 4   \   | 6   \   |
1000 $  |       \ |       \ |
1001 $ 11----6---12----7----13
1002 $  |\        |\        |
1003 $  | \    1  | \     3 |
1004 $ 10   0    11    1    12
1005 $  | 0   \   | 2   \   |
1006 $  |       \ |       \ |
1007 $  8----4----9----5----10
1008 
1009   Level: beginner
1010 
1011 .keywords: DM, create
1012 .seealso: DMPlexCreateFromFile(), DMPlexCreateHexCylinderMesh(), DMSetType(), DMCreate()
1013 @*/
1014 PetscErrorCode DMPlexCreateBoxMesh(MPI_Comm comm, PetscInt dim, PetscBool simplex, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate, DM *dm)
1015 {
1016   PetscInt       i;
1017   PetscInt       fac[3] = {0, 0, 0};
1018   PetscReal      low[3] = {0, 0, 0};
1019   PetscReal      upp[3] = {1, 1, 1};
1020   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1021   PetscErrorCode ierr;
1022 
1023   PetscFunctionBegin;
1024   for (i = 0; i < dim; ++i) fac[i] = faces ? faces[i] : (dim == 1 ? 1 : 4-dim);
1025   if (lower) for (i = 0; i < dim; ++i) low[i] = lower[i];
1026   if (upper) for (i = 0; i < dim; ++i) upp[i] = upper[i];
1027   if (periodicity) for (i = 0; i < dim; ++i) bdt[i] = periodicity[i];
1028 
1029   if (dim == 1)      {ierr = DMPlexCreateLineMesh_Internal(comm, fac[0], low[0], upp[0], bdt[0], dm);CHKERRQ(ierr);}
1030   else if (simplex)  {ierr = DMPlexCreateBoxMesh_Simplex_Internal(comm, dim, fac, low, upp, bdt, interpolate, dm);CHKERRQ(ierr);}
1031   else               {ierr = DMPlexCreateBoxMesh_Tensor_Internal(comm, dim, fac, low, upp, bdt, interpolate, dm);CHKERRQ(ierr);}
1032   PetscFunctionReturn(0);
1033 }
1034 
1035 /*@C
1036   DMPlexSetOptionsPrefix - Sets the prefix used for searching for all DM options in the database.
1037 
1038   Logically Collective on DM
1039 
1040   Input Parameters:
1041 + dm - the DM context
1042 - prefix - the prefix to prepend to all option names
1043 
1044   Notes:
1045   A hyphen (-) must NOT be given at the beginning of the prefix name.
1046   The first character of all runtime options is AUTOMATICALLY the hyphen.
1047 
1048   Level: advanced
1049 
1050 .seealso: SNESSetFromOptions()
1051 @*/
1052 PetscErrorCode DMPlexSetOptionsPrefix(DM dm, const char prefix[])
1053 {
1054   DM_Plex       *mesh = (DM_Plex *) dm->data;
1055   PetscErrorCode ierr;
1056 
1057   PetscFunctionBegin;
1058   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1059   ierr = PetscObjectSetOptionsPrefix((PetscObject) dm, prefix);CHKERRQ(ierr);
1060   ierr = PetscObjectSetOptionsPrefix((PetscObject) mesh->partitioner, prefix);CHKERRQ(ierr);
1061   PetscFunctionReturn(0);
1062 }
1063 
1064 /*@
1065   DMPlexCreateHexCylinderMesh - Creates a mesh on the tensor product of the unit interval with the circle (cylinder) using hexahedra.
1066 
1067   Collective on MPI_Comm
1068 
1069   Input Parameters:
1070 + comm      - The communicator for the DM object
1071 . numRefine - The number of regular refinements to the basic 5 cell structure
1072 - periodicZ - The boundary type for the Z direction
1073 
1074   Output Parameter:
1075 . dm  - The DM object
1076 
1077   Note: Here is the output numbering looking from the bottom of the cylinder:
1078 $       17-----14
1079 $        |     |
1080 $        |  2  |
1081 $        |     |
1082 $ 17-----8-----7-----14
1083 $  |     |     |     |
1084 $  |  3  |  0  |  1  |
1085 $  |     |     |     |
1086 $ 19-----5-----6-----13
1087 $        |     |
1088 $        |  4  |
1089 $        |     |
1090 $       19-----13
1091 $
1092 $ and up through the top
1093 $
1094 $       18-----16
1095 $        |     |
1096 $        |  2  |
1097 $        |     |
1098 $ 18----10----11-----16
1099 $  |     |     |     |
1100 $  |  3  |  0  |  1  |
1101 $  |     |     |     |
1102 $ 20-----9----12-----15
1103 $        |     |
1104 $        |  4  |
1105 $        |     |
1106 $       20-----15
1107 
1108   Level: beginner
1109 
1110 .keywords: DM, create
1111 .seealso: DMPlexCreateBoxMesh(), DMSetType(), DMCreate()
1112 @*/
1113 PetscErrorCode DMPlexCreateHexCylinderMesh(MPI_Comm comm, PetscInt numRefine, DMBoundaryType periodicZ, DM *dm)
1114 {
1115   const PetscInt dim = 3;
1116   PetscInt       numCells, numVertices, r;
1117   PetscMPIInt    rank;
1118   PetscErrorCode ierr;
1119 
1120   PetscFunctionBegin;
1121   PetscValidPointer(dm, 4);
1122   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
1123   if (numRefine < 0) SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Number of refinements %D cannot be negative", numRefine);
1124   ierr = DMCreate(comm, dm);CHKERRQ(ierr);
1125   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
1126   ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
1127   /* Create topology */
1128   {
1129     PetscInt cone[8], c;
1130 
1131     numCells    = !rank ?  5 : 0;
1132     numVertices = !rank ? 16 : 0;
1133     if (periodicZ == DM_BOUNDARY_PERIODIC) {
1134       numCells   *= 3;
1135       numVertices = !rank ? 24 : 0;
1136     }
1137     ierr = DMPlexSetChart(*dm, 0, numCells+numVertices);CHKERRQ(ierr);
1138     for (c = 0; c < numCells; c++) {ierr = DMPlexSetConeSize(*dm, c, 8);CHKERRQ(ierr);}
1139     ierr = DMSetUp(*dm);CHKERRQ(ierr);
1140     if (!rank) {
1141       if (periodicZ == DM_BOUNDARY_PERIODIC) {
1142         cone[0] = 15; cone[1] = 18; cone[2] = 17; cone[3] = 16;
1143         cone[4] = 31; cone[5] = 32; cone[6] = 33; cone[7] = 34;
1144         ierr = DMPlexSetCone(*dm, 0, cone);CHKERRQ(ierr);
1145         cone[0] = 16; cone[1] = 17; cone[2] = 24; cone[3] = 23;
1146         cone[4] = 32; cone[5] = 36; cone[6] = 37; cone[7] = 33; /* 22 25 26 21 */
1147         ierr = DMPlexSetCone(*dm, 1, cone);CHKERRQ(ierr);
1148         cone[0] = 18; cone[1] = 27; cone[2] = 24; cone[3] = 17;
1149         cone[4] = 34; cone[5] = 33; cone[6] = 37; cone[7] = 38;
1150         ierr = DMPlexSetCone(*dm, 2, cone);CHKERRQ(ierr);
1151         cone[0] = 29; cone[1] = 27; cone[2] = 18; cone[3] = 15;
1152         cone[4] = 35; cone[5] = 31; cone[6] = 34; cone[7] = 38;
1153         ierr = DMPlexSetCone(*dm, 3, cone);CHKERRQ(ierr);
1154         cone[0] = 29; cone[1] = 15; cone[2] = 16; cone[3] = 23;
1155         cone[4] = 35; cone[5] = 36; cone[6] = 32; cone[7] = 31;
1156         ierr = DMPlexSetCone(*dm, 4, cone);CHKERRQ(ierr);
1157 
1158         cone[0] = 31; cone[1] = 34; cone[2] = 33; cone[3] = 32;
1159         cone[4] = 19; cone[5] = 22; cone[6] = 21; cone[7] = 20;
1160         ierr = DMPlexSetCone(*dm, 5, cone);CHKERRQ(ierr);
1161         cone[0] = 32; cone[1] = 33; cone[2] = 37; cone[3] = 36;
1162         cone[4] = 22; cone[5] = 25; cone[6] = 26; cone[7] = 21;
1163         ierr = DMPlexSetCone(*dm, 6, cone);CHKERRQ(ierr);
1164         cone[0] = 34; cone[1] = 38; cone[2] = 37; cone[3] = 33;
1165         cone[4] = 20; cone[5] = 21; cone[6] = 26; cone[7] = 28;
1166         ierr = DMPlexSetCone(*dm, 7, cone);CHKERRQ(ierr);
1167         cone[0] = 35; cone[1] = 38; cone[2] = 34; cone[3] = 31;
1168         cone[4] = 30; cone[5] = 19; cone[6] = 20; cone[7] = 28;
1169         ierr = DMPlexSetCone(*dm, 8, cone);CHKERRQ(ierr);
1170         cone[0] = 35; cone[1] = 31; cone[2] = 32; cone[3] = 36;
1171         cone[4] = 30; cone[5] = 25; cone[6] = 22; cone[7] = 19;
1172         ierr = DMPlexSetCone(*dm, 9, cone);CHKERRQ(ierr);
1173 
1174         cone[0] = 19; cone[1] = 20; cone[2] = 21; cone[3] = 22;
1175         cone[4] = 15; cone[5] = 16; cone[6] = 17; cone[7] = 18;
1176         ierr = DMPlexSetCone(*dm, 10, cone);CHKERRQ(ierr);
1177         cone[0] = 22; cone[1] = 21; cone[2] = 26; cone[3] = 25;
1178         cone[4] = 16; cone[5] = 23; cone[6] = 24; cone[7] = 17;
1179         ierr = DMPlexSetCone(*dm, 11, cone);CHKERRQ(ierr);
1180         cone[0] = 20; cone[1] = 28; cone[2] = 26; cone[3] = 21;
1181         cone[4] = 18; cone[5] = 17; cone[6] = 24; cone[7] = 27;
1182         ierr = DMPlexSetCone(*dm, 12, cone);CHKERRQ(ierr);
1183         cone[0] = 30; cone[1] = 28; cone[2] = 20; cone[3] = 19;
1184         cone[4] = 29; cone[5] = 15; cone[6] = 18; cone[7] = 27;
1185         ierr = DMPlexSetCone(*dm, 13, cone);CHKERRQ(ierr);
1186         cone[0] = 30; cone[1] = 19; cone[2] = 22; cone[3] = 25;
1187         cone[4] = 29; cone[5] = 23; cone[6] = 16; cone[7] = 15;
1188         ierr = DMPlexSetCone(*dm, 14, cone);CHKERRQ(ierr);
1189       } else {
1190         cone[0] =  5; cone[1] =  8; cone[2] =  7; cone[3] =  6;
1191         cone[4] =  9; cone[5] = 12; cone[6] = 11; cone[7] = 10;
1192         ierr = DMPlexSetCone(*dm, 0, cone);CHKERRQ(ierr);
1193         cone[0] =  6; cone[1] =  7; cone[2] = 14; cone[3] = 13;
1194         cone[4] = 12; cone[5] = 15; cone[6] = 16; cone[7] = 11;
1195         ierr = DMPlexSetCone(*dm, 1, cone);CHKERRQ(ierr);
1196         cone[0] =  8; cone[1] = 17; cone[2] = 14; cone[3] =  7;
1197         cone[4] = 10; cone[5] = 11; cone[6] = 16; cone[7] = 18;
1198         ierr = DMPlexSetCone(*dm, 2, cone);CHKERRQ(ierr);
1199         cone[0] = 19; cone[1] = 17; cone[2] =  8; cone[3] =  5;
1200         cone[4] = 20; cone[5] =  9; cone[6] = 10; cone[7] = 18;
1201         ierr = DMPlexSetCone(*dm, 3, cone);CHKERRQ(ierr);
1202         cone[0] = 19; cone[1] =  5; cone[2] =  6; cone[3] = 13;
1203         cone[4] = 20; cone[5] = 15; cone[6] = 12; cone[7] =  9;
1204         ierr = DMPlexSetCone(*dm, 4, cone);CHKERRQ(ierr);
1205       }
1206     }
1207     ierr = DMPlexSymmetrize(*dm);CHKERRQ(ierr);
1208     ierr = DMPlexStratify(*dm);CHKERRQ(ierr);
1209   }
1210   /* Interpolate */
1211   {
1212     DM idm;
1213 
1214     ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
1215     ierr = DMDestroy(dm);CHKERRQ(ierr);
1216     *dm  = idm;
1217   }
1218   /* Create cube geometry */
1219   {
1220     Vec             coordinates;
1221     PetscSection    coordSection;
1222     PetscScalar    *coords;
1223     PetscInt        coordSize, v;
1224     const PetscReal dis = 1.0/PetscSqrtReal(2.0);
1225     const PetscReal ds2 = dis/2.0;
1226 
1227     /* Build coordinates */
1228     ierr = DMGetCoordinateSection(*dm, &coordSection);CHKERRQ(ierr);
1229     ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
1230     ierr = PetscSectionSetFieldComponents(coordSection, 0, dim);CHKERRQ(ierr);
1231     ierr = PetscSectionSetChart(coordSection, numCells, numCells+numVertices);CHKERRQ(ierr);
1232     for (v = numCells; v < numCells+numVertices; ++v) {
1233       ierr = PetscSectionSetDof(coordSection, v, dim);CHKERRQ(ierr);
1234       ierr = PetscSectionSetFieldDof(coordSection, v, 0, dim);CHKERRQ(ierr);
1235     }
1236     ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
1237     ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
1238     ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
1239     ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
1240     ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
1241     ierr = VecSetBlockSize(coordinates, dim);CHKERRQ(ierr);
1242     ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
1243     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
1244     if (!rank) {
1245       coords[0*dim+0] = -ds2; coords[0*dim+1] = -ds2; coords[0*dim+2] = 0.0;
1246       coords[1*dim+0] =  ds2; coords[1*dim+1] = -ds2; coords[1*dim+2] = 0.0;
1247       coords[2*dim+0] =  ds2; coords[2*dim+1] =  ds2; coords[2*dim+2] = 0.0;
1248       coords[3*dim+0] = -ds2; coords[3*dim+1] =  ds2; coords[3*dim+2] = 0.0;
1249       coords[4*dim+0] = -ds2; coords[4*dim+1] = -ds2; coords[4*dim+2] = 1.0;
1250       coords[5*dim+0] = -ds2; coords[5*dim+1] =  ds2; coords[5*dim+2] = 1.0;
1251       coords[6*dim+0] =  ds2; coords[6*dim+1] =  ds2; coords[6*dim+2] = 1.0;
1252       coords[7*dim+0] =  ds2; coords[7*dim+1] = -ds2; coords[7*dim+2] = 1.0;
1253       coords[ 8*dim+0] =  dis; coords[ 8*dim+1] = -dis; coords[ 8*dim+2] = 0.0;
1254       coords[ 9*dim+0] =  dis; coords[ 9*dim+1] =  dis; coords[ 9*dim+2] = 0.0;
1255       coords[10*dim+0] =  dis; coords[10*dim+1] = -dis; coords[10*dim+2] = 1.0;
1256       coords[11*dim+0] =  dis; coords[11*dim+1] =  dis; coords[11*dim+2] = 1.0;
1257       coords[12*dim+0] = -dis; coords[12*dim+1] =  dis; coords[12*dim+2] = 0.0;
1258       coords[13*dim+0] = -dis; coords[13*dim+1] =  dis; coords[13*dim+2] = 1.0;
1259       coords[14*dim+0] = -dis; coords[14*dim+1] = -dis; coords[14*dim+2] = 0.0;
1260       coords[15*dim+0] = -dis; coords[15*dim+1] = -dis; coords[15*dim+2] = 1.0;
1261       if (periodicZ == DM_BOUNDARY_PERIODIC) {
1262         /* 15 31 19 */ coords[16*dim+0] = -ds2; coords[16*dim+1] = -ds2; coords[16*dim+2] = 0.5;
1263         /* 16 32 22 */ coords[17*dim+0] =  ds2; coords[17*dim+1] = -ds2; coords[17*dim+2] = 0.5;
1264         /* 17 33 21 */ coords[18*dim+0] =  ds2; coords[18*dim+1] =  ds2; coords[18*dim+2] = 0.5;
1265         /* 18 34 20 */ coords[19*dim+0] = -ds2; coords[19*dim+1] =  ds2; coords[19*dim+2] = 0.5;
1266         /* 29 35 30 */ coords[20*dim+0] = -dis; coords[20*dim+1] = -dis; coords[20*dim+2] = 0.5;
1267         /* 23 36 25 */ coords[21*dim+0] =  dis; coords[21*dim+1] = -dis; coords[21*dim+2] = 0.5;
1268         /* 24 37 26 */ coords[22*dim+0] =  dis; coords[22*dim+1] =  dis; coords[22*dim+2] = 0.5;
1269         /* 27 38 28 */ coords[23*dim+0] = -dis; coords[23*dim+1] =  dis; coords[23*dim+2] = 0.5;
1270       }
1271     }
1272     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
1273     ierr = DMSetCoordinatesLocal(*dm, coordinates);CHKERRQ(ierr);
1274     ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
1275   }
1276   /* Create periodicity */
1277   if (periodicZ == DM_BOUNDARY_PERIODIC || periodicZ == DM_BOUNDARY_TWIST) {
1278     PetscReal      L[3];
1279     PetscReal      maxCell[3];
1280     DMBoundaryType bdType[3];
1281     PetscReal      lower[3] = {0.0, 0.0, 0.0};
1282     PetscReal      upper[3] = {1.0, 1.0, 1.5};
1283     PetscInt       i, numZCells = 3;
1284 
1285     bdType[0] = DM_BOUNDARY_NONE;
1286     bdType[1] = DM_BOUNDARY_NONE;
1287     bdType[2] = periodicZ;
1288     for (i = 0; i < dim; i++) {
1289       L[i]       = upper[i] - lower[i];
1290       maxCell[i] = 1.1 * (L[i] / numZCells);
1291     }
1292     ierr = DMSetPeriodicity(*dm, PETSC_TRUE, maxCell, L, bdType);CHKERRQ(ierr);
1293   }
1294   /* Refine topology */
1295   for (r = 0; r < numRefine; ++r) {
1296     DM rdm = NULL;
1297 
1298     ierr = DMRefine(*dm, comm, &rdm);CHKERRQ(ierr);
1299     ierr = DMDestroy(dm);CHKERRQ(ierr);
1300     *dm  = rdm;
1301   }
1302   /* Remap geometry to cylinder
1303        Interior square: Linear interpolation is correct
1304        The other cells all have vertices on rays from the origin. We want to uniformly expand the spacing
1305        such that the last vertex is on the unit circle. So the closest and farthest vertices are at distance
1306 
1307          phi     = arctan(y/x)
1308          d_close = sqrt(1/8 + 1/4 sin^2(phi))
1309          d_far   = sqrt(1/2 + sin^2(phi))
1310 
1311        so we remap them using
1312 
1313          x_new = x_close + (x - x_close) (1 - d_close) / (d_far - d_close)
1314          y_new = y_close + (y - y_close) (1 - d_close) / (d_far - d_close)
1315 
1316        If pi/4 < phi < 3pi/4 or -3pi/4 < phi < -pi/4, then we switch x and y.
1317   */
1318   {
1319     Vec           coordinates;
1320     PetscSection  coordSection;
1321     PetscScalar  *coords;
1322     PetscInt      vStart, vEnd, v;
1323     const PetscReal dis = 1.0/PetscSqrtReal(2.0);
1324     const PetscReal ds2 = 0.5*dis;
1325 
1326     ierr = DMPlexGetDepthStratum(*dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
1327     ierr = DMGetCoordinateSection(*dm, &coordSection);CHKERRQ(ierr);
1328     ierr = DMGetCoordinatesLocal(*dm, &coordinates);CHKERRQ(ierr);
1329     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
1330     for (v = vStart; v < vEnd; ++v) {
1331       PetscReal phi, sinp, cosp, dc, df, x, y, xc, yc;
1332       PetscInt  off;
1333 
1334       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
1335       if ((PetscAbsScalar(coords[off+0]) <= ds2) && (PetscAbsScalar(coords[off+1]) <= ds2)) continue;
1336       x    = PetscRealPart(coords[off]);
1337       y    = PetscRealPart(coords[off+1]);
1338       phi  = PetscAtan2Real(y, x);
1339       sinp = PetscSinReal(phi);
1340       cosp = PetscCosReal(phi);
1341       if ((PetscAbsReal(phi) > PETSC_PI/4.0) && (PetscAbsReal(phi) < 3.0*PETSC_PI/4.0)) {
1342         dc = PetscAbsReal(ds2/sinp);
1343         df = PetscAbsReal(dis/sinp);
1344         xc = ds2*x/PetscAbsReal(y);
1345         yc = ds2*PetscSignReal(y);
1346       } else {
1347         dc = PetscAbsReal(ds2/cosp);
1348         df = PetscAbsReal(dis/cosp);
1349         xc = ds2*PetscSignReal(x);
1350         yc = ds2*y/PetscAbsReal(x);
1351       }
1352       coords[off+0] = xc + (coords[off+0] - xc)*(1.0 - dc)/(df - dc);
1353       coords[off+1] = yc + (coords[off+1] - yc)*(1.0 - dc)/(df - dc);
1354     }
1355     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
1356     if (periodicZ == DM_BOUNDARY_PERIODIC || periodicZ == DM_BOUNDARY_TWIST) {
1357       ierr = DMLocalizeCoordinates(*dm);CHKERRQ(ierr);
1358     }
1359   }
1360   PetscFunctionReturn(0);
1361 }
1362 
1363 /*@
1364   DMPlexCreateWedgeCylinderMesh - Creates a mesh on the tensor product of the unit interval with the circle (cylinder) using wedges.
1365 
1366   Collective on MPI_Comm
1367 
1368   Input Parameters:
1369 + comm - The communicator for the DM object
1370 . n    - The number of wedges around the origin
1371 - interpolate - Create edges and faces
1372 
1373   Output Parameter:
1374 . dm  - The DM object
1375 
1376   Level: beginner
1377 
1378 .keywords: DM, create
1379 .seealso: DMPlexCreateHexCylinderMesh(), DMPlexCreateBoxMesh(), DMSetType(), DMCreate()
1380 @*/
1381 PetscErrorCode DMPlexCreateWedgeCylinderMesh(MPI_Comm comm, PetscInt n, PetscBool interpolate, DM *dm)
1382 {
1383   const PetscInt dim = 3;
1384   PetscInt       numCells, numVertices;
1385   PetscMPIInt    rank;
1386   PetscErrorCode ierr;
1387 
1388   PetscFunctionBegin;
1389   PetscValidPointer(dm, 3);
1390   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
1391   if (n < 0) SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Number of wedges %D cannot be negative", n);
1392   ierr = DMCreate(comm, dm);CHKERRQ(ierr);
1393   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
1394   ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
1395   /* Create topology */
1396   {
1397     PetscInt cone[6], c;
1398 
1399     numCells    = !rank ?        n : 0;
1400     numVertices = !rank ?  2*(n+1) : 0;
1401     ierr = DMPlexSetChart(*dm, 0, numCells+numVertices);CHKERRQ(ierr);
1402     for (c = 0; c < numCells; c++) {ierr = DMPlexSetConeSize(*dm, c, 6);CHKERRQ(ierr);}
1403     ierr = DMSetUp(*dm);CHKERRQ(ierr);
1404     for (c = 0; c < numCells; c++) {
1405       cone[0] =  c+n*1; cone[1] = (c+1)%n+n*1; cone[2] = 0+3*n;
1406       cone[3] =  c+n*2; cone[4] = (c+1)%n+n*2; cone[5] = 1+3*n;
1407       ierr = DMPlexSetCone(*dm, c, cone);CHKERRQ(ierr);
1408     }
1409     ierr = DMPlexSymmetrize(*dm);CHKERRQ(ierr);
1410     ierr = DMPlexStratify(*dm);CHKERRQ(ierr);
1411   }
1412   /* Interpolate */
1413   if (interpolate) {
1414     DM idm;
1415 
1416     ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
1417     ierr = DMDestroy(dm);CHKERRQ(ierr);
1418     *dm  = idm;
1419   }
1420   /* Create cylinder geometry */
1421   {
1422     Vec          coordinates;
1423     PetscSection coordSection;
1424     PetscScalar *coords;
1425     PetscInt     coordSize, v, c;
1426 
1427     /* Build coordinates */
1428     ierr = DMGetCoordinateSection(*dm, &coordSection);CHKERRQ(ierr);
1429     ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
1430     ierr = PetscSectionSetFieldComponents(coordSection, 0, dim);CHKERRQ(ierr);
1431     ierr = PetscSectionSetChart(coordSection, numCells, numCells+numVertices);CHKERRQ(ierr);
1432     for (v = numCells; v < numCells+numVertices; ++v) {
1433       ierr = PetscSectionSetDof(coordSection, v, dim);CHKERRQ(ierr);
1434       ierr = PetscSectionSetFieldDof(coordSection, v, 0, dim);CHKERRQ(ierr);
1435     }
1436     ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
1437     ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
1438     ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
1439     ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
1440     ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
1441     ierr = VecSetBlockSize(coordinates, dim);CHKERRQ(ierr);
1442     ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
1443     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
1444     for (c = 0; c < numCells; c++) {
1445       coords[(c+0*n)*dim+0] = PetscCosReal(2.0*c*PETSC_PI/n); coords[(c+0*n)*dim+1] = PetscSinReal(2.0*c*PETSC_PI/n); coords[(c+0*n)*dim+2] = 1.0;
1446       coords[(c+1*n)*dim+0] = PetscCosReal(2.0*c*PETSC_PI/n); coords[(c+1*n)*dim+1] = PetscSinReal(2.0*c*PETSC_PI/n); coords[(c+1*n)*dim+2] = 0.0;
1447     }
1448     if (!rank) {
1449       coords[(2*n+0)*dim+0] = 0.0; coords[(2*n+0)*dim+1] = 0.0; coords[(2*n+0)*dim+2] = 1.0;
1450       coords[(2*n+1)*dim+0] = 0.0; coords[(2*n+1)*dim+1] = 0.0; coords[(2*n+1)*dim+2] = 0.0;
1451     }
1452     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
1453     ierr = DMSetCoordinatesLocal(*dm, coordinates);CHKERRQ(ierr);
1454     ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
1455   }
1456   PetscFunctionReturn(0);
1457 }
1458 
1459 PETSC_STATIC_INLINE PetscReal DiffNormReal(PetscInt dim, const PetscReal x[], const PetscReal y[])
1460 {
1461   PetscReal prod = 0.0;
1462   PetscInt  i;
1463   for (i = 0; i < dim; ++i) prod += PetscSqr(x[i] - y[i]);
1464   return PetscSqrtReal(prod);
1465 }
1466 PETSC_STATIC_INLINE PetscReal DotReal(PetscInt dim, const PetscReal x[], const PetscReal y[])
1467 {
1468   PetscReal prod = 0.0;
1469   PetscInt  i;
1470   for (i = 0; i < dim; ++i) prod += x[i]*y[i];
1471   return prod;
1472 }
1473 
1474 /*@
1475   DMPlexCreateSphereMesh - Creates a mesh on the d-dimensional sphere, S^d.
1476 
1477   Collective on MPI_Comm
1478 
1479   Input Parameters:
1480 . comm  - The communicator for the DM object
1481 . dim   - The dimension
1482 - simplex - Use simplices, or tensor product cells
1483 
1484   Output Parameter:
1485 . dm  - The DM object
1486 
1487   Level: beginner
1488 
1489 .keywords: DM, create
1490 .seealso: DMPlexCreateBoxMesh(), DMSetType(), DMCreate()
1491 @*/
1492 PetscErrorCode DMPlexCreateSphereMesh(MPI_Comm comm, PetscInt dim, PetscBool simplex, DM *dm)
1493 {
1494   const PetscInt  embedDim = dim+1;
1495   PetscSection    coordSection;
1496   Vec             coordinates;
1497   PetscScalar    *coords;
1498   PetscReal      *coordsIn;
1499   PetscInt        numCells, numEdges, numVerts, firstVertex, v, firstEdge, coordSize, d, c, e;
1500   PetscMPIInt     rank;
1501   PetscErrorCode  ierr;
1502 
1503   PetscFunctionBegin;
1504   PetscValidPointer(dm, 4);
1505   ierr = DMCreate(comm, dm);CHKERRQ(ierr);
1506   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
1507   ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
1508   ierr = DMSetCoordinateDim(*dm, dim+1);CHKERRQ(ierr);
1509   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) *dm), &rank);CHKERRQ(ierr);
1510   switch (dim) {
1511   case 2:
1512     if (simplex) {
1513       DM              idm;
1514       const PetscReal edgeLen     = 2.0/(1.0 + PETSC_PHI);
1515       const PetscReal vertex[3]   = {0.0, 1.0/(1.0 + PETSC_PHI), PETSC_PHI/(1.0 + PETSC_PHI)};
1516       const PetscInt  degree      = 5;
1517       PetscInt        s[3]        = {1, 1, 1};
1518       PetscInt        cone[3];
1519       PetscInt       *graph, p, i, j, k;
1520 
1521       numCells    = !rank ? 20 : 0;
1522       numVerts    = !rank ? 12 : 0;
1523       firstVertex = numCells;
1524       /* Use icosahedron, which for a unit sphere has coordinates which are all cyclic permutations of
1525 
1526            (0, \pm 1/\phi+1, \pm \phi/\phi+1)
1527 
1528          where \phi^2 - \phi - 1 = 0, meaning \phi is the golden ratio \frac{1 + \sqrt{5}}{2}. The edge
1529          length is then given by 2/\phi = 2 * 2.73606 = 5.47214.
1530        */
1531       /* Construct vertices */
1532       ierr = PetscCalloc1(numVerts * embedDim, &coordsIn);CHKERRQ(ierr);
1533       for (p = 0, i = 0; p < embedDim; ++p) {
1534         for (s[1] = -1; s[1] < 2; s[1] += 2) {
1535           for (s[2] = -1; s[2] < 2; s[2] += 2) {
1536             for (d = 0; d < embedDim; ++d) coordsIn[i*embedDim+d] = s[(d+p)%embedDim]*vertex[(d+p)%embedDim];
1537             ++i;
1538           }
1539         }
1540       }
1541       /* Construct graph */
1542       ierr = PetscCalloc1(numVerts * numVerts, &graph);CHKERRQ(ierr);
1543       for (i = 0; i < numVerts; ++i) {
1544         for (j = 0, k = 0; j < numVerts; ++j) {
1545           if (PetscAbsReal(DiffNormReal(embedDim, &coordsIn[i*embedDim], &coordsIn[j*embedDim]) - edgeLen) < PETSC_SMALL) {graph[i*numVerts+j] = 1; ++k;}
1546         }
1547         if (k != degree) SETERRQ3(comm, PETSC_ERR_PLIB, "Invalid icosahedron, vertex %D degree %D != %D", i, k, degree);
1548       }
1549       /* Build Topology */
1550       ierr = DMPlexSetChart(*dm, 0, numCells+numVerts);CHKERRQ(ierr);
1551       for (c = 0; c < numCells; c++) {
1552         ierr = DMPlexSetConeSize(*dm, c, embedDim);CHKERRQ(ierr);
1553       }
1554       ierr = DMSetUp(*dm);CHKERRQ(ierr); /* Allocate space for cones */
1555       /* Cells */
1556       for (i = 0, c = 0; i < numVerts; ++i) {
1557         for (j = 0; j < i; ++j) {
1558           for (k = 0; k < j; ++k) {
1559             if (graph[i*numVerts+j] && graph[j*numVerts+k] && graph[k*numVerts+i]) {
1560               cone[0] = firstVertex+i; cone[1] = firstVertex+j; cone[2] = firstVertex+k;
1561               /* Check orientation */
1562               {
1563                 const PetscInt epsilon[3][3][3] = {{{0, 0, 0}, {0, 0, 1}, {0, -1, 0}}, {{0, 0, -1}, {0, 0, 0}, {1, 0, 0}}, {{0, 1, 0}, {-1, 0, 0}, {0, 0, 0}}};
1564                 PetscReal normal[3];
1565                 PetscInt  e, f;
1566 
1567                 for (d = 0; d < embedDim; ++d) {
1568                   normal[d] = 0.0;
1569                   for (e = 0; e < embedDim; ++e) {
1570                     for (f = 0; f < embedDim; ++f) {
1571                       normal[d] += epsilon[d][e][f]*(coordsIn[j*embedDim+e] - coordsIn[i*embedDim+e])*(coordsIn[k*embedDim+f] - coordsIn[i*embedDim+f]);
1572                     }
1573                   }
1574                 }
1575                 if (DotReal(embedDim, normal, &coordsIn[i*embedDim]) < 0) {PetscInt tmp = cone[1]; cone[1] = cone[2]; cone[2] = tmp;}
1576               }
1577               ierr = DMPlexSetCone(*dm, c++, cone);CHKERRQ(ierr);
1578             }
1579           }
1580         }
1581       }
1582       ierr = DMPlexSymmetrize(*dm);CHKERRQ(ierr);
1583       ierr = DMPlexStratify(*dm);CHKERRQ(ierr);
1584       ierr = PetscFree(graph);CHKERRQ(ierr);
1585       /* Interpolate mesh */
1586       ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
1587       ierr = DMDestroy(dm);CHKERRQ(ierr);
1588       *dm  = idm;
1589     } else {
1590       /*
1591         12-21--13
1592          |     |
1593         25  4  24
1594          |     |
1595   12-25--9-16--8-24--13
1596    |     |     |     |
1597   23  5 17  0 15  3  22
1598    |     |     |     |
1599   10-20--6-14--7-19--11
1600          |     |
1601         20  1  19
1602          |     |
1603         10-18--11
1604          |     |
1605         23  2  22
1606          |     |
1607         12-21--13
1608        */
1609       const PetscReal dist = 1.0/PetscSqrtReal(3.0);
1610       PetscInt        cone[4], ornt[4];
1611 
1612       numCells    = !rank ?  6 : 0;
1613       numEdges    = !rank ? 12 : 0;
1614       numVerts    = !rank ?  8 : 0;
1615       firstVertex = numCells;
1616       firstEdge   = numCells + numVerts;
1617       /* Build Topology */
1618       ierr = DMPlexSetChart(*dm, 0, numCells+numEdges+numVerts);CHKERRQ(ierr);
1619       for (c = 0; c < numCells; c++) {
1620         ierr = DMPlexSetConeSize(*dm, c, 4);CHKERRQ(ierr);
1621       }
1622       for (e = firstEdge; e < firstEdge+numEdges; ++e) {
1623         ierr = DMPlexSetConeSize(*dm, e, 2);CHKERRQ(ierr);
1624       }
1625       ierr = DMSetUp(*dm);CHKERRQ(ierr); /* Allocate space for cones */
1626       /* Cell 0 */
1627       cone[0] = 14; cone[1] = 15; cone[2] = 16; cone[3] = 17;
1628       ierr = DMPlexSetCone(*dm, 0, cone);CHKERRQ(ierr);
1629       ornt[0] = 0; ornt[1] = 0; ornt[2] = 0; ornt[3] = 0;
1630       ierr = DMPlexSetConeOrientation(*dm, 0, ornt);CHKERRQ(ierr);
1631       /* Cell 1 */
1632       cone[0] = 18; cone[1] = 19; cone[2] = 14; cone[3] = 20;
1633       ierr = DMPlexSetCone(*dm, 1, cone);CHKERRQ(ierr);
1634       ornt[0] = 0; ornt[1] = 0; ornt[2] = -2; ornt[3] = 0;
1635       ierr = DMPlexSetConeOrientation(*dm, 1, ornt);CHKERRQ(ierr);
1636       /* Cell 2 */
1637       cone[0] = 21; cone[1] = 22; cone[2] = 18; cone[3] = 23;
1638       ierr = DMPlexSetCone(*dm, 2, cone);CHKERRQ(ierr);
1639       ornt[0] = 0; ornt[1] = 0; ornt[2] = -2; ornt[3] = 0;
1640       ierr = DMPlexSetConeOrientation(*dm, 2, ornt);CHKERRQ(ierr);
1641       /* Cell 3 */
1642       cone[0] = 19; cone[1] = 22; cone[2] = 24; cone[3] = 15;
1643       ierr = DMPlexSetCone(*dm, 3, cone);CHKERRQ(ierr);
1644       ornt[0] = -2; ornt[1] = -2; ornt[2] = 0; ornt[3] = -2;
1645       ierr = DMPlexSetConeOrientation(*dm, 3, ornt);CHKERRQ(ierr);
1646       /* Cell 4 */
1647       cone[0] = 16; cone[1] = 24; cone[2] = 21; cone[3] = 25;
1648       ierr = DMPlexSetCone(*dm, 4, cone);CHKERRQ(ierr);
1649       ornt[0] = -2; ornt[1] = -2; ornt[2] = -2; ornt[3] = 0;
1650       ierr = DMPlexSetConeOrientation(*dm, 4, ornt);CHKERRQ(ierr);
1651       /* Cell 5 */
1652       cone[0] = 20; cone[1] = 17; cone[2] = 25; cone[3] = 23;
1653       ierr = DMPlexSetCone(*dm, 5, cone);CHKERRQ(ierr);
1654       ornt[0] = -2; ornt[1] = -2; ornt[2] = -2; ornt[3] = -2;
1655       ierr = DMPlexSetConeOrientation(*dm, 5, ornt);CHKERRQ(ierr);
1656       /* Edges */
1657       cone[0] =  6; cone[1] =  7;
1658       ierr = DMPlexSetCone(*dm, 14, cone);CHKERRQ(ierr);
1659       cone[0] =  7; cone[1] =  8;
1660       ierr = DMPlexSetCone(*dm, 15, cone);CHKERRQ(ierr);
1661       cone[0] =  8; cone[1] =  9;
1662       ierr = DMPlexSetCone(*dm, 16, cone);CHKERRQ(ierr);
1663       cone[0] =  9; cone[1] =  6;
1664       ierr = DMPlexSetCone(*dm, 17, cone);CHKERRQ(ierr);
1665       cone[0] = 10; cone[1] = 11;
1666       ierr = DMPlexSetCone(*dm, 18, cone);CHKERRQ(ierr);
1667       cone[0] = 11; cone[1] =  7;
1668       ierr = DMPlexSetCone(*dm, 19, cone);CHKERRQ(ierr);
1669       cone[0] =  6; cone[1] = 10;
1670       ierr = DMPlexSetCone(*dm, 20, cone);CHKERRQ(ierr);
1671       cone[0] = 12; cone[1] = 13;
1672       ierr = DMPlexSetCone(*dm, 21, cone);CHKERRQ(ierr);
1673       cone[0] = 13; cone[1] = 11;
1674       ierr = DMPlexSetCone(*dm, 22, cone);CHKERRQ(ierr);
1675       cone[0] = 10; cone[1] = 12;
1676       ierr = DMPlexSetCone(*dm, 23, cone);CHKERRQ(ierr);
1677       cone[0] = 13; cone[1] =  8;
1678       ierr = DMPlexSetCone(*dm, 24, cone);CHKERRQ(ierr);
1679       cone[0] = 12; cone[1] =  9;
1680       ierr = DMPlexSetCone(*dm, 25, cone);CHKERRQ(ierr);
1681       ierr = DMPlexSymmetrize(*dm);CHKERRQ(ierr);
1682       ierr = DMPlexStratify(*dm);CHKERRQ(ierr);
1683       /* Build coordinates */
1684       ierr = PetscCalloc1(numVerts * embedDim, &coordsIn);CHKERRQ(ierr);
1685       coordsIn[0*embedDim+0] = -dist; coordsIn[0*embedDim+1] =  dist; coordsIn[0*embedDim+2] = -dist;
1686       coordsIn[1*embedDim+0] =  dist; coordsIn[1*embedDim+1] =  dist; coordsIn[1*embedDim+2] = -dist;
1687       coordsIn[2*embedDim+0] =  dist; coordsIn[2*embedDim+1] = -dist; coordsIn[2*embedDim+2] = -dist;
1688       coordsIn[3*embedDim+0] = -dist; coordsIn[3*embedDim+1] = -dist; coordsIn[3*embedDim+2] = -dist;
1689       coordsIn[4*embedDim+0] = -dist; coordsIn[4*embedDim+1] =  dist; coordsIn[4*embedDim+2] =  dist;
1690       coordsIn[5*embedDim+0] =  dist; coordsIn[5*embedDim+1] =  dist; coordsIn[5*embedDim+2] =  dist;
1691       coordsIn[6*embedDim+0] = -dist; coordsIn[6*embedDim+1] = -dist; coordsIn[6*embedDim+2] =  dist;
1692       coordsIn[7*embedDim+0] =  dist; coordsIn[7*embedDim+1] = -dist; coordsIn[7*embedDim+2] =  dist;
1693     }
1694     break;
1695   case 3:
1696     if (simplex) {
1697       DM              idm;
1698       const PetscReal edgeLen         = 1.0/PETSC_PHI;
1699       const PetscReal vertexA[4]      = {0.5, 0.5, 0.5, 0.5};
1700       const PetscReal vertexB[4]      = {1.0, 0.0, 0.0, 0.0};
1701       const PetscReal vertexC[4]      = {0.5, 0.5*PETSC_PHI, 0.5/PETSC_PHI, 0.0};
1702       const PetscInt  degree          = 12;
1703       PetscInt        s[4]            = {1, 1, 1};
1704       PetscInt        evenPerm[12][4] = {{0, 1, 2, 3}, {0, 2, 3, 1}, {0, 3, 1, 2}, {1, 0, 3, 2}, {1, 2, 0, 3}, {1, 3, 2, 0},
1705                                          {2, 0, 1, 3}, {2, 1, 3, 0}, {2, 3, 0, 1}, {3, 0, 2, 1}, {3, 1, 0, 2}, {3, 2, 1, 0}};
1706       PetscInt        cone[4];
1707       PetscInt       *graph, p, i, j, k, l;
1708 
1709       numCells    = !rank ? 600 : 0;
1710       numVerts    = !rank ? 120 : 0;
1711       firstVertex = numCells;
1712       /* Use the 600-cell, which for a unit sphere has coordinates which are
1713 
1714            1/2 (\pm 1, \pm 1,    \pm 1, \pm 1)                          16
1715                (\pm 1,    0,       0,      0)  all cyclic permutations   8
1716            1/2 (\pm 1, \pm phi, \pm 1/phi, 0)  all even permutations    96
1717 
1718          where \phi^2 - \phi - 1 = 0, meaning \phi is the golden ratio \frac{1 + \sqrt{5}}{2}. The edge
1719          length is then given by 1/\phi = 2.73606.
1720 
1721          http://buzzard.pugetsound.edu/sage-practice/ch03s03.html
1722          http://mathworld.wolfram.com/600-Cell.html
1723        */
1724       /* Construct vertices */
1725       ierr = PetscCalloc1(numVerts * embedDim, &coordsIn);CHKERRQ(ierr);
1726       i    = 0;
1727       for (s[0] = -1; s[0] < 2; s[0] += 2) {
1728         for (s[1] = -1; s[1] < 2; s[1] += 2) {
1729           for (s[2] = -1; s[2] < 2; s[2] += 2) {
1730             for (s[3] = -1; s[3] < 2; s[3] += 2) {
1731               for (d = 0; d < embedDim; ++d) coordsIn[i*embedDim+d] = s[d]*vertexA[d];
1732               ++i;
1733             }
1734           }
1735         }
1736       }
1737       for (p = 0; p < embedDim; ++p) {
1738         s[1] = s[2] = s[3] = 1;
1739         for (s[0] = -1; s[0] < 2; s[0] += 2) {
1740           for (d = 0; d < embedDim; ++d) coordsIn[i*embedDim+d] = s[(d+p)%embedDim]*vertexB[(d+p)%embedDim];
1741           ++i;
1742         }
1743       }
1744       for (p = 0; p < 12; ++p) {
1745         s[3] = 1;
1746         for (s[0] = -1; s[0] < 2; s[0] += 2) {
1747           for (s[1] = -1; s[1] < 2; s[1] += 2) {
1748             for (s[2] = -1; s[2] < 2; s[2] += 2) {
1749               for (d = 0; d < embedDim; ++d) coordsIn[i*embedDim+d] = s[evenPerm[p][d]]*vertexC[evenPerm[p][d]];
1750               ++i;
1751             }
1752           }
1753         }
1754       }
1755       if (i != numVerts) SETERRQ2(comm, PETSC_ERR_PLIB, "Invalid 600-cell, vertices %D != %D", i, numVerts);
1756       /* Construct graph */
1757       ierr = PetscCalloc1(numVerts * numVerts, &graph);CHKERRQ(ierr);
1758       for (i = 0; i < numVerts; ++i) {
1759         for (j = 0, k = 0; j < numVerts; ++j) {
1760           if (PetscAbsReal(DiffNormReal(embedDim, &coordsIn[i*embedDim], &coordsIn[j*embedDim]) - edgeLen) < PETSC_SMALL) {graph[i*numVerts+j] = 1; ++k;}
1761         }
1762         if (k != degree) SETERRQ3(comm, PETSC_ERR_PLIB, "Invalid 600-cell, vertex %D degree %D != %D", i, k, degree);
1763       }
1764       /* Build Topology */
1765       ierr = DMPlexSetChart(*dm, 0, numCells+numVerts);CHKERRQ(ierr);
1766       for (c = 0; c < numCells; c++) {
1767         ierr = DMPlexSetConeSize(*dm, c, embedDim);CHKERRQ(ierr);
1768       }
1769       ierr = DMSetUp(*dm);CHKERRQ(ierr); /* Allocate space for cones */
1770       /* Cells */
1771       for (i = 0, c = 0; i < numVerts; ++i) {
1772         for (j = 0; j < i; ++j) {
1773           for (k = 0; k < j; ++k) {
1774             for (l = 0; l < k; ++l) {
1775               if (graph[i*numVerts+j] && graph[j*numVerts+k] && graph[k*numVerts+i] &&
1776                   graph[l*numVerts+i] && graph[l*numVerts+j] && graph[l*numVerts+k]) {
1777                 cone[0] = firstVertex+i; cone[1] = firstVertex+j; cone[2] = firstVertex+k; cone[3] = firstVertex+l;
1778                 /* Check orientation: https://ef.gy/linear-algebra:normal-vectors-in-higher-dimensional-spaces */
1779                 {
1780                   const PetscInt epsilon[4][4][4][4] = {{{{0,  0,  0,  0}, { 0, 0,  0,  0}, { 0,  0, 0,  0}, { 0,  0,  0, 0}},
1781                                                          {{0,  0,  0,  0}, { 0, 0,  0,  0}, { 0,  0, 0,  1}, { 0,  0, -1, 0}},
1782                                                          {{0,  0,  0,  0}, { 0, 0,  0, -1}, { 0,  0, 0,  0}, { 0,  1,  0, 0}},
1783                                                          {{0,  0,  0,  0}, { 0, 0,  1,  0}, { 0, -1, 0,  0}, { 0,  0,  0, 0}}},
1784 
1785                                                         {{{0,  0,  0,  0}, { 0, 0,  0,  0}, { 0,  0, 0, -1}, { 0,  0,  1, 0}},
1786                                                          {{0,  0,  0,  0}, { 0, 0,  0,  0}, { 0,  0, 0,  0}, { 0,  0,  0, 0}},
1787                                                          {{0,  0,  0,  1}, { 0, 0,  0,  0}, { 0,  0, 0,  0}, {-1,  0,  0, 0}},
1788                                                          {{0,  0, -1,  0}, { 0, 0,  0,  0}, { 1,  0, 0,  0}, { 0,  0,  0, 0}}},
1789 
1790                                                         {{{0,  0,  0,  0}, { 0, 0,  0,  1}, { 0,  0, 0,  0}, { 0, -1,  0, 0}},
1791                                                          {{0,  0,  0, -1}, { 0, 0,  0,  0}, { 0,  0, 0,  0}, { 1,  0,  0, 0}},
1792                                                          {{0,  0,  0,  0}, { 0, 0,  0,  0}, { 0,  0, 0,  0}, { 0,  0,  0, 0}},
1793                                                          {{0,  1,  0,  0}, {-1, 0,  0,  0}, { 0,  0, 0,  0}, { 0,  0,  0, 0}}},
1794 
1795                                                         {{{0,  0,  0,  0}, { 0, 0, -1,  0}, { 0,  1, 0,  0}, { 0,  0,  0, 0}},
1796                                                          {{0,  0,  1,  0}, { 0, 0,  0,  0}, {-1,  0, 0,  0}, { 0,  0,  0, 0}},
1797                                                          {{0, -1,  0,  0}, { 1, 0,  0,  0}, { 0,  0, 0,  0}, { 0,  0,  0, 0}},
1798                                                          {{0,  0,  0,  0}, { 0, 0,  0,  0}, { 0,  0, 0,  0}, { 0,  0,  0, 0}}}};
1799                   PetscReal normal[4];
1800                   PetscInt  e, f, g;
1801 
1802                   for (d = 0; d < embedDim; ++d) {
1803                     normal[d] = 0.0;
1804                     for (e = 0; e < embedDim; ++e) {
1805                       for (f = 0; f < embedDim; ++f) {
1806                         for (g = 0; g < embedDim; ++g) {
1807                           normal[d] += epsilon[d][e][f][g]*(coordsIn[j*embedDim+e] - coordsIn[i*embedDim+e])*(coordsIn[k*embedDim+f] - coordsIn[i*embedDim+f])*(coordsIn[l*embedDim+f] - coordsIn[i*embedDim+f]);
1808                         }
1809                       }
1810                     }
1811                   }
1812                   if (DotReal(embedDim, normal, &coordsIn[i*embedDim]) < 0) {PetscInt tmp = cone[1]; cone[1] = cone[2]; cone[2] = tmp;}
1813                 }
1814                 ierr = DMPlexSetCone(*dm, c++, cone);CHKERRQ(ierr);
1815               }
1816             }
1817           }
1818         }
1819       }
1820       ierr = DMPlexSymmetrize(*dm);CHKERRQ(ierr);
1821       ierr = DMPlexStratify(*dm);CHKERRQ(ierr);
1822       ierr = PetscFree(graph);CHKERRQ(ierr);
1823       /* Interpolate mesh */
1824       ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
1825       ierr = DMDestroy(dm);CHKERRQ(ierr);
1826       *dm  = idm;
1827       break;
1828     }
1829   default: SETERRQ1(comm, PETSC_ERR_SUP, "Unsupported dimension for sphere: %D", dim);
1830   }
1831   /* Create coordinates */
1832   ierr = DMGetCoordinateSection(*dm, &coordSection);CHKERRQ(ierr);
1833   ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
1834   ierr = PetscSectionSetFieldComponents(coordSection, 0, embedDim);CHKERRQ(ierr);
1835   ierr = PetscSectionSetChart(coordSection, firstVertex, firstVertex+numVerts);CHKERRQ(ierr);
1836   for (v = firstVertex; v < firstVertex+numVerts; ++v) {
1837     ierr = PetscSectionSetDof(coordSection, v, embedDim);CHKERRQ(ierr);
1838     ierr = PetscSectionSetFieldDof(coordSection, v, 0, embedDim);CHKERRQ(ierr);
1839   }
1840   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
1841   ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
1842   ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
1843   ierr = VecSetBlockSize(coordinates, embedDim);CHKERRQ(ierr);
1844   ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
1845   ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
1846   ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
1847   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
1848   for (v = 0; v < numVerts; ++v) for (d = 0; d < embedDim; ++d) {coords[v*embedDim+d] = coordsIn[v*embedDim+d];}
1849   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
1850   ierr = DMSetCoordinatesLocal(*dm, coordinates);CHKERRQ(ierr);
1851   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
1852   ierr = PetscFree(coordsIn);CHKERRQ(ierr);
1853   PetscFunctionReturn(0);
1854 }
1855 
1856 /* External function declarations here */
1857 extern PetscErrorCode DMCreateInterpolation_Plex(DM dmCoarse, DM dmFine, Mat *interpolation, Vec *scaling);
1858 extern PetscErrorCode DMCreateInjection_Plex(DM dmCoarse, DM dmFine, Mat *mat);
1859 extern PetscErrorCode DMCreateMassMatrix_Plex(DM dmCoarse, DM dmFine, Mat *mat);
1860 extern PetscErrorCode DMCreateDefaultSection_Plex(DM dm);
1861 extern PetscErrorCode DMCreateDefaultConstraints_Plex(DM dm);
1862 extern PetscErrorCode DMCreateMatrix_Plex(DM dm,  Mat *J);
1863 extern PetscErrorCode DMCreateCoordinateDM_Plex(DM dm, DM *cdm);
1864 PETSC_INTERN PetscErrorCode DMClone_Plex(DM dm, DM *newdm);
1865 extern PetscErrorCode DMSetUp_Plex(DM dm);
1866 extern PetscErrorCode DMDestroy_Plex(DM dm);
1867 extern PetscErrorCode DMView_Plex(DM dm, PetscViewer viewer);
1868 extern PetscErrorCode DMLoad_Plex(DM dm, PetscViewer viewer);
1869 extern PetscErrorCode DMCreateSubDM_Plex(DM dm, PetscInt numFields, const PetscInt fields[], IS *is, DM *subdm);
1870 extern PetscErrorCode DMCreateSuperDM_Plex(DM dms[], PetscInt len, IS **is, DM *superdm);
1871 static PetscErrorCode DMInitialize_Plex(DM dm);
1872 
1873 /* Replace dm with the contents of dmNew
1874    - Share the DM_Plex structure
1875    - Share the coordinates
1876    - Share the SF
1877 */
1878 static PetscErrorCode DMPlexReplace_Static(DM dm, DM dmNew)
1879 {
1880   PetscSF               sf;
1881   DM                    coordDM, coarseDM;
1882   Vec                   coords;
1883   PetscBool             isper;
1884   const PetscReal      *maxCell, *L;
1885   const DMBoundaryType *bd;
1886   PetscErrorCode        ierr;
1887 
1888   PetscFunctionBegin;
1889   ierr = DMGetPointSF(dmNew, &sf);CHKERRQ(ierr);
1890   ierr = DMSetPointSF(dm, sf);CHKERRQ(ierr);
1891   ierr = DMGetCoordinateDM(dmNew, &coordDM);CHKERRQ(ierr);
1892   ierr = DMGetCoordinatesLocal(dmNew, &coords);CHKERRQ(ierr);
1893   ierr = DMSetCoordinateDM(dm, coordDM);CHKERRQ(ierr);
1894   ierr = DMSetCoordinatesLocal(dm, coords);CHKERRQ(ierr);
1895   ierr = DMGetPeriodicity(dm, &isper, &maxCell, &L, &bd);CHKERRQ(ierr);
1896   ierr = DMSetPeriodicity(dmNew, isper, maxCell, L, bd);CHKERRQ(ierr);
1897   ierr = DMDestroy_Plex(dm);CHKERRQ(ierr);
1898   ierr = DMInitialize_Plex(dm);CHKERRQ(ierr);
1899   dm->data = dmNew->data;
1900   ((DM_Plex *) dmNew->data)->refct++;
1901   dmNew->labels->refct++;
1902   if (!--(dm->labels->refct)) {
1903     DMLabelLink next = dm->labels->next;
1904 
1905     /* destroy the labels */
1906     while (next) {
1907       DMLabelLink tmp = next->next;
1908 
1909       ierr = DMLabelDestroy(&next->label);CHKERRQ(ierr);
1910       ierr = PetscFree(next);CHKERRQ(ierr);
1911       next = tmp;
1912     }
1913     ierr = PetscFree(dm->labels);CHKERRQ(ierr);
1914   }
1915   dm->labels = dmNew->labels;
1916   dm->depthLabel = dmNew->depthLabel;
1917   ierr = DMGetCoarseDM(dmNew,&coarseDM);CHKERRQ(ierr);
1918   ierr = DMSetCoarseDM(dm,coarseDM);CHKERRQ(ierr);
1919   PetscFunctionReturn(0);
1920 }
1921 
1922 /* Swap dm with the contents of dmNew
1923    - Swap the DM_Plex structure
1924    - Swap the coordinates
1925    - Swap the point PetscSF
1926 */
1927 static PetscErrorCode DMPlexSwap_Static(DM dmA, DM dmB)
1928 {
1929   DM              coordDMA, coordDMB;
1930   Vec             coordsA,  coordsB;
1931   PetscSF         sfA,      sfB;
1932   void            *tmp;
1933   DMLabelLinkList listTmp;
1934   DMLabel         depthTmp;
1935   PetscErrorCode  ierr;
1936 
1937   PetscFunctionBegin;
1938   ierr = DMGetPointSF(dmA, &sfA);CHKERRQ(ierr);
1939   ierr = DMGetPointSF(dmB, &sfB);CHKERRQ(ierr);
1940   ierr = PetscObjectReference((PetscObject) sfA);CHKERRQ(ierr);
1941   ierr = DMSetPointSF(dmA, sfB);CHKERRQ(ierr);
1942   ierr = DMSetPointSF(dmB, sfA);CHKERRQ(ierr);
1943   ierr = PetscObjectDereference((PetscObject) sfA);CHKERRQ(ierr);
1944 
1945   ierr = DMGetCoordinateDM(dmA, &coordDMA);CHKERRQ(ierr);
1946   ierr = DMGetCoordinateDM(dmB, &coordDMB);CHKERRQ(ierr);
1947   ierr = PetscObjectReference((PetscObject) coordDMA);CHKERRQ(ierr);
1948   ierr = DMSetCoordinateDM(dmA, coordDMB);CHKERRQ(ierr);
1949   ierr = DMSetCoordinateDM(dmB, coordDMA);CHKERRQ(ierr);
1950   ierr = PetscObjectDereference((PetscObject) coordDMA);CHKERRQ(ierr);
1951 
1952   ierr = DMGetCoordinatesLocal(dmA, &coordsA);CHKERRQ(ierr);
1953   ierr = DMGetCoordinatesLocal(dmB, &coordsB);CHKERRQ(ierr);
1954   ierr = PetscObjectReference((PetscObject) coordsA);CHKERRQ(ierr);
1955   ierr = DMSetCoordinatesLocal(dmA, coordsB);CHKERRQ(ierr);
1956   ierr = DMSetCoordinatesLocal(dmB, coordsA);CHKERRQ(ierr);
1957   ierr = PetscObjectDereference((PetscObject) coordsA);CHKERRQ(ierr);
1958 
1959   tmp       = dmA->data;
1960   dmA->data = dmB->data;
1961   dmB->data = tmp;
1962   listTmp   = dmA->labels;
1963   dmA->labels = dmB->labels;
1964   dmB->labels = listTmp;
1965   depthTmp  = dmA->depthLabel;
1966   dmA->depthLabel = dmB->depthLabel;
1967   dmB->depthLabel = depthTmp;
1968   PetscFunctionReturn(0);
1969 }
1970 
1971 PetscErrorCode DMSetFromOptions_NonRefinement_Plex(PetscOptionItems *PetscOptionsObject,DM dm)
1972 {
1973   DM_Plex       *mesh = (DM_Plex*) dm->data;
1974   PetscErrorCode ierr;
1975 
1976   PetscFunctionBegin;
1977   /* Handle viewing */
1978   ierr = PetscOptionsBool("-dm_plex_print_set_values", "Output all set values info", "DMView", PETSC_FALSE, &mesh->printSetValues, NULL);CHKERRQ(ierr);
1979   ierr = PetscOptionsInt("-dm_plex_print_fem", "Debug output level all fem computations", "DMView", 0, &mesh->printFEM, NULL);CHKERRQ(ierr);
1980   ierr = PetscOptionsReal("-dm_plex_print_tol", "Tolerance for FEM output", "DMView", mesh->printTol, &mesh->printTol, NULL);CHKERRQ(ierr);
1981   /* Point Location */
1982   ierr = PetscOptionsBool("-dm_plex_hash_location", "Use grid hashing for point location", "DMView", PETSC_FALSE, &mesh->useHashLocation, NULL);CHKERRQ(ierr);
1983   /* Generation and remeshing */
1984   ierr = PetscOptionsBool("-dm_plex_remesh_bd", "Allow changes to the boundary on remeshing", "DMView", PETSC_FALSE, &mesh->remeshBd, NULL);CHKERRQ(ierr);
1985   /* Projection behavior */
1986   ierr = PetscOptionsInt("-dm_plex_max_projection_height", "Maxmimum mesh point height used to project locally", "DMPlexSetMaxProjectionHeight", 0, &mesh->maxProjectionHeight, NULL);CHKERRQ(ierr);
1987   ierr = PetscOptionsBool("-dm_plex_regular_refinement", "Use special nested projection algorithm for regular refinement", "DMPlexSetRegularRefinement", mesh->regularRefinement, &mesh->regularRefinement, NULL);CHKERRQ(ierr);
1988 
1989   ierr = PetscPartitionerSetFromOptions(mesh->partitioner);CHKERRQ(ierr);
1990   PetscFunctionReturn(0);
1991 }
1992 
1993 static PetscErrorCode DMSetFromOptions_Plex(PetscOptionItems *PetscOptionsObject,DM dm)
1994 {
1995   PetscInt       refine = 0, coarsen = 0, r;
1996   PetscBool      isHierarchy;
1997   PetscErrorCode ierr;
1998 
1999   PetscFunctionBegin;
2000   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2001   ierr = PetscOptionsHead(PetscOptionsObject,"DMPlex Options");CHKERRQ(ierr);
2002   /* Handle DMPlex refinement */
2003   ierr = PetscOptionsInt("-dm_refine", "The number of uniform refinements", "DMCreate", refine, &refine, NULL);CHKERRQ(ierr);
2004   ierr = PetscOptionsInt("-dm_refine_hierarchy", "The number of uniform refinements", "DMCreate", refine, &refine, &isHierarchy);CHKERRQ(ierr);
2005   if (refine) {ierr = DMPlexSetRefinementUniform(dm, PETSC_TRUE);CHKERRQ(ierr);}
2006   if (refine && isHierarchy) {
2007     DM *dms, coarseDM;
2008 
2009     ierr = DMGetCoarseDM(dm, &coarseDM);CHKERRQ(ierr);
2010     ierr = PetscObjectReference((PetscObject)coarseDM);CHKERRQ(ierr);
2011     ierr = PetscMalloc1(refine,&dms);CHKERRQ(ierr);
2012     ierr = DMRefineHierarchy(dm, refine, dms);CHKERRQ(ierr);
2013     /* Total hack since we do not pass in a pointer */
2014     ierr = DMPlexSwap_Static(dm, dms[refine-1]);CHKERRQ(ierr);
2015     if (refine == 1) {
2016       ierr = DMSetCoarseDM(dm, dms[0]);CHKERRQ(ierr);
2017       ierr = DMPlexSetRegularRefinement(dm, PETSC_TRUE);CHKERRQ(ierr);
2018     } else {
2019       ierr = DMSetCoarseDM(dm, dms[refine-2]);CHKERRQ(ierr);
2020       ierr = DMPlexSetRegularRefinement(dm, PETSC_TRUE);CHKERRQ(ierr);
2021       ierr = DMSetCoarseDM(dms[0], dms[refine-1]);CHKERRQ(ierr);
2022       ierr = DMPlexSetRegularRefinement(dms[0], PETSC_TRUE);CHKERRQ(ierr);
2023     }
2024     ierr = DMSetCoarseDM(dms[refine-1], coarseDM);CHKERRQ(ierr);
2025     ierr = PetscObjectDereference((PetscObject)coarseDM);CHKERRQ(ierr);
2026     /* Free DMs */
2027     for (r = 0; r < refine; ++r) {
2028       ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dms[r]);CHKERRQ(ierr);
2029       ierr = DMDestroy(&dms[r]);CHKERRQ(ierr);
2030     }
2031     ierr = PetscFree(dms);CHKERRQ(ierr);
2032   } else {
2033     for (r = 0; r < refine; ++r) {
2034       DM refinedMesh;
2035 
2036       ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dm);CHKERRQ(ierr);
2037       ierr = DMRefine(dm, PetscObjectComm((PetscObject) dm), &refinedMesh);CHKERRQ(ierr);
2038       /* Total hack since we do not pass in a pointer */
2039       ierr = DMPlexReplace_Static(dm, refinedMesh);CHKERRQ(ierr);
2040       ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dm);CHKERRQ(ierr);
2041       ierr = DMDestroy(&refinedMesh);CHKERRQ(ierr);
2042     }
2043   }
2044   /* Handle DMPlex coarsening */
2045   ierr = PetscOptionsInt("-dm_coarsen", "Coarsen the mesh", "DMCreate", coarsen, &coarsen, NULL);CHKERRQ(ierr);
2046   ierr = PetscOptionsInt("-dm_coarsen_hierarchy", "The number of coarsenings", "DMCreate", coarsen, &coarsen, &isHierarchy);CHKERRQ(ierr);
2047   if (coarsen && isHierarchy) {
2048     DM *dms;
2049 
2050     ierr = PetscMalloc1(coarsen, &dms);CHKERRQ(ierr);
2051     ierr = DMCoarsenHierarchy(dm, coarsen, dms);CHKERRQ(ierr);
2052     /* Free DMs */
2053     for (r = 0; r < coarsen; ++r) {
2054       ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dms[r]);CHKERRQ(ierr);
2055       ierr = DMDestroy(&dms[r]);CHKERRQ(ierr);
2056     }
2057     ierr = PetscFree(dms);CHKERRQ(ierr);
2058   } else {
2059     for (r = 0; r < coarsen; ++r) {
2060       DM coarseMesh;
2061 
2062       ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dm);CHKERRQ(ierr);
2063       ierr = DMCoarsen(dm, PetscObjectComm((PetscObject) dm), &coarseMesh);CHKERRQ(ierr);
2064       /* Total hack since we do not pass in a pointer */
2065       ierr = DMPlexReplace_Static(dm, coarseMesh);CHKERRQ(ierr);
2066       ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dm);CHKERRQ(ierr);
2067       ierr = DMDestroy(&coarseMesh);CHKERRQ(ierr);
2068     }
2069   }
2070   /* Handle */
2071   ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dm);CHKERRQ(ierr);
2072   ierr = PetscOptionsTail();CHKERRQ(ierr);
2073   PetscFunctionReturn(0);
2074 }
2075 
2076 static PetscErrorCode DMCreateGlobalVector_Plex(DM dm,Vec *vec)
2077 {
2078   PetscErrorCode ierr;
2079 
2080   PetscFunctionBegin;
2081   ierr = DMCreateGlobalVector_Section_Private(dm,vec);CHKERRQ(ierr);
2082   /* ierr = VecSetOperation(*vec, VECOP_DUPLICATE, (void(*)(void)) VecDuplicate_MPI_DM);CHKERRQ(ierr); */
2083   ierr = VecSetOperation(*vec, VECOP_VIEW, (void (*)(void)) VecView_Plex);CHKERRQ(ierr);
2084   ierr = VecSetOperation(*vec, VECOP_VIEWNATIVE, (void (*)(void)) VecView_Plex_Native);CHKERRQ(ierr);
2085   ierr = VecSetOperation(*vec, VECOP_LOAD, (void (*)(void)) VecLoad_Plex);CHKERRQ(ierr);
2086   ierr = VecSetOperation(*vec, VECOP_LOADNATIVE, (void (*)(void)) VecLoad_Plex_Native);CHKERRQ(ierr);
2087   PetscFunctionReturn(0);
2088 }
2089 
2090 static PetscErrorCode DMCreateLocalVector_Plex(DM dm,Vec *vec)
2091 {
2092   PetscErrorCode ierr;
2093 
2094   PetscFunctionBegin;
2095   ierr = DMCreateLocalVector_Section_Private(dm,vec);CHKERRQ(ierr);
2096   ierr = VecSetOperation(*vec, VECOP_VIEW, (void (*)(void)) VecView_Plex_Local);CHKERRQ(ierr);
2097   ierr = VecSetOperation(*vec, VECOP_LOAD, (void (*)(void)) VecLoad_Plex_Local);CHKERRQ(ierr);
2098   PetscFunctionReturn(0);
2099 }
2100 
2101 static PetscErrorCode DMGetDimPoints_Plex(DM dm, PetscInt dim, PetscInt *pStart, PetscInt *pEnd)
2102 {
2103   PetscInt       depth, d;
2104   PetscErrorCode ierr;
2105 
2106   PetscFunctionBegin;
2107   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
2108   if (depth == 1) {
2109     ierr = DMGetDimension(dm, &d);CHKERRQ(ierr);
2110     if (dim == 0)      {ierr = DMPlexGetDepthStratum(dm, dim, pStart, pEnd);CHKERRQ(ierr);}
2111     else if (dim == d) {ierr = DMPlexGetDepthStratum(dm, 1, pStart, pEnd);CHKERRQ(ierr);}
2112     else               {*pStart = 0; *pEnd = 0;}
2113   } else {
2114     ierr = DMPlexGetDepthStratum(dm, dim, pStart, pEnd);CHKERRQ(ierr);
2115   }
2116   PetscFunctionReturn(0);
2117 }
2118 
2119 static PetscErrorCode DMGetNeighors_Plex(DM dm, PetscInt *nranks, const PetscMPIInt *ranks[])
2120 {
2121   PetscSF        sf;
2122   PetscErrorCode ierr;
2123 
2124   PetscFunctionBegin;
2125   ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr);
2126   ierr = PetscSFGetRanks(sf, nranks, ranks, NULL, NULL, NULL);CHKERRQ(ierr);
2127   PetscFunctionReturn(0);
2128 }
2129 
2130 static PetscErrorCode DMHasCreateInjection_Plex(DM dm, PetscBool *flg)
2131 {
2132   PetscFunctionBegin;
2133   PetscValidHeaderSpecific(dm,DM_CLASSID,1);
2134   PetscValidPointer(flg,2);
2135   *flg = PETSC_TRUE;
2136   PetscFunctionReturn(0);
2137 }
2138 
2139 static PetscErrorCode DMInitialize_Plex(DM dm)
2140 {
2141   PetscErrorCode ierr;
2142 
2143   PetscFunctionBegin;
2144   dm->ops->view                            = DMView_Plex;
2145   dm->ops->load                            = DMLoad_Plex;
2146   dm->ops->setfromoptions                  = DMSetFromOptions_Plex;
2147   dm->ops->clone                           = DMClone_Plex;
2148   dm->ops->setup                           = DMSetUp_Plex;
2149   dm->ops->createdefaultsection            = DMCreateDefaultSection_Plex;
2150   dm->ops->createdefaultconstraints        = DMCreateDefaultConstraints_Plex;
2151   dm->ops->createglobalvector              = DMCreateGlobalVector_Plex;
2152   dm->ops->createlocalvector               = DMCreateLocalVector_Plex;
2153   dm->ops->getlocaltoglobalmapping         = NULL;
2154   dm->ops->createfieldis                   = NULL;
2155   dm->ops->createcoordinatedm              = DMCreateCoordinateDM_Plex;
2156   dm->ops->getcoloring                     = NULL;
2157   dm->ops->creatematrix                    = DMCreateMatrix_Plex;
2158   dm->ops->createinterpolation             = DMCreateInterpolation_Plex;
2159   dm->ops->createmassmatrix                = DMCreateMassMatrix_Plex;
2160   dm->ops->getaggregates                   = NULL;
2161   dm->ops->getinjection                    = DMCreateInjection_Plex;
2162   dm->ops->hascreateinjection              = DMHasCreateInjection_Plex;
2163   dm->ops->refine                          = DMRefine_Plex;
2164   dm->ops->coarsen                         = DMCoarsen_Plex;
2165   dm->ops->refinehierarchy                 = DMRefineHierarchy_Plex;
2166   dm->ops->coarsenhierarchy                = DMCoarsenHierarchy_Plex;
2167   dm->ops->adaptlabel                      = DMAdaptLabel_Plex;
2168   dm->ops->adaptmetric                     = DMAdaptMetric_Plex;
2169   dm->ops->globaltolocalbegin              = NULL;
2170   dm->ops->globaltolocalend                = NULL;
2171   dm->ops->localtoglobalbegin              = NULL;
2172   dm->ops->localtoglobalend                = NULL;
2173   dm->ops->destroy                         = DMDestroy_Plex;
2174   dm->ops->createsubdm                     = DMCreateSubDM_Plex;
2175   dm->ops->createsuperdm                   = DMCreateSuperDM_Plex;
2176   dm->ops->getdimpoints                    = DMGetDimPoints_Plex;
2177   dm->ops->locatepoints                    = DMLocatePoints_Plex;
2178   dm->ops->projectfunctionlocal            = DMProjectFunctionLocal_Plex;
2179   dm->ops->projectfunctionlabellocal       = DMProjectFunctionLabelLocal_Plex;
2180   dm->ops->projectfieldlocal               = DMProjectFieldLocal_Plex;
2181   dm->ops->projectfieldlabellocal          = DMProjectFieldLabelLocal_Plex;
2182   dm->ops->computel2diff                   = DMComputeL2Diff_Plex;
2183   dm->ops->computel2gradientdiff           = DMComputeL2GradientDiff_Plex;
2184   dm->ops->computel2fielddiff              = DMComputeL2FieldDiff_Plex;
2185   dm->ops->getneighbors                    = DMGetNeighors_Plex;
2186   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMPlexInsertBoundaryValues_C",DMPlexInsertBoundaryValues_Plex);CHKERRQ(ierr);
2187   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMSetUpGLVisViewer_C",DMSetUpGLVisViewer_Plex);CHKERRQ(ierr);
2188   PetscFunctionReturn(0);
2189 }
2190 
2191 PETSC_INTERN PetscErrorCode DMClone_Plex(DM dm, DM *newdm)
2192 {
2193   DM_Plex        *mesh = (DM_Plex *) dm->data;
2194   PetscErrorCode ierr;
2195 
2196   PetscFunctionBegin;
2197   mesh->refct++;
2198   (*newdm)->data = mesh;
2199   ierr = PetscObjectChangeTypeName((PetscObject) *newdm, DMPLEX);CHKERRQ(ierr);
2200   ierr = DMInitialize_Plex(*newdm);CHKERRQ(ierr);
2201   PetscFunctionReturn(0);
2202 }
2203 
2204 /*MC
2205   DMPLEX = "plex" - A DM object that encapsulates an unstructured mesh, or CW Complex, which can be expressed using a Hasse Diagram.
2206                     In the local representation, Vecs contain all unknowns in the interior and shared boundary. This is
2207                     specified by a PetscSection object. Ownership in the global representation is determined by
2208                     ownership of the underlying DMPlex points. This is specified by another PetscSection object.
2209 
2210   Level: intermediate
2211 
2212 .seealso: DMType, DMPlexCreate(), DMCreate(), DMSetType()
2213 M*/
2214 
2215 PETSC_EXTERN PetscErrorCode DMCreate_Plex(DM dm)
2216 {
2217   DM_Plex        *mesh;
2218   PetscInt       unit, d;
2219   PetscErrorCode ierr;
2220 
2221   PetscFunctionBegin;
2222   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2223   ierr     = PetscNewLog(dm,&mesh);CHKERRQ(ierr);
2224   dm->dim  = 0;
2225   dm->data = mesh;
2226 
2227   mesh->refct             = 1;
2228   ierr                    = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &mesh->coneSection);CHKERRQ(ierr);
2229   mesh->maxConeSize       = 0;
2230   mesh->cones             = NULL;
2231   mesh->coneOrientations  = NULL;
2232   ierr                    = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &mesh->supportSection);CHKERRQ(ierr);
2233   mesh->maxSupportSize    = 0;
2234   mesh->supports          = NULL;
2235   mesh->refinementUniform = PETSC_TRUE;
2236   mesh->refinementLimit   = -1.0;
2237 
2238   mesh->facesTmp = NULL;
2239 
2240   mesh->tetgenOpts   = NULL;
2241   mesh->triangleOpts = NULL;
2242   ierr = PetscPartitionerCreate(PetscObjectComm((PetscObject)dm), &mesh->partitioner);CHKERRQ(ierr);
2243   mesh->remeshBd     = PETSC_FALSE;
2244 
2245   mesh->subpointMap = NULL;
2246 
2247   for (unit = 0; unit < NUM_PETSC_UNITS; ++unit) mesh->scale[unit] = 1.0;
2248 
2249   mesh->regularRefinement   = PETSC_FALSE;
2250   mesh->depthState          = -1;
2251   mesh->globalVertexNumbers = NULL;
2252   mesh->globalCellNumbers   = NULL;
2253   mesh->anchorSection       = NULL;
2254   mesh->anchorIS            = NULL;
2255   mesh->createanchors       = NULL;
2256   mesh->computeanchormatrix = NULL;
2257   mesh->parentSection       = NULL;
2258   mesh->parents             = NULL;
2259   mesh->childIDs            = NULL;
2260   mesh->childSection        = NULL;
2261   mesh->children            = NULL;
2262   mesh->referenceTree       = NULL;
2263   mesh->getchildsymmetry    = NULL;
2264   for (d = 0; d < 8; ++d) mesh->hybridPointMax[d] = PETSC_DETERMINE;
2265   mesh->vtkCellHeight       = 0;
2266   mesh->useAnchors          = PETSC_FALSE;
2267 
2268   mesh->maxProjectionHeight = 0;
2269 
2270   mesh->printSetValues = PETSC_FALSE;
2271   mesh->printFEM       = 0;
2272   mesh->printTol       = 1.0e-10;
2273 
2274   ierr = DMInitialize_Plex(dm);CHKERRQ(ierr);
2275   PetscFunctionReturn(0);
2276 }
2277 
2278 /*@
2279   DMPlexCreate - Creates a DMPlex object, which encapsulates an unstructured mesh, or CW complex, which can be expressed using a Hasse Diagram.
2280 
2281   Collective on MPI_Comm
2282 
2283   Input Parameter:
2284 . comm - The communicator for the DMPlex object
2285 
2286   Output Parameter:
2287 . mesh  - The DMPlex object
2288 
2289   Level: beginner
2290 
2291 .keywords: DMPlex, create
2292 @*/
2293 PetscErrorCode DMPlexCreate(MPI_Comm comm, DM *mesh)
2294 {
2295   PetscErrorCode ierr;
2296 
2297   PetscFunctionBegin;
2298   PetscValidPointer(mesh,2);
2299   ierr = DMCreate(comm, mesh);CHKERRQ(ierr);
2300   ierr = DMSetType(*mesh, DMPLEX);CHKERRQ(ierr);
2301   PetscFunctionReturn(0);
2302 }
2303 
2304 /*
2305   This takes as input the common mesh generator output, a list of the vertices for each cell, but vertex numbers are global and an SF is built for them
2306 */
2307 static PetscErrorCode DMPlexBuildFromCellList_Parallel_Private(DM dm, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, const int cells[], PetscSF *sfVert)
2308 {
2309   PetscSF         sfPoint;
2310   PetscLayout     vLayout;
2311   PetscHashI      vhash;
2312   PetscSFNode    *remoteVerticesAdj, *vertexLocal, *vertexOwner, *remoteVertex;
2313   const PetscInt *vrange;
2314   PetscInt        numVerticesAdj, off, *verticesAdj, numVerticesGhost = 0, *localVertex, *cone, c, p, v, g;
2315   PETSC_UNUSED PetscHashIIter ret, iter;
2316   PetscMPIInt     rank, size;
2317   PetscErrorCode  ierr;
2318 
2319   PetscFunctionBegin;
2320   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRQ(ierr);
2321   ierr = MPI_Comm_size(PetscObjectComm((PetscObject) dm), &size);CHKERRQ(ierr);
2322   /* Partition vertices */
2323   ierr = PetscLayoutCreate(PetscObjectComm((PetscObject) dm), &vLayout);CHKERRQ(ierr);
2324   ierr = PetscLayoutSetLocalSize(vLayout, numVertices);CHKERRQ(ierr);
2325   ierr = PetscLayoutSetBlockSize(vLayout, 1);CHKERRQ(ierr);
2326   ierr = PetscLayoutSetUp(vLayout);CHKERRQ(ierr);
2327   ierr = PetscLayoutGetRanges(vLayout, &vrange);CHKERRQ(ierr);
2328   /* Count vertices and map them to procs */
2329   PetscHashICreate(vhash);
2330   for (c = 0; c < numCells; ++c) {
2331     for (p = 0; p < numCorners; ++p) {
2332       PetscHashIPut(vhash, cells[c*numCorners+p], ret, iter);
2333     }
2334   }
2335   PetscHashISize(vhash, numVerticesAdj);
2336   ierr = PetscMalloc1(numVerticesAdj, &verticesAdj);CHKERRQ(ierr);
2337   off = 0; ierr = PetscHashIGetKeys(vhash, &off, verticesAdj);CHKERRQ(ierr);
2338   if (off != numVerticesAdj) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid number of local vertices %D should be %D", off, numVerticesAdj);
2339   ierr = PetscSortInt(numVerticesAdj, verticesAdj);CHKERRQ(ierr);
2340   ierr = PetscMalloc1(numVerticesAdj, &remoteVerticesAdj);CHKERRQ(ierr);
2341   for (v = 0; v < numVerticesAdj; ++v) {
2342     const PetscInt gv = verticesAdj[v];
2343     PetscInt       vrank;
2344 
2345     ierr = PetscFindInt(gv, size+1, vrange, &vrank);CHKERRQ(ierr);
2346     vrank = vrank < 0 ? -(vrank+2) : vrank;
2347     remoteVerticesAdj[v].index = gv - vrange[vrank];
2348     remoteVerticesAdj[v].rank  = vrank;
2349   }
2350   /* Create cones */
2351   ierr = DMPlexSetChart(dm, 0, numCells+numVerticesAdj);CHKERRQ(ierr);
2352   for (c = 0; c < numCells; ++c) {ierr = DMPlexSetConeSize(dm, c, numCorners);CHKERRQ(ierr);}
2353   ierr = DMSetUp(dm);CHKERRQ(ierr);
2354   ierr = DMGetWorkArray(dm, numCorners, MPIU_INT, &cone);CHKERRQ(ierr);
2355   for (c = 0; c < numCells; ++c) {
2356     for (p = 0; p < numCorners; ++p) {
2357       const PetscInt gv = cells[c*numCorners+p];
2358       PetscInt       lv;
2359 
2360       ierr = PetscFindInt(gv, numVerticesAdj, verticesAdj, &lv);CHKERRQ(ierr);
2361       if (lv < 0) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Could not find global vertex %D in local connectivity", gv);
2362       cone[p] = lv+numCells;
2363     }
2364     ierr = DMPlexSetCone(dm, c, cone);CHKERRQ(ierr);
2365   }
2366   ierr = DMRestoreWorkArray(dm, numCorners, MPIU_INT, &cone);CHKERRQ(ierr);
2367   /* Create SF for vertices */
2368   ierr = PetscSFCreate(PetscObjectComm((PetscObject)dm), sfVert);CHKERRQ(ierr);
2369   ierr = PetscObjectSetName((PetscObject) *sfVert, "Vertex Ownership SF");CHKERRQ(ierr);
2370   ierr = PetscSFSetFromOptions(*sfVert);CHKERRQ(ierr);
2371   ierr = PetscSFSetGraph(*sfVert, numVertices, numVerticesAdj, NULL, PETSC_OWN_POINTER, remoteVerticesAdj, PETSC_OWN_POINTER);CHKERRQ(ierr);
2372   ierr = PetscFree(verticesAdj);CHKERRQ(ierr);
2373   /* Build pointSF */
2374   ierr = PetscMalloc2(numVerticesAdj, &vertexLocal, numVertices, &vertexOwner);CHKERRQ(ierr);
2375   for (v = 0; v < numVerticesAdj; ++v) {vertexLocal[v].index = v+numCells; vertexLocal[v].rank = rank;}
2376   for (v = 0; v < numVertices;    ++v) {vertexOwner[v].index = -1;         vertexOwner[v].rank = -1;}
2377   ierr = PetscSFReduceBegin(*sfVert, MPIU_2INT, vertexLocal, vertexOwner, MPI_MAXLOC);CHKERRQ(ierr);
2378   ierr = PetscSFReduceEnd(*sfVert, MPIU_2INT, vertexLocal, vertexOwner, MPI_MAXLOC);CHKERRQ(ierr);
2379   for (v = 0; v < numVertices;    ++v) if (vertexOwner[v].rank < 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Global vertex %d on rank %d was unclaimed", v + vrange[rank], rank);
2380   ierr = PetscSFBcastBegin(*sfVert, MPIU_2INT, vertexOwner, vertexLocal);CHKERRQ(ierr);
2381   ierr = PetscSFBcastEnd(*sfVert, MPIU_2INT, vertexOwner, vertexLocal);CHKERRQ(ierr);
2382   for (v = 0; v < numVerticesAdj; ++v) if (vertexLocal[v].rank != rank) ++numVerticesGhost;
2383   ierr = PetscMalloc1(numVerticesGhost, &localVertex);CHKERRQ(ierr);
2384   ierr = PetscMalloc1(numVerticesGhost, &remoteVertex);CHKERRQ(ierr);
2385   for (v = 0, g = 0; v < numVerticesAdj; ++v) {
2386     if (vertexLocal[v].rank != rank) {
2387       localVertex[g]        = v+numCells;
2388       remoteVertex[g].index = vertexLocal[v].index;
2389       remoteVertex[g].rank  = vertexLocal[v].rank;
2390       ++g;
2391     }
2392   }
2393   ierr = PetscFree2(vertexLocal, vertexOwner);CHKERRQ(ierr);
2394   if (g != numVerticesGhost) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid number of vertex ghosts %D should be %D", g, numVerticesGhost);
2395   ierr = DMGetPointSF(dm, &sfPoint);CHKERRQ(ierr);
2396   ierr = PetscObjectSetName((PetscObject) sfPoint, "point SF");CHKERRQ(ierr);
2397   ierr = PetscSFSetGraph(sfPoint, numCells+numVerticesAdj, numVerticesGhost, localVertex, PETSC_OWN_POINTER, remoteVertex, PETSC_OWN_POINTER);CHKERRQ(ierr);
2398   ierr = PetscLayoutDestroy(&vLayout);CHKERRQ(ierr);
2399   PetscHashIDestroy(vhash);
2400   /* Fill in the rest of the topology structure */
2401   ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
2402   ierr = DMPlexStratify(dm);CHKERRQ(ierr);
2403   PetscFunctionReturn(0);
2404 }
2405 
2406 /*
2407   This takes as input the coordinates for each owned vertex
2408 */
2409 static PetscErrorCode DMPlexBuildCoordinates_Parallel_Private(DM dm, PetscInt spaceDim, PetscInt numCells, PetscInt numV, PetscSF sfVert, const PetscReal vertexCoords[])
2410 {
2411   PetscSection   coordSection;
2412   Vec            coordinates;
2413   PetscScalar   *coords;
2414   PetscInt       numVertices, numVerticesAdj, coordSize, v;
2415   PetscErrorCode ierr;
2416 
2417   PetscFunctionBegin;
2418   ierr = DMSetCoordinateDim(dm, spaceDim);CHKERRQ(ierr);
2419   ierr = PetscSFGetGraph(sfVert, &numVertices, &numVerticesAdj, NULL, NULL);CHKERRQ(ierr);
2420   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
2421   ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
2422   ierr = PetscSectionSetFieldComponents(coordSection, 0, spaceDim);CHKERRQ(ierr);
2423   ierr = PetscSectionSetChart(coordSection, numCells, numCells + numVerticesAdj);CHKERRQ(ierr);
2424   for (v = numCells; v < numCells+numVerticesAdj; ++v) {
2425     ierr = PetscSectionSetDof(coordSection, v, spaceDim);CHKERRQ(ierr);
2426     ierr = PetscSectionSetFieldDof(coordSection, v, 0, spaceDim);CHKERRQ(ierr);
2427   }
2428   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
2429   ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
2430   ierr = VecCreate(PetscObjectComm((PetscObject)dm), &coordinates);CHKERRQ(ierr);
2431   ierr = VecSetBlockSize(coordinates, spaceDim);CHKERRQ(ierr);
2432   ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
2433   ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
2434   ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
2435   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
2436   {
2437     MPI_Datatype coordtype;
2438 
2439     /* Need a temp buffer for coords if we have complex/single */
2440     ierr = MPI_Type_contiguous(spaceDim, MPIU_SCALAR, &coordtype);CHKERRQ(ierr);
2441     ierr = MPI_Type_commit(&coordtype);CHKERRQ(ierr);
2442 #if defined(PETSC_USE_COMPLEX)
2443     {
2444     PetscScalar *svertexCoords;
2445     PetscInt    i;
2446     ierr = PetscMalloc1(numV*spaceDim,&svertexCoords);CHKERRQ(ierr);
2447     for (i=0; i<numV*spaceDim; i++) svertexCoords[i] = vertexCoords[i];
2448     ierr = PetscSFBcastBegin(sfVert, coordtype, svertexCoords, coords);CHKERRQ(ierr);
2449     ierr = PetscSFBcastEnd(sfVert, coordtype, svertexCoords, coords);CHKERRQ(ierr);
2450     ierr = PetscFree(svertexCoords);CHKERRQ(ierr);
2451     }
2452 #else
2453     ierr = PetscSFBcastBegin(sfVert, coordtype, vertexCoords, coords);CHKERRQ(ierr);
2454     ierr = PetscSFBcastEnd(sfVert, coordtype, vertexCoords, coords);CHKERRQ(ierr);
2455 #endif
2456     ierr = MPI_Type_free(&coordtype);CHKERRQ(ierr);
2457   }
2458   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
2459   ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
2460   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
2461   PetscFunctionReturn(0);
2462 }
2463 
2464 /*@C
2465   DMPlexCreateFromCellListParallel - This takes as input common mesh generator output, a list of the vertices for each cell, and produces a DM
2466 
2467   Input Parameters:
2468 + comm - The communicator
2469 . dim - The topological dimension of the mesh
2470 . numCells - The number of cells owned by this process
2471 . numVertices - The number of vertices owned by this process
2472 . numCorners - The number of vertices for each cell
2473 . interpolate - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
2474 . cells - An array of numCells*numCorners numbers, the global vertex numbers for each cell
2475 . spaceDim - The spatial dimension used for coordinates
2476 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
2477 
2478   Output Parameter:
2479 + dm - The DM
2480 - vertexSF - Optional, SF describing complete vertex ownership
2481 
2482   Note: Two triangles sharing a face
2483 $
2484 $        2
2485 $      / | \
2486 $     /  |  \
2487 $    /   |   \
2488 $   0  0 | 1  3
2489 $    \   |   /
2490 $     \  |  /
2491 $      \ | /
2492 $        1
2493 would have input
2494 $  numCells = 2, numVertices = 4
2495 $  cells = [0 1 2  1 3 2]
2496 $
2497 which would result in the DMPlex
2498 $
2499 $        4
2500 $      / | \
2501 $     /  |  \
2502 $    /   |   \
2503 $   2  0 | 1  5
2504 $    \   |   /
2505 $     \  |  /
2506 $      \ | /
2507 $        3
2508 
2509   Level: beginner
2510 
2511 .seealso: DMPlexCreateFromCellList(), DMPlexCreateFromDAG(), DMPlexCreate()
2512 @*/
2513 PetscErrorCode DMPlexCreateFromCellListParallel(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, PetscBool interpolate, const int cells[], PetscInt spaceDim, const PetscReal vertexCoords[], PetscSF *vertexSF, DM *dm)
2514 {
2515   PetscSF        sfVert;
2516   PetscErrorCode ierr;
2517 
2518   PetscFunctionBegin;
2519   ierr = DMCreate(comm, dm);CHKERRQ(ierr);
2520   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
2521   PetscValidLogicalCollectiveInt(*dm, dim, 2);
2522   PetscValidLogicalCollectiveInt(*dm, spaceDim, 8);
2523   ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
2524   ierr = DMPlexBuildFromCellList_Parallel_Private(*dm, numCells, numVertices, numCorners, cells, &sfVert);CHKERRQ(ierr);
2525   if (interpolate) {
2526     DM idm;
2527 
2528     ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
2529     ierr = DMDestroy(dm);CHKERRQ(ierr);
2530     *dm  = idm;
2531   }
2532   ierr = DMPlexBuildCoordinates_Parallel_Private(*dm, spaceDim, numCells, numVertices,sfVert, vertexCoords);CHKERRQ(ierr);
2533   if (vertexSF) *vertexSF = sfVert;
2534   else {ierr = PetscSFDestroy(&sfVert);CHKERRQ(ierr);}
2535   PetscFunctionReturn(0);
2536 }
2537 
2538 /*
2539   This takes as input the common mesh generator output, a list of the vertices for each cell
2540 */
2541 static PetscErrorCode DMPlexBuildFromCellList_Private(DM dm, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, const int cells[])
2542 {
2543   PetscInt      *cone, c, p;
2544   PetscErrorCode ierr;
2545 
2546   PetscFunctionBegin;
2547   ierr = DMPlexSetChart(dm, 0, numCells+numVertices);CHKERRQ(ierr);
2548   for (c = 0; c < numCells; ++c) {
2549     ierr = DMPlexSetConeSize(dm, c, numCorners);CHKERRQ(ierr);
2550   }
2551   ierr = DMSetUp(dm);CHKERRQ(ierr);
2552   ierr = DMGetWorkArray(dm, numCorners, MPIU_INT, &cone);CHKERRQ(ierr);
2553   for (c = 0; c < numCells; ++c) {
2554     for (p = 0; p < numCorners; ++p) {
2555       cone[p] = cells[c*numCorners+p]+numCells;
2556     }
2557     ierr = DMPlexSetCone(dm, c, cone);CHKERRQ(ierr);
2558   }
2559   ierr = DMRestoreWorkArray(dm, numCorners, MPIU_INT, &cone);CHKERRQ(ierr);
2560   ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
2561   ierr = DMPlexStratify(dm);CHKERRQ(ierr);
2562   PetscFunctionReturn(0);
2563 }
2564 
2565 /*
2566   This takes as input the coordinates for each vertex
2567 */
2568 static PetscErrorCode DMPlexBuildCoordinates_Private(DM dm, PetscInt spaceDim, PetscInt numCells, PetscInt numVertices, const double vertexCoords[])
2569 {
2570   PetscSection   coordSection;
2571   Vec            coordinates;
2572   DM             cdm;
2573   PetscScalar   *coords;
2574   PetscInt       v, d;
2575   PetscErrorCode ierr;
2576 
2577   PetscFunctionBegin;
2578   ierr = DMSetCoordinateDim(dm, spaceDim);CHKERRQ(ierr);
2579   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
2580   ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
2581   ierr = PetscSectionSetFieldComponents(coordSection, 0, spaceDim);CHKERRQ(ierr);
2582   ierr = PetscSectionSetChart(coordSection, numCells, numCells + numVertices);CHKERRQ(ierr);
2583   for (v = numCells; v < numCells+numVertices; ++v) {
2584     ierr = PetscSectionSetDof(coordSection, v, spaceDim);CHKERRQ(ierr);
2585     ierr = PetscSectionSetFieldDof(coordSection, v, 0, spaceDim);CHKERRQ(ierr);
2586   }
2587   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
2588 
2589   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
2590   ierr = DMCreateLocalVector(cdm, &coordinates);CHKERRQ(ierr);
2591   ierr = VecSetBlockSize(coordinates, spaceDim);CHKERRQ(ierr);
2592   ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
2593   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
2594   for (v = 0; v < numVertices; ++v) {
2595     for (d = 0; d < spaceDim; ++d) {
2596       coords[v*spaceDim+d] = vertexCoords[v*spaceDim+d];
2597     }
2598   }
2599   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
2600   ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
2601   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
2602   PetscFunctionReturn(0);
2603 }
2604 
2605 /*@C
2606   DMPlexCreateFromCellList - This takes as input common mesh generator output, a list of the vertices for each cell, and produces a DM
2607 
2608   Input Parameters:
2609 + comm - The communicator
2610 . dim - The topological dimension of the mesh
2611 . numCells - The number of cells
2612 . numVertices - The number of vertices
2613 . numCorners - The number of vertices for each cell
2614 . interpolate - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
2615 . cells - An array of numCells*numCorners numbers, the vertices for each cell
2616 . spaceDim - The spatial dimension used for coordinates
2617 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
2618 
2619   Output Parameter:
2620 . dm - The DM
2621 
2622   Note: Two triangles sharing a face
2623 $
2624 $        2
2625 $      / | \
2626 $     /  |  \
2627 $    /   |   \
2628 $   0  0 | 1  3
2629 $    \   |   /
2630 $     \  |  /
2631 $      \ | /
2632 $        1
2633 would have input
2634 $  numCells = 2, numVertices = 4
2635 $  cells = [0 1 2  1 3 2]
2636 $
2637 which would result in the DMPlex
2638 $
2639 $        4
2640 $      / | \
2641 $     /  |  \
2642 $    /   |   \
2643 $   2  0 | 1  5
2644 $    \   |   /
2645 $     \  |  /
2646 $      \ | /
2647 $        3
2648 
2649   Level: beginner
2650 
2651 .seealso: DMPlexCreateFromDAG(), DMPlexCreate()
2652 @*/
2653 PetscErrorCode DMPlexCreateFromCellList(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, PetscBool interpolate, const int cells[], PetscInt spaceDim, const double vertexCoords[], DM *dm)
2654 {
2655   PetscErrorCode ierr;
2656 
2657   PetscFunctionBegin;
2658   ierr = DMCreate(comm, dm);CHKERRQ(ierr);
2659   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
2660   ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
2661   ierr = DMPlexBuildFromCellList_Private(*dm, numCells, numVertices, numCorners, cells);CHKERRQ(ierr);
2662   if (interpolate) {
2663     DM idm;
2664 
2665     ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
2666     ierr = DMDestroy(dm);CHKERRQ(ierr);
2667     *dm  = idm;
2668   }
2669   ierr = DMPlexBuildCoordinates_Private(*dm, spaceDim, numCells, numVertices, vertexCoords);CHKERRQ(ierr);
2670   PetscFunctionReturn(0);
2671 }
2672 
2673 /*@
2674   DMPlexCreateFromDAG - This takes as input the adjacency-list representation of the Directed Acyclic Graph (Hasse Diagram) encoding a mesh, and produces a DM
2675 
2676   Input Parameters:
2677 + dm - The empty DM object, usually from DMCreate() and DMSetDimension()
2678 . depth - The depth of the DAG
2679 . numPoints - The number of points at each depth
2680 . coneSize - The cone size of each point
2681 . cones - The concatenation of the cone points for each point, the cone list must be oriented correctly for each point
2682 . coneOrientations - The orientation of each cone point
2683 - vertexCoords - An array of numVertices*dim numbers, the coordinates of each vertex
2684 
2685   Output Parameter:
2686 . dm - The DM
2687 
2688   Note: Two triangles sharing a face would have input
2689 $  depth = 1, numPoints = [4 2], coneSize = [3 3 0 0 0 0]
2690 $  cones = [2 3 4  3 5 4], coneOrientations = [0 0 0  0 0 0]
2691 $ vertexCoords = [-1.0 0.0  0.0 -1.0  0.0 1.0  1.0 0.0]
2692 $
2693 which would result in the DMPlex
2694 $
2695 $        4
2696 $      / | \
2697 $     /  |  \
2698 $    /   |   \
2699 $   2  0 | 1  5
2700 $    \   |   /
2701 $     \  |  /
2702 $      \ | /
2703 $        3
2704 $
2705 $ Notice that all points are numbered consecutively, unlikely DMPlexCreateFromCellList()
2706 
2707   Level: advanced
2708 
2709 .seealso: DMPlexCreateFromCellList(), DMPlexCreate()
2710 @*/
2711 PetscErrorCode DMPlexCreateFromDAG(DM dm, PetscInt depth, const PetscInt numPoints[], const PetscInt coneSize[], const PetscInt cones[], const PetscInt coneOrientations[], const PetscScalar vertexCoords[])
2712 {
2713   Vec            coordinates;
2714   PetscSection   coordSection;
2715   PetscScalar    *coords;
2716   PetscInt       coordSize, firstVertex = -1, pStart = 0, pEnd = 0, p, v, dim, dimEmbed, d, off;
2717   PetscErrorCode ierr;
2718 
2719   PetscFunctionBegin;
2720   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
2721   ierr = DMGetCoordinateDim(dm, &dimEmbed);CHKERRQ(ierr);
2722   if (dimEmbed < dim) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Embedding dimension %d cannot be less than intrinsic dimension %d",dimEmbed,dim);
2723   for (d = 0; d <= depth; ++d) pEnd += numPoints[d];
2724   ierr = DMPlexSetChart(dm, pStart, pEnd);CHKERRQ(ierr);
2725   for (p = pStart; p < pEnd; ++p) {
2726     ierr = DMPlexSetConeSize(dm, p, coneSize[p-pStart]);CHKERRQ(ierr);
2727     if (firstVertex < 0 && !coneSize[p - pStart]) {
2728       firstVertex = p - pStart;
2729     }
2730   }
2731   if (firstVertex < 0 && numPoints[0]) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONG,"Expected %d vertices but could not find any", numPoints[0]);
2732   ierr = DMSetUp(dm);CHKERRQ(ierr); /* Allocate space for cones */
2733   for (p = pStart, off = 0; p < pEnd; off += coneSize[p-pStart], ++p) {
2734     ierr = DMPlexSetCone(dm, p, &cones[off]);CHKERRQ(ierr);
2735     ierr = DMPlexSetConeOrientation(dm, p, &coneOrientations[off]);CHKERRQ(ierr);
2736   }
2737   ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
2738   ierr = DMPlexStratify(dm);CHKERRQ(ierr);
2739   /* Build coordinates */
2740   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
2741   ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
2742   ierr = PetscSectionSetFieldComponents(coordSection, 0, dimEmbed);CHKERRQ(ierr);
2743   ierr = PetscSectionSetChart(coordSection, firstVertex, firstVertex+numPoints[0]);CHKERRQ(ierr);
2744   for (v = firstVertex; v < firstVertex+numPoints[0]; ++v) {
2745     ierr = PetscSectionSetDof(coordSection, v, dimEmbed);CHKERRQ(ierr);
2746     ierr = PetscSectionSetFieldDof(coordSection, v, 0, dimEmbed);CHKERRQ(ierr);
2747   }
2748   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
2749   ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
2750   ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
2751   ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
2752   ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
2753   ierr = VecSetBlockSize(coordinates, dimEmbed);CHKERRQ(ierr);
2754   ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
2755   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
2756   for (v = 0; v < numPoints[0]; ++v) {
2757     PetscInt off;
2758 
2759     ierr = PetscSectionGetOffset(coordSection, v+firstVertex, &off);CHKERRQ(ierr);
2760     for (d = 0; d < dimEmbed; ++d) {
2761       coords[off+d] = vertexCoords[v*dimEmbed+d];
2762     }
2763   }
2764   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
2765   ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
2766   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
2767   PetscFunctionReturn(0);
2768 }
2769 
2770 /*@C
2771   DMPlexCreateCellVertexFromFile - Create a DMPlex mesh from a simple cell-vertex file.
2772 
2773 + comm        - The MPI communicator
2774 . filename    - Name of the .dat file
2775 - interpolate - Create faces and edges in the mesh
2776 
2777   Output Parameter:
2778 . dm  - The DM object representing the mesh
2779 
2780   Note: The format is the simplest possible:
2781 $ Ne
2782 $ v0 v1 ... vk
2783 $ Nv
2784 $ x y z marker
2785 
2786   Level: beginner
2787 
2788 .seealso: DMPlexCreateFromFile(), DMPlexCreateMedFromFile(), DMPlexCreateGmsh(), DMPlexCreate()
2789 @*/
2790 PetscErrorCode DMPlexCreateCellVertexFromFile(MPI_Comm comm, const char filename[], PetscBool interpolate, DM *dm)
2791 {
2792   DMLabel         marker;
2793   PetscViewer     viewer;
2794   Vec             coordinates;
2795   PetscSection    coordSection;
2796   PetscScalar    *coords;
2797   char            line[PETSC_MAX_PATH_LEN];
2798   PetscInt        dim = 3, cdim = 3, coordSize, v, c, d;
2799   PetscMPIInt     rank;
2800   int             snum, Nv, Nc;
2801   PetscErrorCode  ierr;
2802 
2803   PetscFunctionBegin;
2804   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
2805   ierr = PetscViewerCreate(comm, &viewer);CHKERRQ(ierr);
2806   ierr = PetscViewerSetType(viewer, PETSCVIEWERASCII);CHKERRQ(ierr);
2807   ierr = PetscViewerFileSetMode(viewer, FILE_MODE_READ);CHKERRQ(ierr);
2808   ierr = PetscViewerFileSetName(viewer, filename);CHKERRQ(ierr);
2809   if (!rank) {
2810     ierr = PetscViewerRead(viewer, line, 2, NULL, PETSC_STRING);CHKERRQ(ierr);
2811     snum = sscanf(line, "%d %d", &Nc, &Nv);
2812     if (snum != 2) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
2813   } else {
2814     Nc = Nv = 0;
2815   }
2816   ierr = DMCreate(comm, dm);CHKERRQ(ierr);
2817   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
2818   ierr = DMPlexSetChart(*dm, 0, Nc+Nv);CHKERRQ(ierr);
2819   ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
2820   ierr = DMSetCoordinateDim(*dm, cdim);CHKERRQ(ierr);
2821   /* Read topology */
2822   if (!rank) {
2823     PetscInt cone[8], corners = 8;
2824     int      vbuf[8], v;
2825 
2826     for (c = 0; c < Nc; ++c) {ierr = DMPlexSetConeSize(*dm, c, corners);CHKERRQ(ierr);}
2827     ierr = DMSetUp(*dm);CHKERRQ(ierr);
2828     for (c = 0; c < Nc; ++c) {
2829       ierr = PetscViewerRead(viewer, line, corners, NULL, PETSC_STRING);CHKERRQ(ierr);
2830       snum = sscanf(line, "%d %d %d %d %d %d %d %d", &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3], &vbuf[4], &vbuf[5], &vbuf[6], &vbuf[7]);
2831       if (snum != corners) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
2832       for (v = 0; v < corners; ++v) cone[v] = vbuf[v] + Nc;
2833       /* Hexahedra are inverted */
2834       {
2835         PetscInt tmp = cone[1];
2836         cone[1] = cone[3];
2837         cone[3] = tmp;
2838       }
2839       ierr = DMPlexSetCone(*dm, c, cone);CHKERRQ(ierr);
2840     }
2841   }
2842   ierr = DMPlexSymmetrize(*dm);CHKERRQ(ierr);
2843   ierr = DMPlexStratify(*dm);CHKERRQ(ierr);
2844   /* Read coordinates */
2845   ierr = DMGetCoordinateSection(*dm, &coordSection);CHKERRQ(ierr);
2846   ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
2847   ierr = PetscSectionSetFieldComponents(coordSection, 0, cdim);CHKERRQ(ierr);
2848   ierr = PetscSectionSetChart(coordSection, Nc, Nc + Nv);CHKERRQ(ierr);
2849   for (v = Nc; v < Nc+Nv; ++v) {
2850     ierr = PetscSectionSetDof(coordSection, v, cdim);CHKERRQ(ierr);
2851     ierr = PetscSectionSetFieldDof(coordSection, v, 0, cdim);CHKERRQ(ierr);
2852   }
2853   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
2854   ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
2855   ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
2856   ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
2857   ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
2858   ierr = VecSetBlockSize(coordinates, cdim);CHKERRQ(ierr);
2859   ierr = VecSetType(coordinates, VECSTANDARD);CHKERRQ(ierr);
2860   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
2861   if (!rank) {
2862     double x[3];
2863     int    val;
2864 
2865     ierr = DMCreateLabel(*dm, "marker");CHKERRQ(ierr);
2866     ierr = DMGetLabel(*dm, "marker", &marker);CHKERRQ(ierr);
2867     for (v = 0; v < Nv; ++v) {
2868       ierr = PetscViewerRead(viewer, line, 4, NULL, PETSC_STRING);CHKERRQ(ierr);
2869       snum = sscanf(line, "%lg %lg %lg %d", &x[0], &x[1], &x[2], &val);
2870       if (snum != 4) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
2871       for (d = 0; d < cdim; ++d) coords[v*cdim+d] = x[d];
2872       if (val) {ierr = DMLabelSetValue(marker, v+Nc, val);CHKERRQ(ierr);}
2873     }
2874   }
2875   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
2876   ierr = DMSetCoordinatesLocal(*dm, coordinates);CHKERRQ(ierr);
2877   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
2878   ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
2879   if (interpolate) {
2880     DM      idm;
2881     DMLabel bdlabel;
2882 
2883     ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
2884     ierr = DMDestroy(dm);CHKERRQ(ierr);
2885     *dm  = idm;
2886 
2887     ierr = DMGetLabel(*dm, "marker", &bdlabel);CHKERRQ(ierr);
2888     ierr = DMPlexMarkBoundaryFaces(*dm, PETSC_DETERMINE, bdlabel);CHKERRQ(ierr);
2889     ierr = DMPlexLabelComplete(*dm, bdlabel);CHKERRQ(ierr);
2890   }
2891   PetscFunctionReturn(0);
2892 }
2893 
2894 /*@C
2895   DMPlexCreateFromFile - This takes a filename and produces a DM
2896 
2897   Input Parameters:
2898 + comm - The communicator
2899 . filename - A file name
2900 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
2901 
2902   Output Parameter:
2903 . dm - The DM
2904 
2905   Level: beginner
2906 
2907 .seealso: DMPlexCreateFromDAG(), DMPlexCreateFromCellList(), DMPlexCreate()
2908 @*/
2909 PetscErrorCode DMPlexCreateFromFile(MPI_Comm comm, const char filename[], PetscBool interpolate, DM *dm)
2910 {
2911   const char    *extGmsh    = ".msh";
2912   const char    *extCGNS    = ".cgns";
2913   const char    *extExodus  = ".exo";
2914   const char    *extGenesis = ".gen";
2915   const char    *extFluent  = ".cas";
2916   const char    *extHDF5    = ".h5";
2917   const char    *extMed     = ".med";
2918   const char    *extPLY     = ".ply";
2919   const char    *extCV      = ".dat";
2920   size_t         len;
2921   PetscBool      isGmsh, isCGNS, isExodus, isGenesis, isFluent, isHDF5, isMed, isPLY, isCV;
2922   PetscMPIInt    rank;
2923   PetscErrorCode ierr;
2924 
2925   PetscFunctionBegin;
2926   PetscValidPointer(filename, 2);
2927   PetscValidPointer(dm, 4);
2928   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
2929   ierr = PetscStrlen(filename, &len);CHKERRQ(ierr);
2930   if (!len) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Filename must be a valid path");
2931   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)], extGmsh,    4, &isGmsh);CHKERRQ(ierr);
2932   ierr = PetscStrncmp(&filename[PetscMax(0,len-5)], extCGNS,    5, &isCGNS);CHKERRQ(ierr);
2933   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)], extExodus,  4, &isExodus);CHKERRQ(ierr);
2934   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)], extGenesis, 4, &isGenesis);CHKERRQ(ierr);
2935   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)], extFluent,  4, &isFluent);CHKERRQ(ierr);
2936   ierr = PetscStrncmp(&filename[PetscMax(0,len-3)], extHDF5,    3, &isHDF5);CHKERRQ(ierr);
2937   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)], extMed,     4, &isMed);CHKERRQ(ierr);
2938   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)], extPLY,     4, &isPLY);CHKERRQ(ierr);
2939   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)], extCV,      4, &isCV);CHKERRQ(ierr);
2940   if (isGmsh) {
2941     ierr = DMPlexCreateGmshFromFile(comm, filename, interpolate, dm);CHKERRQ(ierr);
2942   } else if (isCGNS) {
2943     ierr = DMPlexCreateCGNSFromFile(comm, filename, interpolate, dm);CHKERRQ(ierr);
2944   } else if (isExodus || isGenesis) {
2945     ierr = DMPlexCreateExodusFromFile(comm, filename, interpolate, dm);CHKERRQ(ierr);
2946   } else if (isFluent) {
2947     ierr = DMPlexCreateFluentFromFile(comm, filename, interpolate, dm);CHKERRQ(ierr);
2948   } else if (isHDF5) {
2949     PetscViewer viewer;
2950 
2951     ierr = PetscViewerCreate(comm, &viewer);CHKERRQ(ierr);
2952     ierr = PetscViewerSetType(viewer, PETSCVIEWERHDF5);CHKERRQ(ierr);
2953     ierr = PetscViewerFileSetMode(viewer, FILE_MODE_READ);CHKERRQ(ierr);
2954     ierr = PetscViewerFileSetName(viewer, filename);CHKERRQ(ierr);
2955     ierr = DMCreate(comm, dm);CHKERRQ(ierr);
2956     ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
2957     ierr = DMLoad(*dm, viewer);CHKERRQ(ierr);
2958     ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
2959 
2960     if (interpolate) {
2961       DM idm;
2962 
2963       ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
2964       ierr = DMDestroy(dm);CHKERRQ(ierr);
2965       *dm  = idm;
2966     }
2967   } else if (isMed) {
2968     ierr = DMPlexCreateMedFromFile(comm, filename, interpolate, dm);CHKERRQ(ierr);
2969   } else if (isPLY) {
2970     ierr = DMPlexCreatePLYFromFile(comm, filename, interpolate, dm);CHKERRQ(ierr);
2971   } else if (isCV) {
2972     ierr = DMPlexCreateCellVertexFromFile(comm, filename, interpolate, dm);CHKERRQ(ierr);
2973   } else SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cannot load file %s: unrecognized extension", filename);
2974   PetscFunctionReturn(0);
2975 }
2976 
2977 /*@
2978   DMPlexCreateReferenceCell - Create a DMPLEX with the appropriate FEM reference cell
2979 
2980   Collective on comm
2981 
2982   Input Parameters:
2983 + comm    - The communicator
2984 . dim     - The spatial dimension
2985 - simplex - Flag for simplex, otherwise use a tensor-product cell
2986 
2987   Output Parameter:
2988 . refdm - The reference cell
2989 
2990   Level: intermediate
2991 
2992 .keywords: reference cell
2993 .seealso:
2994 @*/
2995 PetscErrorCode DMPlexCreateReferenceCell(MPI_Comm comm, PetscInt dim, PetscBool simplex, DM *refdm)
2996 {
2997   DM             rdm;
2998   Vec            coords;
2999   PetscErrorCode ierr;
3000 
3001   PetscFunctionBeginUser;
3002   ierr = DMCreate(comm, &rdm);CHKERRQ(ierr);
3003   ierr = DMSetType(rdm, DMPLEX);CHKERRQ(ierr);
3004   ierr = DMSetDimension(rdm, dim);CHKERRQ(ierr);
3005   switch (dim) {
3006   case 0:
3007   {
3008     PetscInt    numPoints[1]        = {1};
3009     PetscInt    coneSize[1]         = {0};
3010     PetscInt    cones[1]            = {0};
3011     PetscInt    coneOrientations[1] = {0};
3012     PetscScalar vertexCoords[1]     = {0.0};
3013 
3014     ierr = DMPlexCreateFromDAG(rdm, 0, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
3015   }
3016   break;
3017   case 1:
3018   {
3019     PetscInt    numPoints[2]        = {2, 1};
3020     PetscInt    coneSize[3]         = {2, 0, 0};
3021     PetscInt    cones[2]            = {1, 2};
3022     PetscInt    coneOrientations[2] = {0, 0};
3023     PetscScalar vertexCoords[2]     = {-1.0,  1.0};
3024 
3025     ierr = DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
3026   }
3027   break;
3028   case 2:
3029     if (simplex) {
3030       PetscInt    numPoints[2]        = {3, 1};
3031       PetscInt    coneSize[4]         = {3, 0, 0, 0};
3032       PetscInt    cones[3]            = {1, 2, 3};
3033       PetscInt    coneOrientations[3] = {0, 0, 0};
3034       PetscScalar vertexCoords[6]     = {-1.0, -1.0,  1.0, -1.0,  -1.0, 1.0};
3035 
3036       ierr = DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
3037     } else {
3038       PetscInt    numPoints[2]        = {4, 1};
3039       PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
3040       PetscInt    cones[4]            = {1, 2, 3, 4};
3041       PetscInt    coneOrientations[4] = {0, 0, 0, 0};
3042       PetscScalar vertexCoords[8]     = {-1.0, -1.0,  1.0, -1.0,  1.0, 1.0,  -1.0, 1.0};
3043 
3044       ierr = DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
3045     }
3046   break;
3047   case 3:
3048     if (simplex) {
3049       PetscInt    numPoints[2]        = {4, 1};
3050       PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
3051       PetscInt    cones[4]            = {1, 3, 2, 4};
3052       PetscInt    coneOrientations[4] = {0, 0, 0, 0};
3053       PetscScalar vertexCoords[12]    = {-1.0, -1.0, -1.0,  1.0, -1.0, -1.0,  -1.0, 1.0, -1.0,  -1.0, -1.0, 1.0};
3054 
3055       ierr = DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
3056     } else {
3057       PetscInt    numPoints[2]        = {8, 1};
3058       PetscInt    coneSize[9]         = {8, 0, 0, 0, 0, 0, 0, 0, 0};
3059       PetscInt    cones[8]            = {1, 4, 3, 2, 5, 6, 7, 8};
3060       PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
3061       PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0,  1.0, -1.0, -1.0,  1.0, 1.0, -1.0,  -1.0, 1.0, -1.0,
3062                                          -1.0, -1.0,  1.0,  1.0, -1.0,  1.0,  1.0, 1.0,  1.0,  -1.0, 1.0,  1.0};
3063 
3064       ierr = DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
3065     }
3066   break;
3067   default:
3068     SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Cannot create reference cell for dimension %d", dim);
3069   }
3070   ierr = DMPlexInterpolate(rdm, refdm);CHKERRQ(ierr);
3071   if (rdm->coordinateDM) {
3072     DM           ncdm;
3073     PetscSection cs;
3074     PetscInt     pEnd = -1;
3075 
3076     ierr = DMGetDefaultSection(rdm->coordinateDM, &cs);CHKERRQ(ierr);
3077     if (cs) {ierr = PetscSectionGetChart(cs, NULL, &pEnd);CHKERRQ(ierr);}
3078     if (pEnd >= 0) {
3079       ierr = DMClone(rdm->coordinateDM, &ncdm);CHKERRQ(ierr);
3080       ierr = DMSetDefaultSection(ncdm, cs);CHKERRQ(ierr);
3081       ierr = DMSetCoordinateDM(*refdm, ncdm);CHKERRQ(ierr);
3082       ierr = DMDestroy(&ncdm);CHKERRQ(ierr);
3083     }
3084   }
3085   ierr = DMGetCoordinatesLocal(rdm, &coords);CHKERRQ(ierr);
3086   if (coords) {
3087     ierr = DMSetCoordinatesLocal(*refdm, coords);CHKERRQ(ierr);
3088   } else {
3089     ierr = DMGetCoordinates(rdm, &coords);CHKERRQ(ierr);
3090     if (coords) {ierr = DMSetCoordinates(*refdm, coords);CHKERRQ(ierr);}
3091   }
3092   ierr = DMDestroy(&rdm);CHKERRQ(ierr);
3093   PetscFunctionReturn(0);
3094 }
3095