xref: /petsc/src/dm/impls/plex/plexcreate.c (revision 276c5506312304d9c58828fa177e3f5c76c17b23)
1 #define PETSCDM_DLL
2 #include <petsc/private/dmpleximpl.h>    /*I   "petscdmplex.h"   I*/
3 #include <petscdmda.h>
4 #include <petscsf.h>
5 
6 /*@
7   DMPlexCreateDoublet - Creates a mesh of two cells of the specified type, optionally with later refinement.
8 
9   Collective on MPI_Comm
10 
11   Input Parameters:
12 + comm - The communicator for the DM object
13 . dim - The spatial dimension
14 . simplex - Flag for simplicial cells, otherwise they are tensor product cells
15 . interpolate - Flag to create intermediate mesh pieces (edges, faces)
16 . refinementUniform - Flag for uniform parallel refinement
17 - refinementLimit - A nonzero number indicates the largest admissible volume for a refined cell
18 
19   Output Parameter:
20 . dm - The DM object
21 
22   Level: beginner
23 
24 .keywords: DM, create
25 .seealso: DMSetType(), DMCreate()
26 @*/
27 PetscErrorCode DMPlexCreateDoublet(MPI_Comm comm, PetscInt dim, PetscBool simplex, PetscBool interpolate, PetscBool refinementUniform, PetscReal refinementLimit, DM *newdm)
28 {
29   DM             dm;
30   PetscInt       p;
31   PetscMPIInt    rank;
32   PetscErrorCode ierr;
33 
34   PetscFunctionBegin;
35   ierr = DMCreate(comm, &dm);CHKERRQ(ierr);
36   ierr = DMSetType(dm, DMPLEX);CHKERRQ(ierr);
37   ierr = DMSetDimension(dm, dim);CHKERRQ(ierr);
38   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
39   switch (dim) {
40   case 2:
41     if (simplex) {ierr = PetscObjectSetName((PetscObject) dm, "triangular");CHKERRQ(ierr);}
42     else         {ierr = PetscObjectSetName((PetscObject) dm, "quadrilateral");CHKERRQ(ierr);}
43     break;
44   case 3:
45     if (simplex) {ierr = PetscObjectSetName((PetscObject) dm, "tetrahedral");CHKERRQ(ierr);}
46     else         {ierr = PetscObjectSetName((PetscObject) dm, "hexahedral");CHKERRQ(ierr);}
47     break;
48   default:
49     SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make meshes for dimension %d", dim);
50   }
51   if (rank) {
52     PetscInt numPoints[2] = {0, 0};
53     ierr = DMPlexCreateFromDAG(dm, 1, numPoints, NULL, NULL, NULL, NULL);CHKERRQ(ierr);
54   } else {
55     switch (dim) {
56     case 2:
57       if (simplex) {
58         PetscInt    numPoints[2]        = {4, 2};
59         PetscInt    coneSize[6]         = {3, 3, 0, 0, 0, 0};
60         PetscInt    cones[6]            = {2, 3, 4,  5, 4, 3};
61         PetscInt    coneOrientations[6] = {0, 0, 0,  0, 0, 0};
62         PetscScalar vertexCoords[8]     = {-0.5, 0.5, 0.0, 0.0, 0.0, 1.0, 0.5, 0.5};
63         PetscInt    markerPoints[8]     = {2, 1, 3, 1, 4, 1, 5, 1};
64 
65         ierr = DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
66         for (p = 0; p < 4; ++p) {ierr = DMSetLabelValue(dm, "marker", markerPoints[p*2], markerPoints[p*2+1]);CHKERRQ(ierr);}
67       } else {
68         PetscInt    numPoints[2]        = {6, 2};
69         PetscInt    coneSize[8]         = {4, 4, 0, 0, 0, 0, 0, 0};
70         PetscInt    cones[8]            = {2, 3, 4, 5,  3, 6, 7, 4};
71         PetscInt    coneOrientations[8] = {0, 0, 0, 0,  0, 0, 0, 0};
72         PetscScalar vertexCoords[12]    = {-1.0, -0.5,  0.0, -0.5,  0.0, 0.5,  -1.0, 0.5,  1.0, -0.5,  1.0, 0.5};
73 
74         ierr = DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
75       }
76       break;
77     case 3:
78       if (simplex) {
79         PetscInt    numPoints[2]        = {5, 2};
80         PetscInt    coneSize[7]         = {4, 4, 0, 0, 0, 0, 0};
81         PetscInt    cones[8]            = {4, 3, 5, 2,  5, 3, 4, 6};
82         PetscInt    coneOrientations[8] = {0, 0, 0, 0,  0, 0, 0, 0};
83         PetscScalar vertexCoords[15]    = {-1.0, 0.0, 0.0,  0.0, -1.0, 0.0,  0.0, 0.0, 1.0,  0.0, 1.0, 0.0,  1.0, 0.0, 0.0};
84         PetscInt    markerPoints[10]    = {2, 1, 3, 1, 4, 1, 5, 1, 6, 1};
85 
86         ierr = DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
87         for (p = 0; p < 5; ++p) {ierr = DMSetLabelValue(dm, "marker", markerPoints[p*2], markerPoints[p*2+1]);CHKERRQ(ierr);}
88       } else {
89         PetscInt    numPoints[2]         = {12, 2};
90         PetscInt    coneSize[14]         = {8, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
91         PetscInt    cones[16]            = {2, 3, 4, 5, 6, 7, 8, 9,  5, 4, 10, 11, 7, 12, 13, 8};
92         PetscInt    coneOrientations[16] = {0, 0, 0, 0, 0, 0, 0, 0,  0, 0,  0,  0, 0,  0,  0, 0};
93         PetscScalar vertexCoords[36]     = {-1.0, -0.5, -0.5,  -1.0,  0.5, -0.5,  0.0,  0.5, -0.5,   0.0, -0.5, -0.5,
94                                             -1.0, -0.5,  0.5,   0.0, -0.5,  0.5,  0.0,  0.5,  0.5,  -1.0,  0.5,  0.5,
95                                              1.0,  0.5, -0.5,   1.0, -0.5, -0.5,  1.0, -0.5,  0.5,   1.0,  0.5,  0.5};
96 
97         ierr = DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
98       }
99       break;
100     default:
101       SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make meshes for dimension %d", dim);
102     }
103   }
104   *newdm = dm;
105   if (refinementLimit > 0.0) {
106     DM rdm;
107     const char *name;
108 
109     ierr = DMPlexSetRefinementUniform(*newdm, PETSC_FALSE);CHKERRQ(ierr);
110     ierr = DMPlexSetRefinementLimit(*newdm, refinementLimit);CHKERRQ(ierr);
111     ierr = DMRefine(*newdm, comm, &rdm);CHKERRQ(ierr);
112     ierr = PetscObjectGetName((PetscObject) *newdm, &name);CHKERRQ(ierr);
113     ierr = PetscObjectSetName((PetscObject)    rdm,  name);CHKERRQ(ierr);
114     ierr = DMDestroy(newdm);CHKERRQ(ierr);
115     *newdm = rdm;
116   }
117   if (interpolate) {
118     DM idm = NULL;
119     const char *name;
120 
121     ierr = DMPlexInterpolate(*newdm, &idm);CHKERRQ(ierr);
122     ierr = PetscObjectGetName((PetscObject) *newdm, &name);CHKERRQ(ierr);
123     ierr = PetscObjectSetName((PetscObject)    idm,  name);CHKERRQ(ierr);
124     ierr = DMPlexCopyCoordinates(*newdm, idm);CHKERRQ(ierr);
125     ierr = DMCopyLabels(*newdm, idm);CHKERRQ(ierr);
126     ierr = DMDestroy(newdm);CHKERRQ(ierr);
127     *newdm = idm;
128   }
129   {
130     DM refinedMesh     = NULL;
131     DM distributedMesh = NULL;
132 
133     /* Distribute mesh over processes */
134     ierr = DMPlexDistribute(*newdm, 0, NULL, &distributedMesh);CHKERRQ(ierr);
135     if (distributedMesh) {
136       ierr = DMDestroy(newdm);CHKERRQ(ierr);
137       *newdm = distributedMesh;
138     }
139     if (refinementUniform) {
140       ierr = DMPlexSetRefinementUniform(*newdm, refinementUniform);CHKERRQ(ierr);
141       ierr = DMRefine(*newdm, comm, &refinedMesh);CHKERRQ(ierr);
142       if (refinedMesh) {
143         ierr = DMDestroy(newdm);CHKERRQ(ierr);
144         *newdm = refinedMesh;
145       }
146     }
147   }
148   PetscFunctionReturn(0);
149 }
150 
151 /*@
152   DMPlexCreateSquareBoundary - Creates a 1D mesh the is the boundary of a square lattice.
153 
154   Collective on MPI_Comm
155 
156   Input Parameters:
157 + comm  - The communicator for the DM object
158 . lower - The lower left corner coordinates
159 . upper - The upper right corner coordinates
160 - edges - The number of cells in each direction
161 
162   Output Parameter:
163 . dm  - The DM object
164 
165   Note: Here is the numbering returned for 2 cells in each direction:
166 $ 18--5-17--4--16
167 $  |     |     |
168 $  6    10     3
169 $  |     |     |
170 $ 19-11-20--9--15
171 $  |     |     |
172 $  7     8     2
173 $  |     |     |
174 $ 12--0-13--1--14
175 
176   Level: beginner
177 
178 .keywords: DM, create
179 .seealso: DMPlexCreateBoxMesh(), DMPlexCreateCubeBoundary(), DMSetType(), DMCreate()
180 @*/
181 PetscErrorCode DMPlexCreateSquareBoundary(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[])
182 {
183   const PetscInt numVertices    = (edges[0]+1)*(edges[1]+1);
184   const PetscInt numEdges       = edges[0]*(edges[1]+1) + (edges[0]+1)*edges[1];
185   PetscInt       markerTop      = 1;
186   PetscInt       markerBottom   = 1;
187   PetscInt       markerRight    = 1;
188   PetscInt       markerLeft     = 1;
189   PetscBool      markerSeparate = PETSC_FALSE;
190   Vec            coordinates;
191   PetscSection   coordSection;
192   PetscScalar    *coords;
193   PetscInt       coordSize;
194   PetscMPIInt    rank;
195   PetscInt       v, vx, vy;
196   PetscErrorCode ierr;
197 
198   PetscFunctionBegin;
199   ierr = PetscOptionsGetBool(((PetscObject) dm)->options,((PetscObject) dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL);CHKERRQ(ierr);
200   if (markerSeparate) {
201     markerTop    = 3;
202     markerBottom = 1;
203     markerRight  = 2;
204     markerLeft   = 4;
205   }
206   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRQ(ierr);
207   if (!rank) {
208     PetscInt e, ex, ey;
209 
210     ierr = DMPlexSetChart(dm, 0, numEdges+numVertices);CHKERRQ(ierr);
211     for (e = 0; e < numEdges; ++e) {
212       ierr = DMPlexSetConeSize(dm, e, 2);CHKERRQ(ierr);
213     }
214     ierr = DMSetUp(dm);CHKERRQ(ierr); /* Allocate space for cones */
215     for (vx = 0; vx <= edges[0]; vx++) {
216       for (ey = 0; ey < edges[1]; ey++) {
217         PetscInt edge   = vx*edges[1] + ey + edges[0]*(edges[1]+1);
218         PetscInt vertex = ey*(edges[0]+1) + vx + numEdges;
219         PetscInt cone[2];
220 
221         cone[0] = vertex; cone[1] = vertex+edges[0]+1;
222         ierr    = DMPlexSetCone(dm, edge, cone);CHKERRQ(ierr);
223         if (vx == edges[0]) {
224           ierr = DMSetLabelValue(dm, "marker", edge,    markerRight);CHKERRQ(ierr);
225           ierr = DMSetLabelValue(dm, "marker", cone[0], markerRight);CHKERRQ(ierr);
226           if (ey == edges[1]-1) {
227             ierr = DMSetLabelValue(dm, "marker", cone[1], markerRight);CHKERRQ(ierr);
228             ierr = DMSetLabelValue(dm, "Face Sets", cone[1], markerRight);CHKERRQ(ierr);
229           }
230         } else if (vx == 0) {
231           ierr = DMSetLabelValue(dm, "marker", edge,    markerLeft);CHKERRQ(ierr);
232           ierr = DMSetLabelValue(dm, "marker", cone[0], markerLeft);CHKERRQ(ierr);
233           if (ey == edges[1]-1) {
234             ierr = DMSetLabelValue(dm, "marker", cone[1], markerLeft);CHKERRQ(ierr);
235             ierr = DMSetLabelValue(dm, "Face Sets", cone[1], markerLeft);CHKERRQ(ierr);
236           }
237         }
238       }
239     }
240     for (vy = 0; vy <= edges[1]; vy++) {
241       for (ex = 0; ex < edges[0]; ex++) {
242         PetscInt edge   = vy*edges[0]     + ex;
243         PetscInt vertex = vy*(edges[0]+1) + ex + numEdges;
244         PetscInt cone[2];
245 
246         cone[0] = vertex; cone[1] = vertex+1;
247         ierr    = DMPlexSetCone(dm, edge, cone);CHKERRQ(ierr);
248         if (vy == edges[1]) {
249           ierr = DMSetLabelValue(dm, "marker", edge,    markerTop);CHKERRQ(ierr);
250           ierr = DMSetLabelValue(dm, "marker", cone[0], markerTop);CHKERRQ(ierr);
251           if (ex == edges[0]-1) {
252             ierr = DMSetLabelValue(dm, "marker", cone[1], markerTop);CHKERRQ(ierr);
253             ierr = DMSetLabelValue(dm, "Face Sets", cone[1], markerTop);CHKERRQ(ierr);
254           }
255         } else if (vy == 0) {
256           ierr = DMSetLabelValue(dm, "marker", edge,    markerBottom);CHKERRQ(ierr);
257           ierr = DMSetLabelValue(dm, "marker", cone[0], markerBottom);CHKERRQ(ierr);
258           if (ex == edges[0]-1) {
259             ierr = DMSetLabelValue(dm, "marker", cone[1], markerBottom);CHKERRQ(ierr);
260             ierr = DMSetLabelValue(dm, "Face Sets", cone[1], markerBottom);CHKERRQ(ierr);
261           }
262         }
263       }
264     }
265   }
266   ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
267   ierr = DMPlexStratify(dm);CHKERRQ(ierr);
268   /* Build coordinates */
269   ierr = DMSetCoordinateDim(dm, 2);CHKERRQ(ierr);
270   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
271   ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
272   ierr = PetscSectionSetChart(coordSection, numEdges, numEdges + numVertices);CHKERRQ(ierr);
273   ierr = PetscSectionSetFieldComponents(coordSection, 0, 2);CHKERRQ(ierr);
274   for (v = numEdges; v < numEdges+numVertices; ++v) {
275     ierr = PetscSectionSetDof(coordSection, v, 2);CHKERRQ(ierr);
276     ierr = PetscSectionSetFieldDof(coordSection, v, 0, 2);CHKERRQ(ierr);
277   }
278   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
279   ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
280   ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
281   ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
282   ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
283   ierr = VecSetBlockSize(coordinates, 2);CHKERRQ(ierr);
284   ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
285   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
286   for (vy = 0; vy <= edges[1]; ++vy) {
287     for (vx = 0; vx <= edges[0]; ++vx) {
288       coords[(vy*(edges[0]+1)+vx)*2+0] = lower[0] + ((upper[0] - lower[0])/edges[0])*vx;
289       coords[(vy*(edges[0]+1)+vx)*2+1] = lower[1] + ((upper[1] - lower[1])/edges[1])*vy;
290     }
291   }
292   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
293   ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
294   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
295   PetscFunctionReturn(0);
296 }
297 
298 /*@
299   DMPlexCreateCubeBoundary - Creates a 2D mesh that is the boundary of a cubic lattice.
300 
301   Collective on MPI_Comm
302 
303   Input Parameters:
304 + comm  - The communicator for the DM object
305 . lower - The lower left front corner coordinates
306 . upper - The upper right back corner coordinates
307 - edges - The number of cells in each direction
308 
309   Output Parameter:
310 . dm  - The DM object
311 
312   Level: beginner
313 
314 .keywords: DM, create
315 .seealso: DMPlexCreateBoxMesh(), DMPlexCreateSquareBoundary(), DMSetType(), DMCreate()
316 @*/
317 PetscErrorCode DMPlexCreateCubeBoundary(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt faces[])
318 {
319   PetscInt       vertices[3], numVertices;
320   PetscInt       numFaces    = 2*faces[0]*faces[1] + 2*faces[1]*faces[2] + 2*faces[0]*faces[2];
321   Vec            coordinates;
322   PetscSection   coordSection;
323   PetscScalar    *coords;
324   PetscInt       coordSize;
325   PetscMPIInt    rank;
326   PetscInt       v, vx, vy, vz;
327   PetscInt       voffset, iface=0, cone[4];
328   PetscErrorCode ierr;
329 
330   PetscFunctionBegin;
331   if ((faces[0] < 1) || (faces[1] < 1) || (faces[2] < 1)) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Must have at least 1 face per side");
332   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRQ(ierr);
333   vertices[0] = faces[0]+1; vertices[1] = faces[1]+1; vertices[2] = faces[2]+1;
334   numVertices = vertices[0]*vertices[1]*vertices[2];
335   if (!rank) {
336     PetscInt f;
337 
338     ierr = DMPlexSetChart(dm, 0, numFaces+numVertices);CHKERRQ(ierr);
339     for (f = 0; f < numFaces; ++f) {
340       ierr = DMPlexSetConeSize(dm, f, 4);CHKERRQ(ierr);
341     }
342     ierr = DMSetUp(dm);CHKERRQ(ierr); /* Allocate space for cones */
343     for (v = 0; v < numFaces+numVertices; ++v) {
344       ierr = DMSetLabelValue(dm, "marker", v, 1);CHKERRQ(ierr);
345     }
346 
347     /* Side 0 (Top) */
348     for (vy = 0; vy < faces[1]; vy++) {
349       for (vx = 0; vx < faces[0]; vx++) {
350         voffset = numFaces + vertices[0]*vertices[1]*(vertices[2]-1) + vy*vertices[0] + vx;
351         cone[0] = voffset; cone[1] = voffset+1; cone[2] = voffset+vertices[0]+1; cone[3] = voffset+vertices[0];
352         ierr    = DMPlexSetCone(dm, iface, cone);CHKERRQ(ierr);
353         iface++;
354       }
355     }
356 
357     /* Side 1 (Bottom) */
358     for (vy = 0; vy < faces[1]; vy++) {
359       for (vx = 0; vx < faces[0]; vx++) {
360         voffset = numFaces + vy*(faces[0]+1) + vx;
361         cone[0] = voffset+1; cone[1] = voffset; cone[2] = voffset+vertices[0]; cone[3] = voffset+vertices[0]+1;
362         ierr    = DMPlexSetCone(dm, iface, cone);CHKERRQ(ierr);
363         iface++;
364       }
365     }
366 
367     /* Side 2 (Front) */
368     for (vz = 0; vz < faces[2]; vz++) {
369       for (vx = 0; vx < faces[0]; vx++) {
370         voffset = numFaces + vz*vertices[0]*vertices[1] + vx;
371         cone[0] = voffset; cone[1] = voffset+1; cone[2] = voffset+vertices[0]*vertices[1]+1; cone[3] = voffset+vertices[0]*vertices[1];
372         ierr    = DMPlexSetCone(dm, iface, cone);CHKERRQ(ierr);
373         iface++;
374       }
375     }
376 
377     /* Side 3 (Back) */
378     for (vz = 0; vz < faces[2]; vz++) {
379       for (vx = 0; vx < faces[0]; vx++) {
380         voffset = numFaces + vz*vertices[0]*vertices[1] + vertices[0]*(vertices[1]-1) + vx;
381         cone[0] = voffset+vertices[0]*vertices[1]; cone[1] = voffset+vertices[0]*vertices[1]+1;
382         cone[2] = voffset+1; cone[3] = voffset;
383         ierr    = DMPlexSetCone(dm, iface, cone);CHKERRQ(ierr);
384         iface++;
385       }
386     }
387 
388     /* Side 4 (Left) */
389     for (vz = 0; vz < faces[2]; vz++) {
390       for (vy = 0; vy < faces[1]; vy++) {
391         voffset = numFaces + vz*vertices[0]*vertices[1] + vy*vertices[0];
392         cone[0] = voffset; cone[1] = voffset+vertices[0]*vertices[1];
393         cone[2] = voffset+vertices[0]*vertices[1]+vertices[0]; cone[3] = voffset+vertices[0];
394         ierr    = DMPlexSetCone(dm, iface, cone);CHKERRQ(ierr);
395         iface++;
396       }
397     }
398 
399     /* Side 5 (Right) */
400     for (vz = 0; vz < faces[2]; vz++) {
401       for (vy = 0; vy < faces[1]; vy++) {
402         voffset = numFaces + vz*vertices[0]*vertices[1] + vy*vertices[0] + faces[0];
403         cone[0] = voffset+vertices[0]*vertices[1]; cone[1] = voffset;
404         cone[2] = voffset+vertices[0]; cone[3] = voffset+vertices[0]*vertices[1]+vertices[0];
405         ierr    = DMPlexSetCone(dm, iface, cone);CHKERRQ(ierr);
406         iface++;
407       }
408     }
409   }
410   ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
411   ierr = DMPlexStratify(dm);CHKERRQ(ierr);
412   /* Build coordinates */
413   ierr = DMSetCoordinateDim(dm, 3);CHKERRQ(ierr);
414   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
415   ierr = PetscSectionSetChart(coordSection, numFaces, numFaces + numVertices);CHKERRQ(ierr);
416   for (v = numFaces; v < numFaces+numVertices; ++v) {
417     ierr = PetscSectionSetDof(coordSection, v, 3);CHKERRQ(ierr);
418   }
419   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
420   ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
421   ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
422   ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
423   ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
424   ierr = VecSetBlockSize(coordinates, 3);CHKERRQ(ierr);
425   ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
426   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
427   for (vz = 0; vz <= faces[2]; ++vz) {
428     for (vy = 0; vy <= faces[1]; ++vy) {
429       for (vx = 0; vx <= faces[0]; ++vx) {
430         coords[((vz*(faces[1]+1)+vy)*(faces[0]+1)+vx)*3+0] = lower[0] + ((upper[0] - lower[0])/faces[0])*vx;
431         coords[((vz*(faces[1]+1)+vy)*(faces[0]+1)+vx)*3+1] = lower[1] + ((upper[1] - lower[1])/faces[1])*vy;
432         coords[((vz*(faces[1]+1)+vy)*(faces[0]+1)+vx)*3+2] = lower[2] + ((upper[2] - lower[2])/faces[2])*vz;
433       }
434     }
435   }
436   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
437   ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
438   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
439   PetscFunctionReturn(0);
440 }
441 
442 static PetscErrorCode DMPlexCreateLineMesh_Internal(MPI_Comm comm,PetscInt segments,PetscReal lower,PetscReal upper,DMBoundaryType bd,DM *dm)
443 {
444   PetscInt       i,fStart,fEnd,numCells = 0,numVerts = 0;
445   PetscInt       numPoints[2],*coneSize,*cones,*coneOrientations;
446   PetscScalar    *vertexCoords;
447   PetscReal      L,maxCell;
448   PetscBool      markerSeparate = PETSC_FALSE;
449   PetscInt       markerLeft  = 1, faceMarkerLeft  = 1;
450   PetscInt       markerRight = 1, faceMarkerRight = 2;
451   PetscBool      wrap = (bd == DM_BOUNDARY_PERIODIC || bd == DM_BOUNDARY_TWIST) ? PETSC_TRUE : PETSC_FALSE;
452   PetscMPIInt    rank;
453   PetscErrorCode ierr;
454 
455   PetscFunctionBegin;
456   PetscValidPointer(dm,4);
457 
458   ierr = DMCreate(comm,dm);CHKERRQ(ierr);
459   ierr = DMSetType(*dm,DMPLEX);CHKERRQ(ierr);
460   ierr = DMSetDimension(*dm,1);CHKERRQ(ierr);
461   ierr = DMCreateLabel(*dm,"marker");CHKERRQ(ierr);
462   ierr = DMCreateLabel(*dm,"Face Sets");CHKERRQ(ierr);
463 
464   ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr);
465   if (!rank) numCells = segments;
466   if (!rank) numVerts = segments + (wrap ? 0 : 1);
467 
468   numPoints[0] = numVerts ; numPoints[1] = numCells;
469   ierr = PetscMalloc4(numCells+numVerts,&coneSize,numCells*2,&cones,numCells+numVerts,&coneOrientations,numVerts,&vertexCoords);CHKERRQ(ierr);
470   ierr = PetscMemzero(coneOrientations,(numCells+numVerts)*sizeof(PetscInt));CHKERRQ(ierr);
471   for (i = 0; i < numCells; ++i) { coneSize[i] = 2; }
472   for (i = 0; i < numVerts; ++i) { coneSize[numCells+i] = 0; }
473   for (i = 0; i < numCells; ++i) { cones[2*i] = numCells + i%numVerts; cones[2*i+1] = numCells + (i+1)%numVerts; }
474   for (i = 0; i < numVerts; ++i) { vertexCoords[i] = lower + (upper-lower)*((PetscReal)i/(PetscReal)numCells); }
475   ierr = DMPlexCreateFromDAG(*dm,1,numPoints,coneSize,cones,coneOrientations,vertexCoords);CHKERRQ(ierr);
476   ierr = PetscFree4(coneSize,cones,coneOrientations,vertexCoords);CHKERRQ(ierr);
477 
478   ierr = PetscOptionsGetBool(((PetscObject)*dm)->options,((PetscObject)*dm)->prefix,"-dm_plex_separate_marker",&markerSeparate,NULL);CHKERRQ(ierr);
479   if (markerSeparate) { markerLeft = faceMarkerLeft; markerRight = faceMarkerRight;}
480   if (!wrap && !rank) {
481     ierr = DMPlexGetHeightStratum(*dm,1,&fStart,&fEnd);CHKERRQ(ierr);
482     ierr = DMSetLabelValue(*dm,"marker",fStart,markerLeft);CHKERRQ(ierr);
483     ierr = DMSetLabelValue(*dm,"marker",fEnd-1,markerRight);CHKERRQ(ierr);
484     ierr = DMSetLabelValue(*dm,"Face Sets",fStart,faceMarkerLeft);CHKERRQ(ierr);
485     ierr = DMSetLabelValue(*dm,"Face Sets",fEnd-1,faceMarkerRight);CHKERRQ(ierr);
486   }
487   if (wrap) {
488     L       = upper - lower;
489     maxCell = (PetscReal)1.1*(L/(PetscReal)PetscMax(1,segments));
490     ierr = DMSetPeriodicity(*dm,PETSC_TRUE,&maxCell,&L,&bd);CHKERRQ(ierr);
491   }
492   PetscFunctionReturn(0);
493 }
494 
495 static PetscErrorCode DMPlexCreateBoxMesh_Simplex_Internal(MPI_Comm comm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate, DM *dm)
496 {
497   DM             boundary;
498   PetscInt       i;
499   PetscErrorCode ierr;
500 
501   PetscFunctionBegin;
502   PetscValidPointer(dm, 4);
503   for (i = 0; i < dim; ++i) if (periodicity[i] != DM_BOUNDARY_NONE) SETERRQ(comm, PETSC_ERR_SUP, "Periodicity is not supported for simplex meshes");
504   ierr = DMCreate(comm, &boundary);CHKERRQ(ierr);
505   PetscValidLogicalCollectiveInt(boundary,dim,2);
506   ierr = DMSetType(boundary, DMPLEX);CHKERRQ(ierr);
507   ierr = DMSetDimension(boundary, dim-1);CHKERRQ(ierr);
508   ierr = DMSetCoordinateDim(boundary, dim);CHKERRQ(ierr);
509   switch (dim) {
510   case 2: ierr = DMPlexCreateSquareBoundary(boundary, lower, upper, faces);CHKERRQ(ierr);break;
511   case 3: ierr = DMPlexCreateCubeBoundary(boundary, lower, upper, faces);CHKERRQ(ierr);break;
512   default: SETERRQ1(comm, PETSC_ERR_SUP, "Dimension not supported: %d", dim);
513   }
514   ierr = DMPlexGenerate(boundary, NULL, interpolate, dm);CHKERRQ(ierr);
515   ierr = DMDestroy(&boundary);CHKERRQ(ierr);
516   PetscFunctionReturn(0);
517 }
518 
519 static PetscErrorCode DMPlexCreateCubeMesh_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[], DMBoundaryType bdX, DMBoundaryType bdY, DMBoundaryType bdZ)
520 {
521   DMLabel        cutLabel = NULL;
522   PetscInt       markerTop      = 1, faceMarkerTop      = 1;
523   PetscInt       markerBottom   = 1, faceMarkerBottom   = 1;
524   PetscInt       markerFront    = 1, faceMarkerFront    = 1;
525   PetscInt       markerBack     = 1, faceMarkerBack     = 1;
526   PetscInt       markerRight    = 1, faceMarkerRight    = 1;
527   PetscInt       markerLeft     = 1, faceMarkerLeft     = 1;
528   PetscInt       dim;
529   PetscBool      markerSeparate = PETSC_FALSE, cutMarker = PETSC_FALSE;
530   PetscMPIInt    rank;
531   PetscErrorCode ierr;
532 
533   PetscFunctionBegin;
534   ierr = DMGetDimension(dm,&dim);CHKERRQ(ierr);
535   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRQ(ierr);
536   ierr = DMCreateLabel(dm,"marker");CHKERRQ(ierr);
537   ierr = DMCreateLabel(dm,"Face Sets");CHKERRQ(ierr);
538   ierr = PetscOptionsGetBool(((PetscObject) dm)->options,((PetscObject) dm)->prefix, "-dm_plex_periodic_cut", &cutMarker, NULL);CHKERRQ(ierr);
539   if (bdX == DM_BOUNDARY_PERIODIC || bdX == DM_BOUNDARY_TWIST ||
540       bdY == DM_BOUNDARY_PERIODIC || bdY == DM_BOUNDARY_TWIST ||
541       bdZ == DM_BOUNDARY_PERIODIC || bdZ == DM_BOUNDARY_TWIST) {
542 
543     if (cutMarker) {ierr = DMCreateLabel(dm, "periodic_cut");CHKERRQ(ierr); ierr = DMGetLabel(dm, "periodic_cut", &cutLabel);CHKERRQ(ierr);}
544   }
545   switch (dim) {
546   case 2:
547     faceMarkerTop    = 3;
548     faceMarkerBottom = 1;
549     faceMarkerRight  = 2;
550     faceMarkerLeft   = 4;
551     break;
552   case 3:
553     faceMarkerBottom = 1;
554     faceMarkerTop    = 2;
555     faceMarkerFront  = 3;
556     faceMarkerBack   = 4;
557     faceMarkerRight  = 5;
558     faceMarkerLeft   = 6;
559     break;
560   default:
561     SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SUP,"Dimension %d not supported",dim);
562     break;
563   }
564   ierr = PetscOptionsGetBool(((PetscObject) dm)->options,((PetscObject) dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL);CHKERRQ(ierr);
565   if (markerSeparate) {
566     markerBottom = faceMarkerBottom;
567     markerTop    = faceMarkerTop;
568     markerFront  = faceMarkerFront;
569     markerBack   = faceMarkerBack;
570     markerRight  = faceMarkerRight;
571     markerLeft   = faceMarkerLeft;
572   }
573   {
574     const PetscInt numXEdges    = !rank ? edges[0] : 0;
575     const PetscInt numYEdges    = !rank ? edges[1] : 0;
576     const PetscInt numZEdges    = !rank ? edges[2] : 0;
577     const PetscInt numXVertices = !rank ? (bdX == DM_BOUNDARY_PERIODIC || bdX == DM_BOUNDARY_TWIST ? edges[0] : edges[0]+1) : 0;
578     const PetscInt numYVertices = !rank ? (bdY == DM_BOUNDARY_PERIODIC || bdY == DM_BOUNDARY_TWIST ? edges[1] : edges[1]+1) : 0;
579     const PetscInt numZVertices = !rank ? (bdZ == DM_BOUNDARY_PERIODIC || bdZ == DM_BOUNDARY_TWIST ? edges[2] : edges[2]+1) : 0;
580     const PetscInt numCells     = numXEdges*numYEdges*numZEdges;
581     const PetscInt numXFaces    = numYEdges*numZEdges;
582     const PetscInt numYFaces    = numXEdges*numZEdges;
583     const PetscInt numZFaces    = numXEdges*numYEdges;
584     const PetscInt numTotXFaces = numXVertices*numXFaces;
585     const PetscInt numTotYFaces = numYVertices*numYFaces;
586     const PetscInt numTotZFaces = numZVertices*numZFaces;
587     const PetscInt numFaces     = numTotXFaces + numTotYFaces + numTotZFaces;
588     const PetscInt numTotXEdges = numXEdges*numYVertices*numZVertices;
589     const PetscInt numTotYEdges = numYEdges*numXVertices*numZVertices;
590     const PetscInt numTotZEdges = numZEdges*numXVertices*numYVertices;
591     const PetscInt numVertices  = numXVertices*numYVertices*numZVertices;
592     const PetscInt numEdges     = numTotXEdges + numTotYEdges + numTotZEdges;
593     const PetscInt firstVertex  = (dim == 2) ? numFaces : numCells;
594     const PetscInt firstXFace   = (dim == 2) ? 0 : numCells + numVertices;
595     const PetscInt firstYFace   = firstXFace + numTotXFaces;
596     const PetscInt firstZFace   = firstYFace + numTotYFaces;
597     const PetscInt firstXEdge   = numCells + numFaces + numVertices;
598     const PetscInt firstYEdge   = firstXEdge + numTotXEdges;
599     const PetscInt firstZEdge   = firstYEdge + numTotYEdges;
600     Vec            coordinates;
601     PetscSection   coordSection;
602     PetscScalar   *coords;
603     PetscInt       coordSize;
604     PetscInt       v, vx, vy, vz;
605     PetscInt       c, f, fx, fy, fz, e, ex, ey, ez;
606 
607     ierr = DMPlexSetChart(dm, 0, numCells+numFaces+numEdges+numVertices);CHKERRQ(ierr);
608     for (c = 0; c < numCells; c++) {
609       ierr = DMPlexSetConeSize(dm, c, 6);CHKERRQ(ierr);
610     }
611     for (f = firstXFace; f < firstXFace+numFaces; ++f) {
612       ierr = DMPlexSetConeSize(dm, f, 4);CHKERRQ(ierr);
613     }
614     for (e = firstXEdge; e < firstXEdge+numEdges; ++e) {
615       ierr = DMPlexSetConeSize(dm, e, 2);CHKERRQ(ierr);
616     }
617     ierr = DMSetUp(dm);CHKERRQ(ierr); /* Allocate space for cones */
618     /* Build cells */
619     for (fz = 0; fz < numZEdges; ++fz) {
620       for (fy = 0; fy < numYEdges; ++fy) {
621         for (fx = 0; fx < numXEdges; ++fx) {
622           PetscInt cell    = (fz*numYEdges + fy)*numXEdges + fx;
623           PetscInt faceB   = firstZFace + (fy*numXEdges+fx)*numZVertices +   fz;
624           PetscInt faceT   = firstZFace + (fy*numXEdges+fx)*numZVertices + ((fz+1)%numZVertices);
625           PetscInt faceF   = firstYFace + (fz*numXEdges+fx)*numYVertices +   fy;
626           PetscInt faceK   = firstYFace + (fz*numXEdges+fx)*numYVertices + ((fy+1)%numYVertices);
627           PetscInt faceL   = firstXFace + (fz*numYEdges+fy)*numXVertices +   fx;
628           PetscInt faceR   = firstXFace + (fz*numYEdges+fy)*numXVertices + ((fx+1)%numXVertices);
629                             /* B,  T,  F,  K,  R,  L */
630           PetscInt ornt[6] = {-4,  0,  0, -1,  0, -4}; /* ??? */
631           PetscInt cone[6];
632 
633           /* no boundary twisting in 3D */
634           cone[0] = faceB; cone[1] = faceT; cone[2] = faceF; cone[3] = faceK; cone[4] = faceR; cone[5] = faceL;
635           ierr    = DMPlexSetCone(dm, cell, cone);CHKERRQ(ierr);
636           ierr    = DMPlexSetConeOrientation(dm, cell, ornt);CHKERRQ(ierr);
637           if (bdX != DM_BOUNDARY_NONE && fx == numXEdges-1 && cutLabel) {ierr = DMLabelSetValue(cutLabel, cell, 2);CHKERRQ(ierr);}
638           if (bdY != DM_BOUNDARY_NONE && fy == numYEdges-1 && cutLabel) {ierr = DMLabelSetValue(cutLabel, cell, 2);CHKERRQ(ierr);}
639           if (bdZ != DM_BOUNDARY_NONE && fz == numZEdges-1 && cutLabel) {ierr = DMLabelSetValue(cutLabel, cell, 2);CHKERRQ(ierr);}
640         }
641       }
642     }
643     /* Build x faces */
644     for (fz = 0; fz < numZEdges; ++fz) {
645       for (fy = 0; fy < numYEdges; ++fy) {
646         for (fx = 0; fx < numXVertices; ++fx) {
647           PetscInt face    = firstXFace + (fz*numYEdges+fy)*numXVertices + fx;
648           PetscInt edgeL   = firstZEdge + (  fy*                 numXVertices+fx)*numZEdges + fz;
649           PetscInt edgeR   = firstZEdge + (((fy+1)%numYVertices)*numXVertices+fx)*numZEdges + fz;
650           PetscInt edgeB   = firstYEdge + (  fz*                 numXVertices+fx)*numYEdges + fy;
651           PetscInt edgeT   = firstYEdge + (((fz+1)%numZVertices)*numXVertices+fx)*numYEdges + fy;
652           PetscInt ornt[4] = {0, 0, -2, -2};
653           PetscInt cone[4];
654 
655           if (dim == 3) {
656             /* markers */
657             if (bdX != DM_BOUNDARY_PERIODIC) {
658               if (fx == numXVertices-1) {
659                 ierr = DMSetLabelValue(dm, "Face Sets", face, faceMarkerRight);CHKERRQ(ierr);
660                 ierr = DMSetLabelValue(dm, "marker", face, markerRight);CHKERRQ(ierr);
661               }
662               else if (fx == 0) {
663                 ierr = DMSetLabelValue(dm, "Face Sets", face, faceMarkerLeft);CHKERRQ(ierr);
664                 ierr = DMSetLabelValue(dm, "marker", face, markerLeft);CHKERRQ(ierr);
665               }
666             }
667           }
668           cone[0] = edgeB; cone[1] = edgeR; cone[2] = edgeT; cone[3] = edgeL;
669           ierr    = DMPlexSetCone(dm, face, cone);CHKERRQ(ierr);
670           ierr    = DMPlexSetConeOrientation(dm, face, ornt);CHKERRQ(ierr);
671         }
672       }
673     }
674     /* Build y faces */
675     for (fz = 0; fz < numZEdges; ++fz) {
676       for (fx = 0; fx < numXEdges; ++fx) {
677         for (fy = 0; fy < numYVertices; ++fy) {
678           PetscInt face    = firstYFace + (fz*numXEdges+fx)*numYVertices + fy;
679           PetscInt edgeL   = firstZEdge + (fy*numXVertices+  fx                 )*numZEdges + fz;
680           PetscInt edgeR   = firstZEdge + (fy*numXVertices+((fx+1)%numXVertices))*numZEdges + fz;
681           PetscInt edgeB   = firstXEdge + (  fz                 *numYVertices+fy)*numXEdges + fx;
682           PetscInt edgeT   = firstXEdge + (((fz+1)%numZVertices)*numYVertices+fy)*numXEdges + fx;
683           PetscInt ornt[4] = {0, 0, -2, -2};
684           PetscInt cone[4];
685 
686           if (dim == 3) {
687             /* markers */
688             if (bdY != DM_BOUNDARY_PERIODIC) {
689               if (fy == numYVertices-1) {
690                 ierr = DMSetLabelValue(dm, "Face Sets", face, faceMarkerBack);CHKERRQ(ierr);
691                 ierr = DMSetLabelValue(dm, "marker", face, markerBack);CHKERRQ(ierr);
692               }
693               else if (fy == 0) {
694                 ierr = DMSetLabelValue(dm, "Face Sets", face, faceMarkerFront);CHKERRQ(ierr);
695                 ierr = DMSetLabelValue(dm, "marker", face, markerFront);CHKERRQ(ierr);
696               }
697             }
698           }
699           cone[0] = edgeB; cone[1] = edgeR; cone[2] = edgeT; cone[3] = edgeL;
700           ierr    = DMPlexSetCone(dm, face, cone);CHKERRQ(ierr);
701           ierr    = DMPlexSetConeOrientation(dm, face, ornt);CHKERRQ(ierr);
702         }
703       }
704     }
705     /* Build z faces */
706     for (fy = 0; fy < numYEdges; ++fy) {
707       for (fx = 0; fx < numXEdges; ++fx) {
708         for (fz = 0; fz < numZVertices; fz++) {
709           PetscInt face    = firstZFace + (fy*numXEdges+fx)*numZVertices + fz;
710           PetscInt edgeL   = firstYEdge + (fz*numXVertices+  fx                 )*numYEdges + fy;
711           PetscInt edgeR   = firstYEdge + (fz*numXVertices+((fx+1)%numXVertices))*numYEdges + fy;
712           PetscInt edgeB   = firstXEdge + (fz*numYVertices+  fy                 )*numXEdges + fx;
713           PetscInt edgeT   = firstXEdge + (fz*numYVertices+((fy+1)%numYVertices))*numXEdges + fx;
714           PetscInt ornt[4] = {0, 0, -2, -2};
715           PetscInt cone[4];
716 
717           if (dim == 2) {
718             if (bdX == DM_BOUNDARY_TWIST && fx == numXEdges-1) {edgeR += numYEdges-1-2*fy; ornt[1] = -2;}
719             if (bdY == DM_BOUNDARY_TWIST && fy == numYEdges-1) {edgeT += numXEdges-1-2*fx; ornt[2] =  0;}
720             if (bdX != DM_BOUNDARY_NONE && fx == numXEdges-1 && cutLabel) {ierr = DMLabelSetValue(cutLabel, face, 2);CHKERRQ(ierr);}
721             if (bdY != DM_BOUNDARY_NONE && fy == numYEdges-1 && cutLabel) {ierr = DMLabelSetValue(cutLabel, face, 2);CHKERRQ(ierr);}
722           } else {
723             /* markers */
724             if (bdZ != DM_BOUNDARY_PERIODIC) {
725               if (fz == numZVertices-1) {
726                 ierr = DMSetLabelValue(dm, "Face Sets", face, faceMarkerTop);CHKERRQ(ierr);
727                 ierr = DMSetLabelValue(dm, "marker", face, markerTop);CHKERRQ(ierr);
728               }
729               else if (fz == 0) {
730                 ierr = DMSetLabelValue(dm, "Face Sets", face, faceMarkerBottom);CHKERRQ(ierr);
731                 ierr = DMSetLabelValue(dm, "marker", face, markerBottom);CHKERRQ(ierr);
732               }
733             }
734           }
735           cone[0] = edgeB; cone[1] = edgeR; cone[2] = edgeT; cone[3] = edgeL;
736           ierr    = DMPlexSetCone(dm, face, cone);CHKERRQ(ierr);
737           ierr    = DMPlexSetConeOrientation(dm, face, ornt);CHKERRQ(ierr);
738         }
739       }
740     }
741     /* Build Z edges*/
742     for (vy = 0; vy < numYVertices; vy++) {
743       for (vx = 0; vx < numXVertices; vx++) {
744         for (ez = 0; ez < numZEdges; ez++) {
745           const PetscInt edge    = firstZEdge  + (vy*numXVertices+vx)*numZEdges + ez;
746           const PetscInt vertexB = firstVertex + (  ez                 *numYVertices+vy)*numXVertices + vx;
747           const PetscInt vertexT = firstVertex + (((ez+1)%numZVertices)*numYVertices+vy)*numXVertices + vx;
748           PetscInt       cone[2];
749 
750           if (dim == 3) {
751             if (bdX != DM_BOUNDARY_PERIODIC) {
752               if (vx == numXVertices-1) {
753                 ierr = DMSetLabelValue(dm, "marker", edge, markerRight);CHKERRQ(ierr);
754               }
755               else if (vx == 0) {
756                 ierr = DMSetLabelValue(dm, "marker", edge, markerLeft);CHKERRQ(ierr);
757               }
758             }
759             if (bdY != DM_BOUNDARY_PERIODIC) {
760               if (vy == numYVertices-1) {
761                 ierr = DMSetLabelValue(dm, "marker", edge, markerBack);CHKERRQ(ierr);
762               }
763               else if (vy == 0) {
764                 ierr = DMSetLabelValue(dm, "marker", edge, markerFront);CHKERRQ(ierr);
765               }
766             }
767           }
768           cone[0] = vertexB; cone[1] = vertexT;
769           ierr = DMPlexSetCone(dm, edge, cone);CHKERRQ(ierr);
770         }
771       }
772     }
773     /* Build Y edges*/
774     for (vz = 0; vz < numZVertices; vz++) {
775       for (vx = 0; vx < numXVertices; vx++) {
776         for (ey = 0; ey < numYEdges; ey++) {
777           const PetscInt nextv   = (dim == 2 && bdY == DM_BOUNDARY_TWIST && ey == numYEdges-1) ? (numXVertices-vx-1) : (vz*numYVertices+((ey+1)%numYVertices))*numXVertices + vx;
778           const PetscInt edge    = firstYEdge  + (vz*numXVertices+vx)*numYEdges + ey;
779           const PetscInt vertexF = firstVertex + (vz*numYVertices+ey)*numXVertices + vx;
780           const PetscInt vertexK = firstVertex + nextv;
781           PetscInt       cone[2];
782 
783           cone[0] = vertexF; cone[1] = vertexK;
784           ierr = DMPlexSetCone(dm, edge, cone);CHKERRQ(ierr);
785           if (dim == 2) {
786             if ((bdX != DM_BOUNDARY_PERIODIC) && (bdX != DM_BOUNDARY_TWIST)) {
787               if (vx == numXVertices-1) {
788                 ierr = DMSetLabelValue(dm, "Face Sets", edge, faceMarkerRight);CHKERRQ(ierr);
789                 ierr = DMSetLabelValue(dm, "marker", edge,    markerRight);CHKERRQ(ierr);
790                 ierr = DMSetLabelValue(dm, "marker", cone[0], markerRight);CHKERRQ(ierr);
791                 if (ey == numYEdges-1) {
792                   ierr = DMSetLabelValue(dm, "marker", cone[1], markerRight);CHKERRQ(ierr);
793                 }
794               } else if (vx == 0) {
795                 ierr = DMSetLabelValue(dm, "Face Sets", edge, faceMarkerLeft);CHKERRQ(ierr);
796                 ierr = DMSetLabelValue(dm, "marker", edge,    markerLeft);CHKERRQ(ierr);
797                 ierr = DMSetLabelValue(dm, "marker", cone[0], markerLeft);CHKERRQ(ierr);
798                 if (ey == numYEdges-1) {
799                   ierr = DMSetLabelValue(dm, "marker", cone[1], markerLeft);CHKERRQ(ierr);
800                 }
801               }
802             } else {
803               if (vx == 0 && cutLabel) {
804                 ierr = DMLabelSetValue(cutLabel, edge,    1);CHKERRQ(ierr);
805                 ierr = DMLabelSetValue(cutLabel, cone[0], 1);CHKERRQ(ierr);
806                 if (ey == numYEdges-1) {
807                   ierr = DMLabelSetValue(cutLabel, cone[1], 1);CHKERRQ(ierr);
808                 }
809               }
810             }
811           } else {
812             if (bdX != DM_BOUNDARY_PERIODIC) {
813               if (vx == numXVertices-1) {
814                 ierr = DMSetLabelValue(dm, "marker", edge, markerRight);CHKERRQ(ierr);
815               } else if (vx == 0) {
816                 ierr = DMSetLabelValue(dm, "marker", edge, markerLeft);CHKERRQ(ierr);
817               }
818             }
819             if (bdZ != DM_BOUNDARY_PERIODIC) {
820               if (vz == numZVertices-1) {
821                 ierr = DMSetLabelValue(dm, "marker", edge, markerTop);CHKERRQ(ierr);
822               } else if (vz == 0) {
823                 ierr = DMSetLabelValue(dm, "marker", edge, markerBottom);CHKERRQ(ierr);
824               }
825             }
826           }
827         }
828       }
829     }
830     /* Build X edges*/
831     for (vz = 0; vz < numZVertices; vz++) {
832       for (vy = 0; vy < numYVertices; vy++) {
833         for (ex = 0; ex < numXEdges; ex++) {
834           const PetscInt nextv   = (dim == 2 && bdX == DM_BOUNDARY_TWIST && ex == numXEdges-1) ? (numYVertices-vy-1)*numXVertices : (vz*numYVertices+vy)*numXVertices + (ex+1)%numXVertices;
835           const PetscInt edge    = firstXEdge  + (vz*numYVertices+vy)*numXEdges + ex;
836           const PetscInt vertexL = firstVertex + (vz*numYVertices+vy)*numXVertices + ex;
837           const PetscInt vertexR = firstVertex + nextv;
838           PetscInt       cone[2];
839 
840           cone[0] = vertexL; cone[1] = vertexR;
841           ierr = DMPlexSetCone(dm, edge, cone);CHKERRQ(ierr);
842           if (dim == 2) {
843             if ((bdY != DM_BOUNDARY_PERIODIC) && (bdY != DM_BOUNDARY_TWIST)) {
844               if (vy == numYVertices-1) {
845                 ierr = DMSetLabelValue(dm, "Face Sets", edge, faceMarkerTop);CHKERRQ(ierr);
846                 ierr = DMSetLabelValue(dm, "marker", edge,    markerTop);CHKERRQ(ierr);
847                 ierr = DMSetLabelValue(dm, "marker", cone[0], markerTop);CHKERRQ(ierr);
848                 if (ex == numXEdges-1) {
849                   ierr = DMSetLabelValue(dm, "marker", cone[1], markerTop);CHKERRQ(ierr);
850                 }
851               } else if (vy == 0) {
852                 ierr = DMSetLabelValue(dm, "Face Sets", edge, faceMarkerBottom);CHKERRQ(ierr);
853                 ierr = DMSetLabelValue(dm, "marker", edge,    markerBottom);CHKERRQ(ierr);
854                 ierr = DMSetLabelValue(dm, "marker", cone[0], markerBottom);CHKERRQ(ierr);
855                 if (ex == numXEdges-1) {
856                   ierr = DMSetLabelValue(dm, "marker", cone[1], markerBottom);CHKERRQ(ierr);
857                 }
858               }
859             } else {
860               if (vy == 0 && cutLabel) {
861                 ierr = DMLabelSetValue(cutLabel, edge,    1);CHKERRQ(ierr);
862                 ierr = DMLabelSetValue(cutLabel, cone[0], 1);CHKERRQ(ierr);
863                 if (ex == numXEdges-1) {
864                   ierr = DMLabelSetValue(cutLabel, cone[1], 1);CHKERRQ(ierr);
865                 }
866               }
867             }
868           } else {
869             if (bdY != DM_BOUNDARY_PERIODIC) {
870               if (vy == numYVertices-1) {
871                 ierr = DMSetLabelValue(dm, "marker", edge, markerBack);CHKERRQ(ierr);
872               }
873               else if (vy == 0) {
874                 ierr = DMSetLabelValue(dm, "marker", edge, markerFront);CHKERRQ(ierr);
875               }
876             }
877             if (bdZ != DM_BOUNDARY_PERIODIC) {
878               if (vz == numZVertices-1) {
879                 ierr = DMSetLabelValue(dm, "marker", edge, markerTop);CHKERRQ(ierr);
880               }
881               else if (vz == 0) {
882                 ierr = DMSetLabelValue(dm, "marker", edge, markerBottom);CHKERRQ(ierr);
883               }
884             }
885           }
886         }
887       }
888     }
889     ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
890     ierr = DMPlexStratify(dm);CHKERRQ(ierr);
891     /* Build coordinates */
892     ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
893     ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
894     ierr = PetscSectionSetFieldComponents(coordSection, 0, dim);CHKERRQ(ierr);
895     ierr = PetscSectionSetChart(coordSection, firstVertex, firstVertex+numVertices);CHKERRQ(ierr);
896     for (v = firstVertex; v < firstVertex+numVertices; ++v) {
897       ierr = PetscSectionSetDof(coordSection, v, dim);CHKERRQ(ierr);
898       ierr = PetscSectionSetFieldDof(coordSection, v, 0, dim);CHKERRQ(ierr);
899     }
900     ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
901     ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
902     ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
903     ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
904     ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
905     ierr = VecSetBlockSize(coordinates, dim);CHKERRQ(ierr);
906     ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
907     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
908     for (vz = 0; vz < numZVertices; ++vz) {
909       for (vy = 0; vy < numYVertices; ++vy) {
910         for (vx = 0; vx < numXVertices; ++vx) {
911           coords[((vz*numYVertices+vy)*numXVertices+vx)*dim+0] = lower[0] + ((upper[0] - lower[0])/numXEdges)*vx;
912           coords[((vz*numYVertices+vy)*numXVertices+vx)*dim+1] = lower[1] + ((upper[1] - lower[1])/numYEdges)*vy;
913           if (dim == 3) {
914             coords[((vz*numYVertices+vy)*numXVertices+vx)*dim+2] = lower[2] + ((upper[2] - lower[2])/numZEdges)*vz;
915           }
916         }
917       }
918     }
919     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
920     ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
921     ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
922   }
923   PetscFunctionReturn(0);
924 }
925 
926 static PetscErrorCode DMPlexCreateBoxMesh_Tensor_Internal(MPI_Comm comm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate, DM *dm)
927 {
928   PetscInt       i;
929   PetscErrorCode ierr;
930 
931   PetscFunctionBegin;
932   PetscValidPointer(dm, 7);
933   ierr = DMCreate(comm, dm);CHKERRQ(ierr);
934   PetscValidLogicalCollectiveInt(*dm,dim,2);
935   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
936   ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
937   switch (dim) {
938   case 2: {ierr = DMPlexCreateCubeMesh_Internal(*dm, lower, upper, faces, periodicity[0], periodicity[1], DM_BOUNDARY_NONE);CHKERRQ(ierr);break;}
939   case 3: {ierr = DMPlexCreateCubeMesh_Internal(*dm, lower, upper, faces, periodicity[0], periodicity[1], periodicity[2]);CHKERRQ(ierr);break;}
940   default: SETERRQ1(comm, PETSC_ERR_SUP, "Dimension not supported: %d", dim);
941   }
942   if (periodicity[0] == DM_BOUNDARY_PERIODIC || periodicity[0] == DM_BOUNDARY_TWIST ||
943       periodicity[1] == DM_BOUNDARY_PERIODIC || periodicity[1] == DM_BOUNDARY_TWIST ||
944       (dim > 2 && (periodicity[2] == DM_BOUNDARY_PERIODIC || periodicity[2] == DM_BOUNDARY_TWIST))) {
945     PetscReal L[3];
946     PetscReal maxCell[3];
947 
948     for (i = 0; i < dim; i++) {
949       L[i]       = upper[i] - lower[i];
950       maxCell[i] = 1.1 * (L[i] / PetscMax(1,faces[i]));
951     }
952     ierr = DMSetPeriodicity(*dm,PETSC_TRUE,maxCell,L,periodicity);CHKERRQ(ierr);
953   }
954   if (!interpolate) {
955     DM udm;
956 
957     ierr = DMPlexUninterpolate(*dm, &udm);CHKERRQ(ierr);
958     ierr = DMDestroy(dm);CHKERRQ(ierr);
959     *dm  = udm;
960   }
961   PetscFunctionReturn(0);
962 }
963 
964 /*@C
965   DMPlexCreateBoxMesh - Creates a mesh on the tensor product of unit intervals (box) using simplices or tensor cells (hexahedra).
966 
967   Collective on MPI_Comm
968 
969   Input Parameters:
970 + comm        - The communicator for the DM object
971 . dim         - The spatial dimension
972 . simplex     - PETSC_TRUE for simplices, PETSC_FALSE for tensor cells
973 . faces       - Number of faces per dimension, or NULL for (1,) in 1D and (2, 2) in 2D and (1, 1, 1) in 3D
974 . lower       - The lower left corner, or NULL for (0, 0, 0)
975 . upper       - The upper right corner, or NULL for (1, 1, 1)
976 . periodicity - The boundary type for the X,Y,Z direction, or NULL for DM_BOUNDARY_NONE
977 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
978 
979   Output Parameter:
980 . dm  - The DM object
981 
982   Note: Here is the numbering returned for 2 faces in each direction for tensor cells:
983 $ 10---17---11---18----12
984 $  |         |         |
985 $  |         |         |
986 $ 20    2   22    3    24
987 $  |         |         |
988 $  |         |         |
989 $  7---15----8---16----9
990 $  |         |         |
991 $  |         |         |
992 $ 19    0   21    1   23
993 $  |         |         |
994 $  |         |         |
995 $  4---13----5---14----6
996 
997 and for simplicial cells
998 
999 $ 14----8---15----9----16
1000 $  |\     5  |\      7 |
1001 $  | \       | \       |
1002 $ 13   2    14    3    15
1003 $  | 4   \   | 6   \   |
1004 $  |       \ |       \ |
1005 $ 11----6---12----7----13
1006 $  |\        |\        |
1007 $  | \    1  | \     3 |
1008 $ 10   0    11    1    12
1009 $  | 0   \   | 2   \   |
1010 $  |       \ |       \ |
1011 $  8----4----9----5----10
1012 
1013   Level: beginner
1014 
1015 .keywords: DM, create
1016 .seealso: DMPlexCreateFromFile(), DMPlexCreateHexCylinderMesh(), DMSetType(), DMCreate()
1017 @*/
1018 PetscErrorCode DMPlexCreateBoxMesh(MPI_Comm comm, PetscInt dim, PetscBool simplex, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate, DM *dm)
1019 {
1020   PetscInt       i;
1021   PetscInt       fac[3] = {0, 0, 0};
1022   PetscReal      low[3] = {0, 0, 0};
1023   PetscReal      upp[3] = {1, 1, 1};
1024   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1025   PetscErrorCode ierr;
1026 
1027   PetscFunctionBegin;
1028   for (i = 0; i < dim; ++i) fac[i] = faces ? faces[i] : (dim == 1 ? 1 : 4-dim);
1029   if (lower) for (i = 0; i < dim; ++i) low[i] = lower[i];
1030   if (upper) for (i = 0; i < dim; ++i) upp[i] = upper[i];
1031   if (periodicity) for (i = 0; i < dim; ++i) bdt[i] = periodicity[i];
1032 
1033   if (dim == 1)      {ierr = DMPlexCreateLineMesh_Internal(comm, fac[0], low[0], upp[0], bdt[0], dm);CHKERRQ(ierr);}
1034   else if (simplex)  {ierr = DMPlexCreateBoxMesh_Simplex_Internal(comm, dim, fac, low, upp, bdt, interpolate, dm);CHKERRQ(ierr);}
1035   else               {ierr = DMPlexCreateBoxMesh_Tensor_Internal(comm, dim, fac, low, upp, bdt, interpolate, dm);CHKERRQ(ierr);}
1036   PetscFunctionReturn(0);
1037 }
1038 
1039 /*@C
1040   DMPlexSetOptionsPrefix - Sets the prefix used for searching for all DM options in the database.
1041 
1042   Logically Collective on DM
1043 
1044   Input Parameters:
1045 + dm - the DM context
1046 - prefix - the prefix to prepend to all option names
1047 
1048   Notes:
1049   A hyphen (-) must NOT be given at the beginning of the prefix name.
1050   The first character of all runtime options is AUTOMATICALLY the hyphen.
1051 
1052   Level: advanced
1053 
1054 .seealso: SNESSetFromOptions()
1055 @*/
1056 PetscErrorCode DMPlexSetOptionsPrefix(DM dm, const char prefix[])
1057 {
1058   DM_Plex       *mesh = (DM_Plex *) dm->data;
1059   PetscErrorCode ierr;
1060 
1061   PetscFunctionBegin;
1062   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1063   ierr = PetscObjectSetOptionsPrefix((PetscObject) dm, prefix);CHKERRQ(ierr);
1064   ierr = PetscObjectSetOptionsPrefix((PetscObject) mesh->partitioner, prefix);CHKERRQ(ierr);
1065   PetscFunctionReturn(0);
1066 }
1067 
1068 /*@
1069   DMPlexCreateHexCylinderMesh - Creates a mesh on the tensor product of the unit interval with the circle (cylinder) using hexahedra.
1070 
1071   Collective on MPI_Comm
1072 
1073   Input Parameters:
1074 + comm      - The communicator for the DM object
1075 . numRefine - The number of regular refinements to the basic 5 cell structure
1076 - periodicZ - The boundary type for the Z direction
1077 
1078   Output Parameter:
1079 . dm  - The DM object
1080 
1081   Note: Here is the output numbering looking from the bottom of the cylinder:
1082 $       17-----14
1083 $        |     |
1084 $        |  2  |
1085 $        |     |
1086 $ 17-----8-----7-----14
1087 $  |     |     |     |
1088 $  |  3  |  0  |  1  |
1089 $  |     |     |     |
1090 $ 19-----5-----6-----13
1091 $        |     |
1092 $        |  4  |
1093 $        |     |
1094 $       19-----13
1095 $
1096 $ and up through the top
1097 $
1098 $       18-----16
1099 $        |     |
1100 $        |  2  |
1101 $        |     |
1102 $ 18----10----11-----16
1103 $  |     |     |     |
1104 $  |  3  |  0  |  1  |
1105 $  |     |     |     |
1106 $ 20-----9----12-----15
1107 $        |     |
1108 $        |  4  |
1109 $        |     |
1110 $       20-----15
1111 
1112   Level: beginner
1113 
1114 .keywords: DM, create
1115 .seealso: DMPlexCreateBoxMesh(), DMSetType(), DMCreate()
1116 @*/
1117 PetscErrorCode DMPlexCreateHexCylinderMesh(MPI_Comm comm, PetscInt numRefine, DMBoundaryType periodicZ, DM *dm)
1118 {
1119   const PetscInt dim = 3;
1120   PetscInt       numCells, numVertices, r;
1121   PetscErrorCode ierr;
1122 
1123   PetscFunctionBegin;
1124   PetscValidPointer(dm, 4);
1125   if (numRefine < 0) SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Number of refinements %D cannot be negative", numRefine);
1126   ierr = DMCreate(comm, dm);CHKERRQ(ierr);
1127   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
1128   ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
1129   /* Create topology */
1130   {
1131     PetscInt cone[8], c;
1132 
1133     numCells    = 5;
1134     numVertices = 16;
1135     if (periodicZ == DM_BOUNDARY_PERIODIC) {
1136       numCells   *= 3;
1137       numVertices = 24;
1138     }
1139     ierr = DMPlexSetChart(*dm, 0, numCells+numVertices);CHKERRQ(ierr);
1140     for (c = 0; c < numCells; c++) {ierr = DMPlexSetConeSize(*dm, c, 8);CHKERRQ(ierr);}
1141     ierr = DMSetUp(*dm);CHKERRQ(ierr);
1142     if (periodicZ == DM_BOUNDARY_PERIODIC) {
1143       cone[0] = 15; cone[1] = 18; cone[2] = 17; cone[3] = 16;
1144       cone[4] = 31; cone[5] = 32; cone[6] = 33; cone[7] = 34;
1145       ierr = DMPlexSetCone(*dm, 0, cone);CHKERRQ(ierr);
1146       cone[0] = 16; cone[1] = 17; cone[2] = 24; cone[3] = 23;
1147       cone[4] = 32; cone[5] = 36; cone[6] = 37; cone[7] = 33; /* 22 25 26 21 */
1148       ierr = DMPlexSetCone(*dm, 1, cone);CHKERRQ(ierr);
1149       cone[0] = 18; cone[1] = 27; cone[2] = 24; cone[3] = 17;
1150       cone[4] = 34; cone[5] = 33; cone[6] = 37; cone[7] = 38;
1151       ierr = DMPlexSetCone(*dm, 2, cone);CHKERRQ(ierr);
1152       cone[0] = 29; cone[1] = 27; cone[2] = 18; cone[3] = 15;
1153       cone[4] = 35; cone[5] = 31; cone[6] = 34; cone[7] = 38;
1154       ierr = DMPlexSetCone(*dm, 3, cone);CHKERRQ(ierr);
1155       cone[0] = 29; cone[1] = 15; cone[2] = 16; cone[3] = 23;
1156       cone[4] = 35; cone[5] = 36; cone[6] = 32; cone[7] = 31;
1157       ierr = DMPlexSetCone(*dm, 4, cone);CHKERRQ(ierr);
1158 
1159       cone[0] = 31; cone[1] = 34; cone[2] = 33; cone[3] = 32;
1160       cone[4] = 19; cone[5] = 22; cone[6] = 21; cone[7] = 20;
1161       ierr = DMPlexSetCone(*dm, 5, cone);CHKERRQ(ierr);
1162       cone[0] = 32; cone[1] = 33; cone[2] = 37; cone[3] = 36;
1163       cone[4] = 22; cone[5] = 25; cone[6] = 26; cone[7] = 21;
1164       ierr = DMPlexSetCone(*dm, 6, cone);CHKERRQ(ierr);
1165       cone[0] = 34; cone[1] = 38; cone[2] = 37; cone[3] = 33;
1166       cone[4] = 20; cone[5] = 21; cone[6] = 26; cone[7] = 28;
1167       ierr = DMPlexSetCone(*dm, 7, cone);CHKERRQ(ierr);
1168       cone[0] = 35; cone[1] = 38; cone[2] = 34; cone[3] = 31;
1169       cone[4] = 30; cone[5] = 19; cone[6] = 20; cone[7] = 28;
1170       ierr = DMPlexSetCone(*dm, 8, cone);CHKERRQ(ierr);
1171       cone[0] = 35; cone[1] = 31; cone[2] = 32; cone[3] = 36;
1172       cone[4] = 30; cone[5] = 25; cone[6] = 22; cone[7] = 19;
1173       ierr = DMPlexSetCone(*dm, 9, cone);CHKERRQ(ierr);
1174 
1175       cone[0] = 19; cone[1] = 20; cone[2] = 21; cone[3] = 22;
1176       cone[4] = 15; cone[5] = 16; cone[6] = 17; cone[7] = 18;
1177       ierr = DMPlexSetCone(*dm, 10, cone);CHKERRQ(ierr);
1178       cone[0] = 22; cone[1] = 21; cone[2] = 26; cone[3] = 25;
1179       cone[4] = 16; cone[5] = 23; cone[6] = 24; cone[7] = 17;
1180       ierr = DMPlexSetCone(*dm, 11, cone);CHKERRQ(ierr);
1181       cone[0] = 20; cone[1] = 28; cone[2] = 26; cone[3] = 21;
1182       cone[4] = 18; cone[5] = 17; cone[6] = 24; cone[7] = 27;
1183       ierr = DMPlexSetCone(*dm, 12, cone);CHKERRQ(ierr);
1184       cone[0] = 30; cone[1] = 28; cone[2] = 20; cone[3] = 19;
1185       cone[4] = 29; cone[5] = 15; cone[6] = 18; cone[7] = 27;
1186       ierr = DMPlexSetCone(*dm, 13, cone);CHKERRQ(ierr);
1187       cone[0] = 30; cone[1] = 19; cone[2] = 22; cone[3] = 25;
1188       cone[4] = 29; cone[5] = 23; cone[6] = 16; cone[7] = 15;
1189       ierr = DMPlexSetCone(*dm, 14, cone);CHKERRQ(ierr);
1190     } else {
1191       cone[0] =  5; cone[1] =  8; cone[2] =  7; cone[3] =  6;
1192       cone[4] =  9; cone[5] = 12; cone[6] = 11; cone[7] = 10;
1193       ierr = DMPlexSetCone(*dm, 0, cone);CHKERRQ(ierr);
1194       cone[0] =  6; cone[1] =  7; cone[2] = 14; cone[3] = 13;
1195       cone[4] = 12; cone[5] = 15; cone[6] = 16; cone[7] = 11;
1196       ierr = DMPlexSetCone(*dm, 1, cone);CHKERRQ(ierr);
1197       cone[0] =  8; cone[1] = 17; cone[2] = 14; cone[3] =  7;
1198       cone[4] = 10; cone[5] = 11; cone[6] = 16; cone[7] = 18;
1199       ierr = DMPlexSetCone(*dm, 2, cone);CHKERRQ(ierr);
1200       cone[0] = 19; cone[1] = 17; cone[2] =  8; cone[3] =  5;
1201       cone[4] = 20; cone[5] =  9; cone[6] = 10; cone[7] = 18;
1202       ierr = DMPlexSetCone(*dm, 3, cone);CHKERRQ(ierr);
1203       cone[0] = 19; cone[1] =  5; cone[2] =  6; cone[3] = 13;
1204       cone[4] = 20; cone[5] = 15; cone[6] = 12; cone[7] =  9;
1205       ierr = DMPlexSetCone(*dm, 4, cone);CHKERRQ(ierr);
1206     }
1207     ierr = DMPlexSymmetrize(*dm);CHKERRQ(ierr);
1208     ierr = DMPlexStratify(*dm);CHKERRQ(ierr);
1209   }
1210   /* Interpolate */
1211   {
1212     DM idm = NULL;
1213 
1214     ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
1215     ierr = DMDestroy(dm);CHKERRQ(ierr);
1216     *dm  = idm;
1217   }
1218   /* Create cube geometry */
1219   {
1220     Vec             coordinates;
1221     PetscSection    coordSection;
1222     PetscScalar    *coords;
1223     PetscInt        coordSize, v;
1224     const PetscReal dis = 1.0/PetscSqrtReal(2.0);
1225     const PetscReal ds2 = dis/2.0;
1226 
1227     /* Build coordinates */
1228     ierr = DMGetCoordinateSection(*dm, &coordSection);CHKERRQ(ierr);
1229     ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
1230     ierr = PetscSectionSetFieldComponents(coordSection, 0, dim);CHKERRQ(ierr);
1231     ierr = PetscSectionSetChart(coordSection, numCells, numCells+numVertices);CHKERRQ(ierr);
1232     for (v = numCells; v < numCells+numVertices; ++v) {
1233       ierr = PetscSectionSetDof(coordSection, v, dim);CHKERRQ(ierr);
1234       ierr = PetscSectionSetFieldDof(coordSection, v, 0, dim);CHKERRQ(ierr);
1235     }
1236     ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
1237     ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
1238     ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
1239     ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
1240     ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
1241     ierr = VecSetBlockSize(coordinates, dim);CHKERRQ(ierr);
1242     ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
1243     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
1244     coords[0*dim+0] = -ds2; coords[0*dim+1] = -ds2; coords[0*dim+2] = 0.0;
1245     coords[1*dim+0] =  ds2; coords[1*dim+1] = -ds2; coords[1*dim+2] = 0.0;
1246     coords[2*dim+0] =  ds2; coords[2*dim+1] =  ds2; coords[2*dim+2] = 0.0;
1247     coords[3*dim+0] = -ds2; coords[3*dim+1] =  ds2; coords[3*dim+2] = 0.0;
1248     coords[4*dim+0] = -ds2; coords[4*dim+1] = -ds2; coords[4*dim+2] = 1.0;
1249     coords[5*dim+0] = -ds2; coords[5*dim+1] =  ds2; coords[5*dim+2] = 1.0;
1250     coords[6*dim+0] =  ds2; coords[6*dim+1] =  ds2; coords[6*dim+2] = 1.0;
1251     coords[7*dim+0] =  ds2; coords[7*dim+1] = -ds2; coords[7*dim+2] = 1.0;
1252     coords[ 8*dim+0] =  dis; coords[ 8*dim+1] = -dis; coords[ 8*dim+2] = 0.0;
1253     coords[ 9*dim+0] =  dis; coords[ 9*dim+1] =  dis; coords[ 9*dim+2] = 0.0;
1254     coords[10*dim+0] =  dis; coords[10*dim+1] = -dis; coords[10*dim+2] = 1.0;
1255     coords[11*dim+0] =  dis; coords[11*dim+1] =  dis; coords[11*dim+2] = 1.0;
1256     coords[12*dim+0] = -dis; coords[12*dim+1] =  dis; coords[12*dim+2] = 0.0;
1257     coords[13*dim+0] = -dis; coords[13*dim+1] =  dis; coords[13*dim+2] = 1.0;
1258     coords[14*dim+0] = -dis; coords[14*dim+1] = -dis; coords[14*dim+2] = 0.0;
1259     coords[15*dim+0] = -dis; coords[15*dim+1] = -dis; coords[15*dim+2] = 1.0;
1260     if (periodicZ == DM_BOUNDARY_PERIODIC) {
1261       /* 15 31 19 */ coords[16*dim+0] = -ds2; coords[16*dim+1] = -ds2; coords[16*dim+2] = 0.5;
1262       /* 16 32 22 */ coords[17*dim+0] =  ds2; coords[17*dim+1] = -ds2; coords[17*dim+2] = 0.5;
1263       /* 17 33 21 */ coords[18*dim+0] =  ds2; coords[18*dim+1] =  ds2; coords[18*dim+2] = 0.5;
1264       /* 18 34 20 */ coords[19*dim+0] = -ds2; coords[19*dim+1] =  ds2; coords[19*dim+2] = 0.5;
1265       /* 29 35 30 */ coords[20*dim+0] = -dis; coords[20*dim+1] = -dis; coords[20*dim+2] = 0.5;
1266       /* 23 36 25 */ coords[21*dim+0] =  dis; coords[21*dim+1] = -dis; coords[21*dim+2] = 0.5;
1267       /* 24 37 26 */ coords[22*dim+0] =  dis; coords[22*dim+1] =  dis; coords[22*dim+2] = 0.5;
1268       /* 27 38 28 */ coords[23*dim+0] = -dis; coords[23*dim+1] =  dis; coords[23*dim+2] = 0.5;
1269     }
1270     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
1271     ierr = DMSetCoordinatesLocal(*dm, coordinates);CHKERRQ(ierr);
1272     ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
1273   }
1274   /* Create periodicity */
1275   if (periodicZ == DM_BOUNDARY_PERIODIC || periodicZ == DM_BOUNDARY_TWIST) {
1276     PetscReal      L[3];
1277     PetscReal      maxCell[3];
1278     DMBoundaryType bdType[3];
1279     PetscReal      lower[3] = {0.0, 0.0, 0.0};
1280     PetscReal      upper[3] = {1.0, 1.0, 1.5};
1281     PetscInt       i, numZCells = 3;
1282 
1283     bdType[0] = DM_BOUNDARY_NONE;
1284     bdType[1] = DM_BOUNDARY_NONE;
1285     bdType[2] = periodicZ;
1286     for (i = 0; i < dim; i++) {
1287       L[i]       = upper[i] - lower[i];
1288       maxCell[i] = 1.1 * (L[i] / numZCells);
1289     }
1290     ierr = DMSetPeriodicity(*dm, PETSC_TRUE, maxCell, L, bdType);CHKERRQ(ierr);
1291   }
1292   /* Refine topology */
1293   for (r = 0; r < numRefine; ++r) {
1294     DM rdm = NULL;
1295 
1296     ierr = DMRefine(*dm, comm, &rdm);CHKERRQ(ierr);
1297     ierr = DMDestroy(dm);CHKERRQ(ierr);
1298     *dm  = rdm;
1299   }
1300   /* Remap geometry to cylinder
1301        Interior square: Linear interpolation is correct
1302        The other cells all have vertices on rays from the origin. We want to uniformly expand the spacing
1303        such that the last vertex is on the unit circle. So the closest and farthest vertices are at distance
1304 
1305          phi     = arctan(y/x)
1306          d_close = sqrt(1/8 + 1/4 sin^2(phi))
1307          d_far   = sqrt(1/2 + sin^2(phi))
1308 
1309        so we remap them using
1310 
1311          x_new = x_close + (x - x_close) (1 - d_close) / (d_far - d_close)
1312          y_new = y_close + (y - y_close) (1 - d_close) / (d_far - d_close)
1313 
1314        If pi/4 < phi < 3pi/4 or -3pi/4 < phi < -pi/4, then we switch x and y.
1315   */
1316   {
1317     Vec           coordinates;
1318     PetscSection  coordSection;
1319     PetscScalar  *coords;
1320     PetscInt      vStart, vEnd, v;
1321     const PetscReal dis = 1.0/PetscSqrtReal(2.0);
1322     const PetscReal ds2 = 0.5*dis;
1323 
1324     ierr = DMPlexGetDepthStratum(*dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
1325     ierr = DMGetCoordinateSection(*dm, &coordSection);CHKERRQ(ierr);
1326     ierr = DMGetCoordinatesLocal(*dm, &coordinates);CHKERRQ(ierr);
1327     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
1328     for (v = vStart; v < vEnd; ++v) {
1329       PetscReal phi, sinp, cosp, dc, df, x, y, xc, yc;
1330       PetscInt  off;
1331 
1332       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
1333       if ((PetscAbsScalar(coords[off+0]) <= ds2) && (PetscAbsScalar(coords[off+1]) <= ds2)) continue;
1334       x    = PetscRealPart(coords[off]);
1335       y    = PetscRealPart(coords[off+1]);
1336       phi  = PetscAtan2Real(y, x);
1337       sinp = PetscSinReal(phi);
1338       cosp = PetscCosReal(phi);
1339       if ((PetscAbsReal(phi) > PETSC_PI/4.0) && (PetscAbsReal(phi) < 3.0*PETSC_PI/4.0)) {
1340         dc = PetscAbsReal(ds2/sinp);
1341         df = PetscAbsReal(dis/sinp);
1342         xc = ds2*x/PetscAbsReal(y);
1343         yc = ds2*PetscSignReal(y);
1344       } else {
1345         dc = PetscAbsReal(ds2/cosp);
1346         df = PetscAbsReal(dis/cosp);
1347         xc = ds2*PetscSignReal(x);
1348         yc = ds2*y/PetscAbsReal(x);
1349       }
1350       coords[off+0] = xc + (coords[off+0] - xc)*(1.0 - dc)/(df - dc);
1351       coords[off+1] = yc + (coords[off+1] - yc)*(1.0 - dc)/(df - dc);
1352     }
1353     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
1354     if (periodicZ == DM_BOUNDARY_PERIODIC || periodicZ == DM_BOUNDARY_TWIST) {
1355       ierr = DMLocalizeCoordinates(*dm);CHKERRQ(ierr);
1356     }
1357   }
1358   PetscFunctionReturn(0);
1359 }
1360 
1361 /*@
1362   DMPlexCreateWedgeCylinderMesh - Creates a mesh on the tensor product of the unit interval with the circle (cylinder) using wedges.
1363 
1364   Collective on MPI_Comm
1365 
1366   Input Parameters:
1367 + comm - The communicator for the DM object
1368 . n    - The number of wedges around the origin
1369 - interpolate - Create edges and faces
1370 
1371   Output Parameter:
1372 . dm  - The DM object
1373 
1374   Level: beginner
1375 
1376 .keywords: DM, create
1377 .seealso: DMPlexCreateHexCylinderMesh(), DMPlexCreateBoxMesh(), DMSetType(), DMCreate()
1378 @*/
1379 PetscErrorCode DMPlexCreateWedgeCylinderMesh(MPI_Comm comm, PetscInt n, PetscBool interpolate, DM *dm)
1380 {
1381   const PetscInt dim = 3;
1382   PetscInt       numCells, numVertices;
1383   PetscErrorCode ierr;
1384 
1385   PetscFunctionBegin;
1386   PetscValidPointer(dm, 3);
1387   if (n < 0) SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Number of wedges %D cannot be negative", n);
1388   ierr = DMCreate(comm, dm);CHKERRQ(ierr);
1389   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
1390   ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
1391   /* Create topology */
1392   {
1393     PetscInt cone[6], c;
1394 
1395     numCells    = n;
1396     numVertices = 2*(n+1);
1397     ierr = DMPlexSetChart(*dm, 0, numCells+numVertices);CHKERRQ(ierr);
1398     for (c = 0; c < numCells; c++) {ierr = DMPlexSetConeSize(*dm, c, 6);CHKERRQ(ierr);}
1399     ierr = DMSetUp(*dm);CHKERRQ(ierr);
1400     for (c = 0; c < numCells; c++) {
1401       cone[0] =  c+n*1; cone[1] = (c+1)%n+n*1; cone[2] = 0+3*n;
1402       cone[3] =  c+n*2; cone[4] = (c+1)%n+n*2; cone[5] = 1+3*n;
1403       ierr = DMPlexSetCone(*dm, c, cone);CHKERRQ(ierr);
1404     }
1405     ierr = DMPlexSymmetrize(*dm);CHKERRQ(ierr);
1406     ierr = DMPlexStratify(*dm);CHKERRQ(ierr);
1407   }
1408   /* Interpolate */
1409   if (interpolate) {
1410     DM idm = NULL;
1411 
1412     ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
1413     ierr = DMDestroy(dm);CHKERRQ(ierr);
1414     *dm  = idm;
1415   }
1416   /* Create cylinder geometry */
1417   {
1418     Vec          coordinates;
1419     PetscSection coordSection;
1420     PetscScalar *coords;
1421     PetscInt     coordSize, v, c;
1422 
1423     /* Build coordinates */
1424     ierr = DMGetCoordinateSection(*dm, &coordSection);CHKERRQ(ierr);
1425     ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
1426     ierr = PetscSectionSetFieldComponents(coordSection, 0, dim);CHKERRQ(ierr);
1427     ierr = PetscSectionSetChart(coordSection, numCells, numCells+numVertices);CHKERRQ(ierr);
1428     for (v = numCells; v < numCells+numVertices; ++v) {
1429       ierr = PetscSectionSetDof(coordSection, v, dim);CHKERRQ(ierr);
1430       ierr = PetscSectionSetFieldDof(coordSection, v, 0, dim);CHKERRQ(ierr);
1431     }
1432     ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
1433     ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
1434     ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
1435     ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
1436     ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
1437     ierr = VecSetBlockSize(coordinates, dim);CHKERRQ(ierr);
1438     ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
1439     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
1440     for (c = 0; c < numCells; c++) {
1441       coords[(c+0*n)*dim+0] = PetscCosReal(2.0*c*PETSC_PI/n); coords[(c+0*n)*dim+1] = PetscSinReal(2.0*c*PETSC_PI/n); coords[(c+0*n)*dim+2] = 1.0;
1442       coords[(c+1*n)*dim+0] = PetscCosReal(2.0*c*PETSC_PI/n); coords[(c+1*n)*dim+1] = PetscSinReal(2.0*c*PETSC_PI/n); coords[(c+1*n)*dim+2] = 0.0;
1443     }
1444     coords[(2*n+0)*dim+0] = 0.0; coords[(2*n+0)*dim+1] = 0.0; coords[(2*n+0)*dim+2] = 1.0;
1445     coords[(2*n+1)*dim+0] = 0.0; coords[(2*n+1)*dim+1] = 0.0; coords[(2*n+1)*dim+2] = 0.0;
1446     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
1447     ierr = DMSetCoordinatesLocal(*dm, coordinates);CHKERRQ(ierr);
1448     ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
1449   }
1450   PetscFunctionReturn(0);
1451 }
1452 
1453 PETSC_STATIC_INLINE PetscReal DiffNormReal(PetscInt dim, const PetscReal x[], const PetscReal y[])
1454 {
1455   PetscReal prod = 0.0;
1456   PetscInt  i;
1457   for (i = 0; i < dim; ++i) prod += PetscSqr(x[i] - y[i]);
1458   return PetscSqrtReal(prod);
1459 }
1460 PETSC_STATIC_INLINE PetscReal DotReal(PetscInt dim, const PetscReal x[], const PetscReal y[])
1461 {
1462   PetscReal prod = 0.0;
1463   PetscInt  i;
1464   for (i = 0; i < dim; ++i) prod += x[i]*y[i];
1465   return prod;
1466 }
1467 
1468 /*@
1469   DMPlexCreateSphereMesh - Creates a mesh on the d-dimensional sphere, S^d.
1470 
1471   Collective on MPI_Comm
1472 
1473   Input Parameters:
1474 . comm  - The communicator for the DM object
1475 . dim   - The dimension
1476 - simplex - Use simplices, or tensor product cells
1477 
1478   Output Parameter:
1479 . dm  - The DM object
1480 
1481   Level: beginner
1482 
1483 .keywords: DM, create
1484 .seealso: DMPlexCreateBoxMesh(), DMSetType(), DMCreate()
1485 @*/
1486 PetscErrorCode DMPlexCreateSphereMesh(MPI_Comm comm, PetscInt dim, PetscBool simplex, DM *dm)
1487 {
1488   const PetscInt  embedDim = dim+1;
1489   PetscSection    coordSection;
1490   Vec             coordinates;
1491   PetscScalar    *coords;
1492   PetscReal      *coordsIn;
1493   PetscInt        numCells, numEdges, numVerts, firstVertex, v, firstEdge, coordSize, d, c, e;
1494   PetscMPIInt     rank;
1495   PetscErrorCode  ierr;
1496 
1497   PetscFunctionBegin;
1498   PetscValidPointer(dm, 4);
1499   ierr = DMCreate(comm, dm);CHKERRQ(ierr);
1500   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
1501   ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
1502   ierr = DMSetCoordinateDim(*dm, dim+1);CHKERRQ(ierr);
1503   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) *dm), &rank);CHKERRQ(ierr);
1504   switch (dim) {
1505   case 2:
1506     if (simplex) {
1507       DM              idm         = NULL;
1508       const PetscReal edgeLen     = 2.0/(1.0 + PETSC_PHI);
1509       const PetscReal vertex[3]   = {0.0, 1.0/(1.0 + PETSC_PHI), PETSC_PHI/(1.0 + PETSC_PHI)};
1510       const PetscInt  degree      = 5;
1511       PetscInt        s[3]        = {1, 1, 1};
1512       PetscInt        cone[3];
1513       PetscInt       *graph, p, i, j, k;
1514 
1515       numCells    = !rank ? 20 : 0;
1516       numVerts    = !rank ? 12 : 0;
1517       firstVertex = numCells;
1518       /* Use icosahedron, which for a unit sphere has coordinates which are all cyclic permutations of
1519 
1520            (0, \pm 1/\phi+1, \pm \phi/\phi+1)
1521 
1522          where \phi^2 - \phi - 1 = 0, meaning \phi is the golden ratio \frac{1 + \sqrt{5}}{2}. The edge
1523          length is then given by 2/\phi = 2 * 2.73606 = 5.47214.
1524        */
1525       /* Construct vertices */
1526       ierr = PetscCalloc1(numVerts * embedDim, &coordsIn);CHKERRQ(ierr);
1527       for (p = 0, i = 0; p < embedDim; ++p) {
1528         for (s[1] = -1; s[1] < 2; s[1] += 2) {
1529           for (s[2] = -1; s[2] < 2; s[2] += 2) {
1530             for (d = 0; d < embedDim; ++d) coordsIn[i*embedDim+d] = s[(d+p)%embedDim]*vertex[(d+p)%embedDim];
1531             ++i;
1532           }
1533         }
1534       }
1535       /* Construct graph */
1536       ierr = PetscCalloc1(numVerts * numVerts, &graph);CHKERRQ(ierr);
1537       for (i = 0; i < numVerts; ++i) {
1538         for (j = 0, k = 0; j < numVerts; ++j) {
1539           if (PetscAbsReal(DiffNormReal(embedDim, &coordsIn[i*embedDim], &coordsIn[j*embedDim]) - edgeLen) < PETSC_SMALL) {graph[i*numVerts+j] = 1; ++k;}
1540         }
1541         if (k != degree) SETERRQ3(comm, PETSC_ERR_PLIB, "Invalid icosahedron, vertex %D degree %D != %D", i, k, degree);
1542       }
1543       /* Build Topology */
1544       ierr = DMPlexSetChart(*dm, 0, numCells+numVerts);CHKERRQ(ierr);
1545       for (c = 0; c < numCells; c++) {
1546         ierr = DMPlexSetConeSize(*dm, c, embedDim);CHKERRQ(ierr);
1547       }
1548       ierr = DMSetUp(*dm);CHKERRQ(ierr); /* Allocate space for cones */
1549       /* Cells */
1550       for (i = 0, c = 0; i < numVerts; ++i) {
1551         for (j = 0; j < i; ++j) {
1552           for (k = 0; k < j; ++k) {
1553             if (graph[i*numVerts+j] && graph[j*numVerts+k] && graph[k*numVerts+i]) {
1554               cone[0] = firstVertex+i; cone[1] = firstVertex+j; cone[2] = firstVertex+k;
1555               /* Check orientation */
1556               {
1557                 const PetscInt epsilon[3][3][3] = {{{0, 0, 0}, {0, 0, 1}, {0, -1, 0}}, {{0, 0, -1}, {0, 0, 0}, {1, 0, 0}}, {{0, 1, 0}, {-1, 0, 0}, {0, 0, 0}}};
1558                 PetscReal normal[3];
1559                 PetscInt  e, f;
1560 
1561                 for (d = 0; d < embedDim; ++d) {
1562                   normal[d] = 0.0;
1563                   for (e = 0; e < embedDim; ++e) {
1564                     for (f = 0; f < embedDim; ++f) {
1565                       normal[d] += epsilon[d][e][f]*(coordsIn[j*embedDim+e] - coordsIn[i*embedDim+e])*(coordsIn[k*embedDim+f] - coordsIn[i*embedDim+f]);
1566                     }
1567                   }
1568                 }
1569                 if (DotReal(embedDim, normal, &coordsIn[i*embedDim]) < 0) {PetscInt tmp = cone[1]; cone[1] = cone[2]; cone[2] = tmp;}
1570               }
1571               ierr = DMPlexSetCone(*dm, c++, cone);CHKERRQ(ierr);
1572             }
1573           }
1574         }
1575       }
1576       ierr = DMPlexSymmetrize(*dm);CHKERRQ(ierr);
1577       ierr = DMPlexStratify(*dm);CHKERRQ(ierr);
1578       ierr = PetscFree(graph);CHKERRQ(ierr);
1579       /* Interpolate mesh */
1580       ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
1581       ierr = DMDestroy(dm);CHKERRQ(ierr);
1582       *dm  = idm;
1583     } else {
1584       /*
1585         12-21--13
1586          |     |
1587         25  4  24
1588          |     |
1589   12-25--9-16--8-24--13
1590    |     |     |     |
1591   23  5 17  0 15  3  22
1592    |     |     |     |
1593   10-20--6-14--7-19--11
1594          |     |
1595         20  1  19
1596          |     |
1597         10-18--11
1598          |     |
1599         23  2  22
1600          |     |
1601         12-21--13
1602        */
1603       const PetscReal dist = 1.0/PetscSqrtReal(3.0);
1604       PetscInt        cone[4], ornt[4];
1605 
1606       numCells    = !rank ?  6 : 0;
1607       numEdges    = !rank ? 12 : 0;
1608       numVerts    = !rank ?  8 : 0;
1609       firstVertex = numCells;
1610       firstEdge   = numCells + numVerts;
1611       /* Build Topology */
1612       ierr = DMPlexSetChart(*dm, 0, numCells+numEdges+numVerts);CHKERRQ(ierr);
1613       for (c = 0; c < numCells; c++) {
1614         ierr = DMPlexSetConeSize(*dm, c, 4);CHKERRQ(ierr);
1615       }
1616       for (e = firstEdge; e < firstEdge+numEdges; ++e) {
1617         ierr = DMPlexSetConeSize(*dm, e, 2);CHKERRQ(ierr);
1618       }
1619       ierr = DMSetUp(*dm);CHKERRQ(ierr); /* Allocate space for cones */
1620       /* Cell 0 */
1621       cone[0] = 14; cone[1] = 15; cone[2] = 16; cone[3] = 17;
1622       ierr = DMPlexSetCone(*dm, 0, cone);CHKERRQ(ierr);
1623       ornt[0] = 0; ornt[1] = 0; ornt[2] = 0; ornt[3] = 0;
1624       ierr = DMPlexSetConeOrientation(*dm, 0, ornt);CHKERRQ(ierr);
1625       /* Cell 1 */
1626       cone[0] = 18; cone[1] = 19; cone[2] = 14; cone[3] = 20;
1627       ierr = DMPlexSetCone(*dm, 1, cone);CHKERRQ(ierr);
1628       ornt[0] = 0; ornt[1] = 0; ornt[2] = -2; ornt[3] = 0;
1629       ierr = DMPlexSetConeOrientation(*dm, 1, ornt);CHKERRQ(ierr);
1630       /* Cell 2 */
1631       cone[0] = 21; cone[1] = 22; cone[2] = 18; cone[3] = 23;
1632       ierr = DMPlexSetCone(*dm, 2, cone);CHKERRQ(ierr);
1633       ornt[0] = 0; ornt[1] = 0; ornt[2] = -2; ornt[3] = 0;
1634       ierr = DMPlexSetConeOrientation(*dm, 2, ornt);CHKERRQ(ierr);
1635       /* Cell 3 */
1636       cone[0] = 19; cone[1] = 22; cone[2] = 24; cone[3] = 15;
1637       ierr = DMPlexSetCone(*dm, 3, cone);CHKERRQ(ierr);
1638       ornt[0] = -2; ornt[1] = -2; ornt[2] = 0; ornt[3] = -2;
1639       ierr = DMPlexSetConeOrientation(*dm, 3, ornt);CHKERRQ(ierr);
1640       /* Cell 4 */
1641       cone[0] = 16; cone[1] = 24; cone[2] = 21; cone[3] = 25;
1642       ierr = DMPlexSetCone(*dm, 4, cone);CHKERRQ(ierr);
1643       ornt[0] = -2; ornt[1] = -2; ornt[2] = -2; ornt[3] = 0;
1644       ierr = DMPlexSetConeOrientation(*dm, 4, ornt);CHKERRQ(ierr);
1645       /* Cell 5 */
1646       cone[0] = 20; cone[1] = 17; cone[2] = 25; cone[3] = 23;
1647       ierr = DMPlexSetCone(*dm, 5, cone);CHKERRQ(ierr);
1648       ornt[0] = -2; ornt[1] = -2; ornt[2] = -2; ornt[3] = -2;
1649       ierr = DMPlexSetConeOrientation(*dm, 5, ornt);CHKERRQ(ierr);
1650       /* Edges */
1651       cone[0] =  6; cone[1] =  7;
1652       ierr = DMPlexSetCone(*dm, 14, cone);CHKERRQ(ierr);
1653       cone[0] =  7; cone[1] =  8;
1654       ierr = DMPlexSetCone(*dm, 15, cone);CHKERRQ(ierr);
1655       cone[0] =  8; cone[1] =  9;
1656       ierr = DMPlexSetCone(*dm, 16, cone);CHKERRQ(ierr);
1657       cone[0] =  9; cone[1] =  6;
1658       ierr = DMPlexSetCone(*dm, 17, cone);CHKERRQ(ierr);
1659       cone[0] = 10; cone[1] = 11;
1660       ierr = DMPlexSetCone(*dm, 18, cone);CHKERRQ(ierr);
1661       cone[0] = 11; cone[1] =  7;
1662       ierr = DMPlexSetCone(*dm, 19, cone);CHKERRQ(ierr);
1663       cone[0] =  6; cone[1] = 10;
1664       ierr = DMPlexSetCone(*dm, 20, cone);CHKERRQ(ierr);
1665       cone[0] = 12; cone[1] = 13;
1666       ierr = DMPlexSetCone(*dm, 21, cone);CHKERRQ(ierr);
1667       cone[0] = 13; cone[1] = 11;
1668       ierr = DMPlexSetCone(*dm, 22, cone);CHKERRQ(ierr);
1669       cone[0] = 10; cone[1] = 12;
1670       ierr = DMPlexSetCone(*dm, 23, cone);CHKERRQ(ierr);
1671       cone[0] = 13; cone[1] =  8;
1672       ierr = DMPlexSetCone(*dm, 24, cone);CHKERRQ(ierr);
1673       cone[0] = 12; cone[1] =  9;
1674       ierr = DMPlexSetCone(*dm, 25, cone);CHKERRQ(ierr);
1675       ierr = DMPlexSymmetrize(*dm);CHKERRQ(ierr);
1676       ierr = DMPlexStratify(*dm);CHKERRQ(ierr);
1677       /* Build coordinates */
1678       ierr = PetscCalloc1(numVerts * embedDim, &coordsIn);CHKERRQ(ierr);
1679       coordsIn[0*embedDim+0] = -dist; coordsIn[0*embedDim+1] =  dist; coordsIn[0*embedDim+2] = -dist;
1680       coordsIn[1*embedDim+0] =  dist; coordsIn[1*embedDim+1] =  dist; coordsIn[1*embedDim+2] = -dist;
1681       coordsIn[2*embedDim+0] =  dist; coordsIn[2*embedDim+1] = -dist; coordsIn[2*embedDim+2] = -dist;
1682       coordsIn[3*embedDim+0] = -dist; coordsIn[3*embedDim+1] = -dist; coordsIn[3*embedDim+2] = -dist;
1683       coordsIn[4*embedDim+0] = -dist; coordsIn[4*embedDim+1] =  dist; coordsIn[4*embedDim+2] =  dist;
1684       coordsIn[5*embedDim+0] =  dist; coordsIn[5*embedDim+1] =  dist; coordsIn[5*embedDim+2] =  dist;
1685       coordsIn[6*embedDim+0] = -dist; coordsIn[6*embedDim+1] = -dist; coordsIn[6*embedDim+2] =  dist;
1686       coordsIn[7*embedDim+0] =  dist; coordsIn[7*embedDim+1] = -dist; coordsIn[7*embedDim+2] =  dist;
1687     }
1688     break;
1689   case 3:
1690     if (simplex) {
1691       DM              idm             = NULL;
1692       const PetscReal edgeLen         = 1.0/PETSC_PHI;
1693       const PetscReal vertexA[4]      = {0.5, 0.5, 0.5, 0.5};
1694       const PetscReal vertexB[4]      = {1.0, 0.0, 0.0, 0.0};
1695       const PetscReal vertexC[4]      = {0.5, 0.5*PETSC_PHI, 0.5/PETSC_PHI, 0.0};
1696       const PetscInt  degree          = 12;
1697       PetscInt        s[4]            = {1, 1, 1};
1698       PetscInt        evenPerm[12][4] = {{0, 1, 2, 3}, {0, 2, 3, 1}, {0, 3, 1, 2}, {1, 0, 3, 2}, {1, 2, 0, 3}, {1, 3, 2, 0},
1699                                          {2, 0, 1, 3}, {2, 1, 3, 0}, {2, 3, 0, 1}, {3, 0, 2, 1}, {3, 1, 0, 2}, {3, 2, 1, 0}};
1700       PetscInt        cone[4];
1701       PetscInt       *graph, p, i, j, k, l;
1702 
1703       numCells    = !rank ? 600 : 0;
1704       numVerts    = !rank ? 120 : 0;
1705       firstVertex = numCells;
1706       /* Use the 600-cell, which for a unit sphere has coordinates which are
1707 
1708            1/2 (\pm 1, \pm 1,    \pm 1, \pm 1)                          16
1709                (\pm 1,    0,       0,      0)  all cyclic permutations   8
1710            1/2 (\pm 1, \pm phi, \pm 1/phi, 0)  all even permutations    96
1711 
1712          where \phi^2 - \phi - 1 = 0, meaning \phi is the golden ratio \frac{1 + \sqrt{5}}{2}. The edge
1713          length is then given by 1/\phi = 2.73606.
1714 
1715          http://buzzard.pugetsound.edu/sage-practice/ch03s03.html
1716          http://mathworld.wolfram.com/600-Cell.html
1717        */
1718       /* Construct vertices */
1719       ierr = PetscCalloc1(numVerts * embedDim, &coordsIn);CHKERRQ(ierr);
1720       i    = 0;
1721       for (s[0] = -1; s[0] < 2; s[0] += 2) {
1722         for (s[1] = -1; s[1] < 2; s[1] += 2) {
1723           for (s[2] = -1; s[2] < 2; s[2] += 2) {
1724             for (s[3] = -1; s[3] < 2; s[3] += 2) {
1725               for (d = 0; d < embedDim; ++d) coordsIn[i*embedDim+d] = s[d]*vertexA[d];
1726               ++i;
1727             }
1728           }
1729         }
1730       }
1731       for (p = 0; p < embedDim; ++p) {
1732         s[1] = s[2] = s[3] = 1;
1733         for (s[0] = -1; s[0] < 2; s[0] += 2) {
1734           for (d = 0; d < embedDim; ++d) coordsIn[i*embedDim+d] = s[(d+p)%embedDim]*vertexB[(d+p)%embedDim];
1735           ++i;
1736         }
1737       }
1738       for (p = 0; p < 12; ++p) {
1739         s[3] = 1;
1740         for (s[0] = -1; s[0] < 2; s[0] += 2) {
1741           for (s[1] = -1; s[1] < 2; s[1] += 2) {
1742             for (s[2] = -1; s[2] < 2; s[2] += 2) {
1743               for (d = 0; d < embedDim; ++d) coordsIn[i*embedDim+d] = s[evenPerm[p][d]]*vertexC[evenPerm[p][d]];
1744               ++i;
1745             }
1746           }
1747         }
1748       }
1749       if (i != numVerts) SETERRQ2(comm, PETSC_ERR_PLIB, "Invalid 600-cell, vertices %D != %D", i, numVerts);
1750       /* Construct graph */
1751       ierr = PetscCalloc1(numVerts * numVerts, &graph);CHKERRQ(ierr);
1752       for (i = 0; i < numVerts; ++i) {
1753         for (j = 0, k = 0; j < numVerts; ++j) {
1754           if (PetscAbsReal(DiffNormReal(embedDim, &coordsIn[i*embedDim], &coordsIn[j*embedDim]) - edgeLen) < PETSC_SMALL) {graph[i*numVerts+j] = 1; ++k;}
1755         }
1756         if (k != degree) SETERRQ3(comm, PETSC_ERR_PLIB, "Invalid 600-cell, vertex %D degree %D != %D", i, k, degree);
1757       }
1758       /* Build Topology */
1759       ierr = DMPlexSetChart(*dm, 0, numCells+numVerts);CHKERRQ(ierr);
1760       for (c = 0; c < numCells; c++) {
1761         ierr = DMPlexSetConeSize(*dm, c, embedDim);CHKERRQ(ierr);
1762       }
1763       ierr = DMSetUp(*dm);CHKERRQ(ierr); /* Allocate space for cones */
1764       /* Cells */
1765       for (i = 0, c = 0; i < numVerts; ++i) {
1766         for (j = 0; j < i; ++j) {
1767           for (k = 0; k < j; ++k) {
1768             for (l = 0; l < k; ++l) {
1769               if (graph[i*numVerts+j] && graph[j*numVerts+k] && graph[k*numVerts+i] &&
1770                   graph[l*numVerts+i] && graph[l*numVerts+j] && graph[l*numVerts+k]) {
1771                 cone[0] = firstVertex+i; cone[1] = firstVertex+j; cone[2] = firstVertex+k; cone[3] = firstVertex+l;
1772                 /* Check orientation: https://ef.gy/linear-algebra:normal-vectors-in-higher-dimensional-spaces */
1773                 {
1774                   const PetscInt epsilon[4][4][4][4] = {{{{0,  0,  0,  0}, { 0, 0,  0,  0}, { 0,  0, 0,  0}, { 0,  0,  0, 0}},
1775                                                          {{0,  0,  0,  0}, { 0, 0,  0,  0}, { 0,  0, 0,  1}, { 0,  0, -1, 0}},
1776                                                          {{0,  0,  0,  0}, { 0, 0,  0, -1}, { 0,  0, 0,  0}, { 0,  1,  0, 0}},
1777                                                          {{0,  0,  0,  0}, { 0, 0,  1,  0}, { 0, -1, 0,  0}, { 0,  0,  0, 0}}},
1778 
1779                                                         {{{0,  0,  0,  0}, { 0, 0,  0,  0}, { 0,  0, 0, -1}, { 0,  0,  1, 0}},
1780                                                          {{0,  0,  0,  0}, { 0, 0,  0,  0}, { 0,  0, 0,  0}, { 0,  0,  0, 0}},
1781                                                          {{0,  0,  0,  1}, { 0, 0,  0,  0}, { 0,  0, 0,  0}, {-1,  0,  0, 0}},
1782                                                          {{0,  0, -1,  0}, { 0, 0,  0,  0}, { 1,  0, 0,  0}, { 0,  0,  0, 0}}},
1783 
1784                                                         {{{0,  0,  0,  0}, { 0, 0,  0,  1}, { 0,  0, 0,  0}, { 0, -1,  0, 0}},
1785                                                          {{0,  0,  0, -1}, { 0, 0,  0,  0}, { 0,  0, 0,  0}, { 1,  0,  0, 0}},
1786                                                          {{0,  0,  0,  0}, { 0, 0,  0,  0}, { 0,  0, 0,  0}, { 0,  0,  0, 0}},
1787                                                          {{0,  1,  0,  0}, {-1, 0,  0,  0}, { 0,  0, 0,  0}, { 0,  0,  0, 0}}},
1788 
1789                                                         {{{0,  0,  0,  0}, { 0, 0, -1,  0}, { 0,  1, 0,  0}, { 0,  0,  0, 0}},
1790                                                          {{0,  0,  1,  0}, { 0, 0,  0,  0}, {-1,  0, 0,  0}, { 0,  0,  0, 0}},
1791                                                          {{0, -1,  0,  0}, { 1, 0,  0,  0}, { 0,  0, 0,  0}, { 0,  0,  0, 0}},
1792                                                          {{0,  0,  0,  0}, { 0, 0,  0,  0}, { 0,  0, 0,  0}, { 0,  0,  0, 0}}}};
1793                   PetscReal normal[4];
1794                   PetscInt  e, f, g;
1795 
1796                   for (d = 0; d < embedDim; ++d) {
1797                     normal[d] = 0.0;
1798                     for (e = 0; e < embedDim; ++e) {
1799                       for (f = 0; f < embedDim; ++f) {
1800                         for (g = 0; g < embedDim; ++g) {
1801                           normal[d] += epsilon[d][e][f][g]*(coordsIn[j*embedDim+e] - coordsIn[i*embedDim+e])*(coordsIn[k*embedDim+f] - coordsIn[i*embedDim+f])*(coordsIn[l*embedDim+f] - coordsIn[i*embedDim+f]);
1802                         }
1803                       }
1804                     }
1805                   }
1806                   if (DotReal(embedDim, normal, &coordsIn[i*embedDim]) < 0) {PetscInt tmp = cone[1]; cone[1] = cone[2]; cone[2] = tmp;}
1807                 }
1808                 ierr = DMPlexSetCone(*dm, c++, cone);CHKERRQ(ierr);
1809               }
1810             }
1811           }
1812         }
1813       }
1814       ierr = DMPlexSymmetrize(*dm);CHKERRQ(ierr);
1815       ierr = DMPlexStratify(*dm);CHKERRQ(ierr);
1816       ierr = PetscFree(graph);CHKERRQ(ierr);
1817       /* Interpolate mesh */
1818       ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
1819       ierr = DMDestroy(dm);CHKERRQ(ierr);
1820       *dm  = idm;
1821       break;
1822     }
1823   default: SETERRQ1(comm, PETSC_ERR_SUP, "Unsupported dimension for sphere: %D", dim);
1824   }
1825   /* Create coordinates */
1826   ierr = DMGetCoordinateSection(*dm, &coordSection);CHKERRQ(ierr);
1827   ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
1828   ierr = PetscSectionSetFieldComponents(coordSection, 0, embedDim);CHKERRQ(ierr);
1829   ierr = PetscSectionSetChart(coordSection, firstVertex, firstVertex+numVerts);CHKERRQ(ierr);
1830   for (v = firstVertex; v < firstVertex+numVerts; ++v) {
1831     ierr = PetscSectionSetDof(coordSection, v, embedDim);CHKERRQ(ierr);
1832     ierr = PetscSectionSetFieldDof(coordSection, v, 0, embedDim);CHKERRQ(ierr);
1833   }
1834   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
1835   ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
1836   ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
1837   ierr = VecSetBlockSize(coordinates, embedDim);CHKERRQ(ierr);
1838   ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
1839   ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
1840   ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
1841   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
1842   for (v = 0; v < numVerts; ++v) for (d = 0; d < embedDim; ++d) {coords[v*embedDim+d] = coordsIn[v*embedDim+d];}
1843   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
1844   ierr = DMSetCoordinatesLocal(*dm, coordinates);CHKERRQ(ierr);
1845   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
1846   ierr = PetscFree(coordsIn);CHKERRQ(ierr);
1847   PetscFunctionReturn(0);
1848 }
1849 
1850 /* External function declarations here */
1851 extern PetscErrorCode DMCreateInterpolation_Plex(DM dmCoarse, DM dmFine, Mat *interpolation, Vec *scaling);
1852 extern PetscErrorCode DMCreateInjection_Plex(DM dmCoarse, DM dmFine, Mat *mat);
1853 extern PetscErrorCode DMCreateMassMatrix_Plex(DM dmCoarse, DM dmFine, Mat *mat);
1854 extern PetscErrorCode DMCreateDefaultSection_Plex(DM dm);
1855 extern PetscErrorCode DMCreateDefaultConstraints_Plex(DM dm);
1856 extern PetscErrorCode DMCreateMatrix_Plex(DM dm,  Mat *J);
1857 extern PetscErrorCode DMCreateCoordinateDM_Plex(DM dm, DM *cdm);
1858 PETSC_INTERN PetscErrorCode DMClone_Plex(DM dm, DM *newdm);
1859 extern PetscErrorCode DMSetUp_Plex(DM dm);
1860 extern PetscErrorCode DMDestroy_Plex(DM dm);
1861 extern PetscErrorCode DMView_Plex(DM dm, PetscViewer viewer);
1862 extern PetscErrorCode DMLoad_Plex(DM dm, PetscViewer viewer);
1863 extern PetscErrorCode DMCreateSubDM_Plex(DM dm, PetscInt numFields, const PetscInt fields[], IS *is, DM *subdm);
1864 static PetscErrorCode DMInitialize_Plex(DM dm);
1865 
1866 /* Replace dm with the contents of dmNew
1867    - Share the DM_Plex structure
1868    - Share the coordinates
1869    - Share the SF
1870 */
1871 static PetscErrorCode DMPlexReplace_Static(DM dm, DM dmNew)
1872 {
1873   PetscSF               sf;
1874   DM                    coordDM, coarseDM;
1875   Vec                   coords;
1876   PetscBool             isper;
1877   const PetscReal      *maxCell, *L;
1878   const DMBoundaryType *bd;
1879   PetscErrorCode        ierr;
1880 
1881   PetscFunctionBegin;
1882   ierr = DMGetPointSF(dmNew, &sf);CHKERRQ(ierr);
1883   ierr = DMSetPointSF(dm, sf);CHKERRQ(ierr);
1884   ierr = DMGetCoordinateDM(dmNew, &coordDM);CHKERRQ(ierr);
1885   ierr = DMGetCoordinatesLocal(dmNew, &coords);CHKERRQ(ierr);
1886   ierr = DMSetCoordinateDM(dm, coordDM);CHKERRQ(ierr);
1887   ierr = DMSetCoordinatesLocal(dm, coords);CHKERRQ(ierr);
1888   ierr = DMGetPeriodicity(dm, &isper, &maxCell, &L, &bd);CHKERRQ(ierr);
1889   ierr = DMSetPeriodicity(dmNew, isper, maxCell, L, bd);CHKERRQ(ierr);
1890   ierr = DMDestroy_Plex(dm);CHKERRQ(ierr);
1891   ierr = DMInitialize_Plex(dm);CHKERRQ(ierr);
1892   dm->data = dmNew->data;
1893   ((DM_Plex *) dmNew->data)->refct++;
1894   dmNew->labels->refct++;
1895   if (!--(dm->labels->refct)) {
1896     DMLabelLink next = dm->labels->next;
1897 
1898     /* destroy the labels */
1899     while (next) {
1900       DMLabelLink tmp = next->next;
1901 
1902       ierr = DMLabelDestroy(&next->label);CHKERRQ(ierr);
1903       ierr = PetscFree(next);CHKERRQ(ierr);
1904       next = tmp;
1905     }
1906     ierr = PetscFree(dm->labels);CHKERRQ(ierr);
1907   }
1908   dm->labels = dmNew->labels;
1909   dm->depthLabel = dmNew->depthLabel;
1910   ierr = DMGetCoarseDM(dmNew,&coarseDM);CHKERRQ(ierr);
1911   ierr = DMSetCoarseDM(dm,coarseDM);CHKERRQ(ierr);
1912   PetscFunctionReturn(0);
1913 }
1914 
1915 /* Swap dm with the contents of dmNew
1916    - Swap the DM_Plex structure
1917    - Swap the coordinates
1918    - Swap the point PetscSF
1919 */
1920 static PetscErrorCode DMPlexSwap_Static(DM dmA, DM dmB)
1921 {
1922   DM              coordDMA, coordDMB;
1923   Vec             coordsA,  coordsB;
1924   PetscSF         sfA,      sfB;
1925   void            *tmp;
1926   DMLabelLinkList listTmp;
1927   DMLabel         depthTmp;
1928   PetscErrorCode  ierr;
1929 
1930   PetscFunctionBegin;
1931   ierr = DMGetPointSF(dmA, &sfA);CHKERRQ(ierr);
1932   ierr = DMGetPointSF(dmB, &sfB);CHKERRQ(ierr);
1933   ierr = PetscObjectReference((PetscObject) sfA);CHKERRQ(ierr);
1934   ierr = DMSetPointSF(dmA, sfB);CHKERRQ(ierr);
1935   ierr = DMSetPointSF(dmB, sfA);CHKERRQ(ierr);
1936   ierr = PetscObjectDereference((PetscObject) sfA);CHKERRQ(ierr);
1937 
1938   ierr = DMGetCoordinateDM(dmA, &coordDMA);CHKERRQ(ierr);
1939   ierr = DMGetCoordinateDM(dmB, &coordDMB);CHKERRQ(ierr);
1940   ierr = PetscObjectReference((PetscObject) coordDMA);CHKERRQ(ierr);
1941   ierr = DMSetCoordinateDM(dmA, coordDMB);CHKERRQ(ierr);
1942   ierr = DMSetCoordinateDM(dmB, coordDMA);CHKERRQ(ierr);
1943   ierr = PetscObjectDereference((PetscObject) coordDMA);CHKERRQ(ierr);
1944 
1945   ierr = DMGetCoordinatesLocal(dmA, &coordsA);CHKERRQ(ierr);
1946   ierr = DMGetCoordinatesLocal(dmB, &coordsB);CHKERRQ(ierr);
1947   ierr = PetscObjectReference((PetscObject) coordsA);CHKERRQ(ierr);
1948   ierr = DMSetCoordinatesLocal(dmA, coordsB);CHKERRQ(ierr);
1949   ierr = DMSetCoordinatesLocal(dmB, coordsA);CHKERRQ(ierr);
1950   ierr = PetscObjectDereference((PetscObject) coordsA);CHKERRQ(ierr);
1951 
1952   tmp       = dmA->data;
1953   dmA->data = dmB->data;
1954   dmB->data = tmp;
1955   listTmp   = dmA->labels;
1956   dmA->labels = dmB->labels;
1957   dmB->labels = listTmp;
1958   depthTmp  = dmA->depthLabel;
1959   dmA->depthLabel = dmB->depthLabel;
1960   dmB->depthLabel = depthTmp;
1961   PetscFunctionReturn(0);
1962 }
1963 
1964 PetscErrorCode DMSetFromOptions_NonRefinement_Plex(PetscOptionItems *PetscOptionsObject,DM dm)
1965 {
1966   DM_Plex       *mesh = (DM_Plex*) dm->data;
1967   PetscErrorCode ierr;
1968 
1969   PetscFunctionBegin;
1970   /* Handle viewing */
1971   ierr = PetscOptionsBool("-dm_plex_print_set_values", "Output all set values info", "DMView", PETSC_FALSE, &mesh->printSetValues, NULL);CHKERRQ(ierr);
1972   ierr = PetscOptionsInt("-dm_plex_print_fem", "Debug output level all fem computations", "DMView", 0, &mesh->printFEM, NULL);CHKERRQ(ierr);
1973   ierr = PetscOptionsReal("-dm_plex_print_tol", "Tolerance for FEM output", "DMView", mesh->printTol, &mesh->printTol, NULL);CHKERRQ(ierr);
1974   /* Point Location */
1975   ierr = PetscOptionsBool("-dm_plex_hash_location", "Use grid hashing for point location", "DMView", PETSC_FALSE, &mesh->useHashLocation, NULL);CHKERRQ(ierr);
1976   /* Generation and remeshing */
1977   ierr = PetscOptionsBool("-dm_plex_remesh_bd", "Allow changes to the boundary on remeshing", "DMView", PETSC_FALSE, &mesh->remeshBd, NULL);CHKERRQ(ierr);
1978   /* Projection behavior */
1979   ierr = PetscOptionsInt("-dm_plex_max_projection_height", "Maxmimum mesh point height used to project locally", "DMPlexSetMaxProjectionHeight", 0, &mesh->maxProjectionHeight, NULL);CHKERRQ(ierr);
1980   ierr = PetscOptionsBool("-dm_plex_regular_refinement", "Use special nested projection algorithm for regular refinement", "DMPlexSetRegularRefinement", mesh->regularRefinement, &mesh->regularRefinement, NULL);CHKERRQ(ierr);
1981 
1982   ierr = PetscPartitionerSetFromOptions(mesh->partitioner);CHKERRQ(ierr);
1983   PetscFunctionReturn(0);
1984 }
1985 
1986 static PetscErrorCode DMSetFromOptions_Plex(PetscOptionItems *PetscOptionsObject,DM dm)
1987 {
1988   PetscInt       refine = 0, coarsen = 0, r;
1989   PetscBool      isHierarchy;
1990   PetscErrorCode ierr;
1991 
1992   PetscFunctionBegin;
1993   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1994   ierr = PetscOptionsHead(PetscOptionsObject,"DMPlex Options");CHKERRQ(ierr);
1995   /* Handle DMPlex refinement */
1996   ierr = PetscOptionsInt("-dm_refine", "The number of uniform refinements", "DMCreate", refine, &refine, NULL);CHKERRQ(ierr);
1997   ierr = PetscOptionsInt("-dm_refine_hierarchy", "The number of uniform refinements", "DMCreate", refine, &refine, &isHierarchy);CHKERRQ(ierr);
1998   if (refine) {ierr = DMPlexSetRefinementUniform(dm, PETSC_TRUE);CHKERRQ(ierr);}
1999   if (refine && isHierarchy) {
2000     DM *dms, coarseDM;
2001 
2002     ierr = DMGetCoarseDM(dm, &coarseDM);CHKERRQ(ierr);
2003     ierr = PetscObjectReference((PetscObject)coarseDM);CHKERRQ(ierr);
2004     ierr = PetscMalloc1(refine,&dms);CHKERRQ(ierr);
2005     ierr = DMRefineHierarchy(dm, refine, dms);CHKERRQ(ierr);
2006     /* Total hack since we do not pass in a pointer */
2007     ierr = DMPlexSwap_Static(dm, dms[refine-1]);CHKERRQ(ierr);
2008     if (refine == 1) {
2009       ierr = DMSetCoarseDM(dm, dms[0]);CHKERRQ(ierr);
2010       ierr = DMPlexSetRegularRefinement(dm, PETSC_TRUE);CHKERRQ(ierr);
2011     } else {
2012       ierr = DMSetCoarseDM(dm, dms[refine-2]);CHKERRQ(ierr);
2013       ierr = DMPlexSetRegularRefinement(dm, PETSC_TRUE);CHKERRQ(ierr);
2014       ierr = DMSetCoarseDM(dms[0], dms[refine-1]);CHKERRQ(ierr);
2015       ierr = DMPlexSetRegularRefinement(dms[0], PETSC_TRUE);CHKERRQ(ierr);
2016     }
2017     ierr = DMSetCoarseDM(dms[refine-1], coarseDM);CHKERRQ(ierr);
2018     ierr = PetscObjectDereference((PetscObject)coarseDM);CHKERRQ(ierr);
2019     /* Free DMs */
2020     for (r = 0; r < refine; ++r) {
2021       ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dms[r]);CHKERRQ(ierr);
2022       ierr = DMDestroy(&dms[r]);CHKERRQ(ierr);
2023     }
2024     ierr = PetscFree(dms);CHKERRQ(ierr);
2025   } else {
2026     for (r = 0; r < refine; ++r) {
2027       DM refinedMesh;
2028 
2029       ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dm);CHKERRQ(ierr);
2030       ierr = DMRefine(dm, PetscObjectComm((PetscObject) dm), &refinedMesh);CHKERRQ(ierr);
2031       /* Total hack since we do not pass in a pointer */
2032       ierr = DMPlexReplace_Static(dm, refinedMesh);CHKERRQ(ierr);
2033       ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dm);CHKERRQ(ierr);
2034       ierr = DMDestroy(&refinedMesh);CHKERRQ(ierr);
2035     }
2036   }
2037   /* Handle DMPlex coarsening */
2038   ierr = PetscOptionsInt("-dm_coarsen", "Coarsen the mesh", "DMCreate", coarsen, &coarsen, NULL);CHKERRQ(ierr);
2039   ierr = PetscOptionsInt("-dm_coarsen_hierarchy", "The number of coarsenings", "DMCreate", coarsen, &coarsen, &isHierarchy);CHKERRQ(ierr);
2040   if (coarsen && isHierarchy) {
2041     DM *dms;
2042 
2043     ierr = PetscMalloc1(coarsen, &dms);CHKERRQ(ierr);
2044     ierr = DMCoarsenHierarchy(dm, coarsen, dms);CHKERRQ(ierr);
2045     /* Free DMs */
2046     for (r = 0; r < coarsen; ++r) {
2047       ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dms[r]);CHKERRQ(ierr);
2048       ierr = DMDestroy(&dms[r]);CHKERRQ(ierr);
2049     }
2050     ierr = PetscFree(dms);CHKERRQ(ierr);
2051   } else {
2052     for (r = 0; r < coarsen; ++r) {
2053       DM coarseMesh;
2054 
2055       ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dm);CHKERRQ(ierr);
2056       ierr = DMCoarsen(dm, PetscObjectComm((PetscObject) dm), &coarseMesh);CHKERRQ(ierr);
2057       /* Total hack since we do not pass in a pointer */
2058       ierr = DMPlexReplace_Static(dm, coarseMesh);CHKERRQ(ierr);
2059       ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dm);CHKERRQ(ierr);
2060       ierr = DMDestroy(&coarseMesh);CHKERRQ(ierr);
2061     }
2062   }
2063   /* Handle */
2064   ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dm);CHKERRQ(ierr);
2065   ierr = PetscOptionsTail();CHKERRQ(ierr);
2066   PetscFunctionReturn(0);
2067 }
2068 
2069 static PetscErrorCode DMCreateGlobalVector_Plex(DM dm,Vec *vec)
2070 {
2071   PetscErrorCode ierr;
2072 
2073   PetscFunctionBegin;
2074   ierr = DMCreateGlobalVector_Section_Private(dm,vec);CHKERRQ(ierr);
2075   /* ierr = VecSetOperation(*vec, VECOP_DUPLICATE, (void(*)(void)) VecDuplicate_MPI_DM);CHKERRQ(ierr); */
2076   ierr = VecSetOperation(*vec, VECOP_VIEW, (void (*)(void)) VecView_Plex);CHKERRQ(ierr);
2077   ierr = VecSetOperation(*vec, VECOP_VIEWNATIVE, (void (*)(void)) VecView_Plex_Native);CHKERRQ(ierr);
2078   ierr = VecSetOperation(*vec, VECOP_LOAD, (void (*)(void)) VecLoad_Plex);CHKERRQ(ierr);
2079   ierr = VecSetOperation(*vec, VECOP_LOADNATIVE, (void (*)(void)) VecLoad_Plex_Native);CHKERRQ(ierr);
2080   PetscFunctionReturn(0);
2081 }
2082 
2083 static PetscErrorCode DMCreateLocalVector_Plex(DM dm,Vec *vec)
2084 {
2085   PetscErrorCode ierr;
2086 
2087   PetscFunctionBegin;
2088   ierr = DMCreateLocalVector_Section_Private(dm,vec);CHKERRQ(ierr);
2089   ierr = VecSetOperation(*vec, VECOP_VIEW, (void (*)(void)) VecView_Plex_Local);CHKERRQ(ierr);
2090   ierr = VecSetOperation(*vec, VECOP_LOAD, (void (*)(void)) VecLoad_Plex_Local);CHKERRQ(ierr);
2091   PetscFunctionReturn(0);
2092 }
2093 
2094 static PetscErrorCode DMGetDimPoints_Plex(DM dm, PetscInt dim, PetscInt *pStart, PetscInt *pEnd)
2095 {
2096   PetscInt       depth, d;
2097   PetscErrorCode ierr;
2098 
2099   PetscFunctionBegin;
2100   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
2101   if (depth == 1) {
2102     ierr = DMGetDimension(dm, &d);CHKERRQ(ierr);
2103     if (dim == 0)      {ierr = DMPlexGetDepthStratum(dm, dim, pStart, pEnd);CHKERRQ(ierr);}
2104     else if (dim == d) {ierr = DMPlexGetDepthStratum(dm, 1, pStart, pEnd);CHKERRQ(ierr);}
2105     else               {*pStart = 0; *pEnd = 0;}
2106   } else {
2107     ierr = DMPlexGetDepthStratum(dm, dim, pStart, pEnd);CHKERRQ(ierr);
2108   }
2109   PetscFunctionReturn(0);
2110 }
2111 
2112 static PetscErrorCode DMGetNeighors_Plex(DM dm, PetscInt *nranks, const PetscMPIInt *ranks[])
2113 {
2114   PetscSF        sf;
2115   PetscErrorCode ierr;
2116 
2117   PetscFunctionBegin;
2118   ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr);
2119   ierr = PetscSFGetRanks(sf, nranks, ranks, NULL, NULL, NULL);CHKERRQ(ierr);
2120   PetscFunctionReturn(0);
2121 }
2122 
2123 static PetscErrorCode DMInitialize_Plex(DM dm)
2124 {
2125   PetscErrorCode ierr;
2126 
2127   PetscFunctionBegin;
2128   dm->ops->view                            = DMView_Plex;
2129   dm->ops->load                            = DMLoad_Plex;
2130   dm->ops->setfromoptions                  = DMSetFromOptions_Plex;
2131   dm->ops->clone                           = DMClone_Plex;
2132   dm->ops->setup                           = DMSetUp_Plex;
2133   dm->ops->createdefaultsection            = DMCreateDefaultSection_Plex;
2134   dm->ops->createdefaultconstraints        = DMCreateDefaultConstraints_Plex;
2135   dm->ops->createglobalvector              = DMCreateGlobalVector_Plex;
2136   dm->ops->createlocalvector               = DMCreateLocalVector_Plex;
2137   dm->ops->getlocaltoglobalmapping         = NULL;
2138   dm->ops->createfieldis                   = NULL;
2139   dm->ops->createcoordinatedm              = DMCreateCoordinateDM_Plex;
2140   dm->ops->getcoloring                     = NULL;
2141   dm->ops->creatematrix                    = DMCreateMatrix_Plex;
2142   dm->ops->createinterpolation             = DMCreateInterpolation_Plex;
2143   dm->ops->createmassmatrix                = DMCreateMassMatrix_Plex;
2144   dm->ops->getaggregates                   = NULL;
2145   dm->ops->getinjection                    = DMCreateInjection_Plex;
2146   dm->ops->refine                          = DMRefine_Plex;
2147   dm->ops->coarsen                         = DMCoarsen_Plex;
2148   dm->ops->refinehierarchy                 = DMRefineHierarchy_Plex;
2149   dm->ops->coarsenhierarchy                = DMCoarsenHierarchy_Plex;
2150   dm->ops->adaptlabel                      = DMAdaptLabel_Plex;
2151   dm->ops->adaptmetric                     = DMAdaptMetric_Plex;
2152   dm->ops->globaltolocalbegin              = NULL;
2153   dm->ops->globaltolocalend                = NULL;
2154   dm->ops->localtoglobalbegin              = NULL;
2155   dm->ops->localtoglobalend                = NULL;
2156   dm->ops->destroy                         = DMDestroy_Plex;
2157   dm->ops->createsubdm                     = DMCreateSubDM_Plex;
2158   dm->ops->getdimpoints                    = DMGetDimPoints_Plex;
2159   dm->ops->locatepoints                    = DMLocatePoints_Plex;
2160   dm->ops->projectfunctionlocal            = DMProjectFunctionLocal_Plex;
2161   dm->ops->projectfunctionlabellocal       = DMProjectFunctionLabelLocal_Plex;
2162   dm->ops->projectfieldlocal               = DMProjectFieldLocal_Plex;
2163   dm->ops->projectfieldlabellocal          = DMProjectFieldLabelLocal_Plex;
2164   dm->ops->computel2diff                   = DMComputeL2Diff_Plex;
2165   dm->ops->computel2gradientdiff           = DMComputeL2GradientDiff_Plex;
2166   dm->ops->computel2fielddiff              = DMComputeL2FieldDiff_Plex;
2167   dm->ops->getneighbors                    = DMGetNeighors_Plex;
2168   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMPlexInsertBoundaryValues_C",DMPlexInsertBoundaryValues_Plex);CHKERRQ(ierr);
2169   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMSetUpGLVisViewer_C",DMSetUpGLVisViewer_Plex);CHKERRQ(ierr);
2170   PetscFunctionReturn(0);
2171 }
2172 
2173 PETSC_INTERN PetscErrorCode DMClone_Plex(DM dm, DM *newdm)
2174 {
2175   DM_Plex        *mesh = (DM_Plex *) dm->data;
2176   PetscErrorCode ierr;
2177 
2178   PetscFunctionBegin;
2179   mesh->refct++;
2180   (*newdm)->data = mesh;
2181   ierr = PetscObjectChangeTypeName((PetscObject) *newdm, DMPLEX);CHKERRQ(ierr);
2182   ierr = DMInitialize_Plex(*newdm);CHKERRQ(ierr);
2183   PetscFunctionReturn(0);
2184 }
2185 
2186 /*MC
2187   DMPLEX = "plex" - A DM object that encapsulates an unstructured mesh, or CW Complex, which can be expressed using a Hasse Diagram.
2188                     In the local representation, Vecs contain all unknowns in the interior and shared boundary. This is
2189                     specified by a PetscSection object. Ownership in the global representation is determined by
2190                     ownership of the underlying DMPlex points. This is specified by another PetscSection object.
2191 
2192   Level: intermediate
2193 
2194 .seealso: DMType, DMPlexCreate(), DMCreate(), DMSetType()
2195 M*/
2196 
2197 PETSC_EXTERN PetscErrorCode DMCreate_Plex(DM dm)
2198 {
2199   DM_Plex        *mesh;
2200   PetscInt       unit, d;
2201   PetscErrorCode ierr;
2202 
2203   PetscFunctionBegin;
2204   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2205   ierr     = PetscNewLog(dm,&mesh);CHKERRQ(ierr);
2206   dm->dim  = 0;
2207   dm->data = mesh;
2208 
2209   mesh->refct             = 1;
2210   ierr                    = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &mesh->coneSection);CHKERRQ(ierr);
2211   mesh->maxConeSize       = 0;
2212   mesh->cones             = NULL;
2213   mesh->coneOrientations  = NULL;
2214   ierr                    = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &mesh->supportSection);CHKERRQ(ierr);
2215   mesh->maxSupportSize    = 0;
2216   mesh->supports          = NULL;
2217   mesh->refinementUniform = PETSC_TRUE;
2218   mesh->refinementLimit   = -1.0;
2219 
2220   mesh->facesTmp = NULL;
2221 
2222   mesh->tetgenOpts   = NULL;
2223   mesh->triangleOpts = NULL;
2224   ierr = PetscPartitionerCreate(PetscObjectComm((PetscObject)dm), &mesh->partitioner);CHKERRQ(ierr);
2225   mesh->remeshBd     = PETSC_FALSE;
2226 
2227   mesh->subpointMap = NULL;
2228 
2229   for (unit = 0; unit < NUM_PETSC_UNITS; ++unit) mesh->scale[unit] = 1.0;
2230 
2231   mesh->regularRefinement   = PETSC_FALSE;
2232   mesh->depthState          = -1;
2233   mesh->globalVertexNumbers = NULL;
2234   mesh->globalCellNumbers   = NULL;
2235   mesh->anchorSection       = NULL;
2236   mesh->anchorIS            = NULL;
2237   mesh->createanchors       = NULL;
2238   mesh->computeanchormatrix = NULL;
2239   mesh->parentSection       = NULL;
2240   mesh->parents             = NULL;
2241   mesh->childIDs            = NULL;
2242   mesh->childSection        = NULL;
2243   mesh->children            = NULL;
2244   mesh->referenceTree       = NULL;
2245   mesh->getchildsymmetry    = NULL;
2246   for (d = 0; d < 8; ++d) mesh->hybridPointMax[d] = PETSC_DETERMINE;
2247   mesh->vtkCellHeight       = 0;
2248   mesh->useCone             = PETSC_FALSE;
2249   mesh->useClosure          = PETSC_TRUE;
2250   mesh->useAnchors          = PETSC_FALSE;
2251 
2252   mesh->maxProjectionHeight = 0;
2253 
2254   mesh->printSetValues = PETSC_FALSE;
2255   mesh->printFEM       = 0;
2256   mesh->printTol       = 1.0e-10;
2257 
2258   ierr = DMInitialize_Plex(dm);CHKERRQ(ierr);
2259   PetscFunctionReturn(0);
2260 }
2261 
2262 /*@
2263   DMPlexCreate - Creates a DMPlex object, which encapsulates an unstructured mesh, or CW complex, which can be expressed using a Hasse Diagram.
2264 
2265   Collective on MPI_Comm
2266 
2267   Input Parameter:
2268 . comm - The communicator for the DMPlex object
2269 
2270   Output Parameter:
2271 . mesh  - The DMPlex object
2272 
2273   Level: beginner
2274 
2275 .keywords: DMPlex, create
2276 @*/
2277 PetscErrorCode DMPlexCreate(MPI_Comm comm, DM *mesh)
2278 {
2279   PetscErrorCode ierr;
2280 
2281   PetscFunctionBegin;
2282   PetscValidPointer(mesh,2);
2283   ierr = DMCreate(comm, mesh);CHKERRQ(ierr);
2284   ierr = DMSetType(*mesh, DMPLEX);CHKERRQ(ierr);
2285   PetscFunctionReturn(0);
2286 }
2287 
2288 /*
2289   This takes as input the common mesh generator output, a list of the vertices for each cell, but vertex numbers are global and an SF is built for them
2290 */
2291 static PetscErrorCode DMPlexBuildFromCellList_Parallel_Private(DM dm, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, const int cells[], PetscSF *sfVert)
2292 {
2293   PetscSF         sfPoint;
2294   PetscLayout     vLayout;
2295   PetscHashI      vhash;
2296   PetscSFNode    *remoteVerticesAdj, *vertexLocal, *vertexOwner, *remoteVertex;
2297   const PetscInt *vrange;
2298   PetscInt        numVerticesAdj, off, *verticesAdj, numVerticesGhost = 0, *localVertex, *cone, c, p, v, g;
2299   PETSC_UNUSED PetscHashIIter ret, iter;
2300   PetscMPIInt     rank, size;
2301   PetscErrorCode  ierr;
2302 
2303   PetscFunctionBegin;
2304   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRQ(ierr);
2305   ierr = MPI_Comm_size(PetscObjectComm((PetscObject) dm), &size);CHKERRQ(ierr);
2306   /* Partition vertices */
2307   ierr = PetscLayoutCreate(PetscObjectComm((PetscObject) dm), &vLayout);CHKERRQ(ierr);
2308   ierr = PetscLayoutSetLocalSize(vLayout, numVertices);CHKERRQ(ierr);
2309   ierr = PetscLayoutSetBlockSize(vLayout, 1);CHKERRQ(ierr);
2310   ierr = PetscLayoutSetUp(vLayout);CHKERRQ(ierr);
2311   ierr = PetscLayoutGetRanges(vLayout, &vrange);CHKERRQ(ierr);
2312   /* Count vertices and map them to procs */
2313   PetscHashICreate(vhash);
2314   for (c = 0; c < numCells; ++c) {
2315     for (p = 0; p < numCorners; ++p) {
2316       PetscHashIPut(vhash, cells[c*numCorners+p], ret, iter);
2317     }
2318   }
2319   PetscHashISize(vhash, numVerticesAdj);
2320   ierr = PetscMalloc1(numVerticesAdj, &verticesAdj);CHKERRQ(ierr);
2321   off = 0; ierr = PetscHashIGetKeys(vhash, &off, verticesAdj);CHKERRQ(ierr);
2322   if (off != numVerticesAdj) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid number of local vertices %D should be %D", off, numVerticesAdj);
2323   ierr = PetscSortInt(numVerticesAdj, verticesAdj);CHKERRQ(ierr);
2324   ierr = PetscMalloc1(numVerticesAdj, &remoteVerticesAdj);CHKERRQ(ierr);
2325   for (v = 0; v < numVerticesAdj; ++v) {
2326     const PetscInt gv = verticesAdj[v];
2327     PetscInt       vrank;
2328 
2329     ierr = PetscFindInt(gv, size+1, vrange, &vrank);CHKERRQ(ierr);
2330     vrank = vrank < 0 ? -(vrank+2) : vrank;
2331     remoteVerticesAdj[v].index = gv - vrange[vrank];
2332     remoteVerticesAdj[v].rank  = vrank;
2333   }
2334   /* Create cones */
2335   ierr = DMPlexSetChart(dm, 0, numCells+numVerticesAdj);CHKERRQ(ierr);
2336   for (c = 0; c < numCells; ++c) {ierr = DMPlexSetConeSize(dm, c, numCorners);CHKERRQ(ierr);}
2337   ierr = DMSetUp(dm);CHKERRQ(ierr);
2338   ierr = DMGetWorkArray(dm, numCorners, MPIU_INT, &cone);CHKERRQ(ierr);
2339   for (c = 0; c < numCells; ++c) {
2340     for (p = 0; p < numCorners; ++p) {
2341       const PetscInt gv = cells[c*numCorners+p];
2342       PetscInt       lv;
2343 
2344       ierr = PetscFindInt(gv, numVerticesAdj, verticesAdj, &lv);CHKERRQ(ierr);
2345       if (lv < 0) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Could not find global vertex %D in local connectivity", gv);
2346       cone[p] = lv+numCells;
2347     }
2348     ierr = DMPlexSetCone(dm, c, cone);CHKERRQ(ierr);
2349   }
2350   ierr = DMRestoreWorkArray(dm, numCorners, MPIU_INT, &cone);CHKERRQ(ierr);
2351   /* Create SF for vertices */
2352   ierr = PetscSFCreate(PetscObjectComm((PetscObject)dm), sfVert);CHKERRQ(ierr);
2353   ierr = PetscObjectSetName((PetscObject) *sfVert, "Vertex Ownership SF");CHKERRQ(ierr);
2354   ierr = PetscSFSetFromOptions(*sfVert);CHKERRQ(ierr);
2355   ierr = PetscSFSetGraph(*sfVert, numVertices, numVerticesAdj, NULL, PETSC_OWN_POINTER, remoteVerticesAdj, PETSC_OWN_POINTER);CHKERRQ(ierr);
2356   ierr = PetscFree(verticesAdj);CHKERRQ(ierr);
2357   /* Build pointSF */
2358   ierr = PetscMalloc2(numVerticesAdj, &vertexLocal, numVertices, &vertexOwner);CHKERRQ(ierr);
2359   for (v = 0; v < numVerticesAdj; ++v) {vertexLocal[v].index = v+numCells; vertexLocal[v].rank = rank;}
2360   for (v = 0; v < numVertices;    ++v) {vertexOwner[v].index = -1;         vertexOwner[v].rank = -1;}
2361   ierr = PetscSFReduceBegin(*sfVert, MPIU_2INT, vertexLocal, vertexOwner, MPI_MAXLOC);CHKERRQ(ierr);
2362   ierr = PetscSFReduceEnd(*sfVert, MPIU_2INT, vertexLocal, vertexOwner, MPI_MAXLOC);CHKERRQ(ierr);
2363   for (v = 0; v < numVertices;    ++v) if (vertexOwner[v].rank < 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Global vertex %d on rank %d was unclaimed", v + vrange[rank], rank);
2364   ierr = PetscSFBcastBegin(*sfVert, MPIU_2INT, vertexOwner, vertexLocal);CHKERRQ(ierr);
2365   ierr = PetscSFBcastEnd(*sfVert, MPIU_2INT, vertexOwner, vertexLocal);CHKERRQ(ierr);
2366   for (v = 0; v < numVerticesAdj; ++v) if (vertexLocal[v].rank != rank) ++numVerticesGhost;
2367   ierr = PetscMalloc1(numVerticesGhost, &localVertex);CHKERRQ(ierr);
2368   ierr = PetscMalloc1(numVerticesGhost, &remoteVertex);CHKERRQ(ierr);
2369   for (v = 0, g = 0; v < numVerticesAdj; ++v) {
2370     if (vertexLocal[v].rank != rank) {
2371       localVertex[g]        = v+numCells;
2372       remoteVertex[g].index = vertexLocal[v].index;
2373       remoteVertex[g].rank  = vertexLocal[v].rank;
2374       ++g;
2375     }
2376   }
2377   ierr = PetscFree2(vertexLocal, vertexOwner);CHKERRQ(ierr);
2378   if (g != numVerticesGhost) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid number of vertex ghosts %D should be %D", g, numVerticesGhost);
2379   ierr = DMGetPointSF(dm, &sfPoint);CHKERRQ(ierr);
2380   ierr = PetscObjectSetName((PetscObject) sfPoint, "point SF");CHKERRQ(ierr);
2381   ierr = PetscSFSetGraph(sfPoint, numCells+numVerticesAdj, numVerticesGhost, localVertex, PETSC_OWN_POINTER, remoteVertex, PETSC_OWN_POINTER);CHKERRQ(ierr);
2382   ierr = PetscLayoutDestroy(&vLayout);CHKERRQ(ierr);
2383   PetscHashIDestroy(vhash);
2384   /* Fill in the rest of the topology structure */
2385   ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
2386   ierr = DMPlexStratify(dm);CHKERRQ(ierr);
2387   PetscFunctionReturn(0);
2388 }
2389 
2390 /*
2391   This takes as input the coordinates for each owned vertex
2392 */
2393 static PetscErrorCode DMPlexBuildCoordinates_Parallel_Private(DM dm, PetscInt spaceDim, PetscInt numCells, PetscInt numV, PetscSF sfVert, const PetscReal vertexCoords[])
2394 {
2395   PetscSection   coordSection;
2396   Vec            coordinates;
2397   PetscScalar   *coords;
2398   PetscInt       numVertices, numVerticesAdj, coordSize, v;
2399   PetscErrorCode ierr;
2400 
2401   PetscFunctionBegin;
2402   ierr = DMSetCoordinateDim(dm, spaceDim);CHKERRQ(ierr);
2403   ierr = PetscSFGetGraph(sfVert, &numVertices, &numVerticesAdj, NULL, NULL);CHKERRQ(ierr);
2404   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
2405   ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
2406   ierr = PetscSectionSetFieldComponents(coordSection, 0, spaceDim);CHKERRQ(ierr);
2407   ierr = PetscSectionSetChart(coordSection, numCells, numCells + numVerticesAdj);CHKERRQ(ierr);
2408   for (v = numCells; v < numCells+numVerticesAdj; ++v) {
2409     ierr = PetscSectionSetDof(coordSection, v, spaceDim);CHKERRQ(ierr);
2410     ierr = PetscSectionSetFieldDof(coordSection, v, 0, spaceDim);CHKERRQ(ierr);
2411   }
2412   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
2413   ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
2414   ierr = VecCreate(PetscObjectComm((PetscObject)dm), &coordinates);CHKERRQ(ierr);
2415   ierr = VecSetBlockSize(coordinates, spaceDim);CHKERRQ(ierr);
2416   ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
2417   ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
2418   ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
2419   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
2420   {
2421     MPI_Datatype coordtype;
2422 
2423     /* Need a temp buffer for coords if we have complex/single */
2424     ierr = MPI_Type_contiguous(spaceDim, MPIU_SCALAR, &coordtype);CHKERRQ(ierr);
2425     ierr = MPI_Type_commit(&coordtype);CHKERRQ(ierr);
2426 #if defined(PETSC_USE_COMPLEX)
2427     {
2428     PetscScalar *svertexCoords;
2429     PetscInt    i;
2430     ierr = PetscMalloc1(numV*spaceDim,&svertexCoords);CHKERRQ(ierr);
2431     for (i=0; i<numV*spaceDim; i++) svertexCoords[i] = vertexCoords[i];
2432     ierr = PetscSFBcastBegin(sfVert, coordtype, svertexCoords, coords);CHKERRQ(ierr);
2433     ierr = PetscSFBcastEnd(sfVert, coordtype, svertexCoords, coords);CHKERRQ(ierr);
2434     ierr = PetscFree(svertexCoords);CHKERRQ(ierr);
2435     }
2436 #else
2437     ierr = PetscSFBcastBegin(sfVert, coordtype, vertexCoords, coords);CHKERRQ(ierr);
2438     ierr = PetscSFBcastEnd(sfVert, coordtype, vertexCoords, coords);CHKERRQ(ierr);
2439 #endif
2440     ierr = MPI_Type_free(&coordtype);CHKERRQ(ierr);
2441   }
2442   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
2443   ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
2444   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
2445   PetscFunctionReturn(0);
2446 }
2447 
2448 /*@C
2449   DMPlexCreateFromCellListParallel - This takes as input common mesh generator output, a list of the vertices for each cell, and produces a DM
2450 
2451   Input Parameters:
2452 + comm - The communicator
2453 . dim - The topological dimension of the mesh
2454 . numCells - The number of cells owned by this process
2455 . numVertices - The number of vertices owned by this process
2456 . numCorners - The number of vertices for each cell
2457 . interpolate - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
2458 . cells - An array of numCells*numCorners numbers, the global vertex numbers for each cell
2459 . spaceDim - The spatial dimension used for coordinates
2460 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
2461 
2462   Output Parameter:
2463 + dm - The DM
2464 - vertexSF - Optional, SF describing complete vertex ownership
2465 
2466   Note: Two triangles sharing a face
2467 $
2468 $        2
2469 $      / | \
2470 $     /  |  \
2471 $    /   |   \
2472 $   0  0 | 1  3
2473 $    \   |   /
2474 $     \  |  /
2475 $      \ | /
2476 $        1
2477 would have input
2478 $  numCells = 2, numVertices = 4
2479 $  cells = [0 1 2  1 3 2]
2480 $
2481 which would result in the DMPlex
2482 $
2483 $        4
2484 $      / | \
2485 $     /  |  \
2486 $    /   |   \
2487 $   2  0 | 1  5
2488 $    \   |   /
2489 $     \  |  /
2490 $      \ | /
2491 $        3
2492 
2493   Level: beginner
2494 
2495 .seealso: DMPlexCreateFromCellList(), DMPlexCreateFromDAG(), DMPlexCreate()
2496 @*/
2497 PetscErrorCode DMPlexCreateFromCellListParallel(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, PetscBool interpolate, const int cells[], PetscInt spaceDim, const PetscReal vertexCoords[], PetscSF *vertexSF, DM *dm)
2498 {
2499   PetscSF        sfVert;
2500   PetscErrorCode ierr;
2501 
2502   PetscFunctionBegin;
2503   ierr = DMCreate(comm, dm);CHKERRQ(ierr);
2504   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
2505   PetscValidLogicalCollectiveInt(*dm, dim, 2);
2506   PetscValidLogicalCollectiveInt(*dm, spaceDim, 8);
2507   ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
2508   ierr = DMPlexBuildFromCellList_Parallel_Private(*dm, numCells, numVertices, numCorners, cells, &sfVert);CHKERRQ(ierr);
2509   if (interpolate) {
2510     DM idm = NULL;
2511 
2512     ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
2513     ierr = DMDestroy(dm);CHKERRQ(ierr);
2514     *dm  = idm;
2515   }
2516   ierr = DMPlexBuildCoordinates_Parallel_Private(*dm, spaceDim, numCells, numVertices,sfVert, vertexCoords);CHKERRQ(ierr);
2517   if (vertexSF) *vertexSF = sfVert;
2518   else {ierr = PetscSFDestroy(&sfVert);CHKERRQ(ierr);}
2519   PetscFunctionReturn(0);
2520 }
2521 
2522 /*
2523   This takes as input the common mesh generator output, a list of the vertices for each cell
2524 */
2525 static PetscErrorCode DMPlexBuildFromCellList_Private(DM dm, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, const int cells[])
2526 {
2527   PetscInt      *cone, c, p;
2528   PetscErrorCode ierr;
2529 
2530   PetscFunctionBegin;
2531   ierr = DMPlexSetChart(dm, 0, numCells+numVertices);CHKERRQ(ierr);
2532   for (c = 0; c < numCells; ++c) {
2533     ierr = DMPlexSetConeSize(dm, c, numCorners);CHKERRQ(ierr);
2534   }
2535   ierr = DMSetUp(dm);CHKERRQ(ierr);
2536   ierr = DMGetWorkArray(dm, numCorners, MPIU_INT, &cone);CHKERRQ(ierr);
2537   for (c = 0; c < numCells; ++c) {
2538     for (p = 0; p < numCorners; ++p) {
2539       cone[p] = cells[c*numCorners+p]+numCells;
2540     }
2541     ierr = DMPlexSetCone(dm, c, cone);CHKERRQ(ierr);
2542   }
2543   ierr = DMRestoreWorkArray(dm, numCorners, MPIU_INT, &cone);CHKERRQ(ierr);
2544   ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
2545   ierr = DMPlexStratify(dm);CHKERRQ(ierr);
2546   PetscFunctionReturn(0);
2547 }
2548 
2549 /*
2550   This takes as input the coordinates for each vertex
2551 */
2552 static PetscErrorCode DMPlexBuildCoordinates_Private(DM dm, PetscInt spaceDim, PetscInt numCells, PetscInt numVertices, const double vertexCoords[])
2553 {
2554   PetscSection   coordSection;
2555   Vec            coordinates;
2556   DM             cdm;
2557   PetscScalar   *coords;
2558   PetscInt       v, d;
2559   PetscErrorCode ierr;
2560 
2561   PetscFunctionBegin;
2562   ierr = DMSetCoordinateDim(dm, spaceDim);CHKERRQ(ierr);
2563   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
2564   ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
2565   ierr = PetscSectionSetFieldComponents(coordSection, 0, spaceDim);CHKERRQ(ierr);
2566   ierr = PetscSectionSetChart(coordSection, numCells, numCells + numVertices);CHKERRQ(ierr);
2567   for (v = numCells; v < numCells+numVertices; ++v) {
2568     ierr = PetscSectionSetDof(coordSection, v, spaceDim);CHKERRQ(ierr);
2569     ierr = PetscSectionSetFieldDof(coordSection, v, 0, spaceDim);CHKERRQ(ierr);
2570   }
2571   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
2572 
2573   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
2574   ierr = DMCreateLocalVector(cdm, &coordinates);CHKERRQ(ierr);
2575   ierr = VecSetBlockSize(coordinates, spaceDim);CHKERRQ(ierr);
2576   ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
2577   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
2578   for (v = 0; v < numVertices; ++v) {
2579     for (d = 0; d < spaceDim; ++d) {
2580       coords[v*spaceDim+d] = vertexCoords[v*spaceDim+d];
2581     }
2582   }
2583   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
2584   ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
2585   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
2586   PetscFunctionReturn(0);
2587 }
2588 
2589 /*@C
2590   DMPlexCreateFromCellList - This takes as input common mesh generator output, a list of the vertices for each cell, and produces a DM
2591 
2592   Input Parameters:
2593 + comm - The communicator
2594 . dim - The topological dimension of the mesh
2595 . numCells - The number of cells
2596 . numVertices - The number of vertices
2597 . numCorners - The number of vertices for each cell
2598 . interpolate - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
2599 . cells - An array of numCells*numCorners numbers, the vertices for each cell
2600 . spaceDim - The spatial dimension used for coordinates
2601 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
2602 
2603   Output Parameter:
2604 . dm - The DM
2605 
2606   Note: Two triangles sharing a face
2607 $
2608 $        2
2609 $      / | \
2610 $     /  |  \
2611 $    /   |   \
2612 $   0  0 | 1  3
2613 $    \   |   /
2614 $     \  |  /
2615 $      \ | /
2616 $        1
2617 would have input
2618 $  numCells = 2, numVertices = 4
2619 $  cells = [0 1 2  1 3 2]
2620 $
2621 which would result in the DMPlex
2622 $
2623 $        4
2624 $      / | \
2625 $     /  |  \
2626 $    /   |   \
2627 $   2  0 | 1  5
2628 $    \   |   /
2629 $     \  |  /
2630 $      \ | /
2631 $        3
2632 
2633   Level: beginner
2634 
2635 .seealso: DMPlexCreateFromDAG(), DMPlexCreate()
2636 @*/
2637 PetscErrorCode DMPlexCreateFromCellList(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, PetscBool interpolate, const int cells[], PetscInt spaceDim, const double vertexCoords[], DM *dm)
2638 {
2639   PetscErrorCode ierr;
2640 
2641   PetscFunctionBegin;
2642   ierr = DMCreate(comm, dm);CHKERRQ(ierr);
2643   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
2644   ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
2645   ierr = DMPlexBuildFromCellList_Private(*dm, numCells, numVertices, numCorners, cells);CHKERRQ(ierr);
2646   if (interpolate) {
2647     DM idm = NULL;
2648 
2649     ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
2650     ierr = DMDestroy(dm);CHKERRQ(ierr);
2651     *dm  = idm;
2652   }
2653   ierr = DMPlexBuildCoordinates_Private(*dm, spaceDim, numCells, numVertices, vertexCoords);CHKERRQ(ierr);
2654   PetscFunctionReturn(0);
2655 }
2656 
2657 /*@
2658   DMPlexCreateFromDAG - This takes as input the adjacency-list representation of the Directed Acyclic Graph (Hasse Diagram) encoding a mesh, and produces a DM
2659 
2660   Input Parameters:
2661 + dm - The empty DM object, usually from DMCreate() and DMSetDimension()
2662 . depth - The depth of the DAG
2663 . numPoints - The number of points at each depth
2664 . coneSize - The cone size of each point
2665 . cones - The concatenation of the cone points for each point, the cone list must be oriented correctly for each point
2666 . coneOrientations - The orientation of each cone point
2667 - vertexCoords - An array of numVertices*dim numbers, the coordinates of each vertex
2668 
2669   Output Parameter:
2670 . dm - The DM
2671 
2672   Note: Two triangles sharing a face would have input
2673 $  depth = 1, numPoints = [4 2], coneSize = [3 3 0 0 0 0]
2674 $  cones = [2 3 4  3 5 4], coneOrientations = [0 0 0  0 0 0]
2675 $ vertexCoords = [-1.0 0.0  0.0 -1.0  0.0 1.0  1.0 0.0]
2676 $
2677 which would result in the DMPlex
2678 $
2679 $        4
2680 $      / | \
2681 $     /  |  \
2682 $    /   |   \
2683 $   2  0 | 1  5
2684 $    \   |   /
2685 $     \  |  /
2686 $      \ | /
2687 $        3
2688 $
2689 $ Notice that all points are numbered consecutively, unlikely DMPlexCreateFromCellList()
2690 
2691   Level: advanced
2692 
2693 .seealso: DMPlexCreateFromCellList(), DMPlexCreate()
2694 @*/
2695 PetscErrorCode DMPlexCreateFromDAG(DM dm, PetscInt depth, const PetscInt numPoints[], const PetscInt coneSize[], const PetscInt cones[], const PetscInt coneOrientations[], const PetscScalar vertexCoords[])
2696 {
2697   Vec            coordinates;
2698   PetscSection   coordSection;
2699   PetscScalar    *coords;
2700   PetscInt       coordSize, firstVertex = -1, pStart = 0, pEnd = 0, p, v, dim, dimEmbed, d, off;
2701   PetscErrorCode ierr;
2702 
2703   PetscFunctionBegin;
2704   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
2705   ierr = DMGetCoordinateDim(dm, &dimEmbed);CHKERRQ(ierr);
2706   if (dimEmbed < dim) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Embedding dimension %d cannot be less than intrinsic dimension %d",dimEmbed,dim);
2707   for (d = 0; d <= depth; ++d) pEnd += numPoints[d];
2708   ierr = DMPlexSetChart(dm, pStart, pEnd);CHKERRQ(ierr);
2709   for (p = pStart; p < pEnd; ++p) {
2710     ierr = DMPlexSetConeSize(dm, p, coneSize[p-pStart]);CHKERRQ(ierr);
2711     if (firstVertex < 0 && !coneSize[p - pStart]) {
2712       firstVertex = p - pStart;
2713     }
2714   }
2715   if (firstVertex < 0 && numPoints[0]) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONG,"Expected %d vertices but could not find any", numPoints[0]);
2716   ierr = DMSetUp(dm);CHKERRQ(ierr); /* Allocate space for cones */
2717   for (p = pStart, off = 0; p < pEnd; off += coneSize[p-pStart], ++p) {
2718     ierr = DMPlexSetCone(dm, p, &cones[off]);CHKERRQ(ierr);
2719     ierr = DMPlexSetConeOrientation(dm, p, &coneOrientations[off]);CHKERRQ(ierr);
2720   }
2721   ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
2722   ierr = DMPlexStratify(dm);CHKERRQ(ierr);
2723   /* Build coordinates */
2724   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
2725   ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
2726   ierr = PetscSectionSetFieldComponents(coordSection, 0, dimEmbed);CHKERRQ(ierr);
2727   ierr = PetscSectionSetChart(coordSection, firstVertex, firstVertex+numPoints[0]);CHKERRQ(ierr);
2728   for (v = firstVertex; v < firstVertex+numPoints[0]; ++v) {
2729     ierr = PetscSectionSetDof(coordSection, v, dimEmbed);CHKERRQ(ierr);
2730     ierr = PetscSectionSetFieldDof(coordSection, v, 0, dimEmbed);CHKERRQ(ierr);
2731   }
2732   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
2733   ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
2734   ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
2735   ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
2736   ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
2737   ierr = VecSetBlockSize(coordinates, dimEmbed);CHKERRQ(ierr);
2738   ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
2739   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
2740   for (v = 0; v < numPoints[0]; ++v) {
2741     PetscInt off;
2742 
2743     ierr = PetscSectionGetOffset(coordSection, v+firstVertex, &off);CHKERRQ(ierr);
2744     for (d = 0; d < dimEmbed; ++d) {
2745       coords[off+d] = vertexCoords[v*dimEmbed+d];
2746     }
2747   }
2748   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
2749   ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
2750   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
2751   PetscFunctionReturn(0);
2752 }
2753 
2754 /*@C
2755   DMPlexCreateFromFile - This takes a filename and produces a DM
2756 
2757   Input Parameters:
2758 + comm - The communicator
2759 . filename - A file name
2760 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
2761 
2762   Output Parameter:
2763 . dm - The DM
2764 
2765   Level: beginner
2766 
2767 .seealso: DMPlexCreateFromDAG(), DMPlexCreateFromCellList(), DMPlexCreate()
2768 @*/
2769 PetscErrorCode DMPlexCreateFromFile(MPI_Comm comm, const char filename[], PetscBool interpolate, DM *dm)
2770 {
2771   const char    *extGmsh    = ".msh";
2772   const char    *extCGNS    = ".cgns";
2773   const char    *extExodus  = ".exo";
2774   const char    *extGenesis = ".gen";
2775   const char    *extFluent  = ".cas";
2776   const char    *extHDF5    = ".h5";
2777   const char    *extMed     = ".med";
2778   const char    *extPLY     = ".ply";
2779   size_t         len;
2780   PetscBool      isGmsh, isCGNS, isExodus, isGenesis, isFluent, isHDF5, isMed, isPLY;
2781   PetscMPIInt    rank;
2782   PetscErrorCode ierr;
2783 
2784   PetscFunctionBegin;
2785   PetscValidPointer(filename, 2);
2786   PetscValidPointer(dm, 4);
2787   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
2788   ierr = PetscStrlen(filename, &len);CHKERRQ(ierr);
2789   if (!len) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Filename must be a valid path");
2790   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)], extGmsh,    4, &isGmsh);CHKERRQ(ierr);
2791   ierr = PetscStrncmp(&filename[PetscMax(0,len-5)], extCGNS,    5, &isCGNS);CHKERRQ(ierr);
2792   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)], extExodus,  4, &isExodus);CHKERRQ(ierr);
2793   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)], extGenesis, 4, &isGenesis);CHKERRQ(ierr);
2794   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)], extFluent,  4, &isFluent);CHKERRQ(ierr);
2795   ierr = PetscStrncmp(&filename[PetscMax(0,len-3)], extHDF5,    3, &isHDF5);CHKERRQ(ierr);
2796   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)], extMed,     4, &isMed);CHKERRQ(ierr);
2797   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)], extPLY,     4, &isPLY);CHKERRQ(ierr);
2798   if (isGmsh) {
2799     ierr = DMPlexCreateGmshFromFile(comm, filename, interpolate, dm);CHKERRQ(ierr);
2800   } else if (isCGNS) {
2801     ierr = DMPlexCreateCGNSFromFile(comm, filename, interpolate, dm);CHKERRQ(ierr);
2802   } else if (isExodus || isGenesis) {
2803     ierr = DMPlexCreateExodusFromFile(comm, filename, interpolate, dm);CHKERRQ(ierr);
2804   } else if (isFluent) {
2805     ierr = DMPlexCreateFluentFromFile(comm, filename, interpolate, dm);CHKERRQ(ierr);
2806   } else if (isHDF5) {
2807     PetscViewer viewer;
2808 
2809     ierr = PetscViewerCreate(comm, &viewer);CHKERRQ(ierr);
2810     ierr = PetscViewerSetType(viewer, PETSCVIEWERHDF5);CHKERRQ(ierr);
2811     ierr = PetscViewerFileSetMode(viewer, FILE_MODE_READ);CHKERRQ(ierr);
2812     ierr = PetscViewerFileSetName(viewer, filename);CHKERRQ(ierr);
2813     ierr = DMCreate(comm, dm);CHKERRQ(ierr);
2814     ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
2815     ierr = DMLoad(*dm, viewer);CHKERRQ(ierr);
2816     ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
2817   } else if (isMed) {
2818     ierr = DMPlexCreateMedFromFile(comm, filename, interpolate, dm);CHKERRQ(ierr);
2819   } else if (isPLY) {
2820     ierr = DMPlexCreatePLYFromFile(comm, filename, interpolate, dm);CHKERRQ(ierr);
2821   } else SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cannot load file %s: unrecognized extension", filename);
2822   PetscFunctionReturn(0);
2823 }
2824 
2825 /*@
2826   DMPlexCreateReferenceCell - Create a DMPLEX with the appropriate FEM reference cell
2827 
2828   Collective on comm
2829 
2830   Input Parameters:
2831 + comm    - The communicator
2832 . dim     - The spatial dimension
2833 - simplex - Flag for simplex, otherwise use a tensor-product cell
2834 
2835   Output Parameter:
2836 . refdm - The reference cell
2837 
2838   Level: intermediate
2839 
2840 .keywords: reference cell
2841 .seealso:
2842 @*/
2843 PetscErrorCode DMPlexCreateReferenceCell(MPI_Comm comm, PetscInt dim, PetscBool simplex, DM *refdm)
2844 {
2845   DM             rdm;
2846   Vec            coords;
2847   PetscErrorCode ierr;
2848 
2849   PetscFunctionBeginUser;
2850   ierr = DMCreate(comm, &rdm);CHKERRQ(ierr);
2851   ierr = DMSetType(rdm, DMPLEX);CHKERRQ(ierr);
2852   ierr = DMSetDimension(rdm, dim);CHKERRQ(ierr);
2853   switch (dim) {
2854   case 0:
2855   {
2856     PetscInt    numPoints[1]        = {1};
2857     PetscInt    coneSize[1]         = {0};
2858     PetscInt    cones[1]            = {0};
2859     PetscInt    coneOrientations[1] = {0};
2860     PetscScalar vertexCoords[1]     = {0.0};
2861 
2862     ierr = DMPlexCreateFromDAG(rdm, 0, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
2863   }
2864   break;
2865   case 1:
2866   {
2867     PetscInt    numPoints[2]        = {2, 1};
2868     PetscInt    coneSize[3]         = {2, 0, 0};
2869     PetscInt    cones[2]            = {1, 2};
2870     PetscInt    coneOrientations[2] = {0, 0};
2871     PetscScalar vertexCoords[2]     = {-1.0,  1.0};
2872 
2873     ierr = DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
2874   }
2875   break;
2876   case 2:
2877     if (simplex) {
2878       PetscInt    numPoints[2]        = {3, 1};
2879       PetscInt    coneSize[4]         = {3, 0, 0, 0};
2880       PetscInt    cones[3]            = {1, 2, 3};
2881       PetscInt    coneOrientations[3] = {0, 0, 0};
2882       PetscScalar vertexCoords[6]     = {-1.0, -1.0,  1.0, -1.0,  -1.0, 1.0};
2883 
2884       ierr = DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
2885     } else {
2886       PetscInt    numPoints[2]        = {4, 1};
2887       PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
2888       PetscInt    cones[4]            = {1, 2, 3, 4};
2889       PetscInt    coneOrientations[4] = {0, 0, 0, 0};
2890       PetscScalar vertexCoords[8]     = {-1.0, -1.0,  1.0, -1.0,  1.0, 1.0,  -1.0, 1.0};
2891 
2892       ierr = DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
2893     }
2894   break;
2895   case 3:
2896     if (simplex) {
2897       PetscInt    numPoints[2]        = {4, 1};
2898       PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
2899       PetscInt    cones[4]            = {1, 3, 2, 4};
2900       PetscInt    coneOrientations[4] = {0, 0, 0, 0};
2901       PetscScalar vertexCoords[12]    = {-1.0, -1.0, -1.0,  1.0, -1.0, -1.0,  -1.0, 1.0, -1.0,  -1.0, -1.0, 1.0};
2902 
2903       ierr = DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
2904     } else {
2905       PetscInt    numPoints[2]        = {8, 1};
2906       PetscInt    coneSize[9]         = {8, 0, 0, 0, 0, 0, 0, 0, 0};
2907       PetscInt    cones[8]            = {1, 4, 3, 2, 5, 6, 7, 8};
2908       PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
2909       PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0,  1.0, -1.0, -1.0,  1.0, 1.0, -1.0,  -1.0, 1.0, -1.0,
2910                                          -1.0, -1.0,  1.0,  1.0, -1.0,  1.0,  1.0, 1.0,  1.0,  -1.0, 1.0,  1.0};
2911 
2912       ierr = DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
2913     }
2914   break;
2915   default:
2916     SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Cannot create reference cell for dimension %d", dim);
2917   }
2918   *refdm = NULL;
2919   ierr = DMPlexInterpolate(rdm, refdm);CHKERRQ(ierr);
2920   if (rdm->coordinateDM) {
2921     DM           ncdm;
2922     PetscSection cs;
2923     PetscInt     pEnd = -1;
2924 
2925     ierr = DMGetDefaultSection(rdm->coordinateDM, &cs);CHKERRQ(ierr);
2926     if (cs) {ierr = PetscSectionGetChart(cs, NULL, &pEnd);CHKERRQ(ierr);}
2927     if (pEnd >= 0) {
2928       ierr = DMClone(rdm->coordinateDM, &ncdm);CHKERRQ(ierr);
2929       ierr = DMSetDefaultSection(ncdm, cs);CHKERRQ(ierr);
2930       ierr = DMSetCoordinateDM(*refdm, ncdm);CHKERRQ(ierr);
2931       ierr = DMDestroy(&ncdm);CHKERRQ(ierr);
2932     }
2933   }
2934   ierr = DMGetCoordinatesLocal(rdm, &coords);CHKERRQ(ierr);
2935   if (coords) {
2936     ierr = DMSetCoordinatesLocal(*refdm, coords);CHKERRQ(ierr);
2937   } else {
2938     ierr = DMGetCoordinates(rdm, &coords);CHKERRQ(ierr);
2939     if (coords) {ierr = DMSetCoordinates(*refdm, coords);CHKERRQ(ierr);}
2940   }
2941   ierr = DMDestroy(&rdm);CHKERRQ(ierr);
2942   PetscFunctionReturn(0);
2943 }
2944