xref: /petsc/src/dm/impls/plex/plexcreate.c (revision 0ecfd9fc350141c372c751a4adf2797bf0a87e35)
1 #define PETSCDM_DLL
2 #include <petsc/private/dmpleximpl.h>    /*I   "petscdmplex.h"   I*/
3 #include <petscdmda.h>
4 #include <petscsf.h>
5 
6 /*@
7   DMPlexCreateDoublet - Creates a mesh of two cells of the specified type, optionally with later refinement.
8 
9   Collective on MPI_Comm
10 
11   Input Parameters:
12 + comm - The communicator for the DM object
13 . dim - The spatial dimension
14 . simplex - Flag for simplicial cells, otherwise they are tensor product cells
15 . interpolate - Flag to create intermediate mesh pieces (edges, faces)
16 . refinementUniform - Flag for uniform parallel refinement
17 - refinementLimit - A nonzero number indicates the largest admissible volume for a refined cell
18 
19   Output Parameter:
20 . dm  - The DM object
21 
22   Level: beginner
23 
24 .keywords: DM, create
25 .seealso: DMSetType(), DMCreate()
26 @*/
27 PetscErrorCode DMPlexCreateDoublet(MPI_Comm comm, PetscInt dim, PetscBool simplex, PetscBool interpolate, PetscBool refinementUniform, PetscReal refinementLimit, DM *newdm)
28 {
29   DM             dm;
30   PetscInt       p;
31   PetscMPIInt    rank;
32   PetscErrorCode ierr;
33 
34   PetscFunctionBegin;
35   ierr = DMCreate(comm, &dm);CHKERRQ(ierr);
36   ierr = DMSetType(dm, DMPLEX);CHKERRQ(ierr);
37   ierr = DMSetDimension(dm, dim);CHKERRQ(ierr);
38   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
39   switch (dim) {
40   case 2:
41     if (simplex) {ierr = PetscObjectSetName((PetscObject) dm, "triangular");CHKERRQ(ierr);}
42     else         {ierr = PetscObjectSetName((PetscObject) dm, "quadrilateral");CHKERRQ(ierr);}
43     break;
44   case 3:
45     if (simplex) {ierr = PetscObjectSetName((PetscObject) dm, "tetrahedral");CHKERRQ(ierr);}
46     else         {ierr = PetscObjectSetName((PetscObject) dm, "hexahedral");CHKERRQ(ierr);}
47     break;
48   default:
49     SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make meshes for dimension %d", dim);
50   }
51   if (rank) {
52     PetscInt numPoints[2] = {0, 0};
53     ierr = DMPlexCreateFromDAG(dm, 1, numPoints, NULL, NULL, NULL, NULL);CHKERRQ(ierr);
54   } else {
55     switch (dim) {
56     case 2:
57       if (simplex) {
58         PetscInt    numPoints[2]        = {4, 2};
59         PetscInt    coneSize[6]         = {3, 3, 0, 0, 0, 0};
60         PetscInt    cones[6]            = {2, 3, 4,  5, 4, 3};
61         PetscInt    coneOrientations[6] = {0, 0, 0,  0, 0, 0};
62         PetscScalar vertexCoords[8]     = {-0.5, 0.5, 0.0, 0.0, 0.0, 1.0, 0.5, 0.5};
63         PetscInt    markerPoints[8]     = {2, 1, 3, 1, 4, 1, 5, 1};
64 
65         ierr = DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
66         for (p = 0; p < 4; ++p) {ierr = DMSetLabelValue(dm, "marker", markerPoints[p*2], markerPoints[p*2+1]);CHKERRQ(ierr);}
67       } else {
68         PetscInt    numPoints[2]        = {6, 2};
69         PetscInt    coneSize[8]         = {4, 4, 0, 0, 0, 0, 0, 0};
70         PetscInt    cones[8]            = {2, 3, 4, 5,  3, 6, 7, 4};
71         PetscInt    coneOrientations[8] = {0, 0, 0, 0,  0, 0, 0, 0};
72         PetscScalar vertexCoords[12]    = {-1.0, -0.5,  0.0, -0.5,  0.0, 0.5,  -1.0, 0.5,  1.0, -0.5,  1.0, 0.5};
73 
74         ierr = DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
75       }
76       break;
77     case 3:
78       if (simplex) {
79         PetscInt    numPoints[2]        = {5, 2};
80         PetscInt    coneSize[7]         = {4, 4, 0, 0, 0, 0, 0};
81         PetscInt    cones[8]            = {4, 3, 5, 2,  5, 3, 4, 6};
82         PetscInt    coneOrientations[8] = {0, 0, 0, 0,  0, 0, 0, 0};
83         PetscScalar vertexCoords[15]    = {-1.0, 0.0, 0.0,  0.0, -1.0, 0.0,  0.0, 0.0, 1.0,  0.0, 1.0, 0.0,  1.0, 0.0, 0.0};
84         PetscInt    markerPoints[10]    = {2, 1, 3, 1, 4, 1, 5, 1, 6, 1};
85 
86         ierr = DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
87         for (p = 0; p < 5; ++p) {ierr = DMSetLabelValue(dm, "marker", markerPoints[p*2], markerPoints[p*2+1]);CHKERRQ(ierr);}
88       } else {
89         PetscInt    numPoints[2]         = {12, 2};
90         PetscInt    coneSize[14]         = {8, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
91         PetscInt    cones[16]            = {2, 3, 4, 5, 6, 7, 8, 9,  5, 4, 10, 11, 7, 12, 13, 8};
92         PetscInt    coneOrientations[16] = {0, 0, 0, 0, 0, 0, 0, 0,  0, 0,  0,  0, 0,  0,  0, 0};
93         PetscScalar vertexCoords[36]     = {-1.0, -0.5, -0.5,  -1.0,  0.5, -0.5,  0.0,  0.5, -0.5,   0.0, -0.5, -0.5,
94                                             -1.0, -0.5,  0.5,   0.0, -0.5,  0.5,  0.0,  0.5,  0.5,  -1.0,  0.5,  0.5,
95                                              1.0,  0.5, -0.5,   1.0, -0.5, -0.5,  1.0, -0.5,  0.5,   1.0,  0.5,  0.5};
96 
97         ierr = DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
98       }
99       break;
100     default:
101       SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make meshes for dimension %d", dim);
102     }
103   }
104   *newdm = dm;
105   if (refinementLimit > 0.0) {
106     DM rdm;
107     const char *name;
108 
109     ierr = DMPlexSetRefinementUniform(*newdm, PETSC_FALSE);CHKERRQ(ierr);
110     ierr = DMPlexSetRefinementLimit(*newdm, refinementLimit);CHKERRQ(ierr);
111     ierr = DMRefine(*newdm, comm, &rdm);CHKERRQ(ierr);
112     ierr = PetscObjectGetName((PetscObject) *newdm, &name);CHKERRQ(ierr);
113     ierr = PetscObjectSetName((PetscObject)    rdm,  name);CHKERRQ(ierr);
114     ierr = DMDestroy(newdm);CHKERRQ(ierr);
115     *newdm = rdm;
116   }
117   if (interpolate) {
118     DM idm = NULL;
119     const char *name;
120 
121     ierr = DMPlexInterpolate(*newdm, &idm);CHKERRQ(ierr);
122     ierr = PetscObjectGetName((PetscObject) *newdm, &name);CHKERRQ(ierr);
123     ierr = PetscObjectSetName((PetscObject)    idm,  name);CHKERRQ(ierr);
124     ierr = DMPlexCopyCoordinates(*newdm, idm);CHKERRQ(ierr);
125     ierr = DMCopyLabels(*newdm, idm);CHKERRQ(ierr);
126     ierr = DMDestroy(newdm);CHKERRQ(ierr);
127     *newdm = idm;
128   }
129   {
130     DM refinedMesh     = NULL;
131     DM distributedMesh = NULL;
132 
133     /* Distribute mesh over processes */
134     ierr = DMPlexDistribute(*newdm, 0, NULL, &distributedMesh);CHKERRQ(ierr);
135     if (distributedMesh) {
136       ierr = DMDestroy(newdm);CHKERRQ(ierr);
137       *newdm = distributedMesh;
138     }
139     if (refinementUniform) {
140       ierr = DMPlexSetRefinementUniform(*newdm, refinementUniform);CHKERRQ(ierr);
141       ierr = DMRefine(*newdm, comm, &refinedMesh);CHKERRQ(ierr);
142       if (refinedMesh) {
143         ierr = DMDestroy(newdm);CHKERRQ(ierr);
144         *newdm = refinedMesh;
145       }
146     }
147   }
148   PetscFunctionReturn(0);
149 }
150 
151 /*@
152   DMPlexCreateSquareBoundary - Creates a 1D mesh the is the boundary of a square lattice.
153 
154   Collective on MPI_Comm
155 
156   Input Parameters:
157 + comm  - The communicator for the DM object
158 . lower - The lower left corner coordinates
159 . upper - The upper right corner coordinates
160 - edges - The number of cells in each direction
161 
162   Output Parameter:
163 . dm  - The DM object
164 
165   Note: Here is the numbering returned for 2 cells in each direction:
166 $ 18--5-17--4--16
167 $  |     |     |
168 $  6    10     3
169 $  |     |     |
170 $ 19-11-20--9--15
171 $  |     |     |
172 $  7     8     2
173 $  |     |     |
174 $ 12--0-13--1--14
175 
176   Level: beginner
177 
178 .keywords: DM, create
179 .seealso: DMPlexCreateBoxMesh(), DMPlexCreateCubeBoundary(), DMSetType(), DMCreate()
180 @*/
181 PetscErrorCode DMPlexCreateSquareBoundary(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[])
182 {
183   const PetscInt numVertices    = (edges[0]+1)*(edges[1]+1);
184   const PetscInt numEdges       = edges[0]*(edges[1]+1) + (edges[0]+1)*edges[1];
185   PetscInt       markerTop      = 1;
186   PetscInt       markerBottom   = 1;
187   PetscInt       markerRight    = 1;
188   PetscInt       markerLeft     = 1;
189   PetscBool      markerSeparate = PETSC_FALSE;
190   Vec            coordinates;
191   PetscSection   coordSection;
192   PetscScalar    *coords;
193   PetscInt       coordSize;
194   PetscMPIInt    rank;
195   PetscInt       v, vx, vy;
196   PetscErrorCode ierr;
197 
198   PetscFunctionBegin;
199   ierr = PetscOptionsGetBool(((PetscObject) dm)->options,((PetscObject) dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL);CHKERRQ(ierr);
200   if (markerSeparate) {
201     markerTop    = 3;
202     markerBottom = 1;
203     markerRight  = 2;
204     markerLeft   = 4;
205   }
206   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRQ(ierr);
207   if (!rank) {
208     PetscInt e, ex, ey;
209 
210     ierr = DMPlexSetChart(dm, 0, numEdges+numVertices);CHKERRQ(ierr);
211     for (e = 0; e < numEdges; ++e) {
212       ierr = DMPlexSetConeSize(dm, e, 2);CHKERRQ(ierr);
213     }
214     ierr = DMSetUp(dm);CHKERRQ(ierr); /* Allocate space for cones */
215     for (vx = 0; vx <= edges[0]; vx++) {
216       for (ey = 0; ey < edges[1]; ey++) {
217         PetscInt edge   = vx*edges[1] + ey + edges[0]*(edges[1]+1);
218         PetscInt vertex = ey*(edges[0]+1) + vx + numEdges;
219         PetscInt cone[2];
220 
221         cone[0] = vertex; cone[1] = vertex+edges[0]+1;
222         ierr    = DMPlexSetCone(dm, edge, cone);CHKERRQ(ierr);
223         if (vx == edges[0]) {
224           ierr = DMSetLabelValue(dm, "marker", edge,    markerRight);CHKERRQ(ierr);
225           ierr = DMSetLabelValue(dm, "marker", cone[0], markerRight);CHKERRQ(ierr);
226           if (ey == edges[1]-1) {
227             ierr = DMSetLabelValue(dm, "marker", cone[1], markerRight);CHKERRQ(ierr);
228             ierr = DMSetLabelValue(dm, "Face Sets", cone[1], markerRight);CHKERRQ(ierr);
229           }
230         } else if (vx == 0) {
231           ierr = DMSetLabelValue(dm, "marker", edge,    markerLeft);CHKERRQ(ierr);
232           ierr = DMSetLabelValue(dm, "marker", cone[0], markerLeft);CHKERRQ(ierr);
233           if (ey == edges[1]-1) {
234             ierr = DMSetLabelValue(dm, "marker", cone[1], markerLeft);CHKERRQ(ierr);
235             ierr = DMSetLabelValue(dm, "Face Sets", cone[1], markerLeft);CHKERRQ(ierr);
236           }
237         }
238       }
239     }
240     for (vy = 0; vy <= edges[1]; vy++) {
241       for (ex = 0; ex < edges[0]; ex++) {
242         PetscInt edge   = vy*edges[0]     + ex;
243         PetscInt vertex = vy*(edges[0]+1) + ex + numEdges;
244         PetscInt cone[2];
245 
246         cone[0] = vertex; cone[1] = vertex+1;
247         ierr    = DMPlexSetCone(dm, edge, cone);CHKERRQ(ierr);
248         if (vy == edges[1]) {
249           ierr = DMSetLabelValue(dm, "marker", edge,    markerTop);CHKERRQ(ierr);
250           ierr = DMSetLabelValue(dm, "marker", cone[0], markerTop);CHKERRQ(ierr);
251           if (ex == edges[0]-1) {
252             ierr = DMSetLabelValue(dm, "marker", cone[1], markerTop);CHKERRQ(ierr);
253             ierr = DMSetLabelValue(dm, "Face Sets", cone[1], markerTop);CHKERRQ(ierr);
254           }
255         } else if (vy == 0) {
256           ierr = DMSetLabelValue(dm, "marker", edge,    markerBottom);CHKERRQ(ierr);
257           ierr = DMSetLabelValue(dm, "marker", cone[0], markerBottom);CHKERRQ(ierr);
258           if (ex == edges[0]-1) {
259             ierr = DMSetLabelValue(dm, "marker", cone[1], markerBottom);CHKERRQ(ierr);
260             ierr = DMSetLabelValue(dm, "Face Sets", cone[1], markerBottom);CHKERRQ(ierr);
261           }
262         }
263       }
264     }
265   }
266   ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
267   ierr = DMPlexStratify(dm);CHKERRQ(ierr);
268   /* Build coordinates */
269   ierr = DMSetCoordinateDim(dm, 2);CHKERRQ(ierr);
270   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
271   ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
272   ierr = PetscSectionSetChart(coordSection, numEdges, numEdges + numVertices);CHKERRQ(ierr);
273   ierr = PetscSectionSetFieldComponents(coordSection, 0, 2);CHKERRQ(ierr);
274   for (v = numEdges; v < numEdges+numVertices; ++v) {
275     ierr = PetscSectionSetDof(coordSection, v, 2);CHKERRQ(ierr);
276     ierr = PetscSectionSetFieldDof(coordSection, v, 0, 2);CHKERRQ(ierr);
277   }
278   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
279   ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
280   ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
281   ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
282   ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
283   ierr = VecSetBlockSize(coordinates, 2);CHKERRQ(ierr);
284   ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
285   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
286   for (vy = 0; vy <= edges[1]; ++vy) {
287     for (vx = 0; vx <= edges[0]; ++vx) {
288       coords[(vy*(edges[0]+1)+vx)*2+0] = lower[0] + ((upper[0] - lower[0])/edges[0])*vx;
289       coords[(vy*(edges[0]+1)+vx)*2+1] = lower[1] + ((upper[1] - lower[1])/edges[1])*vy;
290     }
291   }
292   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
293   ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
294   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
295   PetscFunctionReturn(0);
296 }
297 
298 /*@
299   DMPlexCreateCubeBoundary - Creates a 2D mesh the is the boundary of a cubic lattice.
300 
301   Collective on MPI_Comm
302 
303   Input Parameters:
304 + comm  - The communicator for the DM object
305 . lower - The lower left front corner coordinates
306 . upper - The upper right back corner coordinates
307 - edges - The number of cells in each direction
308 
309   Output Parameter:
310 . dm  - The DM object
311 
312   Level: beginner
313 
314 .keywords: DM, create
315 .seealso: DMPlexCreateBoxMesh(), DMPlexCreateSquareBoundary(), DMSetType(), DMCreate()
316 @*/
317 PetscErrorCode DMPlexCreateCubeBoundary(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt faces[])
318 {
319   PetscInt       vertices[3], numVertices;
320   PetscInt       numFaces    = 2*faces[0]*faces[1] + 2*faces[1]*faces[2] + 2*faces[0]*faces[2];
321   Vec            coordinates;
322   PetscSection   coordSection;
323   PetscScalar    *coords;
324   PetscInt       coordSize;
325   PetscMPIInt    rank;
326   PetscInt       v, vx, vy, vz;
327   PetscInt       voffset, iface=0, cone[4];
328   PetscErrorCode ierr;
329 
330   PetscFunctionBegin;
331   if ((faces[0] < 1) || (faces[1] < 1) || (faces[2] < 1)) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Must have at least 1 face per side");
332   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRQ(ierr);
333   vertices[0] = faces[0]+1; vertices[1] = faces[1]+1; vertices[2] = faces[2]+1;
334   numVertices = vertices[0]*vertices[1]*vertices[2];
335   if (!rank) {
336     PetscInt f;
337 
338     ierr = DMPlexSetChart(dm, 0, numFaces+numVertices);CHKERRQ(ierr);
339     for (f = 0; f < numFaces; ++f) {
340       ierr = DMPlexSetConeSize(dm, f, 4);CHKERRQ(ierr);
341     }
342     ierr = DMSetUp(dm);CHKERRQ(ierr); /* Allocate space for cones */
343     for (v = 0; v < numFaces+numVertices; ++v) {
344       ierr = DMSetLabelValue(dm, "marker", v, 1);CHKERRQ(ierr);
345     }
346 
347     /* Side 0 (Top) */
348     for (vy = 0; vy < faces[1]; vy++) {
349       for (vx = 0; vx < faces[0]; vx++) {
350         voffset = numFaces + vertices[0]*vertices[1]*(vertices[2]-1) + vy*vertices[0] + vx;
351         cone[0] = voffset; cone[1] = voffset+1; cone[2] = voffset+vertices[0]+1; cone[3] = voffset+vertices[0];
352         ierr    = DMPlexSetCone(dm, iface, cone);CHKERRQ(ierr);
353         iface++;
354       }
355     }
356 
357     /* Side 1 (Bottom) */
358     for (vy = 0; vy < faces[1]; vy++) {
359       for (vx = 0; vx < faces[0]; vx++) {
360         voffset = numFaces + vy*(faces[0]+1) + vx;
361         cone[0] = voffset+1; cone[1] = voffset; cone[2] = voffset+vertices[0]; cone[3] = voffset+vertices[0]+1;
362         ierr    = DMPlexSetCone(dm, iface, cone);CHKERRQ(ierr);
363         iface++;
364       }
365     }
366 
367     /* Side 2 (Front) */
368     for (vz = 0; vz < faces[2]; vz++) {
369       for (vx = 0; vx < faces[0]; vx++) {
370         voffset = numFaces + vz*vertices[0]*vertices[1] + vx;
371         cone[0] = voffset; cone[1] = voffset+1; cone[2] = voffset+vertices[0]*vertices[1]+1; cone[3] = voffset+vertices[0]*vertices[1];
372         ierr    = DMPlexSetCone(dm, iface, cone);CHKERRQ(ierr);
373         iface++;
374       }
375     }
376 
377     /* Side 3 (Back) */
378     for (vz = 0; vz < faces[2]; vz++) {
379       for (vx = 0; vx < faces[0]; vx++) {
380         voffset = numFaces + vz*vertices[0]*vertices[1] + vertices[0]*(vertices[1]-1) + vx;
381         cone[0] = voffset+vertices[0]*vertices[1]; cone[1] = voffset+vertices[0]*vertices[1]+1;
382         cone[2] = voffset+1; cone[3] = voffset;
383         ierr    = DMPlexSetCone(dm, iface, cone);CHKERRQ(ierr);
384         iface++;
385       }
386     }
387 
388     /* Side 4 (Left) */
389     for (vz = 0; vz < faces[2]; vz++) {
390       for (vy = 0; vy < faces[1]; vy++) {
391         voffset = numFaces + vz*vertices[0]*vertices[1] + vy*vertices[0];
392         cone[0] = voffset; cone[1] = voffset+vertices[0]*vertices[1];
393         cone[2] = voffset+vertices[0]*vertices[1]+vertices[0]; cone[3] = voffset+vertices[0];
394         ierr    = DMPlexSetCone(dm, iface, cone);CHKERRQ(ierr);
395         iface++;
396       }
397     }
398 
399     /* Side 5 (Right) */
400     for (vz = 0; vz < faces[2]; vz++) {
401       for (vy = 0; vy < faces[1]; vy++) {
402         voffset = numFaces + vz*vertices[0]*vertices[1] + vy*vertices[0] + faces[0];
403         cone[0] = voffset+vertices[0]*vertices[1]; cone[1] = voffset;
404         cone[2] = voffset+vertices[0]; cone[3] = voffset+vertices[0]*vertices[1]+vertices[0];
405         ierr    = DMPlexSetCone(dm, iface, cone);CHKERRQ(ierr);
406         iface++;
407       }
408     }
409   }
410   ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
411   ierr = DMPlexStratify(dm);CHKERRQ(ierr);
412   /* Build coordinates */
413   ierr = DMSetCoordinateDim(dm, 3);CHKERRQ(ierr);
414   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
415   ierr = PetscSectionSetChart(coordSection, numFaces, numFaces + numVertices);CHKERRQ(ierr);
416   for (v = numFaces; v < numFaces+numVertices; ++v) {
417     ierr = PetscSectionSetDof(coordSection, v, 3);CHKERRQ(ierr);
418   }
419   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
420   ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
421   ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
422   ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
423   ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
424   ierr = VecSetBlockSize(coordinates, 3);CHKERRQ(ierr);
425   ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
426   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
427   for (vz = 0; vz <= faces[2]; ++vz) {
428     for (vy = 0; vy <= faces[1]; ++vy) {
429       for (vx = 0; vx <= faces[0]; ++vx) {
430         coords[((vz*(faces[1]+1)+vy)*(faces[0]+1)+vx)*3+0] = lower[0] + ((upper[0] - lower[0])/faces[0])*vx;
431         coords[((vz*(faces[1]+1)+vy)*(faces[0]+1)+vx)*3+1] = lower[1] + ((upper[1] - lower[1])/faces[1])*vy;
432         coords[((vz*(faces[1]+1)+vy)*(faces[0]+1)+vx)*3+2] = lower[2] + ((upper[2] - lower[2])/faces[2])*vz;
433       }
434     }
435   }
436   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
437   ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
438   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
439   PetscFunctionReturn(0);
440 }
441 
442 /*@
443   DMPlexCreateBoxMesh - Creates a mesh on the tensor product of unit intervals (box) using simplices.
444 
445   Collective on MPI_Comm
446 
447   Input Parameters:
448 + comm - The communicator for the DM object
449 . dim - The spatial dimension
450 . numFaces - Number of faces per dimension
451 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
452 
453   Output Parameter:
454 . dm  - The DM object
455 
456   Level: beginner
457 
458 .keywords: DM, create
459 .seealso: DMPlexCreateHexBoxMesh(), DMSetType(), DMCreate()
460 @*/
461 PetscErrorCode DMPlexCreateBoxMesh(MPI_Comm comm, PetscInt dim, PetscInt numFaces, PetscBool interpolate, DM *dm)
462 {
463   DM             boundary;
464   PetscErrorCode ierr;
465 
466   PetscFunctionBegin;
467   PetscValidPointer(dm, 4);
468   ierr = DMCreate(comm, &boundary);CHKERRQ(ierr);
469   PetscValidLogicalCollectiveInt(boundary,dim,2);
470   ierr = DMSetType(boundary, DMPLEX);CHKERRQ(ierr);
471   ierr = DMSetDimension(boundary, dim-1);CHKERRQ(ierr);
472   ierr = DMSetCoordinateDim(boundary, dim);CHKERRQ(ierr);
473   switch (dim) {
474   case 2:
475   {
476     PetscReal lower[2] = {0.0, 0.0};
477     PetscReal upper[2] = {1.0, 1.0};
478     PetscInt  edges[2];
479 
480     edges[0] = numFaces; edges[1] = numFaces;
481     ierr = DMPlexCreateSquareBoundary(boundary, lower, upper, edges);CHKERRQ(ierr);
482     break;
483   }
484   case 3:
485   {
486     PetscReal lower[3] = {0.0, 0.0, 0.0};
487     PetscReal upper[3] = {1.0, 1.0, 1.0};
488     PetscInt  faces[3];
489 
490     faces[0] = numFaces; faces[1] = numFaces; faces[2] = numFaces;
491     ierr = DMPlexCreateCubeBoundary(boundary, lower, upper, faces);CHKERRQ(ierr);
492     break;
493   }
494   default:
495     SETERRQ1(comm, PETSC_ERR_SUP, "Dimension not supported: %d", dim);
496   }
497   ierr = DMPlexGenerate(boundary, NULL, interpolate, dm);CHKERRQ(ierr);
498   ierr = DMDestroy(&boundary);CHKERRQ(ierr);
499   PetscFunctionReturn(0);
500 }
501 
502 static PetscErrorCode DMPlexCreateCubeMesh_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[], DMBoundaryType bdX, DMBoundaryType bdY, DMBoundaryType bdZ)
503 {
504   DMLabel        cutLabel = NULL;
505   PetscInt       markerTop      = 1, faceMarkerTop      = 1;
506   PetscInt       markerBottom   = 1, faceMarkerBottom   = 1;
507   PetscInt       markerFront    = 1, faceMarkerFront    = 1;
508   PetscInt       markerBack     = 1, faceMarkerBack     = 1;
509   PetscInt       markerRight    = 1, faceMarkerRight    = 1;
510   PetscInt       markerLeft     = 1, faceMarkerLeft     = 1;
511   PetscInt       dim;
512   PetscBool      markerSeparate = PETSC_FALSE, cutMarker = PETSC_FALSE;
513   PetscMPIInt    rank;
514   PetscErrorCode ierr;
515 
516   PetscFunctionBegin;
517   ierr = DMGetDimension(dm,&dim);CHKERRQ(ierr);
518   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRQ(ierr);
519   ierr = DMCreateLabel(dm,"marker");CHKERRQ(ierr);
520   ierr = DMCreateLabel(dm,"Face Sets");CHKERRQ(ierr);
521   if (bdX == DM_BOUNDARY_PERIODIC || bdX == DM_BOUNDARY_TWIST ||
522       bdY == DM_BOUNDARY_PERIODIC || bdY == DM_BOUNDARY_TWIST ||
523       bdZ == DM_BOUNDARY_PERIODIC || bdZ == DM_BOUNDARY_TWIST) {
524     ierr = PetscOptionsGetBool(((PetscObject) dm)->options,((PetscObject) dm)->prefix, "-dm_plex_periodic_cut", &cutMarker, NULL);CHKERRQ(ierr);
525     if (cutMarker) {ierr = DMCreateLabel(dm, "periodic_cut");CHKERRQ(ierr); ierr = DMGetLabel(dm, "periodic_cut", &cutLabel);CHKERRQ(ierr);}
526   }
527   switch (dim) {
528   case 2:
529     faceMarkerTop    = 3;
530     faceMarkerBottom = 1;
531     faceMarkerRight  = 2;
532     faceMarkerLeft   = 4;
533     break;
534   case 3:
535     faceMarkerBottom = 1;
536     faceMarkerTop    = 2;
537     faceMarkerFront  = 3;
538     faceMarkerBack   = 4;
539     faceMarkerRight  = 5;
540     faceMarkerLeft   = 6;
541     break;
542   default:
543     SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SUP,"Dimension %d not supported",dim);
544     break;
545   }
546   ierr = PetscOptionsGetBool(((PetscObject) dm)->options,((PetscObject) dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL);CHKERRQ(ierr);
547   if (markerSeparate) {
548     markerBottom = faceMarkerBottom;
549     markerTop    = faceMarkerTop;
550     markerFront  = faceMarkerFront;
551     markerBack   = faceMarkerBack;
552     markerRight  = faceMarkerRight;
553     markerLeft   = faceMarkerLeft;
554   }
555   {
556     const PetscInt numXEdges    = !rank ? edges[0] : 0;
557     const PetscInt numYEdges    = !rank ? edges[1] : 0;
558     const PetscInt numZEdges    = !rank ? edges[2] : 0;
559     const PetscInt numXVertices = !rank ? (bdX == DM_BOUNDARY_PERIODIC || bdX == DM_BOUNDARY_TWIST ? edges[0] : edges[0]+1) : 0;
560     const PetscInt numYVertices = !rank ? (bdY == DM_BOUNDARY_PERIODIC || bdY == DM_BOUNDARY_TWIST ? edges[1] : edges[1]+1) : 0;
561     const PetscInt numZVertices = !rank ? (bdZ == DM_BOUNDARY_PERIODIC || bdZ == DM_BOUNDARY_TWIST ? edges[2] : edges[2]+1) : 0;
562     const PetscInt numCells     = numXEdges*numYEdges*numZEdges;
563     const PetscInt numXFaces    = numYEdges*numZEdges;
564     const PetscInt numYFaces    = numXEdges*numZEdges;
565     const PetscInt numZFaces    = numXEdges*numYEdges;
566     const PetscInt numTotXFaces = numXVertices*numXFaces;
567     const PetscInt numTotYFaces = numYVertices*numYFaces;
568     const PetscInt numTotZFaces = numZVertices*numZFaces;
569     const PetscInt numFaces     = numTotXFaces + numTotYFaces + numTotZFaces;
570     const PetscInt numTotXEdges = numXEdges*numYVertices*numZVertices;
571     const PetscInt numTotYEdges = numYEdges*numXVertices*numZVertices;
572     const PetscInt numTotZEdges = numZEdges*numXVertices*numYVertices;
573     const PetscInt numVertices  = numXVertices*numYVertices*numZVertices;
574     const PetscInt numEdges     = numTotXEdges + numTotYEdges + numTotZEdges;
575     const PetscInt firstVertex  = (dim == 2) ? numFaces : numCells;
576     const PetscInt firstXFace   = (dim == 2) ? 0 : numCells + numVertices;
577     const PetscInt firstYFace   = firstXFace + numTotXFaces;
578     const PetscInt firstZFace   = firstYFace + numTotYFaces;
579     const PetscInt firstXEdge   = numCells + numFaces + numVertices;
580     const PetscInt firstYEdge   = firstXEdge + numTotXEdges;
581     const PetscInt firstZEdge   = firstYEdge + numTotYEdges;
582     Vec            coordinates;
583     PetscSection   coordSection;
584     PetscScalar   *coords;
585     PetscInt       coordSize;
586     PetscInt       v, vx, vy, vz;
587     PetscInt       c, f, fx, fy, fz, e, ex, ey, ez;
588 
589     ierr = DMPlexSetChart(dm, 0, numCells+numFaces+numEdges+numVertices);CHKERRQ(ierr);
590     for (c = 0; c < numCells; c++) {
591       ierr = DMPlexSetConeSize(dm, c, 6);CHKERRQ(ierr);
592     }
593     for (f = firstXFace; f < firstXFace+numFaces; ++f) {
594       ierr = DMPlexSetConeSize(dm, f, 4);CHKERRQ(ierr);
595     }
596     for (e = firstXEdge; e < firstXEdge+numEdges; ++e) {
597       ierr = DMPlexSetConeSize(dm, e, 2);CHKERRQ(ierr);
598     }
599     ierr = DMSetUp(dm);CHKERRQ(ierr); /* Allocate space for cones */
600     /* Build cells */
601     for (fz = 0; fz < numZEdges; ++fz) {
602       for (fy = 0; fy < numYEdges; ++fy) {
603         for (fx = 0; fx < numXEdges; ++fx) {
604           PetscInt cell    = (fz*numYEdges + fy)*numXEdges + fx;
605           PetscInt faceB   = firstZFace + (fy*numXEdges+fx)*numZVertices +   fz;
606           PetscInt faceT   = firstZFace + (fy*numXEdges+fx)*numZVertices + ((fz+1)%numZVertices);
607           PetscInt faceF   = firstYFace + (fz*numXEdges+fx)*numYVertices +   fy;
608           PetscInt faceK   = firstYFace + (fz*numXEdges+fx)*numYVertices + ((fy+1)%numYVertices);
609           PetscInt faceL   = firstXFace + (fz*numYEdges+fy)*numXVertices +   fx;
610           PetscInt faceR   = firstXFace + (fz*numYEdges+fy)*numXVertices + ((fx+1)%numXVertices);
611                             /* B,  T,  F,  K,  R,  L */
612           PetscInt ornt[6] = {-4,  0,  0, -1,  0, -4}; /* ??? */
613           PetscInt cone[6];
614 
615           /* no boundary twisting in 3D */
616           cone[0] = faceB; cone[1] = faceT; cone[2] = faceF; cone[3] = faceK; cone[4] = faceR; cone[5] = faceL;
617           ierr    = DMPlexSetCone(dm, cell, cone);CHKERRQ(ierr);
618           ierr    = DMPlexSetConeOrientation(dm, cell, ornt);CHKERRQ(ierr);
619           if (bdX != DM_BOUNDARY_NONE && fx == numXEdges-1 && cutLabel) {ierr = DMLabelSetValue(cutLabel, cell, 2);CHKERRQ(ierr);}
620           if (bdY != DM_BOUNDARY_NONE && fy == numYEdges-1 && cutLabel) {ierr = DMLabelSetValue(cutLabel, cell, 2);CHKERRQ(ierr);}
621           if (bdZ != DM_BOUNDARY_NONE && fz == numZEdges-1 && cutLabel) {ierr = DMLabelSetValue(cutLabel, cell, 2);CHKERRQ(ierr);}
622         }
623       }
624     }
625     /* Build x faces */
626     for (fz = 0; fz < numZEdges; ++fz) {
627       for (fy = 0; fy < numYEdges; ++fy) {
628         for (fx = 0; fx < numXVertices; ++fx) {
629           PetscInt face    = firstXFace + (fz*numYEdges+fy)*numXVertices + fx;
630           PetscInt edgeL   = firstZEdge + (  fy*                 numXVertices+fx)*numZEdges + fz;
631           PetscInt edgeR   = firstZEdge + (((fy+1)%numYVertices)*numXVertices+fx)*numZEdges + fz;
632           PetscInt edgeB   = firstYEdge + (  fz*                 numXVertices+fx)*numYEdges + fy;
633           PetscInt edgeT   = firstYEdge + (((fz+1)%numZVertices)*numXVertices+fx)*numYEdges + fy;
634           PetscInt ornt[4] = {0, 0, -2, -2};
635           PetscInt cone[4];
636 
637           if (dim == 3) {
638             /* markers */
639             if (bdX != DM_BOUNDARY_PERIODIC) {
640               if (fx == numXVertices-1) {
641                 ierr = DMSetLabelValue(dm, "Face Sets", face, faceMarkerRight);CHKERRQ(ierr);
642                 ierr = DMSetLabelValue(dm, "marker", face, markerRight);CHKERRQ(ierr);
643               }
644               else if (fx == 0) {
645                 ierr = DMSetLabelValue(dm, "Face Sets", face, faceMarkerLeft);CHKERRQ(ierr);
646                 ierr = DMSetLabelValue(dm, "marker", face, markerLeft);CHKERRQ(ierr);
647               }
648             }
649           }
650           cone[0] = edgeB; cone[1] = edgeR; cone[2] = edgeT; cone[3] = edgeL;
651           ierr    = DMPlexSetCone(dm, face, cone);CHKERRQ(ierr);
652           ierr    = DMPlexSetConeOrientation(dm, face, ornt);CHKERRQ(ierr);
653         }
654       }
655     }
656     /* Build y faces */
657     for (fz = 0; fz < numZEdges; ++fz) {
658       for (fx = 0; fx < numXEdges; ++fx) {
659         for (fy = 0; fy < numYVertices; ++fy) {
660           PetscInt face    = firstYFace + (fz*numXEdges+fx)*numYVertices + fy;
661           PetscInt edgeL   = firstZEdge + (fy*numXVertices+  fx                 )*numZEdges + fz;
662           PetscInt edgeR   = firstZEdge + (fy*numXVertices+((fx+1)%numXVertices))*numZEdges + fz;
663           PetscInt edgeB   = firstXEdge + (  fz                 *numYVertices+fy)*numXEdges + fx;
664           PetscInt edgeT   = firstXEdge + (((fz+1)%numZVertices)*numYVertices+fy)*numXEdges + fx;
665           PetscInt ornt[4] = {0, 0, -2, -2};
666           PetscInt cone[4];
667 
668           if (dim == 3) {
669             /* markers */
670             if (bdY != DM_BOUNDARY_PERIODIC) {
671               if (fy == numYVertices-1) {
672                 ierr = DMSetLabelValue(dm, "Face Sets", face, faceMarkerBack);CHKERRQ(ierr);
673                 ierr = DMSetLabelValue(dm, "marker", face, markerBack);CHKERRQ(ierr);
674               }
675               else if (fy == 0) {
676                 ierr = DMSetLabelValue(dm, "Face Sets", face, faceMarkerFront);CHKERRQ(ierr);
677                 ierr = DMSetLabelValue(dm, "marker", face, markerFront);CHKERRQ(ierr);
678               }
679             }
680           }
681           cone[0] = edgeB; cone[1] = edgeR; cone[2] = edgeT; cone[3] = edgeL;
682           ierr    = DMPlexSetCone(dm, face, cone);CHKERRQ(ierr);
683           ierr    = DMPlexSetConeOrientation(dm, face, ornt);CHKERRQ(ierr);
684         }
685       }
686     }
687     /* Build z faces */
688     for (fy = 0; fy < numYEdges; ++fy) {
689       for (fx = 0; fx < numXEdges; ++fx) {
690         for (fz = 0; fz < numZVertices; fz++) {
691           PetscInt face    = firstZFace + (fy*numXEdges+fx)*numZVertices + fz;
692           PetscInt edgeL   = firstYEdge + (fz*numXVertices+  fx                 )*numYEdges + fy;
693           PetscInt edgeR   = firstYEdge + (fz*numXVertices+((fx+1)%numXVertices))*numYEdges + fy;
694           PetscInt edgeB   = firstXEdge + (fz*numYVertices+  fy                 )*numXEdges + fx;
695           PetscInt edgeT   = firstXEdge + (fz*numYVertices+((fy+1)%numYVertices))*numXEdges + fx;
696           PetscInt ornt[4] = {0, 0, -2, -2};
697           PetscInt cone[4];
698 
699           if (dim == 2) {
700             if (bdX == DM_BOUNDARY_TWIST && fx == numXEdges-1) {edgeR += numYEdges-1-2*fy; ornt[1] = -2;}
701             if (bdY == DM_BOUNDARY_TWIST && fy == numYEdges-1) {edgeT += numXEdges-1-2*fx; ornt[2] =  0;}
702             if (bdX != DM_BOUNDARY_NONE && fx == numXEdges-1 && cutLabel) {ierr = DMLabelSetValue(cutLabel, face, 2);CHKERRQ(ierr);}
703             if (bdY != DM_BOUNDARY_NONE && fy == numYEdges-1 && cutLabel) {ierr = DMLabelSetValue(cutLabel, face, 2);CHKERRQ(ierr);}
704           } else {
705             /* markers */
706             if (bdZ != DM_BOUNDARY_PERIODIC) {
707               if (fz == numZVertices-1) {
708                 ierr = DMSetLabelValue(dm, "Face Sets", face, faceMarkerTop);CHKERRQ(ierr);
709                 ierr = DMSetLabelValue(dm, "marker", face, markerTop);CHKERRQ(ierr);
710               }
711               else if (fz == 0) {
712                 ierr = DMSetLabelValue(dm, "Face Sets", face, faceMarkerBottom);CHKERRQ(ierr);
713                 ierr = DMSetLabelValue(dm, "marker", face, markerBottom);CHKERRQ(ierr);
714               }
715             }
716           }
717           cone[0] = edgeB; cone[1] = edgeR; cone[2] = edgeT; cone[3] = edgeL;
718           ierr    = DMPlexSetCone(dm, face, cone);CHKERRQ(ierr);
719           ierr    = DMPlexSetConeOrientation(dm, face, ornt);CHKERRQ(ierr);
720         }
721       }
722     }
723     /* Build Z edges*/
724     for (vy = 0; vy < numYVertices; vy++) {
725       for (vx = 0; vx < numXVertices; vx++) {
726         for (ez = 0; ez < numZEdges; ez++) {
727           const PetscInt edge    = firstZEdge  + (vy*numXVertices+vx)*numZEdges + ez;
728           const PetscInt vertexB = firstVertex + (  ez                 *numYVertices+vy)*numXVertices + vx;
729           const PetscInt vertexT = firstVertex + (((ez+1)%numZVertices)*numYVertices+vy)*numXVertices + vx;
730           PetscInt       cone[2];
731 
732           if (dim == 3) {
733             if (bdX != DM_BOUNDARY_PERIODIC) {
734               if (vx == numXVertices-1) {
735                 ierr = DMSetLabelValue(dm, "marker", edge, markerRight);CHKERRQ(ierr);
736               }
737               else if (vx == 0) {
738                 ierr = DMSetLabelValue(dm, "marker", edge, markerLeft);CHKERRQ(ierr);
739               }
740             }
741             if (bdY != DM_BOUNDARY_PERIODIC) {
742               if (vy == numYVertices-1) {
743                 ierr = DMSetLabelValue(dm, "marker", edge, markerBack);CHKERRQ(ierr);
744               }
745               else if (vy == 0) {
746                 ierr = DMSetLabelValue(dm, "marker", edge, markerFront);CHKERRQ(ierr);
747               }
748             }
749           }
750           cone[0] = vertexB; cone[1] = vertexT;
751           ierr = DMPlexSetCone(dm, edge, cone);CHKERRQ(ierr);
752         }
753       }
754     }
755     /* Build Y edges*/
756     for (vz = 0; vz < numZVertices; vz++) {
757       for (vx = 0; vx < numXVertices; vx++) {
758         for (ey = 0; ey < numYEdges; ey++) {
759           const PetscInt nextv   = (dim == 2 && bdY == DM_BOUNDARY_TWIST && ey == numYEdges-1) ? (numXVertices-vx-1) : (vz*numYVertices+((ey+1)%numYVertices))*numXVertices + vx;
760           const PetscInt edge    = firstYEdge  + (vz*numXVertices+vx)*numYEdges + ey;
761           const PetscInt vertexF = firstVertex + (vz*numYVertices+ey)*numXVertices + vx;
762           const PetscInt vertexK = firstVertex + nextv;
763           PetscInt       cone[2];
764 
765           cone[0] = vertexF; cone[1] = vertexK;
766           ierr = DMPlexSetCone(dm, edge, cone);CHKERRQ(ierr);
767           if (dim == 2) {
768             if ((bdX != DM_BOUNDARY_PERIODIC) && (bdX != DM_BOUNDARY_TWIST)) {
769               if (vx == numXVertices-1) {
770                 ierr = DMSetLabelValue(dm, "Face Sets", edge, faceMarkerRight);CHKERRQ(ierr);
771                 ierr = DMSetLabelValue(dm, "marker", edge,    markerRight);CHKERRQ(ierr);
772                 ierr = DMSetLabelValue(dm, "marker", cone[0], markerRight);CHKERRQ(ierr);
773                 if (ey == numYEdges-1) {
774                   ierr = DMSetLabelValue(dm, "marker", cone[1], markerRight);CHKERRQ(ierr);
775                 }
776               } else if (vx == 0) {
777                 ierr = DMSetLabelValue(dm, "Face Sets", edge, faceMarkerLeft);CHKERRQ(ierr);
778                 ierr = DMSetLabelValue(dm, "marker", edge,    markerLeft);CHKERRQ(ierr);
779                 ierr = DMSetLabelValue(dm, "marker", cone[0], markerLeft);CHKERRQ(ierr);
780                 if (ey == numYEdges-1) {
781                   ierr = DMSetLabelValue(dm, "marker", cone[1], markerLeft);CHKERRQ(ierr);
782                 }
783               }
784             } else {
785               if (vx == 0 && cutMarker) {
786                 ierr = DMLabelSetValue(cutLabel, edge,    1);CHKERRQ(ierr);
787                 ierr = DMLabelSetValue(cutLabel, cone[0], 1);CHKERRQ(ierr);
788                 if (ey == numYEdges-1) {
789                   ierr = DMLabelSetValue(cutLabel, cone[1], 1);CHKERRQ(ierr);
790                 }
791               }
792             }
793           } else {
794             if (bdX != DM_BOUNDARY_PERIODIC) {
795               if (vx == numXVertices-1) {
796                 ierr = DMSetLabelValue(dm, "marker", edge, markerRight);CHKERRQ(ierr);
797               } else if (vx == 0) {
798                 ierr = DMSetLabelValue(dm, "marker", edge, markerLeft);CHKERRQ(ierr);
799               }
800             }
801             if (bdZ != DM_BOUNDARY_PERIODIC) {
802               if (vz == numZVertices-1) {
803                 ierr = DMSetLabelValue(dm, "marker", edge, markerTop);CHKERRQ(ierr);
804               } else if (vz == 0) {
805                 ierr = DMSetLabelValue(dm, "marker", edge, markerBottom);CHKERRQ(ierr);
806               }
807             }
808           }
809         }
810       }
811     }
812     /* Build X edges*/
813     for (vz = 0; vz < numZVertices; vz++) {
814       for (vy = 0; vy < numYVertices; vy++) {
815         for (ex = 0; ex < numXEdges; ex++) {
816           const PetscInt nextv   = (dim == 2 && bdX == DM_BOUNDARY_TWIST && ex == numXEdges-1) ? (numYVertices-vy-1)*numXVertices : (vz*numYVertices+vy)*numXVertices + (ex+1)%numXVertices;
817           const PetscInt edge    = firstXEdge  + (vz*numYVertices+vy)*numXEdges + ex;
818           const PetscInt vertexL = firstVertex + (vz*numYVertices+vy)*numXVertices + ex;
819           const PetscInt vertexR = firstVertex + nextv;
820           PetscInt       cone[2];
821 
822           cone[0] = vertexL; cone[1] = vertexR;
823           ierr = DMPlexSetCone(dm, edge, cone);CHKERRQ(ierr);
824           if (dim == 2) {
825             if ((bdY != DM_BOUNDARY_PERIODIC) && (bdY != DM_BOUNDARY_TWIST)) {
826               if (vy == numYVertices-1) {
827                 ierr = DMSetLabelValue(dm, "Face Sets", edge, faceMarkerTop);CHKERRQ(ierr);
828                 ierr = DMSetLabelValue(dm, "marker", edge,    markerTop);CHKERRQ(ierr);
829                 ierr = DMSetLabelValue(dm, "marker", cone[0], markerTop);CHKERRQ(ierr);
830                 if (ex == numXEdges-1) {
831                   ierr = DMSetLabelValue(dm, "marker", cone[1], markerTop);CHKERRQ(ierr);
832                 }
833               } else if (vy == 0) {
834                 ierr = DMSetLabelValue(dm, "Face Sets", edge, faceMarkerBottom);CHKERRQ(ierr);
835                 ierr = DMSetLabelValue(dm, "marker", edge,    markerBottom);CHKERRQ(ierr);
836                 ierr = DMSetLabelValue(dm, "marker", cone[0], markerBottom);CHKERRQ(ierr);
837                 if (ex == numXEdges-1) {
838                   ierr = DMSetLabelValue(dm, "marker", cone[1], markerBottom);CHKERRQ(ierr);
839                 }
840               }
841             } else {
842               if (vy == 0 && cutMarker) {
843                 ierr = DMLabelSetValue(cutLabel, edge,    1);CHKERRQ(ierr);
844                 ierr = DMLabelSetValue(cutLabel, cone[0], 1);CHKERRQ(ierr);
845                 if (ex == numXEdges-1) {
846                   ierr = DMLabelSetValue(cutLabel, cone[1], 1);CHKERRQ(ierr);
847                 }
848               }
849             }
850           } else {
851             if (bdY != DM_BOUNDARY_PERIODIC) {
852               if (vy == numYVertices-1) {
853                 ierr = DMSetLabelValue(dm, "marker", edge, markerBack);CHKERRQ(ierr);
854               }
855               else if (vy == 0) {
856                 ierr = DMSetLabelValue(dm, "marker", edge, markerFront);CHKERRQ(ierr);
857               }
858             }
859             if (bdZ != DM_BOUNDARY_PERIODIC) {
860               if (vz == numZVertices-1) {
861                 ierr = DMSetLabelValue(dm, "marker", edge, markerTop);CHKERRQ(ierr);
862               }
863               else if (vz == 0) {
864                 ierr = DMSetLabelValue(dm, "marker", edge, markerBottom);CHKERRQ(ierr);
865               }
866             }
867           }
868         }
869       }
870     }
871     ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
872     ierr = DMPlexStratify(dm);CHKERRQ(ierr);
873     /* Build coordinates */
874     ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
875     ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
876     ierr = PetscSectionSetFieldComponents(coordSection, 0, dim);CHKERRQ(ierr);
877     ierr = PetscSectionSetChart(coordSection, firstVertex, firstVertex+numVertices);CHKERRQ(ierr);
878     for (v = firstVertex; v < firstVertex+numVertices; ++v) {
879       ierr = PetscSectionSetDof(coordSection, v, dim);CHKERRQ(ierr);
880       ierr = PetscSectionSetFieldDof(coordSection, v, 0, dim);CHKERRQ(ierr);
881     }
882     ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
883     ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
884     ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
885     ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
886     ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
887     ierr = VecSetBlockSize(coordinates, dim);CHKERRQ(ierr);
888     ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
889     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
890     for (vz = 0; vz < numZVertices; ++vz) {
891       for (vy = 0; vy < numYVertices; ++vy) {
892         for (vx = 0; vx < numXVertices; ++vx) {
893           coords[((vz*numYVertices+vy)*numXVertices+vx)*dim+0] = lower[0] + ((upper[0] - lower[0])/numXEdges)*vx;
894           coords[((vz*numYVertices+vy)*numXVertices+vx)*dim+1] = lower[1] + ((upper[1] - lower[1])/numYEdges)*vy;
895           if (dim == 3) {
896             coords[((vz*numYVertices+vy)*numXVertices+vx)*dim+2] = lower[2] + ((upper[2] - lower[2])/numZEdges)*vz;
897           }
898         }
899       }
900     }
901     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
902     ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
903     ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
904   }
905   PetscFunctionReturn(0);
906 }
907 
908 /*@
909   DMPlexCreateHexBoxMesh - Creates a mesh on the tensor product of unit intervals (box) using hexahedra.
910 
911   Collective on MPI_Comm
912 
913   Input Parameters:
914 + comm  - The communicator for the DM object
915 . dim   - The spatial dimension
916 . periodicX - The boundary type for the X direction
917 . periodicY - The boundary type for the Y direction
918 . periodicZ - The boundary type for the Z direction
919 - cells - The number of cells in each direction
920 
921   Output Parameter:
922 . dm  - The DM object
923 
924   Note: Here is the numbering returned for 2 cells in each direction:
925 $ 22--8-23--9--24
926 $  |     |     |
927 $ 13  2 14  3  15
928 $  |     |     |
929 $ 19--6-20--7--21
930 $  |     |     |
931 $ 10  0 11  1 12
932 $  |     |     |
933 $ 16--4-17--5--18
934 
935   Level: beginner
936 
937 .keywords: DM, create
938 .seealso: DMPlexCreateBoxMesh(), DMSetType(), DMCreate()
939 @*/
940 PetscErrorCode DMPlexCreateHexBoxMesh(MPI_Comm comm, PetscInt dim, const PetscInt cells[], DMBoundaryType periodicX, DMBoundaryType periodicY, DMBoundaryType periodicZ, DM *dm)
941 {
942   PetscInt       i;
943   PetscErrorCode ierr;
944 
945   PetscFunctionBegin;
946   PetscValidPointer(dm, 7);
947   ierr = DMCreate(comm, dm);CHKERRQ(ierr);
948   PetscValidLogicalCollectiveInt(*dm,dim,2);
949   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
950   ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
951   switch (dim) {
952   case 2:
953   {
954     PetscReal lower[3] = {0.0, 0.0, 0.0};
955     PetscReal upper[3] = {1.0, 1.0, 0.0};
956     PetscInt  edges[3];
957 
958     edges[0] = cells[0];
959     edges[1] = cells[1];
960     edges[2] = 0;
961 
962     ierr = DMPlexCreateCubeMesh_Internal(*dm, lower, upper, edges, periodicX, periodicY, DM_BOUNDARY_NONE);CHKERRQ(ierr);
963     if (periodicX == DM_BOUNDARY_PERIODIC || periodicX == DM_BOUNDARY_TWIST ||
964         periodicY == DM_BOUNDARY_PERIODIC || periodicY == DM_BOUNDARY_TWIST) {
965       PetscReal      L[2];
966       PetscReal      maxCell[2];
967       DMBoundaryType bdType[2];
968 
969       bdType[0] = periodicX;
970       bdType[1] = periodicY;
971       for (i = 0; i < dim; i++) {
972         L[i]       = upper[i] - lower[i];
973         maxCell[i] = 1.1 * (L[i] / PetscMax(1,cells[i]));
974       }
975 
976       ierr = DMSetPeriodicity(*dm,PETSC_TRUE,maxCell,L,bdType);CHKERRQ(ierr);
977     }
978     break;
979   }
980   case 3:
981   {
982     PetscReal lower[3] = {0.0, 0.0, 0.0};
983     PetscReal upper[3] = {1.0, 1.0, 1.0};
984 
985     ierr = DMPlexCreateCubeMesh_Internal(*dm, lower, upper, cells, periodicX, periodicY, periodicZ);CHKERRQ(ierr);
986     if (periodicX == DM_BOUNDARY_PERIODIC || periodicX == DM_BOUNDARY_TWIST ||
987         periodicY == DM_BOUNDARY_PERIODIC || periodicY == DM_BOUNDARY_TWIST ||
988         periodicZ == DM_BOUNDARY_PERIODIC || periodicZ == DM_BOUNDARY_TWIST) {
989       PetscReal      L[3];
990       PetscReal      maxCell[3];
991       DMBoundaryType bdType[3];
992 
993       bdType[0] = periodicX;
994       bdType[1] = periodicY;
995       bdType[2] = periodicZ;
996       for (i = 0; i < dim; i++) {
997         L[i]       = upper[i] - lower[i];
998         maxCell[i] = 1.1 * (L[i] / cells[i]);
999       }
1000 
1001       ierr = DMSetPeriodicity(*dm,PETSC_TRUE,maxCell,L,bdType);CHKERRQ(ierr);
1002     }
1003     break;
1004   }
1005   default:
1006     SETERRQ1(comm, PETSC_ERR_SUP, "Dimension not supported: %d", dim);
1007   }
1008   PetscFunctionReturn(0);
1009 }
1010 
1011 /*@C
1012   DMPlexSetOptionsPrefix - Sets the prefix used for searching for all DM options in the database.
1013 
1014   Logically Collective on DM
1015 
1016   Input Parameters:
1017 + dm - the DM context
1018 - prefix - the prefix to prepend to all option names
1019 
1020   Notes:
1021   A hyphen (-) must NOT be given at the beginning of the prefix name.
1022   The first character of all runtime options is AUTOMATICALLY the hyphen.
1023 
1024   Level: advanced
1025 
1026 .seealso: SNESSetFromOptions()
1027 @*/
1028 PetscErrorCode DMPlexSetOptionsPrefix(DM dm, const char prefix[])
1029 {
1030   DM_Plex       *mesh = (DM_Plex *) dm->data;
1031   PetscErrorCode ierr;
1032 
1033   PetscFunctionBegin;
1034   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1035   ierr = PetscObjectSetOptionsPrefix((PetscObject) dm, prefix);CHKERRQ(ierr);
1036   ierr = PetscObjectSetOptionsPrefix((PetscObject) mesh->partitioner, prefix);CHKERRQ(ierr);
1037   PetscFunctionReturn(0);
1038 }
1039 
1040 /*@
1041   DMPlexCreateHexCylinderMesh - Creates a mesh on the tensor product of the unit interval with the circle (cylinder) using hexahedra.
1042 
1043   Collective on MPI_Comm
1044 
1045   Input Parameters:
1046 + comm      - The communicator for the DM object
1047 . numRefine - The number of regular refinements to the basic 5 cell structure
1048 - periodicZ - The boundary type for the Z direction
1049 
1050   Output Parameter:
1051 . dm  - The DM object
1052 
1053   Note: Here is the output numbering looking from the bottom of the cylinder:
1054 $       17-----14
1055 $        |     |
1056 $        |  2  |
1057 $        |     |
1058 $ 17-----8-----7-----14
1059 $  |     |     |     |
1060 $  |  3  |  0  |  1  |
1061 $  |     |     |     |
1062 $ 19-----5-----6-----13
1063 $        |     |
1064 $        |  4  |
1065 $        |     |
1066 $       19-----13
1067 $
1068 $ and up through the top
1069 $
1070 $       18-----16
1071 $        |     |
1072 $        |  2  |
1073 $        |     |
1074 $ 18----10----11-----16
1075 $  |     |     |     |
1076 $  |  3  |  0  |  1  |
1077 $  |     |     |     |
1078 $ 20-----9----12-----15
1079 $        |     |
1080 $        |  4  |
1081 $        |     |
1082 $       20-----15
1083 
1084   Level: beginner
1085 
1086 .keywords: DM, create
1087 .seealso: DMPlexCreateHexBoxMesh(), DMPlexCreateBoxMesh(), DMSetType(), DMCreate()
1088 @*/
1089 PetscErrorCode DMPlexCreateHexCylinderMesh(MPI_Comm comm, PetscInt numRefine, DMBoundaryType periodicZ, DM *dm)
1090 {
1091   const PetscInt dim = 3;
1092   PetscInt       numCells, numVertices, r;
1093   PetscErrorCode ierr;
1094 
1095   PetscFunctionBegin;
1096   PetscValidPointer(dm, 4);
1097   if (numRefine < 0) SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Number of refinements %D cannot be negative", numRefine);
1098   ierr = DMCreate(comm, dm);CHKERRQ(ierr);
1099   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
1100   ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
1101   /* Create topology */
1102   {
1103     PetscInt cone[8], c;
1104 
1105     numCells    = 5;
1106     numVertices = 16;
1107     if (periodicZ == DM_BOUNDARY_PERIODIC) {
1108       numCells   *= 3;
1109       numVertices = 24;
1110     }
1111     ierr = DMPlexSetChart(*dm, 0, numCells+numVertices);CHKERRQ(ierr);
1112     for (c = 0; c < numCells; c++) {ierr = DMPlexSetConeSize(*dm, c, 8);CHKERRQ(ierr);}
1113     ierr = DMSetUp(*dm);CHKERRQ(ierr);
1114     if (periodicZ == DM_BOUNDARY_PERIODIC) {
1115       cone[0] = 15; cone[1] = 18; cone[2] = 17; cone[3] = 16;
1116       cone[4] = 31; cone[5] = 32; cone[6] = 33; cone[7] = 34;
1117       ierr = DMPlexSetCone(*dm, 0, cone);CHKERRQ(ierr);
1118       cone[0] = 16; cone[1] = 17; cone[2] = 24; cone[3] = 23;
1119       cone[4] = 32; cone[5] = 36; cone[6] = 37; cone[7] = 33; /* 22 25 26 21 */
1120       ierr = DMPlexSetCone(*dm, 1, cone);CHKERRQ(ierr);
1121       cone[0] = 18; cone[1] = 27; cone[2] = 24; cone[3] = 17;
1122       cone[4] = 34; cone[5] = 33; cone[6] = 37; cone[7] = 38;
1123       ierr = DMPlexSetCone(*dm, 2, cone);CHKERRQ(ierr);
1124       cone[0] = 29; cone[1] = 27; cone[2] = 18; cone[3] = 15;
1125       cone[4] = 35; cone[5] = 31; cone[6] = 34; cone[7] = 38;
1126       ierr = DMPlexSetCone(*dm, 3, cone);CHKERRQ(ierr);
1127       cone[0] = 29; cone[1] = 15; cone[2] = 16; cone[3] = 23;
1128       cone[4] = 35; cone[5] = 36; cone[6] = 32; cone[7] = 31;
1129       ierr = DMPlexSetCone(*dm, 4, cone);CHKERRQ(ierr);
1130 
1131       cone[0] = 31; cone[1] = 34; cone[2] = 33; cone[3] = 32;
1132       cone[4] = 19; cone[5] = 22; cone[6] = 21; cone[7] = 20;
1133       ierr = DMPlexSetCone(*dm, 5, cone);CHKERRQ(ierr);
1134       cone[0] = 32; cone[1] = 33; cone[2] = 37; cone[3] = 36;
1135       cone[4] = 22; cone[5] = 25; cone[6] = 26; cone[7] = 21;
1136       ierr = DMPlexSetCone(*dm, 6, cone);CHKERRQ(ierr);
1137       cone[0] = 34; cone[1] = 38; cone[2] = 37; cone[3] = 33;
1138       cone[4] = 20; cone[5] = 21; cone[6] = 26; cone[7] = 28;
1139       ierr = DMPlexSetCone(*dm, 7, cone);CHKERRQ(ierr);
1140       cone[0] = 35; cone[1] = 38; cone[2] = 34; cone[3] = 31;
1141       cone[4] = 30; cone[5] = 19; cone[6] = 20; cone[7] = 28;
1142       ierr = DMPlexSetCone(*dm, 8, cone);CHKERRQ(ierr);
1143       cone[0] = 35; cone[1] = 31; cone[2] = 32; cone[3] = 36;
1144       cone[4] = 30; cone[5] = 25; cone[6] = 22; cone[7] = 19;
1145       ierr = DMPlexSetCone(*dm, 9, cone);CHKERRQ(ierr);
1146 
1147       cone[0] = 19; cone[1] = 20; cone[2] = 21; cone[3] = 22;
1148       cone[4] = 15; cone[5] = 16; cone[6] = 17; cone[7] = 18;
1149       ierr = DMPlexSetCone(*dm, 10, cone);CHKERRQ(ierr);
1150       cone[0] = 22; cone[1] = 21; cone[2] = 26; cone[3] = 25;
1151       cone[4] = 16; cone[5] = 23; cone[6] = 24; cone[7] = 17;
1152       ierr = DMPlexSetCone(*dm, 11, cone);CHKERRQ(ierr);
1153       cone[0] = 20; cone[1] = 28; cone[2] = 26; cone[3] = 21;
1154       cone[4] = 18; cone[5] = 17; cone[6] = 24; cone[7] = 27;
1155       ierr = DMPlexSetCone(*dm, 12, cone);CHKERRQ(ierr);
1156       cone[0] = 30; cone[1] = 28; cone[2] = 20; cone[3] = 19;
1157       cone[4] = 29; cone[5] = 15; cone[6] = 18; cone[7] = 27;
1158       ierr = DMPlexSetCone(*dm, 13, cone);CHKERRQ(ierr);
1159       cone[0] = 30; cone[1] = 19; cone[2] = 22; cone[3] = 25;
1160       cone[4] = 29; cone[5] = 23; cone[6] = 16; cone[7] = 15;
1161       ierr = DMPlexSetCone(*dm, 14, cone);CHKERRQ(ierr);
1162     } else {
1163       cone[0] =  5; cone[1] =  8; cone[2] =  7; cone[3] =  6;
1164       cone[4] =  9; cone[5] = 12; cone[6] = 11; cone[7] = 10;
1165       ierr = DMPlexSetCone(*dm, 0, cone);CHKERRQ(ierr);
1166       cone[0] =  6; cone[1] =  7; cone[2] = 14; cone[3] = 13;
1167       cone[4] = 12; cone[5] = 15; cone[6] = 16; cone[7] = 11;
1168       ierr = DMPlexSetCone(*dm, 1, cone);CHKERRQ(ierr);
1169       cone[0] =  8; cone[1] = 17; cone[2] = 14; cone[3] =  7;
1170       cone[4] = 10; cone[5] = 11; cone[6] = 16; cone[7] = 18;
1171       ierr = DMPlexSetCone(*dm, 2, cone);CHKERRQ(ierr);
1172       cone[0] = 19; cone[1] = 17; cone[2] =  8; cone[3] =  5;
1173       cone[4] = 20; cone[5] =  9; cone[6] = 10; cone[7] = 18;
1174       ierr = DMPlexSetCone(*dm, 3, cone);CHKERRQ(ierr);
1175       cone[0] = 19; cone[1] =  5; cone[2] =  6; cone[3] = 13;
1176       cone[4] = 20; cone[5] = 15; cone[6] = 12; cone[7] =  9;
1177       ierr = DMPlexSetCone(*dm, 4, cone);CHKERRQ(ierr);
1178     }
1179     ierr = DMPlexSymmetrize(*dm);CHKERRQ(ierr);
1180     ierr = DMPlexStratify(*dm);CHKERRQ(ierr);
1181   }
1182   /* Interpolate */
1183   {
1184     DM idm = NULL;
1185 
1186     ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
1187     ierr = DMDestroy(dm);CHKERRQ(ierr);
1188     *dm  = idm;
1189   }
1190   /* Create cube geometry */
1191   {
1192     Vec             coordinates;
1193     PetscSection    coordSection;
1194     PetscScalar    *coords;
1195     PetscInt        coordSize, v;
1196     const PetscReal dis = 1.0/PetscSqrtReal(2.0);
1197     const PetscReal ds2 = dis/2.0;
1198 
1199     /* Build coordinates */
1200     ierr = DMGetCoordinateSection(*dm, &coordSection);CHKERRQ(ierr);
1201     ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
1202     ierr = PetscSectionSetFieldComponents(coordSection, 0, dim);CHKERRQ(ierr);
1203     ierr = PetscSectionSetChart(coordSection, numCells, numCells+numVertices);CHKERRQ(ierr);
1204     for (v = numCells; v < numCells+numVertices; ++v) {
1205       ierr = PetscSectionSetDof(coordSection, v, dim);CHKERRQ(ierr);
1206       ierr = PetscSectionSetFieldDof(coordSection, v, 0, dim);CHKERRQ(ierr);
1207     }
1208     ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
1209     ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
1210     ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
1211     ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
1212     ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
1213     ierr = VecSetBlockSize(coordinates, dim);CHKERRQ(ierr);
1214     ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
1215     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
1216     coords[0*dim+0] = -ds2; coords[0*dim+1] = -ds2; coords[0*dim+2] = 0.0;
1217     coords[1*dim+0] =  ds2; coords[1*dim+1] = -ds2; coords[1*dim+2] = 0.0;
1218     coords[2*dim+0] =  ds2; coords[2*dim+1] =  ds2; coords[2*dim+2] = 0.0;
1219     coords[3*dim+0] = -ds2; coords[3*dim+1] =  ds2; coords[3*dim+2] = 0.0;
1220     coords[4*dim+0] = -ds2; coords[4*dim+1] = -ds2; coords[4*dim+2] = 1.0;
1221     coords[5*dim+0] = -ds2; coords[5*dim+1] =  ds2; coords[5*dim+2] = 1.0;
1222     coords[6*dim+0] =  ds2; coords[6*dim+1] =  ds2; coords[6*dim+2] = 1.0;
1223     coords[7*dim+0] =  ds2; coords[7*dim+1] = -ds2; coords[7*dim+2] = 1.0;
1224     coords[ 8*dim+0] =  dis; coords[ 8*dim+1] = -dis; coords[ 8*dim+2] = 0.0;
1225     coords[ 9*dim+0] =  dis; coords[ 9*dim+1] =  dis; coords[ 9*dim+2] = 0.0;
1226     coords[10*dim+0] =  dis; coords[10*dim+1] = -dis; coords[10*dim+2] = 1.0;
1227     coords[11*dim+0] =  dis; coords[11*dim+1] =  dis; coords[11*dim+2] = 1.0;
1228     coords[12*dim+0] = -dis; coords[12*dim+1] =  dis; coords[12*dim+2] = 0.0;
1229     coords[13*dim+0] = -dis; coords[13*dim+1] =  dis; coords[13*dim+2] = 1.0;
1230     coords[14*dim+0] = -dis; coords[14*dim+1] = -dis; coords[14*dim+2] = 0.0;
1231     coords[15*dim+0] = -dis; coords[15*dim+1] = -dis; coords[15*dim+2] = 1.0;
1232     if (periodicZ == DM_BOUNDARY_PERIODIC) {
1233       /* 15 31 19 */ coords[16*dim+0] = -ds2; coords[16*dim+1] = -ds2; coords[16*dim+2] = 0.5;
1234       /* 16 32 22 */ coords[17*dim+0] =  ds2; coords[17*dim+1] = -ds2; coords[17*dim+2] = 0.5;
1235       /* 17 33 21 */ coords[18*dim+0] =  ds2; coords[18*dim+1] =  ds2; coords[18*dim+2] = 0.5;
1236       /* 18 34 20 */ coords[19*dim+0] = -ds2; coords[19*dim+1] =  ds2; coords[19*dim+2] = 0.5;
1237       /* 29 35 30 */ coords[20*dim+0] = -dis; coords[20*dim+1] = -dis; coords[20*dim+2] = 0.5;
1238       /* 23 36 25 */ coords[21*dim+0] =  dis; coords[21*dim+1] = -dis; coords[21*dim+2] = 0.5;
1239       /* 24 37 26 */ coords[22*dim+0] =  dis; coords[22*dim+1] =  dis; coords[22*dim+2] = 0.5;
1240       /* 27 38 28 */ coords[23*dim+0] = -dis; coords[23*dim+1] =  dis; coords[23*dim+2] = 0.5;
1241     }
1242     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
1243     ierr = DMSetCoordinatesLocal(*dm, coordinates);CHKERRQ(ierr);
1244     ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
1245   }
1246   /* Create periodicity */
1247   if (periodicZ == DM_BOUNDARY_PERIODIC || periodicZ == DM_BOUNDARY_TWIST) {
1248     PetscReal      L[3];
1249     PetscReal      maxCell[3];
1250     DMBoundaryType bdType[3];
1251     PetscReal      lower[3] = {0.0, 0.0, 0.0};
1252     PetscReal      upper[3] = {1.0, 1.0, 1.5};
1253     PetscInt       i, numZCells = 3;
1254 
1255     bdType[0] = DM_BOUNDARY_NONE;
1256     bdType[1] = DM_BOUNDARY_NONE;
1257     bdType[2] = periodicZ;
1258     for (i = 0; i < dim; i++) {
1259       L[i]       = upper[i] - lower[i];
1260       maxCell[i] = 1.1 * (L[i] / numZCells);
1261     }
1262     ierr = DMSetPeriodicity(*dm, PETSC_TRUE, maxCell, L, bdType);CHKERRQ(ierr);
1263   }
1264   /* Refine topology */
1265   for (r = 0; r < numRefine; ++r) {
1266     DM rdm = NULL;
1267 
1268     ierr = DMRefine(*dm, comm, &rdm);CHKERRQ(ierr);
1269     ierr = DMDestroy(dm);CHKERRQ(ierr);
1270     *dm  = rdm;
1271   }
1272   /* Remap geometry to cylinder
1273        Interior square: Linear interpolation is correct
1274        The other cells all have vertices on rays from the origin. We want to uniformly expand the spacing
1275        such that the last vertex is on the unit circle. So the closest and farthest vertices are at distance
1276 
1277          phi     = arctan(y/x)
1278          d_close = sqrt(1/8 + 1/4 sin^2(phi))
1279          d_far   = sqrt(1/2 + sin^2(phi))
1280 
1281        so we remap them using
1282 
1283          x_new = x_close + (x - x_close) (1 - d_close) / (d_far - d_close)
1284          y_new = y_close + (y - y_close) (1 - d_close) / (d_far - d_close)
1285 
1286        If pi/4 < phi < 3pi/4 or -3pi/4 < phi < -pi/4, then we switch x and y.
1287   */
1288   {
1289     Vec           coordinates;
1290     PetscSection  coordSection;
1291     PetscScalar  *coords;
1292     PetscInt      vStart, vEnd, v;
1293     const PetscReal dis = 1.0/PetscSqrtReal(2.0);
1294     const PetscReal ds2 = 0.5*dis;
1295 
1296     ierr = DMPlexGetDepthStratum(*dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
1297     ierr = DMGetCoordinateSection(*dm, &coordSection);CHKERRQ(ierr);
1298     ierr = DMGetCoordinatesLocal(*dm, &coordinates);CHKERRQ(ierr);
1299     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
1300     for (v = vStart; v < vEnd; ++v) {
1301       PetscReal phi, sinp, cosp, dc, df, x, y, xc, yc;
1302       PetscInt  off;
1303 
1304       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
1305       if ((PetscAbsScalar(coords[off+0]) <= ds2) && (PetscAbsScalar(coords[off+1]) <= ds2)) continue;
1306       x    = PetscRealPart(coords[off]);
1307       y    = PetscRealPart(coords[off+1]);
1308       phi  = PetscAtan2Real(y, x);
1309       sinp = PetscSinReal(phi);
1310       cosp = PetscCosReal(phi);
1311       if ((PetscAbsReal(phi) > PETSC_PI/4.0) && (PetscAbsReal(phi) < 3.0*PETSC_PI/4.0)) {
1312         dc = PetscAbsReal(ds2/sinp);
1313         df = PetscAbsReal(dis/sinp);
1314         xc = ds2*x/PetscAbsReal(y);
1315         yc = ds2*PetscSignReal(y);
1316       } else {
1317         dc = PetscAbsReal(ds2/cosp);
1318         df = PetscAbsReal(dis/cosp);
1319         xc = ds2*PetscSignReal(x);
1320         yc = ds2*y/PetscAbsReal(x);
1321       }
1322       coords[off+0] = xc + (coords[off+0] - xc)*(1.0 - dc)/(df - dc);
1323       coords[off+1] = yc + (coords[off+1] - yc)*(1.0 - dc)/(df - dc);
1324     }
1325     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
1326     if (periodicZ == DM_BOUNDARY_PERIODIC || periodicZ == DM_BOUNDARY_TWIST) {
1327       ierr = DMLocalizeCoordinates(*dm);CHKERRQ(ierr);
1328     }
1329   }
1330   PetscFunctionReturn(0);
1331 }
1332 
1333 /*@
1334   DMPlexCreateWedgeCylinderMesh - Creates a mesh on the tensor product of the unit interval with the circle (cylinder) using wedges.
1335 
1336   Collective on MPI_Comm
1337 
1338   Input Parameters:
1339 + comm - The communicator for the DM object
1340 . n    - The number of wedges around the origin
1341 - interpolate - Create edges and faces
1342 
1343   Output Parameter:
1344 . dm  - The DM object
1345 
1346   Level: beginner
1347 
1348 .keywords: DM, create
1349 .seealso: DMPlexCreateHexCylinderMesh(), DMPlexCreateHexBoxMesh(), DMPlexCreateBoxMesh(), DMSetType(), DMCreate()
1350 @*/
1351 PetscErrorCode DMPlexCreateWedgeCylinderMesh(MPI_Comm comm, PetscInt n, PetscBool interpolate, DM *dm)
1352 {
1353   const PetscInt dim = 3;
1354   PetscInt       numCells, numVertices;
1355   PetscErrorCode ierr;
1356 
1357   PetscFunctionBegin;
1358   PetscValidPointer(dm, 3);
1359   if (n < 0) SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Number of wedges %D cannot be negative", n);
1360   ierr = DMCreate(comm, dm);CHKERRQ(ierr);
1361   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
1362   ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
1363   /* Create topology */
1364   {
1365     PetscInt cone[6], c;
1366 
1367     numCells    = n;
1368     numVertices = 2*(n+1);
1369     ierr = DMPlexSetChart(*dm, 0, numCells+numVertices);CHKERRQ(ierr);
1370     for (c = 0; c < numCells; c++) {ierr = DMPlexSetConeSize(*dm, c, 6);CHKERRQ(ierr);}
1371     ierr = DMSetUp(*dm);CHKERRQ(ierr);
1372     for (c = 0; c < numCells; c++) {
1373       cone[0] =  c+n*1; cone[1] = (c+1)%n+n*1; cone[2] = 0+3*n;
1374       cone[3] =  c+n*2; cone[4] = (c+1)%n+n*2; cone[5] = 1+3*n;
1375       ierr = DMPlexSetCone(*dm, c, cone);CHKERRQ(ierr);
1376     }
1377     ierr = DMPlexSymmetrize(*dm);CHKERRQ(ierr);
1378     ierr = DMPlexStratify(*dm);CHKERRQ(ierr);
1379   }
1380   /* Interpolate */
1381   if (interpolate) {
1382     DM idm = NULL;
1383 
1384     ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
1385     ierr = DMDestroy(dm);CHKERRQ(ierr);
1386     *dm  = idm;
1387   }
1388   /* Create cylinder geometry */
1389   {
1390     Vec          coordinates;
1391     PetscSection coordSection;
1392     PetscScalar *coords;
1393     PetscInt     coordSize, v, c;
1394 
1395     /* Build coordinates */
1396     ierr = DMGetCoordinateSection(*dm, &coordSection);CHKERRQ(ierr);
1397     ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
1398     ierr = PetscSectionSetFieldComponents(coordSection, 0, dim);CHKERRQ(ierr);
1399     ierr = PetscSectionSetChart(coordSection, numCells, numCells+numVertices);CHKERRQ(ierr);
1400     for (v = numCells; v < numCells+numVertices; ++v) {
1401       ierr = PetscSectionSetDof(coordSection, v, dim);CHKERRQ(ierr);
1402       ierr = PetscSectionSetFieldDof(coordSection, v, 0, dim);CHKERRQ(ierr);
1403     }
1404     ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
1405     ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
1406     ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
1407     ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
1408     ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
1409     ierr = VecSetBlockSize(coordinates, dim);CHKERRQ(ierr);
1410     ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
1411     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
1412     for (c = 0; c < numCells; c++) {
1413       coords[(c+0*n)*dim+0] = PetscCosReal(2.0*c*PETSC_PI/n); coords[(c+0*n)*dim+1] = PetscSinReal(2.0*c*PETSC_PI/n); coords[(c+0*n)*dim+2] = 1.0;
1414       coords[(c+1*n)*dim+0] = PetscCosReal(2.0*c*PETSC_PI/n); coords[(c+1*n)*dim+1] = PetscSinReal(2.0*c*PETSC_PI/n); coords[(c+1*n)*dim+2] = 0.0;
1415     }
1416     coords[(2*n+0)*dim+0] = 0.0; coords[(2*n+0)*dim+1] = 0.0; coords[(2*n+0)*dim+2] = 1.0;
1417     coords[(2*n+1)*dim+0] = 0.0; coords[(2*n+1)*dim+1] = 0.0; coords[(2*n+1)*dim+2] = 0.0;
1418     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
1419     ierr = DMSetCoordinatesLocal(*dm, coordinates);CHKERRQ(ierr);
1420     ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
1421   }
1422   PetscFunctionReturn(0);
1423 }
1424 
1425 PETSC_STATIC_INLINE PetscReal DiffNormReal(PetscInt dim, const PetscReal x[], const PetscReal y[])
1426 {
1427   PetscReal prod = 0.0;
1428   PetscInt  i;
1429   for (i = 0; i < dim; ++i) prod += PetscSqr(x[i] - y[i]);
1430   return PetscSqrtReal(prod);
1431 }
1432 PETSC_STATIC_INLINE PetscReal DotReal(PetscInt dim, const PetscReal x[], const PetscReal y[])
1433 {
1434   PetscReal prod = 0.0;
1435   PetscInt  i;
1436   for (i = 0; i < dim; ++i) prod += x[i]*y[i];
1437   return prod;
1438 }
1439 
1440 /*@
1441   DMPlexCreateSphereMesh - Creates a mesh on the d-dimensional sphere, S^d.
1442 
1443   Collective on MPI_Comm
1444 
1445   Input Parameters:
1446 . comm  - The communicator for the DM object
1447 . dim   - The dimension
1448 - simplex - Use simplices, or tensor product cells
1449 
1450   Output Parameter:
1451 . dm  - The DM object
1452 
1453   Level: beginner
1454 
1455 .keywords: DM, create
1456 .seealso: DMPlexCreateBoxMesh(), DMPlexCreateHexBoxMesh(), DMSetType(), DMCreate()
1457 @*/
1458 PetscErrorCode DMPlexCreateSphereMesh(MPI_Comm comm, PetscInt dim, PetscBool simplex, DM *dm)
1459 {
1460   const PetscInt  embedDim = dim+1;
1461   PetscSection    coordSection;
1462   Vec             coordinates;
1463   PetscScalar    *coords;
1464   PetscReal      *coordsIn;
1465   PetscInt        numCells, numEdges, numVerts, firstVertex, v, firstEdge, coordSize, d, c, e;
1466   PetscMPIInt     rank;
1467   PetscErrorCode  ierr;
1468 
1469   PetscFunctionBegin;
1470   PetscValidPointer(dm, 4);
1471   ierr = DMCreate(comm, dm);CHKERRQ(ierr);
1472   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
1473   ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
1474   ierr = DMSetCoordinateDim(*dm, dim+1);CHKERRQ(ierr);
1475   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) *dm), &rank);CHKERRQ(ierr);
1476   switch (dim) {
1477   case 2:
1478     if (simplex) {
1479       DM              idm         = NULL;
1480       const PetscReal edgeLen     = 2.0/(1.0 + PETSC_PHI);
1481       const PetscReal vertex[3]   = {0.0, 1.0/(1.0 + PETSC_PHI), PETSC_PHI/(1.0 + PETSC_PHI)};
1482       const PetscInt  degree      = 5;
1483       PetscInt        s[3]        = {1, 1, 1};
1484       PetscInt        cone[3];
1485       PetscInt       *graph, p, i, j, k;
1486 
1487       numCells    = !rank ? 20 : 0;
1488       numVerts    = !rank ? 12 : 0;
1489       firstVertex = numCells;
1490       /* Use icosahedron, which for a unit sphere has coordinates which are all cyclic permutations of
1491 
1492            (0, \pm 1/\phi+1, \pm \phi/\phi+1)
1493 
1494          where \phi^2 - \phi - 1 = 0, meaning \phi is the golden ratio \frac{1 + \sqrt{5}}{2}. The edge
1495          length is then given by 2/\phi = 2 * 2.73606 = 5.47214.
1496        */
1497       /* Construct vertices */
1498       ierr = PetscCalloc1(numVerts * embedDim, &coordsIn);CHKERRQ(ierr);
1499       for (p = 0, i = 0; p < embedDim; ++p) {
1500         for (s[1] = -1; s[1] < 2; s[1] += 2) {
1501           for (s[2] = -1; s[2] < 2; s[2] += 2) {
1502             for (d = 0; d < embedDim; ++d) coordsIn[i*embedDim+d] = s[(d+p)%embedDim]*vertex[(d+p)%embedDim];
1503             ++i;
1504           }
1505         }
1506       }
1507       /* Construct graph */
1508       ierr = PetscCalloc1(numVerts * numVerts, &graph);CHKERRQ(ierr);
1509       for (i = 0; i < numVerts; ++i) {
1510         for (j = 0, k = 0; j < numVerts; ++j) {
1511           if (PetscAbsReal(DiffNormReal(embedDim, &coordsIn[i*embedDim], &coordsIn[j*embedDim]) - edgeLen) < PETSC_SMALL) {graph[i*numVerts+j] = 1; ++k;}
1512         }
1513         if (k != degree) SETERRQ3(comm, PETSC_ERR_PLIB, "Invalid icosahedron, vertex %D degree %D != %D", i, k, degree);
1514       }
1515       /* Build Topology */
1516       ierr = DMPlexSetChart(*dm, 0, numCells+numVerts);CHKERRQ(ierr);
1517       for (c = 0; c < numCells; c++) {
1518         ierr = DMPlexSetConeSize(*dm, c, embedDim);CHKERRQ(ierr);
1519       }
1520       ierr = DMSetUp(*dm);CHKERRQ(ierr); /* Allocate space for cones */
1521       /* Cells */
1522       for (i = 0, c = 0; i < numVerts; ++i) {
1523         for (j = 0; j < i; ++j) {
1524           for (k = 0; k < j; ++k) {
1525             if (graph[i*numVerts+j] && graph[j*numVerts+k] && graph[k*numVerts+i]) {
1526               cone[0] = firstVertex+i; cone[1] = firstVertex+j; cone[2] = firstVertex+k;
1527               /* Check orientation */
1528               {
1529                 const PetscInt epsilon[3][3][3] = {{{0, 0, 0}, {0, 0, 1}, {0, -1, 0}}, {{0, 0, -1}, {0, 0, 0}, {1, 0, 0}}, {{0, 1, 0}, {-1, 0, 0}, {0, 0, 0}}};
1530                 PetscReal normal[3];
1531                 PetscInt  e, f;
1532 
1533                 for (d = 0; d < embedDim; ++d) {
1534                   normal[d] = 0.0;
1535                   for (e = 0; e < embedDim; ++e) {
1536                     for (f = 0; f < embedDim; ++f) {
1537                       normal[d] += epsilon[d][e][f]*(coordsIn[j*embedDim+e] - coordsIn[i*embedDim+e])*(coordsIn[k*embedDim+f] - coordsIn[i*embedDim+f]);
1538                     }
1539                   }
1540                 }
1541                 if (DotReal(embedDim, normal, &coordsIn[i*embedDim]) < 0) {PetscInt tmp = cone[1]; cone[1] = cone[2]; cone[2] = tmp;}
1542               }
1543               ierr = DMPlexSetCone(*dm, c++, cone);CHKERRQ(ierr);
1544             }
1545           }
1546         }
1547       }
1548       ierr = DMPlexSymmetrize(*dm);CHKERRQ(ierr);
1549       ierr = DMPlexStratify(*dm);CHKERRQ(ierr);
1550       ierr = PetscFree(graph);CHKERRQ(ierr);
1551       /* Interpolate mesh */
1552       ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
1553       ierr = DMDestroy(dm);CHKERRQ(ierr);
1554       *dm  = idm;
1555     } else {
1556       /*
1557         12-21--13
1558          |     |
1559         25  4  24
1560          |     |
1561   12-25--9-16--8-24--13
1562    |     |     |     |
1563   23  5 17  0 15  3  22
1564    |     |     |     |
1565   10-20--6-14--7-19--11
1566          |     |
1567         20  1  19
1568          |     |
1569         10-18--11
1570          |     |
1571         23  2  22
1572          |     |
1573         12-21--13
1574        */
1575       const PetscReal dist = 1.0/PetscSqrtReal(3.0);
1576       PetscInt        cone[4], ornt[4];
1577 
1578       numCells    = !rank ?  6 : 0;
1579       numEdges    = !rank ? 12 : 0;
1580       numVerts    = !rank ?  8 : 0;
1581       firstVertex = numCells;
1582       firstEdge   = numCells + numVerts;
1583       /* Build Topology */
1584       ierr = DMPlexSetChart(*dm, 0, numCells+numEdges+numVerts);CHKERRQ(ierr);
1585       for (c = 0; c < numCells; c++) {
1586         ierr = DMPlexSetConeSize(*dm, c, 4);CHKERRQ(ierr);
1587       }
1588       for (e = firstEdge; e < firstEdge+numEdges; ++e) {
1589         ierr = DMPlexSetConeSize(*dm, e, 2);CHKERRQ(ierr);
1590       }
1591       ierr = DMSetUp(*dm);CHKERRQ(ierr); /* Allocate space for cones */
1592       /* Cell 0 */
1593       cone[0] = 14; cone[1] = 15; cone[2] = 16; cone[3] = 17;
1594       ierr = DMPlexSetCone(*dm, 0, cone);CHKERRQ(ierr);
1595       ornt[0] = 0; ornt[1] = 0; ornt[2] = 0; ornt[3] = 0;
1596       ierr = DMPlexSetConeOrientation(*dm, 0, ornt);CHKERRQ(ierr);
1597       /* Cell 1 */
1598       cone[0] = 18; cone[1] = 19; cone[2] = 14; cone[3] = 20;
1599       ierr = DMPlexSetCone(*dm, 1, cone);CHKERRQ(ierr);
1600       ornt[0] = 0; ornt[1] = 0; ornt[2] = -2; ornt[3] = 0;
1601       ierr = DMPlexSetConeOrientation(*dm, 1, ornt);CHKERRQ(ierr);
1602       /* Cell 2 */
1603       cone[0] = 21; cone[1] = 22; cone[2] = 18; cone[3] = 23;
1604       ierr = DMPlexSetCone(*dm, 2, cone);CHKERRQ(ierr);
1605       ornt[0] = 0; ornt[1] = 0; ornt[2] = -2; ornt[3] = 0;
1606       ierr = DMPlexSetConeOrientation(*dm, 2, ornt);CHKERRQ(ierr);
1607       /* Cell 3 */
1608       cone[0] = 19; cone[1] = 22; cone[2] = 24; cone[3] = 15;
1609       ierr = DMPlexSetCone(*dm, 3, cone);CHKERRQ(ierr);
1610       ornt[0] = -2; ornt[1] = -2; ornt[2] = 0; ornt[3] = -2;
1611       ierr = DMPlexSetConeOrientation(*dm, 3, ornt);CHKERRQ(ierr);
1612       /* Cell 4 */
1613       cone[0] = 16; cone[1] = 24; cone[2] = 21; cone[3] = 25;
1614       ierr = DMPlexSetCone(*dm, 4, cone);CHKERRQ(ierr);
1615       ornt[0] = -2; ornt[1] = -2; ornt[2] = -2; ornt[3] = 0;
1616       ierr = DMPlexSetConeOrientation(*dm, 4, ornt);CHKERRQ(ierr);
1617       /* Cell 5 */
1618       cone[0] = 20; cone[1] = 17; cone[2] = 25; cone[3] = 23;
1619       ierr = DMPlexSetCone(*dm, 5, cone);CHKERRQ(ierr);
1620       ornt[0] = -2; ornt[1] = -2; ornt[2] = -2; ornt[3] = -2;
1621       ierr = DMPlexSetConeOrientation(*dm, 5, ornt);CHKERRQ(ierr);
1622       /* Edges */
1623       cone[0] =  6; cone[1] =  7;
1624       ierr = DMPlexSetCone(*dm, 14, cone);CHKERRQ(ierr);
1625       cone[0] =  7; cone[1] =  8;
1626       ierr = DMPlexSetCone(*dm, 15, cone);CHKERRQ(ierr);
1627       cone[0] =  8; cone[1] =  9;
1628       ierr = DMPlexSetCone(*dm, 16, cone);CHKERRQ(ierr);
1629       cone[0] =  9; cone[1] =  6;
1630       ierr = DMPlexSetCone(*dm, 17, cone);CHKERRQ(ierr);
1631       cone[0] = 10; cone[1] = 11;
1632       ierr = DMPlexSetCone(*dm, 18, cone);CHKERRQ(ierr);
1633       cone[0] = 11; cone[1] =  7;
1634       ierr = DMPlexSetCone(*dm, 19, cone);CHKERRQ(ierr);
1635       cone[0] =  6; cone[1] = 10;
1636       ierr = DMPlexSetCone(*dm, 20, cone);CHKERRQ(ierr);
1637       cone[0] = 12; cone[1] = 13;
1638       ierr = DMPlexSetCone(*dm, 21, cone);CHKERRQ(ierr);
1639       cone[0] = 13; cone[1] = 11;
1640       ierr = DMPlexSetCone(*dm, 22, cone);CHKERRQ(ierr);
1641       cone[0] = 10; cone[1] = 12;
1642       ierr = DMPlexSetCone(*dm, 23, cone);CHKERRQ(ierr);
1643       cone[0] = 13; cone[1] =  8;
1644       ierr = DMPlexSetCone(*dm, 24, cone);CHKERRQ(ierr);
1645       cone[0] = 12; cone[1] =  9;
1646       ierr = DMPlexSetCone(*dm, 25, cone);CHKERRQ(ierr);
1647       ierr = DMPlexSymmetrize(*dm);CHKERRQ(ierr);
1648       ierr = DMPlexStratify(*dm);CHKERRQ(ierr);
1649       /* Build coordinates */
1650       ierr = PetscCalloc1(numVerts * embedDim, &coordsIn);CHKERRQ(ierr);
1651       coordsIn[0*embedDim+0] = -dist; coordsIn[0*embedDim+1] =  dist; coordsIn[0*embedDim+2] = -dist;
1652       coordsIn[1*embedDim+0] =  dist; coordsIn[1*embedDim+1] =  dist; coordsIn[1*embedDim+2] = -dist;
1653       coordsIn[2*embedDim+0] =  dist; coordsIn[2*embedDim+1] = -dist; coordsIn[2*embedDim+2] = -dist;
1654       coordsIn[3*embedDim+0] = -dist; coordsIn[3*embedDim+1] = -dist; coordsIn[3*embedDim+2] = -dist;
1655       coordsIn[4*embedDim+0] = -dist; coordsIn[4*embedDim+1] =  dist; coordsIn[4*embedDim+2] =  dist;
1656       coordsIn[5*embedDim+0] =  dist; coordsIn[5*embedDim+1] =  dist; coordsIn[5*embedDim+2] =  dist;
1657       coordsIn[6*embedDim+0] = -dist; coordsIn[6*embedDim+1] = -dist; coordsIn[6*embedDim+2] =  dist;
1658       coordsIn[7*embedDim+0] =  dist; coordsIn[7*embedDim+1] = -dist; coordsIn[7*embedDim+2] =  dist;
1659     }
1660     break;
1661   case 3:
1662     if (simplex) {
1663       DM              idm             = NULL;
1664       const PetscReal edgeLen         = 1.0/PETSC_PHI;
1665       const PetscReal vertexA[4]      = {0.5, 0.5, 0.5, 0.5};
1666       const PetscReal vertexB[4]      = {1.0, 0.0, 0.0, 0.0};
1667       const PetscReal vertexC[4]      = {0.5, 0.5*PETSC_PHI, 0.5/PETSC_PHI, 0.0};
1668       const PetscInt  degree          = 12;
1669       PetscInt        s[4]            = {1, 1, 1};
1670       PetscInt        evenPerm[12][4] = {{0, 1, 2, 3}, {0, 2, 3, 1}, {0, 3, 1, 2}, {1, 0, 3, 2}, {1, 2, 0, 3}, {1, 3, 2, 0},
1671                                          {2, 0, 1, 3}, {2, 1, 3, 0}, {2, 3, 0, 1}, {3, 0, 2, 1}, {3, 1, 0, 2}, {3, 2, 1, 0}};
1672       PetscInt        cone[4];
1673       PetscInt       *graph, p, i, j, k, l;
1674 
1675       numCells    = !rank ? 600 : 0;
1676       numVerts    = !rank ? 120 : 0;
1677       firstVertex = numCells;
1678       /* Use the 600-cell, which for a unit sphere has coordinates which are
1679 
1680            1/2 (\pm 1, \pm 1,    \pm 1, \pm 1)                          16
1681                (\pm 1,    0,       0,      0)  all cyclic permutations   8
1682            1/2 (\pm 1, \pm phi, \pm 1/phi, 0)  all even permutations    96
1683 
1684          where \phi^2 - \phi - 1 = 0, meaning \phi is the golden ratio \frac{1 + \sqrt{5}}{2}. The edge
1685          length is then given by 1/\phi = 2.73606.
1686 
1687          http://buzzard.pugetsound.edu/sage-practice/ch03s03.html
1688          http://mathworld.wolfram.com/600-Cell.html
1689        */
1690       /* Construct vertices */
1691       ierr = PetscCalloc1(numVerts * embedDim, &coordsIn);CHKERRQ(ierr);
1692       i    = 0;
1693       for (s[0] = -1; s[0] < 2; s[0] += 2) {
1694         for (s[1] = -1; s[1] < 2; s[1] += 2) {
1695           for (s[2] = -1; s[2] < 2; s[2] += 2) {
1696             for (s[3] = -1; s[3] < 2; s[3] += 2) {
1697               for (d = 0; d < embedDim; ++d) coordsIn[i*embedDim+d] = s[d]*vertexA[d];
1698               ++i;
1699             }
1700           }
1701         }
1702       }
1703       for (p = 0; p < embedDim; ++p) {
1704         s[1] = s[2] = s[3] = 1;
1705         for (s[0] = -1; s[0] < 2; s[0] += 2) {
1706           for (d = 0; d < embedDim; ++d) coordsIn[i*embedDim+d] = s[(d+p)%embedDim]*vertexB[(d+p)%embedDim];
1707           ++i;
1708         }
1709       }
1710       for (p = 0; p < 12; ++p) {
1711         s[3] = 1;
1712         for (s[0] = -1; s[0] < 2; s[0] += 2) {
1713           for (s[1] = -1; s[1] < 2; s[1] += 2) {
1714             for (s[2] = -1; s[2] < 2; s[2] += 2) {
1715               for (d = 0; d < embedDim; ++d) coordsIn[i*embedDim+d] = s[evenPerm[p][d]]*vertexC[evenPerm[p][d]];
1716               ++i;
1717             }
1718           }
1719         }
1720       }
1721       if (i != numVerts) SETERRQ2(comm, PETSC_ERR_PLIB, "Invalid 600-cell, vertices %D != %D", i, numVerts);
1722       /* Construct graph */
1723       ierr = PetscCalloc1(numVerts * numVerts, &graph);CHKERRQ(ierr);
1724       for (i = 0; i < numVerts; ++i) {
1725         for (j = 0, k = 0; j < numVerts; ++j) {
1726           if (PetscAbsReal(DiffNormReal(embedDim, &coordsIn[i*embedDim], &coordsIn[j*embedDim]) - edgeLen) < PETSC_SMALL) {graph[i*numVerts+j] = 1; ++k;}
1727         }
1728         if (k != degree) SETERRQ3(comm, PETSC_ERR_PLIB, "Invalid 600-cell, vertex %D degree %D != %D", i, k, degree);
1729       }
1730       /* Build Topology */
1731       ierr = DMPlexSetChart(*dm, 0, numCells+numVerts);CHKERRQ(ierr);
1732       for (c = 0; c < numCells; c++) {
1733         ierr = DMPlexSetConeSize(*dm, c, embedDim);CHKERRQ(ierr);
1734       }
1735       ierr = DMSetUp(*dm);CHKERRQ(ierr); /* Allocate space for cones */
1736       /* Cells */
1737       for (i = 0, c = 0; i < numVerts; ++i) {
1738         for (j = 0; j < i; ++j) {
1739           for (k = 0; k < j; ++k) {
1740             for (l = 0; l < k; ++l) {
1741               if (graph[i*numVerts+j] && graph[j*numVerts+k] && graph[k*numVerts+i] &&
1742                   graph[l*numVerts+i] && graph[l*numVerts+j] && graph[l*numVerts+k]) {
1743                 cone[0] = firstVertex+i; cone[1] = firstVertex+j; cone[2] = firstVertex+k; cone[3] = firstVertex+l;
1744                 /* Check orientation: https://ef.gy/linear-algebra:normal-vectors-in-higher-dimensional-spaces */
1745                 {
1746                   const PetscInt epsilon[4][4][4][4] = {{{{0,  0,  0,  0}, { 0, 0,  0,  0}, { 0,  0, 0,  0}, { 0,  0,  0, 0}},
1747                                                          {{0,  0,  0,  0}, { 0, 0,  0,  0}, { 0,  0, 0,  1}, { 0,  0, -1, 0}},
1748                                                          {{0,  0,  0,  0}, { 0, 0,  0, -1}, { 0,  0, 0,  0}, { 0,  1,  0, 0}},
1749                                                          {{0,  0,  0,  0}, { 0, 0,  1,  0}, { 0, -1, 0,  0}, { 0,  0,  0, 0}}},
1750 
1751                                                         {{{0,  0,  0,  0}, { 0, 0,  0,  0}, { 0,  0, 0, -1}, { 0,  0,  1, 0}},
1752                                                          {{0,  0,  0,  0}, { 0, 0,  0,  0}, { 0,  0, 0,  0}, { 0,  0,  0, 0}},
1753                                                          {{0,  0,  0,  1}, { 0, 0,  0,  0}, { 0,  0, 0,  0}, {-1,  0,  0, 0}},
1754                                                          {{0,  0, -1,  0}, { 0, 0,  0,  0}, { 1,  0, 0,  0}, { 0,  0,  0, 0}}},
1755 
1756                                                         {{{0,  0,  0,  0}, { 0, 0,  0,  1}, { 0,  0, 0,  0}, { 0, -1,  0, 0}},
1757                                                          {{0,  0,  0, -1}, { 0, 0,  0,  0}, { 0,  0, 0,  0}, { 1,  0,  0, 0}},
1758                                                          {{0,  0,  0,  0}, { 0, 0,  0,  0}, { 0,  0, 0,  0}, { 0,  0,  0, 0}},
1759                                                          {{0,  1,  0,  0}, {-1, 0,  0,  0}, { 0,  0, 0,  0}, { 0,  0,  0, 0}}},
1760 
1761                                                         {{{0,  0,  0,  0}, { 0, 0, -1,  0}, { 0,  1, 0,  0}, { 0,  0,  0, 0}},
1762                                                          {{0,  0,  1,  0}, { 0, 0,  0,  0}, {-1,  0, 0,  0}, { 0,  0,  0, 0}},
1763                                                          {{0, -1,  0,  0}, { 1, 0,  0,  0}, { 0,  0, 0,  0}, { 0,  0,  0, 0}},
1764                                                          {{0,  0,  0,  0}, { 0, 0,  0,  0}, { 0,  0, 0,  0}, { 0,  0,  0, 0}}}};
1765                   PetscReal normal[4];
1766                   PetscInt  e, f, g;
1767 
1768                   for (d = 0; d < embedDim; ++d) {
1769                     normal[d] = 0.0;
1770                     for (e = 0; e < embedDim; ++e) {
1771                       for (f = 0; f < embedDim; ++f) {
1772                         for (g = 0; g < embedDim; ++g) {
1773                           normal[d] += epsilon[d][e][f][g]*(coordsIn[j*embedDim+e] - coordsIn[i*embedDim+e])*(coordsIn[k*embedDim+f] - coordsIn[i*embedDim+f])*(coordsIn[l*embedDim+f] - coordsIn[i*embedDim+f]);
1774                         }
1775                       }
1776                     }
1777                   }
1778                   if (DotReal(embedDim, normal, &coordsIn[i*embedDim]) < 0) {PetscInt tmp = cone[1]; cone[1] = cone[2]; cone[2] = tmp;}
1779                 }
1780                 ierr = DMPlexSetCone(*dm, c++, cone);CHKERRQ(ierr);
1781               }
1782             }
1783           }
1784         }
1785       }
1786       ierr = DMPlexSymmetrize(*dm);CHKERRQ(ierr);
1787       ierr = DMPlexStratify(*dm);CHKERRQ(ierr);
1788       ierr = PetscFree(graph);CHKERRQ(ierr);
1789       /* Interpolate mesh */
1790       ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
1791       ierr = DMDestroy(dm);CHKERRQ(ierr);
1792       *dm  = idm;
1793       break;
1794     }
1795   default: SETERRQ1(comm, PETSC_ERR_SUP, "Unsupported dimension for sphere: %D", dim);
1796   }
1797   /* Create coordinates */
1798   ierr = DMGetCoordinateSection(*dm, &coordSection);CHKERRQ(ierr);
1799   ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
1800   ierr = PetscSectionSetFieldComponents(coordSection, 0, embedDim);CHKERRQ(ierr);
1801   ierr = PetscSectionSetChart(coordSection, firstVertex, firstVertex+numVerts);CHKERRQ(ierr);
1802   for (v = firstVertex; v < firstVertex+numVerts; ++v) {
1803     ierr = PetscSectionSetDof(coordSection, v, embedDim);CHKERRQ(ierr);
1804     ierr = PetscSectionSetFieldDof(coordSection, v, 0, embedDim);CHKERRQ(ierr);
1805   }
1806   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
1807   ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
1808   ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
1809   ierr = VecSetBlockSize(coordinates, embedDim);CHKERRQ(ierr);
1810   ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
1811   ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
1812   ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
1813   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
1814   for (v = 0; v < numVerts; ++v) for (d = 0; d < embedDim; ++d) {coords[v*embedDim+d] = coordsIn[v*embedDim+d];}
1815   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
1816   ierr = DMSetCoordinatesLocal(*dm, coordinates);CHKERRQ(ierr);
1817   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
1818   ierr = PetscFree(coordsIn);CHKERRQ(ierr);
1819   PetscFunctionReturn(0);
1820 }
1821 
1822 /* External function declarations here */
1823 extern PetscErrorCode DMCreateInterpolation_Plex(DM dmCoarse, DM dmFine, Mat *interpolation, Vec *scaling);
1824 extern PetscErrorCode DMCreateInjection_Plex(DM dmCoarse, DM dmFine, Mat *mat);
1825 extern PetscErrorCode DMCreateDefaultSection_Plex(DM dm);
1826 extern PetscErrorCode DMCreateDefaultConstraints_Plex(DM dm);
1827 extern PetscErrorCode DMCreateMatrix_Plex(DM dm,  Mat *J);
1828 extern PetscErrorCode DMCreateCoordinateDM_Plex(DM dm, DM *cdm);
1829 PETSC_INTERN PetscErrorCode DMClone_Plex(DM dm, DM *newdm);
1830 extern PetscErrorCode DMSetUp_Plex(DM dm);
1831 extern PetscErrorCode DMDestroy_Plex(DM dm);
1832 extern PetscErrorCode DMView_Plex(DM dm, PetscViewer viewer);
1833 extern PetscErrorCode DMLoad_Plex(DM dm, PetscViewer viewer);
1834 extern PetscErrorCode DMCreateSubDM_Plex(DM dm, PetscInt numFields, PetscInt fields[], IS *is, DM *subdm);
1835 static PetscErrorCode DMInitialize_Plex(DM dm);
1836 
1837 /* Replace dm with the contents of dmNew
1838    - Share the DM_Plex structure
1839    - Share the coordinates
1840    - Share the SF
1841 */
1842 static PetscErrorCode DMPlexReplace_Static(DM dm, DM dmNew)
1843 {
1844   PetscSF               sf;
1845   DM                    coordDM, coarseDM;
1846   Vec                   coords;
1847   PetscBool             isper;
1848   const PetscReal      *maxCell, *L;
1849   const DMBoundaryType *bd;
1850   PetscErrorCode        ierr;
1851 
1852   PetscFunctionBegin;
1853   ierr = DMGetPointSF(dmNew, &sf);CHKERRQ(ierr);
1854   ierr = DMSetPointSF(dm, sf);CHKERRQ(ierr);
1855   ierr = DMGetCoordinateDM(dmNew, &coordDM);CHKERRQ(ierr);
1856   ierr = DMGetCoordinatesLocal(dmNew, &coords);CHKERRQ(ierr);
1857   ierr = DMSetCoordinateDM(dm, coordDM);CHKERRQ(ierr);
1858   ierr = DMSetCoordinatesLocal(dm, coords);CHKERRQ(ierr);
1859   ierr = DMGetPeriodicity(dm, &isper, &maxCell, &L, &bd);CHKERRQ(ierr);
1860   ierr = DMSetPeriodicity(dmNew, isper, maxCell, L, bd);CHKERRQ(ierr);
1861   ierr = DMDestroy_Plex(dm);CHKERRQ(ierr);
1862   ierr = DMInitialize_Plex(dm);CHKERRQ(ierr);
1863   dm->data = dmNew->data;
1864   ((DM_Plex *) dmNew->data)->refct++;
1865   dmNew->labels->refct++;
1866   if (!--(dm->labels->refct)) {
1867     DMLabelLink next = dm->labels->next;
1868 
1869     /* destroy the labels */
1870     while (next) {
1871       DMLabelLink tmp = next->next;
1872 
1873       ierr = DMLabelDestroy(&next->label);CHKERRQ(ierr);
1874       ierr = PetscFree(next);CHKERRQ(ierr);
1875       next = tmp;
1876     }
1877     ierr = PetscFree(dm->labels);CHKERRQ(ierr);
1878   }
1879   dm->labels = dmNew->labels;
1880   dm->depthLabel = dmNew->depthLabel;
1881   ierr = DMGetCoarseDM(dmNew,&coarseDM);CHKERRQ(ierr);
1882   ierr = DMSetCoarseDM(dm,coarseDM);CHKERRQ(ierr);
1883   PetscFunctionReturn(0);
1884 }
1885 
1886 /* Swap dm with the contents of dmNew
1887    - Swap the DM_Plex structure
1888    - Swap the coordinates
1889    - Swap the point PetscSF
1890 */
1891 static PetscErrorCode DMPlexSwap_Static(DM dmA, DM dmB)
1892 {
1893   DM              coordDMA, coordDMB;
1894   Vec             coordsA,  coordsB;
1895   PetscSF         sfA,      sfB;
1896   void            *tmp;
1897   DMLabelLinkList listTmp;
1898   DMLabel         depthTmp;
1899   PetscErrorCode  ierr;
1900 
1901   PetscFunctionBegin;
1902   ierr = DMGetPointSF(dmA, &sfA);CHKERRQ(ierr);
1903   ierr = DMGetPointSF(dmB, &sfB);CHKERRQ(ierr);
1904   ierr = PetscObjectReference((PetscObject) sfA);CHKERRQ(ierr);
1905   ierr = DMSetPointSF(dmA, sfB);CHKERRQ(ierr);
1906   ierr = DMSetPointSF(dmB, sfA);CHKERRQ(ierr);
1907   ierr = PetscObjectDereference((PetscObject) sfA);CHKERRQ(ierr);
1908 
1909   ierr = DMGetCoordinateDM(dmA, &coordDMA);CHKERRQ(ierr);
1910   ierr = DMGetCoordinateDM(dmB, &coordDMB);CHKERRQ(ierr);
1911   ierr = PetscObjectReference((PetscObject) coordDMA);CHKERRQ(ierr);
1912   ierr = DMSetCoordinateDM(dmA, coordDMB);CHKERRQ(ierr);
1913   ierr = DMSetCoordinateDM(dmB, coordDMA);CHKERRQ(ierr);
1914   ierr = PetscObjectDereference((PetscObject) coordDMA);CHKERRQ(ierr);
1915 
1916   ierr = DMGetCoordinatesLocal(dmA, &coordsA);CHKERRQ(ierr);
1917   ierr = DMGetCoordinatesLocal(dmB, &coordsB);CHKERRQ(ierr);
1918   ierr = PetscObjectReference((PetscObject) coordsA);CHKERRQ(ierr);
1919   ierr = DMSetCoordinatesLocal(dmA, coordsB);CHKERRQ(ierr);
1920   ierr = DMSetCoordinatesLocal(dmB, coordsA);CHKERRQ(ierr);
1921   ierr = PetscObjectDereference((PetscObject) coordsA);CHKERRQ(ierr);
1922 
1923   tmp       = dmA->data;
1924   dmA->data = dmB->data;
1925   dmB->data = tmp;
1926   listTmp   = dmA->labels;
1927   dmA->labels = dmB->labels;
1928   dmB->labels = listTmp;
1929   depthTmp  = dmA->depthLabel;
1930   dmA->depthLabel = dmB->depthLabel;
1931   dmB->depthLabel = depthTmp;
1932   PetscFunctionReturn(0);
1933 }
1934 
1935 PetscErrorCode DMSetFromOptions_NonRefinement_Plex(PetscOptionItems *PetscOptionsObject,DM dm)
1936 {
1937   DM_Plex       *mesh = (DM_Plex*) dm->data;
1938   PetscErrorCode ierr;
1939 
1940   PetscFunctionBegin;
1941   /* Handle viewing */
1942   ierr = PetscOptionsBool("-dm_plex_print_set_values", "Output all set values info", "DMView", PETSC_FALSE, &mesh->printSetValues, NULL);CHKERRQ(ierr);
1943   ierr = PetscOptionsInt("-dm_plex_print_fem", "Debug output level all fem computations", "DMView", 0, &mesh->printFEM, NULL);CHKERRQ(ierr);
1944   ierr = PetscOptionsReal("-dm_plex_print_tol", "Tolerance for FEM output", "DMView", mesh->printTol, &mesh->printTol, NULL);CHKERRQ(ierr);
1945   /* Point Location */
1946   ierr = PetscOptionsBool("-dm_plex_hash_location", "Use grid hashing for point location", "DMView", PETSC_FALSE, &mesh->useHashLocation, NULL);CHKERRQ(ierr);
1947   /* Generation and remeshing */
1948   ierr = PetscOptionsBool("-dm_plex_remesh_bd", "Allow changes to the boundary on remeshing", "DMView", PETSC_FALSE, &mesh->remeshBd, NULL);CHKERRQ(ierr);
1949   /* Projection behavior */
1950   ierr = PetscOptionsInt("-dm_plex_max_projection_height", "Maxmimum mesh point height used to project locally", "DMPlexSetMaxProjectionHeight", 0, &mesh->maxProjectionHeight, NULL);CHKERRQ(ierr);
1951   ierr = PetscOptionsBool("-dm_plex_regular_refinement", "Use special nested projection algorithm for regular refinement", "DMPlexSetRegularRefinement", mesh->regularRefinement, &mesh->regularRefinement, NULL);CHKERRQ(ierr);
1952 
1953   ierr = PetscPartitionerSetFromOptions(mesh->partitioner);CHKERRQ(ierr);
1954   PetscFunctionReturn(0);
1955 }
1956 
1957 static PetscErrorCode DMSetFromOptions_Plex(PetscOptionItems *PetscOptionsObject,DM dm)
1958 {
1959   PetscInt       refine = 0, coarsen = 0, r;
1960   PetscBool      isHierarchy;
1961   PetscErrorCode ierr;
1962 
1963   PetscFunctionBegin;
1964   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1965   ierr = PetscOptionsHead(PetscOptionsObject,"DMPlex Options");CHKERRQ(ierr);
1966   /* Handle DMPlex refinement */
1967   ierr = PetscOptionsInt("-dm_refine", "The number of uniform refinements", "DMCreate", refine, &refine, NULL);CHKERRQ(ierr);
1968   ierr = PetscOptionsInt("-dm_refine_hierarchy", "The number of uniform refinements", "DMCreate", refine, &refine, &isHierarchy);CHKERRQ(ierr);
1969   if (refine) {ierr = DMPlexSetRefinementUniform(dm, PETSC_TRUE);CHKERRQ(ierr);}
1970   if (refine && isHierarchy) {
1971     DM *dms, coarseDM;
1972 
1973     ierr = DMGetCoarseDM(dm, &coarseDM);CHKERRQ(ierr);
1974     ierr = PetscObjectReference((PetscObject)coarseDM);CHKERRQ(ierr);
1975     ierr = PetscMalloc1(refine,&dms);CHKERRQ(ierr);
1976     ierr = DMRefineHierarchy(dm, refine, dms);CHKERRQ(ierr);
1977     /* Total hack since we do not pass in a pointer */
1978     ierr = DMPlexSwap_Static(dm, dms[refine-1]);CHKERRQ(ierr);
1979     if (refine == 1) {
1980       ierr = DMSetCoarseDM(dm, dms[0]);CHKERRQ(ierr);
1981       ierr = DMPlexSetRegularRefinement(dm, PETSC_TRUE);CHKERRQ(ierr);
1982     } else {
1983       ierr = DMSetCoarseDM(dm, dms[refine-2]);CHKERRQ(ierr);
1984       ierr = DMPlexSetRegularRefinement(dm, PETSC_TRUE);CHKERRQ(ierr);
1985       ierr = DMSetCoarseDM(dms[0], dms[refine-1]);CHKERRQ(ierr);
1986       ierr = DMPlexSetRegularRefinement(dms[0], PETSC_TRUE);CHKERRQ(ierr);
1987     }
1988     ierr = DMSetCoarseDM(dms[refine-1], coarseDM);CHKERRQ(ierr);
1989     ierr = PetscObjectDereference((PetscObject)coarseDM);CHKERRQ(ierr);
1990     /* Free DMs */
1991     for (r = 0; r < refine; ++r) {
1992       ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dms[r]);CHKERRQ(ierr);
1993       ierr = DMDestroy(&dms[r]);CHKERRQ(ierr);
1994     }
1995     ierr = PetscFree(dms);CHKERRQ(ierr);
1996   } else {
1997     for (r = 0; r < refine; ++r) {
1998       DM refinedMesh;
1999 
2000       ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dm);CHKERRQ(ierr);
2001       ierr = DMRefine(dm, PetscObjectComm((PetscObject) dm), &refinedMesh);CHKERRQ(ierr);
2002       /* Total hack since we do not pass in a pointer */
2003       ierr = DMPlexReplace_Static(dm, refinedMesh);CHKERRQ(ierr);
2004       ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dm);CHKERRQ(ierr);
2005       ierr = DMDestroy(&refinedMesh);CHKERRQ(ierr);
2006     }
2007   }
2008   /* Handle DMPlex coarsening */
2009   ierr = PetscOptionsInt("-dm_coarsen", "Coarsen the mesh", "DMCreate", coarsen, &coarsen, NULL);CHKERRQ(ierr);
2010   ierr = PetscOptionsInt("-dm_coarsen_hierarchy", "The number of coarsenings", "DMCreate", coarsen, &coarsen, &isHierarchy);CHKERRQ(ierr);
2011   if (coarsen && isHierarchy) {
2012     DM *dms;
2013 
2014     ierr = PetscMalloc1(coarsen, &dms);CHKERRQ(ierr);
2015     ierr = DMCoarsenHierarchy(dm, coarsen, dms);CHKERRQ(ierr);
2016     /* Free DMs */
2017     for (r = 0; r < coarsen; ++r) {
2018       ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dms[r]);CHKERRQ(ierr);
2019       ierr = DMDestroy(&dms[r]);CHKERRQ(ierr);
2020     }
2021     ierr = PetscFree(dms);CHKERRQ(ierr);
2022   } else {
2023     for (r = 0; r < coarsen; ++r) {
2024       DM coarseMesh;
2025 
2026       ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dm);CHKERRQ(ierr);
2027       ierr = DMCoarsen(dm, PetscObjectComm((PetscObject) dm), &coarseMesh);CHKERRQ(ierr);
2028       /* Total hack since we do not pass in a pointer */
2029       ierr = DMPlexReplace_Static(dm, coarseMesh);CHKERRQ(ierr);
2030       ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dm);CHKERRQ(ierr);
2031       ierr = DMDestroy(&coarseMesh);CHKERRQ(ierr);
2032     }
2033   }
2034   /* Handle */
2035   ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dm);CHKERRQ(ierr);
2036   ierr = PetscOptionsTail();CHKERRQ(ierr);
2037   PetscFunctionReturn(0);
2038 }
2039 
2040 static PetscErrorCode DMCreateGlobalVector_Plex(DM dm,Vec *vec)
2041 {
2042   PetscErrorCode ierr;
2043 
2044   PetscFunctionBegin;
2045   ierr = DMCreateGlobalVector_Section_Private(dm,vec);CHKERRQ(ierr);
2046   /* ierr = VecSetOperation(*vec, VECOP_DUPLICATE, (void(*)(void)) VecDuplicate_MPI_DM);CHKERRQ(ierr); */
2047   ierr = VecSetOperation(*vec, VECOP_VIEW, (void (*)(void)) VecView_Plex);CHKERRQ(ierr);
2048   ierr = VecSetOperation(*vec, VECOP_VIEWNATIVE, (void (*)(void)) VecView_Plex_Native);CHKERRQ(ierr);
2049   ierr = VecSetOperation(*vec, VECOP_LOAD, (void (*)(void)) VecLoad_Plex);CHKERRQ(ierr);
2050   ierr = VecSetOperation(*vec, VECOP_LOADNATIVE, (void (*)(void)) VecLoad_Plex_Native);CHKERRQ(ierr);
2051   PetscFunctionReturn(0);
2052 }
2053 
2054 static PetscErrorCode DMCreateLocalVector_Plex(DM dm,Vec *vec)
2055 {
2056   PetscErrorCode ierr;
2057 
2058   PetscFunctionBegin;
2059   ierr = DMCreateLocalVector_Section_Private(dm,vec);CHKERRQ(ierr);
2060   ierr = VecSetOperation(*vec, VECOP_VIEW, (void (*)(void)) VecView_Plex_Local);CHKERRQ(ierr);
2061   ierr = VecSetOperation(*vec, VECOP_LOAD, (void (*)(void)) VecLoad_Plex_Local);CHKERRQ(ierr);
2062   PetscFunctionReturn(0);
2063 }
2064 
2065 static PetscErrorCode DMGetDimPoints_Plex(DM dm, PetscInt dim, PetscInt *pStart, PetscInt *pEnd)
2066 {
2067   PetscInt       depth, d;
2068   PetscErrorCode ierr;
2069 
2070   PetscFunctionBegin;
2071   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
2072   if (depth == 1) {
2073     ierr = DMGetDimension(dm, &d);CHKERRQ(ierr);
2074     if (dim == 0)      {ierr = DMPlexGetDepthStratum(dm, dim, pStart, pEnd);CHKERRQ(ierr);}
2075     else if (dim == d) {ierr = DMPlexGetDepthStratum(dm, 1, pStart, pEnd);CHKERRQ(ierr);}
2076     else               {*pStart = 0; *pEnd = 0;}
2077   } else {
2078     ierr = DMPlexGetDepthStratum(dm, dim, pStart, pEnd);CHKERRQ(ierr);
2079   }
2080   PetscFunctionReturn(0);
2081 }
2082 
2083 static PetscErrorCode DMGetNeighors_Plex(DM dm, PetscInt *nranks, const PetscMPIInt *ranks[])
2084 {
2085   PetscSF        sf;
2086   PetscErrorCode ierr;
2087 
2088   PetscFunctionBegin;
2089   ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr);
2090   ierr = PetscSFGetRanks(sf, nranks, ranks, NULL, NULL, NULL);CHKERRQ(ierr);
2091   PetscFunctionReturn(0);
2092 }
2093 
2094 static PetscErrorCode DMInitialize_Plex(DM dm)
2095 {
2096   PetscErrorCode ierr;
2097 
2098   PetscFunctionBegin;
2099   dm->ops->view                            = DMView_Plex;
2100   dm->ops->load                            = DMLoad_Plex;
2101   dm->ops->setfromoptions                  = DMSetFromOptions_Plex;
2102   dm->ops->clone                           = DMClone_Plex;
2103   dm->ops->setup                           = DMSetUp_Plex;
2104   dm->ops->createdefaultsection            = DMCreateDefaultSection_Plex;
2105   dm->ops->createdefaultconstraints        = DMCreateDefaultConstraints_Plex;
2106   dm->ops->createglobalvector              = DMCreateGlobalVector_Plex;
2107   dm->ops->createlocalvector               = DMCreateLocalVector_Plex;
2108   dm->ops->getlocaltoglobalmapping         = NULL;
2109   dm->ops->createfieldis                   = NULL;
2110   dm->ops->createcoordinatedm              = DMCreateCoordinateDM_Plex;
2111   dm->ops->getcoloring                     = NULL;
2112   dm->ops->creatematrix                    = DMCreateMatrix_Plex;
2113   dm->ops->createinterpolation             = DMCreateInterpolation_Plex;
2114   dm->ops->getaggregates                   = NULL;
2115   dm->ops->getinjection                    = DMCreateInjection_Plex;
2116   dm->ops->refine                          = DMRefine_Plex;
2117   dm->ops->coarsen                         = DMCoarsen_Plex;
2118   dm->ops->refinehierarchy                 = DMRefineHierarchy_Plex;
2119   dm->ops->coarsenhierarchy                = DMCoarsenHierarchy_Plex;
2120   dm->ops->adaptlabel                      = DMAdaptLabel_Plex;
2121   dm->ops->adaptmetric                     = DMAdaptMetric_Plex;
2122   dm->ops->globaltolocalbegin              = NULL;
2123   dm->ops->globaltolocalend                = NULL;
2124   dm->ops->localtoglobalbegin              = NULL;
2125   dm->ops->localtoglobalend                = NULL;
2126   dm->ops->destroy                         = DMDestroy_Plex;
2127   dm->ops->createsubdm                     = DMCreateSubDM_Plex;
2128   dm->ops->getdimpoints                    = DMGetDimPoints_Plex;
2129   dm->ops->locatepoints                    = DMLocatePoints_Plex;
2130   dm->ops->projectfunctionlocal            = DMProjectFunctionLocal_Plex;
2131   dm->ops->projectfunctionlabellocal       = DMProjectFunctionLabelLocal_Plex;
2132   dm->ops->projectfieldlocal               = DMProjectFieldLocal_Plex;
2133   dm->ops->projectfieldlabellocal          = DMProjectFieldLabelLocal_Plex;
2134   dm->ops->computel2diff                   = DMComputeL2Diff_Plex;
2135   dm->ops->computel2gradientdiff           = DMComputeL2GradientDiff_Plex;
2136   dm->ops->computel2fielddiff              = DMComputeL2FieldDiff_Plex;
2137   dm->ops->getneighbors                    = DMGetNeighors_Plex;
2138   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMPlexInsertBoundaryValues_C",DMPlexInsertBoundaryValues_Plex);CHKERRQ(ierr);
2139   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMSetUpGLVisViewer_C",DMSetUpGLVisViewer_Plex);CHKERRQ(ierr);
2140   PetscFunctionReturn(0);
2141 }
2142 
2143 PETSC_INTERN PetscErrorCode DMClone_Plex(DM dm, DM *newdm)
2144 {
2145   DM_Plex        *mesh = (DM_Plex *) dm->data;
2146   PetscErrorCode ierr;
2147 
2148   PetscFunctionBegin;
2149   mesh->refct++;
2150   (*newdm)->data = mesh;
2151   ierr = PetscObjectChangeTypeName((PetscObject) *newdm, DMPLEX);CHKERRQ(ierr);
2152   ierr = DMInitialize_Plex(*newdm);CHKERRQ(ierr);
2153   PetscFunctionReturn(0);
2154 }
2155 
2156 /*MC
2157   DMPLEX = "plex" - A DM object that encapsulates an unstructured mesh, or CW Complex, which can be expressed using a Hasse Diagram.
2158                     In the local representation, Vecs contain all unknowns in the interior and shared boundary. This is
2159                     specified by a PetscSection object. Ownership in the global representation is determined by
2160                     ownership of the underlying DMPlex points. This is specified by another PetscSection object.
2161 
2162   Level: intermediate
2163 
2164 .seealso: DMType, DMPlexCreate(), DMCreate(), DMSetType()
2165 M*/
2166 
2167 PETSC_EXTERN PetscErrorCode DMCreate_Plex(DM dm)
2168 {
2169   DM_Plex        *mesh;
2170   PetscInt       unit, d;
2171   PetscErrorCode ierr;
2172 
2173   PetscFunctionBegin;
2174   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2175   ierr     = PetscNewLog(dm,&mesh);CHKERRQ(ierr);
2176   dm->dim  = 0;
2177   dm->data = mesh;
2178 
2179   mesh->refct             = 1;
2180   ierr                    = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &mesh->coneSection);CHKERRQ(ierr);
2181   mesh->maxConeSize       = 0;
2182   mesh->cones             = NULL;
2183   mesh->coneOrientations  = NULL;
2184   ierr                    = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &mesh->supportSection);CHKERRQ(ierr);
2185   mesh->maxSupportSize    = 0;
2186   mesh->supports          = NULL;
2187   mesh->refinementUniform = PETSC_TRUE;
2188   mesh->refinementLimit   = -1.0;
2189 
2190   mesh->facesTmp = NULL;
2191 
2192   mesh->tetgenOpts   = NULL;
2193   mesh->triangleOpts = NULL;
2194   ierr = PetscPartitionerCreate(PetscObjectComm((PetscObject)dm), &mesh->partitioner);CHKERRQ(ierr);
2195   mesh->remeshBd     = PETSC_FALSE;
2196 
2197   mesh->subpointMap = NULL;
2198 
2199   for (unit = 0; unit < NUM_PETSC_UNITS; ++unit) mesh->scale[unit] = 1.0;
2200 
2201   mesh->regularRefinement   = PETSC_FALSE;
2202   mesh->depthState          = -1;
2203   mesh->globalVertexNumbers = NULL;
2204   mesh->globalCellNumbers   = NULL;
2205   mesh->anchorSection       = NULL;
2206   mesh->anchorIS            = NULL;
2207   mesh->createanchors       = NULL;
2208   mesh->computeanchormatrix = NULL;
2209   mesh->parentSection       = NULL;
2210   mesh->parents             = NULL;
2211   mesh->childIDs            = NULL;
2212   mesh->childSection        = NULL;
2213   mesh->children            = NULL;
2214   mesh->referenceTree       = NULL;
2215   mesh->getchildsymmetry    = NULL;
2216   for (d = 0; d < 8; ++d) mesh->hybridPointMax[d] = PETSC_DETERMINE;
2217   mesh->vtkCellHeight       = 0;
2218   mesh->useCone             = PETSC_FALSE;
2219   mesh->useClosure          = PETSC_TRUE;
2220   mesh->useAnchors          = PETSC_FALSE;
2221 
2222   mesh->maxProjectionHeight = 0;
2223 
2224   mesh->printSetValues = PETSC_FALSE;
2225   mesh->printFEM       = 0;
2226   mesh->printTol       = 1.0e-10;
2227 
2228   ierr = DMInitialize_Plex(dm);CHKERRQ(ierr);
2229   PetscFunctionReturn(0);
2230 }
2231 
2232 /*@
2233   DMPlexCreate - Creates a DMPlex object, which encapsulates an unstructured mesh, or CW complex, which can be expressed using a Hasse Diagram.
2234 
2235   Collective on MPI_Comm
2236 
2237   Input Parameter:
2238 . comm - The communicator for the DMPlex object
2239 
2240   Output Parameter:
2241 . mesh  - The DMPlex object
2242 
2243   Level: beginner
2244 
2245 .keywords: DMPlex, create
2246 @*/
2247 PetscErrorCode DMPlexCreate(MPI_Comm comm, DM *mesh)
2248 {
2249   PetscErrorCode ierr;
2250 
2251   PetscFunctionBegin;
2252   PetscValidPointer(mesh,2);
2253   ierr = DMCreate(comm, mesh);CHKERRQ(ierr);
2254   ierr = DMSetType(*mesh, DMPLEX);CHKERRQ(ierr);
2255   PetscFunctionReturn(0);
2256 }
2257 
2258 /*
2259   This takes as input the common mesh generator output, a list of the vertices for each cell, but vertex numbers are global and an SF is built for them
2260 */
2261 static PetscErrorCode DMPlexBuildFromCellList_Parallel_Private(DM dm, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, const int cells[], PetscSF *sfVert)
2262 {
2263   PetscSF         sfPoint;
2264   PetscLayout     vLayout;
2265   PetscHashI      vhash;
2266   PetscSFNode    *remoteVerticesAdj, *vertexLocal, *vertexOwner, *remoteVertex;
2267   const PetscInt *vrange;
2268   PetscInt        numVerticesAdj, off, *verticesAdj, numVerticesGhost = 0, *localVertex, *cone, c, p, v, g;
2269   PETSC_UNUSED PetscHashIIter ret, iter;
2270   PetscMPIInt     rank, size;
2271   PetscErrorCode  ierr;
2272 
2273   PetscFunctionBegin;
2274   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRQ(ierr);
2275   ierr = MPI_Comm_size(PetscObjectComm((PetscObject) dm), &size);CHKERRQ(ierr);
2276   /* Partition vertices */
2277   ierr = PetscLayoutCreate(PetscObjectComm((PetscObject) dm), &vLayout);CHKERRQ(ierr);
2278   ierr = PetscLayoutSetLocalSize(vLayout, numVertices);CHKERRQ(ierr);
2279   ierr = PetscLayoutSetBlockSize(vLayout, 1);CHKERRQ(ierr);
2280   ierr = PetscLayoutSetUp(vLayout);CHKERRQ(ierr);
2281   ierr = PetscLayoutGetRanges(vLayout, &vrange);CHKERRQ(ierr);
2282   /* Count vertices and map them to procs */
2283   PetscHashICreate(vhash);
2284   for (c = 0; c < numCells; ++c) {
2285     for (p = 0; p < numCorners; ++p) {
2286       PetscHashIPut(vhash, cells[c*numCorners+p], ret, iter);
2287     }
2288   }
2289   PetscHashISize(vhash, numVerticesAdj);
2290   ierr = PetscMalloc1(numVerticesAdj, &verticesAdj);CHKERRQ(ierr);
2291   off = 0; ierr = PetscHashIGetKeys(vhash, &off, verticesAdj);CHKERRQ(ierr);
2292   if (off != numVerticesAdj) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid number of local vertices %D should be %D", off, numVerticesAdj);
2293   ierr = PetscSortInt(numVerticesAdj, verticesAdj);CHKERRQ(ierr);
2294   ierr = PetscMalloc1(numVerticesAdj, &remoteVerticesAdj);CHKERRQ(ierr);
2295   for (v = 0; v < numVerticesAdj; ++v) {
2296     const PetscInt gv = verticesAdj[v];
2297     PetscInt       vrank;
2298 
2299     ierr = PetscFindInt(gv, size+1, vrange, &vrank);CHKERRQ(ierr);
2300     vrank = vrank < 0 ? -(vrank+2) : vrank;
2301     remoteVerticesAdj[v].index = gv - vrange[vrank];
2302     remoteVerticesAdj[v].rank  = vrank;
2303   }
2304   /* Create cones */
2305   ierr = DMPlexSetChart(dm, 0, numCells+numVerticesAdj);CHKERRQ(ierr);
2306   for (c = 0; c < numCells; ++c) {ierr = DMPlexSetConeSize(dm, c, numCorners);CHKERRQ(ierr);}
2307   ierr = DMSetUp(dm);CHKERRQ(ierr);
2308   ierr = DMGetWorkArray(dm, numCorners, PETSC_INT, &cone);CHKERRQ(ierr);
2309   for (c = 0; c < numCells; ++c) {
2310     for (p = 0; p < numCorners; ++p) {
2311       const PetscInt gv = cells[c*numCorners+p];
2312       PetscInt       lv;
2313 
2314       ierr = PetscFindInt(gv, numVerticesAdj, verticesAdj, &lv);CHKERRQ(ierr);
2315       if (lv < 0) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Could not find global vertex %D in local connectivity", gv);
2316       cone[p] = lv+numCells;
2317     }
2318     ierr = DMPlexSetCone(dm, c, cone);CHKERRQ(ierr);
2319   }
2320   ierr = DMRestoreWorkArray(dm, numCorners, PETSC_INT, &cone);CHKERRQ(ierr);
2321   /* Create SF for vertices */
2322   ierr = PetscSFCreate(PetscObjectComm((PetscObject)dm), sfVert);CHKERRQ(ierr);
2323   ierr = PetscObjectSetName((PetscObject) *sfVert, "Vertex Ownership SF");CHKERRQ(ierr);
2324   ierr = PetscSFSetFromOptions(*sfVert);CHKERRQ(ierr);
2325   ierr = PetscSFSetGraph(*sfVert, numVertices, numVerticesAdj, NULL, PETSC_OWN_POINTER, remoteVerticesAdj, PETSC_OWN_POINTER);CHKERRQ(ierr);
2326   ierr = PetscFree(verticesAdj);CHKERRQ(ierr);
2327   /* Build pointSF */
2328   ierr = PetscMalloc2(numVerticesAdj, &vertexLocal, numVertices, &vertexOwner);CHKERRQ(ierr);
2329   for (v = 0; v < numVerticesAdj; ++v) {vertexLocal[v].index = v+numCells; vertexLocal[v].rank = rank;}
2330   for (v = 0; v < numVertices;    ++v) {vertexOwner[v].index = -1;         vertexOwner[v].rank = -1;}
2331   ierr = PetscSFReduceBegin(*sfVert, MPIU_2INT, vertexLocal, vertexOwner, MPI_MAXLOC);CHKERRQ(ierr);
2332   ierr = PetscSFReduceEnd(*sfVert, MPIU_2INT, vertexLocal, vertexOwner, MPI_MAXLOC);CHKERRQ(ierr);
2333   for (v = 0; v < numVertices;    ++v) if (vertexOwner[v].rank < 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Global vertex %d on rank %d was unclaimed", v + vrange[rank], rank);
2334   ierr = PetscSFBcastBegin(*sfVert, MPIU_2INT, vertexOwner, vertexLocal);CHKERRQ(ierr);
2335   ierr = PetscSFBcastEnd(*sfVert, MPIU_2INT, vertexOwner, vertexLocal);CHKERRQ(ierr);
2336   for (v = 0; v < numVerticesAdj; ++v) if (vertexLocal[v].rank != rank) ++numVerticesGhost;
2337   ierr = PetscMalloc1(numVerticesGhost, &localVertex);CHKERRQ(ierr);
2338   ierr = PetscMalloc1(numVerticesGhost, &remoteVertex);CHKERRQ(ierr);
2339   for (v = 0, g = 0; v < numVerticesAdj; ++v) {
2340     if (vertexLocal[v].rank != rank) {
2341       localVertex[g]        = v+numCells;
2342       remoteVertex[g].index = vertexLocal[v].index;
2343       remoteVertex[g].rank  = vertexLocal[v].rank;
2344       ++g;
2345     }
2346   }
2347   ierr = PetscFree2(vertexLocal, vertexOwner);CHKERRQ(ierr);
2348   if (g != numVerticesGhost) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid number of vertex ghosts %D should be %D", g, numVerticesGhost);
2349   ierr = DMGetPointSF(dm, &sfPoint);CHKERRQ(ierr);
2350   ierr = PetscObjectSetName((PetscObject) sfPoint, "point SF");CHKERRQ(ierr);
2351   ierr = PetscSFSetGraph(sfPoint, numCells+numVerticesAdj, numVerticesGhost, localVertex, PETSC_OWN_POINTER, remoteVertex, PETSC_OWN_POINTER);CHKERRQ(ierr);
2352   ierr = PetscLayoutDestroy(&vLayout);CHKERRQ(ierr);
2353   PetscHashIDestroy(vhash);
2354   /* Fill in the rest of the topology structure */
2355   ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
2356   ierr = DMPlexStratify(dm);CHKERRQ(ierr);
2357   PetscFunctionReturn(0);
2358 }
2359 
2360 /*
2361   This takes as input the coordinates for each owned vertex
2362 */
2363 static PetscErrorCode DMPlexBuildCoordinates_Parallel_Private(DM dm, PetscInt spaceDim, PetscInt numCells, PetscInt numV, PetscSF sfVert, const PetscReal vertexCoords[])
2364 {
2365   PetscSection   coordSection;
2366   Vec            coordinates;
2367   PetscScalar   *coords;
2368   PetscInt       numVertices, numVerticesAdj, coordSize, v;
2369   PetscErrorCode ierr;
2370 
2371   PetscFunctionBegin;
2372   ierr = DMSetCoordinateDim(dm, spaceDim);CHKERRQ(ierr);
2373   ierr = PetscSFGetGraph(sfVert, &numVertices, &numVerticesAdj, NULL, NULL);CHKERRQ(ierr);
2374   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
2375   ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
2376   ierr = PetscSectionSetFieldComponents(coordSection, 0, spaceDim);CHKERRQ(ierr);
2377   ierr = PetscSectionSetChart(coordSection, numCells, numCells + numVerticesAdj);CHKERRQ(ierr);
2378   for (v = numCells; v < numCells+numVerticesAdj; ++v) {
2379     ierr = PetscSectionSetDof(coordSection, v, spaceDim);CHKERRQ(ierr);
2380     ierr = PetscSectionSetFieldDof(coordSection, v, 0, spaceDim);CHKERRQ(ierr);
2381   }
2382   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
2383   ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
2384   ierr = VecCreate(PetscObjectComm((PetscObject)dm), &coordinates);CHKERRQ(ierr);
2385   ierr = VecSetBlockSize(coordinates, spaceDim);CHKERRQ(ierr);
2386   ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
2387   ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
2388   ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
2389   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
2390   {
2391     MPI_Datatype coordtype;
2392 
2393     /* Need a temp buffer for coords if we have complex/single */
2394     ierr = MPI_Type_contiguous(spaceDim, MPIU_SCALAR, &coordtype);CHKERRQ(ierr);
2395     ierr = MPI_Type_commit(&coordtype);CHKERRQ(ierr);
2396 #if defined(PETSC_USE_COMPLEX)
2397     {
2398     PetscScalar *svertexCoords;
2399     PetscInt    i;
2400     ierr = PetscMalloc1(numV*spaceDim,&svertexCoords);CHKERRQ(ierr);
2401     for (i=0; i<numV*spaceDim; i++) svertexCoords[i] = vertexCoords[i];
2402     ierr = PetscSFBcastBegin(sfVert, coordtype, svertexCoords, coords);CHKERRQ(ierr);
2403     ierr = PetscSFBcastEnd(sfVert, coordtype, svertexCoords, coords);CHKERRQ(ierr);
2404     ierr = PetscFree(svertexCoords);CHKERRQ(ierr);
2405     }
2406 #else
2407     ierr = PetscSFBcastBegin(sfVert, coordtype, vertexCoords, coords);CHKERRQ(ierr);
2408     ierr = PetscSFBcastEnd(sfVert, coordtype, vertexCoords, coords);CHKERRQ(ierr);
2409 #endif
2410     ierr = MPI_Type_free(&coordtype);CHKERRQ(ierr);
2411   }
2412   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
2413   ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
2414   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
2415   PetscFunctionReturn(0);
2416 }
2417 
2418 /*@C
2419   DMPlexCreateFromCellListParallel - This takes as input common mesh generator output, a list of the vertices for each cell, and produces a DM
2420 
2421   Input Parameters:
2422 + comm - The communicator
2423 . dim - The topological dimension of the mesh
2424 . numCells - The number of cells owned by this process
2425 . numVertices - The number of vertices owned by this process
2426 . numCorners - The number of vertices for each cell
2427 . interpolate - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
2428 . cells - An array of numCells*numCorners numbers, the global vertex numbers for each cell
2429 . spaceDim - The spatial dimension used for coordinates
2430 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
2431 
2432   Output Parameter:
2433 + dm - The DM
2434 - vertexSF - Optional, SF describing complete vertex ownership
2435 
2436   Note: Two triangles sharing a face
2437 $
2438 $        2
2439 $      / | \
2440 $     /  |  \
2441 $    /   |   \
2442 $   0  0 | 1  3
2443 $    \   |   /
2444 $     \  |  /
2445 $      \ | /
2446 $        1
2447 would have input
2448 $  numCells = 2, numVertices = 4
2449 $  cells = [0 1 2  1 3 2]
2450 $
2451 which would result in the DMPlex
2452 $
2453 $        4
2454 $      / | \
2455 $     /  |  \
2456 $    /   |   \
2457 $   2  0 | 1  5
2458 $    \   |   /
2459 $     \  |  /
2460 $      \ | /
2461 $        3
2462 
2463   Level: beginner
2464 
2465 .seealso: DMPlexCreateFromCellList(), DMPlexCreateFromDAG(), DMPlexCreate()
2466 @*/
2467 PetscErrorCode DMPlexCreateFromCellListParallel(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, PetscBool interpolate, const int cells[], PetscInt spaceDim, const PetscReal vertexCoords[], PetscSF *vertexSF, DM *dm)
2468 {
2469   PetscSF        sfVert;
2470   PetscErrorCode ierr;
2471 
2472   PetscFunctionBegin;
2473   ierr = DMCreate(comm, dm);CHKERRQ(ierr);
2474   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
2475   PetscValidLogicalCollectiveInt(*dm, dim, 2);
2476   PetscValidLogicalCollectiveInt(*dm, spaceDim, 8);
2477   ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
2478   ierr = DMPlexBuildFromCellList_Parallel_Private(*dm, numCells, numVertices, numCorners, cells, &sfVert);CHKERRQ(ierr);
2479   if (interpolate) {
2480     DM idm = NULL;
2481 
2482     ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
2483     ierr = DMDestroy(dm);CHKERRQ(ierr);
2484     *dm  = idm;
2485   }
2486   ierr = DMPlexBuildCoordinates_Parallel_Private(*dm, spaceDim, numCells, numVertices,sfVert, vertexCoords);CHKERRQ(ierr);
2487   if (vertexSF) *vertexSF = sfVert;
2488   else {ierr = PetscSFDestroy(&sfVert);CHKERRQ(ierr);}
2489   PetscFunctionReturn(0);
2490 }
2491 
2492 /*
2493   This takes as input the common mesh generator output, a list of the vertices for each cell
2494 */
2495 static PetscErrorCode DMPlexBuildFromCellList_Private(DM dm, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, const int cells[])
2496 {
2497   PetscInt      *cone, c, p;
2498   PetscErrorCode ierr;
2499 
2500   PetscFunctionBegin;
2501   ierr = DMPlexSetChart(dm, 0, numCells+numVertices);CHKERRQ(ierr);
2502   for (c = 0; c < numCells; ++c) {
2503     ierr = DMPlexSetConeSize(dm, c, numCorners);CHKERRQ(ierr);
2504   }
2505   ierr = DMSetUp(dm);CHKERRQ(ierr);
2506   ierr = DMGetWorkArray(dm, numCorners, PETSC_INT, &cone);CHKERRQ(ierr);
2507   for (c = 0; c < numCells; ++c) {
2508     for (p = 0; p < numCorners; ++p) {
2509       cone[p] = cells[c*numCorners+p]+numCells;
2510     }
2511     ierr = DMPlexSetCone(dm, c, cone);CHKERRQ(ierr);
2512   }
2513   ierr = DMRestoreWorkArray(dm, numCorners, PETSC_INT, &cone);CHKERRQ(ierr);
2514   ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
2515   ierr = DMPlexStratify(dm);CHKERRQ(ierr);
2516   PetscFunctionReturn(0);
2517 }
2518 
2519 /*
2520   This takes as input the coordinates for each vertex
2521 */
2522 static PetscErrorCode DMPlexBuildCoordinates_Private(DM dm, PetscInt spaceDim, PetscInt numCells, PetscInt numVertices, const double vertexCoords[])
2523 {
2524   PetscSection   coordSection;
2525   Vec            coordinates;
2526   DM             cdm;
2527   PetscScalar   *coords;
2528   PetscInt       v, d;
2529   PetscErrorCode ierr;
2530 
2531   PetscFunctionBegin;
2532   ierr = DMSetCoordinateDim(dm, spaceDim);CHKERRQ(ierr);
2533   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
2534   ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
2535   ierr = PetscSectionSetFieldComponents(coordSection, 0, spaceDim);CHKERRQ(ierr);
2536   ierr = PetscSectionSetChart(coordSection, numCells, numCells + numVertices);CHKERRQ(ierr);
2537   for (v = numCells; v < numCells+numVertices; ++v) {
2538     ierr = PetscSectionSetDof(coordSection, v, spaceDim);CHKERRQ(ierr);
2539     ierr = PetscSectionSetFieldDof(coordSection, v, 0, spaceDim);CHKERRQ(ierr);
2540   }
2541   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
2542 
2543   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
2544   ierr = DMCreateLocalVector(cdm, &coordinates);CHKERRQ(ierr);
2545   ierr = VecSetBlockSize(coordinates, spaceDim);CHKERRQ(ierr);
2546   ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
2547   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
2548   for (v = 0; v < numVertices; ++v) {
2549     for (d = 0; d < spaceDim; ++d) {
2550       coords[v*spaceDim+d] = vertexCoords[v*spaceDim+d];
2551     }
2552   }
2553   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
2554   ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
2555   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
2556   PetscFunctionReturn(0);
2557 }
2558 
2559 /*@C
2560   DMPlexCreateFromCellList - This takes as input common mesh generator output, a list of the vertices for each cell, and produces a DM
2561 
2562   Input Parameters:
2563 + comm - The communicator
2564 . dim - The topological dimension of the mesh
2565 . numCells - The number of cells
2566 . numVertices - The number of vertices
2567 . numCorners - The number of vertices for each cell
2568 . interpolate - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
2569 . cells - An array of numCells*numCorners numbers, the vertices for each cell
2570 . spaceDim - The spatial dimension used for coordinates
2571 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
2572 
2573   Output Parameter:
2574 . dm - The DM
2575 
2576   Note: Two triangles sharing a face
2577 $
2578 $        2
2579 $      / | \
2580 $     /  |  \
2581 $    /   |   \
2582 $   0  0 | 1  3
2583 $    \   |   /
2584 $     \  |  /
2585 $      \ | /
2586 $        1
2587 would have input
2588 $  numCells = 2, numVertices = 4
2589 $  cells = [0 1 2  1 3 2]
2590 $
2591 which would result in the DMPlex
2592 $
2593 $        4
2594 $      / | \
2595 $     /  |  \
2596 $    /   |   \
2597 $   2  0 | 1  5
2598 $    \   |   /
2599 $     \  |  /
2600 $      \ | /
2601 $        3
2602 
2603   Level: beginner
2604 
2605 .seealso: DMPlexCreateFromDAG(), DMPlexCreate()
2606 @*/
2607 PetscErrorCode DMPlexCreateFromCellList(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, PetscBool interpolate, const int cells[], PetscInt spaceDim, const double vertexCoords[], DM *dm)
2608 {
2609   PetscErrorCode ierr;
2610 
2611   PetscFunctionBegin;
2612   ierr = DMCreate(comm, dm);CHKERRQ(ierr);
2613   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
2614   ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
2615   ierr = DMPlexBuildFromCellList_Private(*dm, numCells, numVertices, numCorners, cells);CHKERRQ(ierr);
2616   if (interpolate) {
2617     DM idm = NULL;
2618 
2619     ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
2620     ierr = DMDestroy(dm);CHKERRQ(ierr);
2621     *dm  = idm;
2622   }
2623   ierr = DMPlexBuildCoordinates_Private(*dm, spaceDim, numCells, numVertices, vertexCoords);CHKERRQ(ierr);
2624   PetscFunctionReturn(0);
2625 }
2626 
2627 /*@
2628   DMPlexCreateFromDAG - This takes as input the adjacency-list representation of the Directed Acyclic Graph (Hasse Diagram) encoding a mesh, and produces a DM
2629 
2630   Input Parameters:
2631 + dm - The empty DM object, usually from DMCreate() and DMSetDimension()
2632 . depth - The depth of the DAG
2633 . numPoints - The number of points at each depth
2634 . coneSize - The cone size of each point
2635 . cones - The concatenation of the cone points for each point, the cone list must be oriented correctly for each point
2636 . coneOrientations - The orientation of each cone point
2637 - vertexCoords - An array of numVertices*dim numbers, the coordinates of each vertex
2638 
2639   Output Parameter:
2640 . dm - The DM
2641 
2642   Note: Two triangles sharing a face would have input
2643 $  depth = 1, numPoints = [4 2], coneSize = [3 3 0 0 0 0]
2644 $  cones = [2 3 4  3 5 4], coneOrientations = [0 0 0  0 0 0]
2645 $ vertexCoords = [-1.0 0.0  0.0 -1.0  0.0 1.0  1.0 0.0]
2646 $
2647 which would result in the DMPlex
2648 $
2649 $        4
2650 $      / | \
2651 $     /  |  \
2652 $    /   |   \
2653 $   2  0 | 1  5
2654 $    \   |   /
2655 $     \  |  /
2656 $      \ | /
2657 $        3
2658 $
2659 $ Notice that all points are numbered consecutively, unlikely DMPlexCreateFromCellList()
2660 
2661   Level: advanced
2662 
2663 .seealso: DMPlexCreateFromCellList(), DMPlexCreate()
2664 @*/
2665 PetscErrorCode DMPlexCreateFromDAG(DM dm, PetscInt depth, const PetscInt numPoints[], const PetscInt coneSize[], const PetscInt cones[], const PetscInt coneOrientations[], const PetscScalar vertexCoords[])
2666 {
2667   Vec            coordinates;
2668   PetscSection   coordSection;
2669   PetscScalar    *coords;
2670   PetscInt       coordSize, firstVertex = -1, pStart = 0, pEnd = 0, p, v, dim, dimEmbed, d, off;
2671   PetscErrorCode ierr;
2672 
2673   PetscFunctionBegin;
2674   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
2675   ierr = DMGetCoordinateDim(dm, &dimEmbed);CHKERRQ(ierr);
2676   if (dimEmbed < dim) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Embedding dimension %d cannot be less than intrinsic dimension %d",dimEmbed,dim);
2677   for (d = 0; d <= depth; ++d) pEnd += numPoints[d];
2678   ierr = DMPlexSetChart(dm, pStart, pEnd);CHKERRQ(ierr);
2679   for (p = pStart; p < pEnd; ++p) {
2680     ierr = DMPlexSetConeSize(dm, p, coneSize[p-pStart]);CHKERRQ(ierr);
2681     if (firstVertex < 0 && !coneSize[p - pStart]) {
2682       firstVertex = p - pStart;
2683     }
2684   }
2685   if (firstVertex < 0 && numPoints[0]) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONG,"Expected %d vertices but could not find any", numPoints[0]);
2686   ierr = DMSetUp(dm);CHKERRQ(ierr); /* Allocate space for cones */
2687   for (p = pStart, off = 0; p < pEnd; off += coneSize[p-pStart], ++p) {
2688     ierr = DMPlexSetCone(dm, p, &cones[off]);CHKERRQ(ierr);
2689     ierr = DMPlexSetConeOrientation(dm, p, &coneOrientations[off]);CHKERRQ(ierr);
2690   }
2691   ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
2692   ierr = DMPlexStratify(dm);CHKERRQ(ierr);
2693   /* Build coordinates */
2694   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
2695   ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
2696   ierr = PetscSectionSetFieldComponents(coordSection, 0, dimEmbed);CHKERRQ(ierr);
2697   ierr = PetscSectionSetChart(coordSection, firstVertex, firstVertex+numPoints[0]);CHKERRQ(ierr);
2698   for (v = firstVertex; v < firstVertex+numPoints[0]; ++v) {
2699     ierr = PetscSectionSetDof(coordSection, v, dimEmbed);CHKERRQ(ierr);
2700     ierr = PetscSectionSetFieldDof(coordSection, v, 0, dimEmbed);CHKERRQ(ierr);
2701   }
2702   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
2703   ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
2704   ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
2705   ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
2706   ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
2707   ierr = VecSetBlockSize(coordinates, dimEmbed);CHKERRQ(ierr);
2708   ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
2709   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
2710   for (v = 0; v < numPoints[0]; ++v) {
2711     PetscInt off;
2712 
2713     ierr = PetscSectionGetOffset(coordSection, v+firstVertex, &off);CHKERRQ(ierr);
2714     for (d = 0; d < dimEmbed; ++d) {
2715       coords[off+d] = vertexCoords[v*dimEmbed+d];
2716     }
2717   }
2718   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
2719   ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
2720   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
2721   PetscFunctionReturn(0);
2722 }
2723 
2724 /*@C
2725   DMPlexCreateFromFile - This takes a filename and produces a DM
2726 
2727   Input Parameters:
2728 + comm - The communicator
2729 . filename - A file name
2730 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
2731 
2732   Output Parameter:
2733 . dm - The DM
2734 
2735   Level: beginner
2736 
2737 .seealso: DMPlexCreateFromDAG(), DMPlexCreateFromCellList(), DMPlexCreate()
2738 @*/
2739 PetscErrorCode DMPlexCreateFromFile(MPI_Comm comm, const char filename[], PetscBool interpolate, DM *dm)
2740 {
2741   const char    *extGmsh    = ".msh";
2742   const char    *extCGNS    = ".cgns";
2743   const char    *extExodus  = ".exo";
2744   const char    *extGenesis = ".gen";
2745   const char    *extFluent  = ".cas";
2746   const char    *extHDF5    = ".h5";
2747   const char    *extMed     = ".med";
2748   const char    *extPLY     = ".ply";
2749   size_t         len;
2750   PetscBool      isGmsh, isCGNS, isExodus, isGenesis, isFluent, isHDF5, isMed, isPLY;
2751   PetscMPIInt    rank;
2752   PetscErrorCode ierr;
2753 
2754   PetscFunctionBegin;
2755   PetscValidPointer(filename, 2);
2756   PetscValidPointer(dm, 4);
2757   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
2758   ierr = PetscStrlen(filename, &len);CHKERRQ(ierr);
2759   if (!len) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Filename must be a valid path");
2760   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)], extGmsh,    4, &isGmsh);CHKERRQ(ierr);
2761   ierr = PetscStrncmp(&filename[PetscMax(0,len-5)], extCGNS,    5, &isCGNS);CHKERRQ(ierr);
2762   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)], extExodus,  4, &isExodus);CHKERRQ(ierr);
2763   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)], extGenesis, 4, &isGenesis);CHKERRQ(ierr);
2764   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)], extFluent,  4, &isFluent);CHKERRQ(ierr);
2765   ierr = PetscStrncmp(&filename[PetscMax(0,len-3)], extHDF5,    3, &isHDF5);CHKERRQ(ierr);
2766   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)], extMed,     4, &isMed);CHKERRQ(ierr);
2767   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)], extPLY,     4, &isPLY);CHKERRQ(ierr);
2768   if (isGmsh) {
2769     ierr = DMPlexCreateGmshFromFile(comm, filename, interpolate, dm);CHKERRQ(ierr);
2770   } else if (isCGNS) {
2771     ierr = DMPlexCreateCGNSFromFile(comm, filename, interpolate, dm);CHKERRQ(ierr);
2772   } else if (isExodus || isGenesis) {
2773     ierr = DMPlexCreateExodusFromFile(comm, filename, interpolate, dm);CHKERRQ(ierr);
2774   } else if (isFluent) {
2775     ierr = DMPlexCreateFluentFromFile(comm, filename, interpolate, dm);CHKERRQ(ierr);
2776   } else if (isHDF5) {
2777     PetscViewer viewer;
2778 
2779     ierr = PetscViewerCreate(comm, &viewer);CHKERRQ(ierr);
2780     ierr = PetscViewerSetType(viewer, PETSCVIEWERHDF5);CHKERRQ(ierr);
2781     ierr = PetscViewerFileSetMode(viewer, FILE_MODE_READ);CHKERRQ(ierr);
2782     ierr = PetscViewerFileSetName(viewer, filename);CHKERRQ(ierr);
2783     ierr = DMCreate(comm, dm);CHKERRQ(ierr);
2784     ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
2785     ierr = DMLoad(*dm, viewer);CHKERRQ(ierr);
2786     ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
2787   } else if (isMed) {
2788     ierr = DMPlexCreateMedFromFile(comm, filename, interpolate, dm);CHKERRQ(ierr);
2789   } else if (isPLY) {
2790     ierr = DMPlexCreatePLYFromFile(comm, filename, interpolate, dm);CHKERRQ(ierr);
2791   } else SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cannot load file %s: unrecognized extension", filename);
2792   PetscFunctionReturn(0);
2793 }
2794 
2795 /*@
2796   DMPlexCreateReferenceCell - Create a DMPLEX with the appropriate FEM reference cell
2797 
2798   Collective on comm
2799 
2800   Input Parameters:
2801 + comm    - The communicator
2802 . dim     - The spatial dimension
2803 - simplex - Flag for simplex, otherwise use a tensor-product cell
2804 
2805   Output Parameter:
2806 . refdm - The reference cell
2807 
2808   Level: intermediate
2809 
2810 .keywords: reference cell
2811 .seealso:
2812 @*/
2813 PetscErrorCode DMPlexCreateReferenceCell(MPI_Comm comm, PetscInt dim, PetscBool simplex, DM *refdm)
2814 {
2815   DM             rdm;
2816   Vec            coords;
2817   PetscErrorCode ierr;
2818 
2819   PetscFunctionBeginUser;
2820   ierr = DMCreate(comm, &rdm);CHKERRQ(ierr);
2821   ierr = DMSetType(rdm, DMPLEX);CHKERRQ(ierr);
2822   ierr = DMSetDimension(rdm, dim);CHKERRQ(ierr);
2823   switch (dim) {
2824   case 0:
2825   {
2826     PetscInt    numPoints[1]        = {1};
2827     PetscInt    coneSize[1]         = {0};
2828     PetscInt    cones[1]            = {0};
2829     PetscInt    coneOrientations[1] = {0};
2830     PetscScalar vertexCoords[1]     = {0.0};
2831 
2832     ierr = DMPlexCreateFromDAG(rdm, 0, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
2833   }
2834   break;
2835   case 1:
2836   {
2837     PetscInt    numPoints[2]        = {2, 1};
2838     PetscInt    coneSize[3]         = {2, 0, 0};
2839     PetscInt    cones[2]            = {1, 2};
2840     PetscInt    coneOrientations[2] = {0, 0};
2841     PetscScalar vertexCoords[2]     = {-1.0,  1.0};
2842 
2843     ierr = DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
2844   }
2845   break;
2846   case 2:
2847     if (simplex) {
2848       PetscInt    numPoints[2]        = {3, 1};
2849       PetscInt    coneSize[4]         = {3, 0, 0, 0};
2850       PetscInt    cones[3]            = {1, 2, 3};
2851       PetscInt    coneOrientations[3] = {0, 0, 0};
2852       PetscScalar vertexCoords[6]     = {-1.0, -1.0,  1.0, -1.0,  -1.0, 1.0};
2853 
2854       ierr = DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
2855     } else {
2856       PetscInt    numPoints[2]        = {4, 1};
2857       PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
2858       PetscInt    cones[4]            = {1, 2, 3, 4};
2859       PetscInt    coneOrientations[4] = {0, 0, 0, 0};
2860       PetscScalar vertexCoords[8]     = {-1.0, -1.0,  1.0, -1.0,  1.0, 1.0,  -1.0, 1.0};
2861 
2862       ierr = DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
2863     }
2864   break;
2865   case 3:
2866     if (simplex) {
2867       PetscInt    numPoints[2]        = {4, 1};
2868       PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
2869       PetscInt    cones[4]            = {1, 3, 2, 4};
2870       PetscInt    coneOrientations[4] = {0, 0, 0, 0};
2871       PetscScalar vertexCoords[12]    = {-1.0, -1.0, -1.0,  1.0, -1.0, -1.0,  -1.0, 1.0, -1.0,  -1.0, -1.0, 1.0};
2872 
2873       ierr = DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
2874     } else {
2875       PetscInt    numPoints[2]        = {8, 1};
2876       PetscInt    coneSize[9]         = {8, 0, 0, 0, 0, 0, 0, 0, 0};
2877       PetscInt    cones[8]            = {1, 4, 3, 2, 5, 6, 7, 8};
2878       PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
2879       PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0,  1.0, -1.0, -1.0,  1.0, 1.0, -1.0,  -1.0, 1.0, -1.0,
2880                                          -1.0, -1.0,  1.0,  1.0, -1.0,  1.0,  1.0, 1.0,  1.0,  -1.0, 1.0,  1.0};
2881 
2882       ierr = DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
2883     }
2884   break;
2885   default:
2886     SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Cannot create reference cell for dimension %d", dim);
2887   }
2888   *refdm = NULL;
2889   ierr = DMPlexInterpolate(rdm, refdm);CHKERRQ(ierr);
2890   if (rdm->coordinateDM) {
2891     DM           ncdm;
2892     PetscSection cs;
2893     PetscInt     pEnd = -1;
2894 
2895     ierr = DMGetDefaultSection(rdm->coordinateDM, &cs);CHKERRQ(ierr);
2896     if (cs) {ierr = PetscSectionGetChart(cs, NULL, &pEnd);CHKERRQ(ierr);}
2897     if (pEnd >= 0) {
2898       ierr = DMClone(rdm->coordinateDM, &ncdm);CHKERRQ(ierr);
2899       ierr = DMSetDefaultSection(ncdm, cs);CHKERRQ(ierr);
2900       ierr = DMSetCoordinateDM(*refdm, ncdm);CHKERRQ(ierr);
2901       ierr = DMDestroy(&ncdm);CHKERRQ(ierr);
2902     }
2903   }
2904   ierr = DMGetCoordinatesLocal(rdm, &coords);CHKERRQ(ierr);
2905   if (coords) {
2906     ierr = DMSetCoordinatesLocal(*refdm, coords);CHKERRQ(ierr);
2907   } else {
2908     ierr = DMGetCoordinates(rdm, &coords);CHKERRQ(ierr);
2909     if (coords) {ierr = DMSetCoordinates(*refdm, coords);CHKERRQ(ierr);}
2910   }
2911   ierr = DMDestroy(&rdm);CHKERRQ(ierr);
2912   PetscFunctionReturn(0);
2913 }
2914