xref: /petsc/src/dm/impls/plex/plexcreate.c (revision 38cfc46e4752cf65941cec0be9bac4d1d6832cc8)
1 #define PETSCDM_DLL
2 #include <petsc/private/dmpleximpl.h>    /*I   "petscdmplex.h"   I*/
3 #include <petsc/private/hashseti.h>          /*I   "petscdmplex.h"   I*/
4 #include <petscsf.h>
5 
6 /*@
7   DMPlexCreateDoublet - Creates a mesh of two cells of the specified type, optionally with later refinement.
8 
9   Collective
10 
11   Input Parameters:
12 + comm - The communicator for the DM object
13 . dim - The spatial dimension
14 . simplex - Flag for simplicial cells, otherwise they are tensor product cells
15 . interpolate - Flag to create intermediate mesh pieces (edges, faces)
16 . refinementUniform - Flag for uniform parallel refinement
17 - refinementLimit - A nonzero number indicates the largest admissible volume for a refined cell
18 
19   Output Parameter:
20 . dm - The DM object
21 
22   Level: beginner
23 
24 .seealso: DMSetType(), DMCreate()
25 @*/
26 PetscErrorCode DMPlexCreateDoublet(MPI_Comm comm, PetscInt dim, PetscBool simplex, PetscBool interpolate, PetscBool refinementUniform, PetscReal refinementLimit, DM *newdm)
27 {
28   DM             dm;
29   PetscInt       p;
30   PetscMPIInt    rank;
31   PetscErrorCode ierr;
32 
33   PetscFunctionBegin;
34   ierr = DMCreate(comm, &dm);CHKERRQ(ierr);
35   ierr = DMSetType(dm, DMPLEX);CHKERRQ(ierr);
36   ierr = DMSetDimension(dm, dim);CHKERRQ(ierr);
37   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
38   switch (dim) {
39   case 2:
40     if (simplex) {ierr = PetscObjectSetName((PetscObject) dm, "triangular");CHKERRQ(ierr);}
41     else         {ierr = PetscObjectSetName((PetscObject) dm, "quadrilateral");CHKERRQ(ierr);}
42     break;
43   case 3:
44     if (simplex) {ierr = PetscObjectSetName((PetscObject) dm, "tetrahedral");CHKERRQ(ierr);}
45     else         {ierr = PetscObjectSetName((PetscObject) dm, "hexahedral");CHKERRQ(ierr);}
46     break;
47   default:
48     SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make meshes for dimension %d", dim);
49   }
50   if (rank) {
51     PetscInt numPoints[2] = {0, 0};
52     ierr = DMPlexCreateFromDAG(dm, 1, numPoints, NULL, NULL, NULL, NULL);CHKERRQ(ierr);
53   } else {
54     switch (dim) {
55     case 2:
56       if (simplex) {
57         PetscInt    numPoints[2]        = {4, 2};
58         PetscInt    coneSize[6]         = {3, 3, 0, 0, 0, 0};
59         PetscInt    cones[6]            = {2, 3, 4,  5, 4, 3};
60         PetscInt    coneOrientations[6] = {0, 0, 0,  0, 0, 0};
61         PetscScalar vertexCoords[8]     = {-0.5, 0.5, 0.0, 0.0, 0.0, 1.0, 0.5, 0.5};
62         PetscInt    markerPoints[8]     = {2, 1, 3, 1, 4, 1, 5, 1};
63 
64         ierr = DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
65         for (p = 0; p < 4; ++p) {ierr = DMSetLabelValue(dm, "marker", markerPoints[p*2], markerPoints[p*2+1]);CHKERRQ(ierr);}
66       } else {
67         PetscInt    numPoints[2]        = {6, 2};
68         PetscInt    coneSize[8]         = {4, 4, 0, 0, 0, 0, 0, 0};
69         PetscInt    cones[8]            = {2, 3, 4, 5,  3, 6, 7, 4};
70         PetscInt    coneOrientations[8] = {0, 0, 0, 0,  0, 0, 0, 0};
71         PetscScalar vertexCoords[12]    = {-1.0, -0.5,  0.0, -0.5,  0.0, 0.5,  -1.0, 0.5,  1.0, -0.5,  1.0, 0.5};
72 
73         ierr = DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
74       }
75       break;
76     case 3:
77       if (simplex) {
78         PetscInt    numPoints[2]        = {5, 2};
79         PetscInt    coneSize[7]         = {4, 4, 0, 0, 0, 0, 0};
80         PetscInt    cones[8]            = {4, 3, 5, 2,  5, 3, 4, 6};
81         PetscInt    coneOrientations[8] = {0, 0, 0, 0,  0, 0, 0, 0};
82         PetscScalar vertexCoords[15]    = {-1.0, 0.0, 0.0,  0.0, -1.0, 0.0,  0.0, 0.0, 1.0,  0.0, 1.0, 0.0,  1.0, 0.0, 0.0};
83         PetscInt    markerPoints[10]    = {2, 1, 3, 1, 4, 1, 5, 1, 6, 1};
84 
85         ierr = DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
86         for (p = 0; p < 5; ++p) {ierr = DMSetLabelValue(dm, "marker", markerPoints[p*2], markerPoints[p*2+1]);CHKERRQ(ierr);}
87       } else {
88         PetscInt    numPoints[2]         = {12, 2};
89         PetscInt    coneSize[14]         = {8, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
90         PetscInt    cones[16]            = {2, 3, 4, 5, 6, 7, 8, 9,  5, 4, 10, 11, 7, 12, 13, 8};
91         PetscInt    coneOrientations[16] = {0, 0, 0, 0, 0, 0, 0, 0,  0, 0,  0,  0, 0,  0,  0, 0};
92         PetscScalar vertexCoords[36]     = {-1.0, -0.5, -0.5,  -1.0,  0.5, -0.5,  0.0,  0.5, -0.5,   0.0, -0.5, -0.5,
93                                             -1.0, -0.5,  0.5,   0.0, -0.5,  0.5,  0.0,  0.5,  0.5,  -1.0,  0.5,  0.5,
94                                              1.0,  0.5, -0.5,   1.0, -0.5, -0.5,  1.0, -0.5,  0.5,   1.0,  0.5,  0.5};
95 
96         ierr = DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
97       }
98       break;
99     default:
100       SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make meshes for dimension %d", dim);
101     }
102   }
103   *newdm = dm;
104   if (refinementLimit > 0.0) {
105     DM rdm;
106     const char *name;
107 
108     ierr = DMPlexSetRefinementUniform(*newdm, PETSC_FALSE);CHKERRQ(ierr);
109     ierr = DMPlexSetRefinementLimit(*newdm, refinementLimit);CHKERRQ(ierr);
110     ierr = DMRefine(*newdm, comm, &rdm);CHKERRQ(ierr);
111     ierr = PetscObjectGetName((PetscObject) *newdm, &name);CHKERRQ(ierr);
112     ierr = PetscObjectSetName((PetscObject)    rdm,  name);CHKERRQ(ierr);
113     ierr = DMDestroy(newdm);CHKERRQ(ierr);
114     *newdm = rdm;
115   }
116   if (interpolate) {
117     DM idm;
118 
119     ierr = DMPlexInterpolate(*newdm, &idm);CHKERRQ(ierr);
120     ierr = DMDestroy(newdm);CHKERRQ(ierr);
121     *newdm = idm;
122   }
123   {
124     DM refinedMesh     = NULL;
125     DM distributedMesh = NULL;
126 
127     /* Distribute mesh over processes */
128     ierr = DMPlexDistribute(*newdm, 0, NULL, &distributedMesh);CHKERRQ(ierr);
129     if (distributedMesh) {
130       ierr = DMDestroy(newdm);CHKERRQ(ierr);
131       *newdm = distributedMesh;
132     }
133     if (refinementUniform) {
134       ierr = DMPlexSetRefinementUniform(*newdm, refinementUniform);CHKERRQ(ierr);
135       ierr = DMRefine(*newdm, comm, &refinedMesh);CHKERRQ(ierr);
136       if (refinedMesh) {
137         ierr = DMDestroy(newdm);CHKERRQ(ierr);
138         *newdm = refinedMesh;
139       }
140     }
141   }
142   PetscFunctionReturn(0);
143 }
144 
145 /*@
146   DMPlexCreateSquareBoundary - Creates a 1D mesh the is the boundary of a square lattice.
147 
148   Collective
149 
150   Input Parameters:
151 + comm  - The communicator for the DM object
152 . lower - The lower left corner coordinates
153 . upper - The upper right corner coordinates
154 - edges - The number of cells in each direction
155 
156   Output Parameter:
157 . dm  - The DM object
158 
159   Note: Here is the numbering returned for 2 cells in each direction:
160 $ 18--5-17--4--16
161 $  |     |     |
162 $  6    10     3
163 $  |     |     |
164 $ 19-11-20--9--15
165 $  |     |     |
166 $  7     8     2
167 $  |     |     |
168 $ 12--0-13--1--14
169 
170   Level: beginner
171 
172 .seealso: DMPlexCreateBoxMesh(), DMPlexCreateCubeBoundary(), DMSetType(), DMCreate()
173 @*/
174 PetscErrorCode DMPlexCreateSquareBoundary(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[])
175 {
176   const PetscInt numVertices    = (edges[0]+1)*(edges[1]+1);
177   const PetscInt numEdges       = edges[0]*(edges[1]+1) + (edges[0]+1)*edges[1];
178   PetscInt       markerTop      = 1;
179   PetscInt       markerBottom   = 1;
180   PetscInt       markerRight    = 1;
181   PetscInt       markerLeft     = 1;
182   PetscBool      markerSeparate = PETSC_FALSE;
183   Vec            coordinates;
184   PetscSection   coordSection;
185   PetscScalar    *coords;
186   PetscInt       coordSize;
187   PetscMPIInt    rank;
188   PetscInt       v, vx, vy;
189   PetscErrorCode ierr;
190 
191   PetscFunctionBegin;
192   ierr = PetscOptionsGetBool(((PetscObject) dm)->options,((PetscObject) dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL);CHKERRQ(ierr);
193   if (markerSeparate) {
194     markerTop    = 3;
195     markerBottom = 1;
196     markerRight  = 2;
197     markerLeft   = 4;
198   }
199   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRQ(ierr);
200   if (!rank) {
201     PetscInt e, ex, ey;
202 
203     ierr = DMPlexSetChart(dm, 0, numEdges+numVertices);CHKERRQ(ierr);
204     for (e = 0; e < numEdges; ++e) {
205       ierr = DMPlexSetConeSize(dm, e, 2);CHKERRQ(ierr);
206     }
207     ierr = DMSetUp(dm);CHKERRQ(ierr); /* Allocate space for cones */
208     for (vx = 0; vx <= edges[0]; vx++) {
209       for (ey = 0; ey < edges[1]; ey++) {
210         PetscInt edge   = vx*edges[1] + ey + edges[0]*(edges[1]+1);
211         PetscInt vertex = ey*(edges[0]+1) + vx + numEdges;
212         PetscInt cone[2];
213 
214         cone[0] = vertex; cone[1] = vertex+edges[0]+1;
215         ierr    = DMPlexSetCone(dm, edge, cone);CHKERRQ(ierr);
216         if (vx == edges[0]) {
217           ierr = DMSetLabelValue(dm, "marker", edge,    markerRight);CHKERRQ(ierr);
218           ierr = DMSetLabelValue(dm, "marker", cone[0], markerRight);CHKERRQ(ierr);
219           if (ey == edges[1]-1) {
220             ierr = DMSetLabelValue(dm, "marker", cone[1], markerRight);CHKERRQ(ierr);
221             ierr = DMSetLabelValue(dm, "Face Sets", cone[1], markerRight);CHKERRQ(ierr);
222           }
223         } else if (vx == 0) {
224           ierr = DMSetLabelValue(dm, "marker", edge,    markerLeft);CHKERRQ(ierr);
225           ierr = DMSetLabelValue(dm, "marker", cone[0], markerLeft);CHKERRQ(ierr);
226           if (ey == edges[1]-1) {
227             ierr = DMSetLabelValue(dm, "marker", cone[1], markerLeft);CHKERRQ(ierr);
228             ierr = DMSetLabelValue(dm, "Face Sets", cone[1], markerLeft);CHKERRQ(ierr);
229           }
230         }
231       }
232     }
233     for (vy = 0; vy <= edges[1]; vy++) {
234       for (ex = 0; ex < edges[0]; ex++) {
235         PetscInt edge   = vy*edges[0]     + ex;
236         PetscInt vertex = vy*(edges[0]+1) + ex + numEdges;
237         PetscInt cone[2];
238 
239         cone[0] = vertex; cone[1] = vertex+1;
240         ierr    = DMPlexSetCone(dm, edge, cone);CHKERRQ(ierr);
241         if (vy == edges[1]) {
242           ierr = DMSetLabelValue(dm, "marker", edge,    markerTop);CHKERRQ(ierr);
243           ierr = DMSetLabelValue(dm, "marker", cone[0], markerTop);CHKERRQ(ierr);
244           if (ex == edges[0]-1) {
245             ierr = DMSetLabelValue(dm, "marker", cone[1], markerTop);CHKERRQ(ierr);
246             ierr = DMSetLabelValue(dm, "Face Sets", cone[1], markerTop);CHKERRQ(ierr);
247           }
248         } else if (vy == 0) {
249           ierr = DMSetLabelValue(dm, "marker", edge,    markerBottom);CHKERRQ(ierr);
250           ierr = DMSetLabelValue(dm, "marker", cone[0], markerBottom);CHKERRQ(ierr);
251           if (ex == edges[0]-1) {
252             ierr = DMSetLabelValue(dm, "marker", cone[1], markerBottom);CHKERRQ(ierr);
253             ierr = DMSetLabelValue(dm, "Face Sets", cone[1], markerBottom);CHKERRQ(ierr);
254           }
255         }
256       }
257     }
258   }
259   ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
260   ierr = DMPlexStratify(dm);CHKERRQ(ierr);
261   /* Build coordinates */
262   ierr = DMSetCoordinateDim(dm, 2);CHKERRQ(ierr);
263   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
264   ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
265   ierr = PetscSectionSetChart(coordSection, numEdges, numEdges + numVertices);CHKERRQ(ierr);
266   ierr = PetscSectionSetFieldComponents(coordSection, 0, 2);CHKERRQ(ierr);
267   for (v = numEdges; v < numEdges+numVertices; ++v) {
268     ierr = PetscSectionSetDof(coordSection, v, 2);CHKERRQ(ierr);
269     ierr = PetscSectionSetFieldDof(coordSection, v, 0, 2);CHKERRQ(ierr);
270   }
271   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
272   ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
273   ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
274   ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
275   ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
276   ierr = VecSetBlockSize(coordinates, 2);CHKERRQ(ierr);
277   ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
278   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
279   for (vy = 0; vy <= edges[1]; ++vy) {
280     for (vx = 0; vx <= edges[0]; ++vx) {
281       coords[(vy*(edges[0]+1)+vx)*2+0] = lower[0] + ((upper[0] - lower[0])/edges[0])*vx;
282       coords[(vy*(edges[0]+1)+vx)*2+1] = lower[1] + ((upper[1] - lower[1])/edges[1])*vy;
283     }
284   }
285   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
286   ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
287   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
288   PetscFunctionReturn(0);
289 }
290 
291 /*@
292   DMPlexCreateCubeBoundary - Creates a 2D mesh that is the boundary of a cubic lattice.
293 
294   Collective
295 
296   Input Parameters:
297 + comm  - The communicator for the DM object
298 . lower - The lower left front corner coordinates
299 . upper - The upper right back corner coordinates
300 - edges - The number of cells in each direction
301 
302   Output Parameter:
303 . dm  - The DM object
304 
305   Level: beginner
306 
307 .seealso: DMPlexCreateBoxMesh(), DMPlexCreateSquareBoundary(), DMSetType(), DMCreate()
308 @*/
309 PetscErrorCode DMPlexCreateCubeBoundary(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt faces[])
310 {
311   PetscInt       vertices[3], numVertices;
312   PetscInt       numFaces    = 2*faces[0]*faces[1] + 2*faces[1]*faces[2] + 2*faces[0]*faces[2];
313   Vec            coordinates;
314   PetscSection   coordSection;
315   PetscScalar    *coords;
316   PetscInt       coordSize;
317   PetscMPIInt    rank;
318   PetscInt       v, vx, vy, vz;
319   PetscInt       voffset, iface=0, cone[4];
320   PetscErrorCode ierr;
321 
322   PetscFunctionBegin;
323   if ((faces[0] < 1) || (faces[1] < 1) || (faces[2] < 1)) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Must have at least 1 face per side");
324   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRQ(ierr);
325   vertices[0] = faces[0]+1; vertices[1] = faces[1]+1; vertices[2] = faces[2]+1;
326   numVertices = vertices[0]*vertices[1]*vertices[2];
327   if (!rank) {
328     PetscInt f;
329 
330     ierr = DMPlexSetChart(dm, 0, numFaces+numVertices);CHKERRQ(ierr);
331     for (f = 0; f < numFaces; ++f) {
332       ierr = DMPlexSetConeSize(dm, f, 4);CHKERRQ(ierr);
333     }
334     ierr = DMSetUp(dm);CHKERRQ(ierr); /* Allocate space for cones */
335 
336     /* Side 0 (Top) */
337     for (vy = 0; vy < faces[1]; vy++) {
338       for (vx = 0; vx < faces[0]; vx++) {
339         voffset = numFaces + vertices[0]*vertices[1]*(vertices[2]-1) + vy*vertices[0] + vx;
340         cone[0] = voffset; cone[1] = voffset+1; cone[2] = voffset+vertices[0]+1; cone[3] = voffset+vertices[0];
341         ierr    = DMPlexSetCone(dm, iface, cone);CHKERRQ(ierr);
342         ierr    = DMSetLabelValue(dm, "marker", iface, 1);CHKERRQ(ierr);
343         ierr    = DMSetLabelValue(dm, "marker", voffset+0, 1);CHKERRQ(ierr);
344         ierr    = DMSetLabelValue(dm, "marker", voffset+1, 1);CHKERRQ(ierr);
345         ierr    = DMSetLabelValue(dm, "marker", voffset+vertices[0]+0, 1);CHKERRQ(ierr);
346         ierr    = DMSetLabelValue(dm, "marker", voffset+vertices[0]+1, 1);CHKERRQ(ierr);
347         iface++;
348       }
349     }
350 
351     /* Side 1 (Bottom) */
352     for (vy = 0; vy < faces[1]; vy++) {
353       for (vx = 0; vx < faces[0]; vx++) {
354         voffset = numFaces + vy*(faces[0]+1) + vx;
355         cone[0] = voffset+1; cone[1] = voffset; cone[2] = voffset+vertices[0]; cone[3] = voffset+vertices[0]+1;
356         ierr    = DMPlexSetCone(dm, iface, cone);CHKERRQ(ierr);
357         ierr    = DMSetLabelValue(dm, "marker", iface, 1);CHKERRQ(ierr);
358         ierr    = DMSetLabelValue(dm, "marker", voffset+0, 1);CHKERRQ(ierr);
359         ierr    = DMSetLabelValue(dm, "marker", voffset+1, 1);CHKERRQ(ierr);
360         ierr    = DMSetLabelValue(dm, "marker", voffset+vertices[0]+0, 1);CHKERRQ(ierr);
361         ierr    = DMSetLabelValue(dm, "marker", voffset+vertices[0]+1, 1);CHKERRQ(ierr);
362         iface++;
363       }
364     }
365 
366     /* Side 2 (Front) */
367     for (vz = 0; vz < faces[2]; vz++) {
368       for (vx = 0; vx < faces[0]; vx++) {
369         voffset = numFaces + vz*vertices[0]*vertices[1] + vx;
370         cone[0] = voffset; cone[1] = voffset+1; cone[2] = voffset+vertices[0]*vertices[1]+1; cone[3] = voffset+vertices[0]*vertices[1];
371         ierr    = DMPlexSetCone(dm, iface, cone);CHKERRQ(ierr);
372         ierr    = DMSetLabelValue(dm, "marker", iface, 1);CHKERRQ(ierr);
373         ierr    = DMSetLabelValue(dm, "marker", voffset+0, 1);CHKERRQ(ierr);
374         ierr    = DMSetLabelValue(dm, "marker", voffset+1, 1);CHKERRQ(ierr);
375         ierr    = DMSetLabelValue(dm, "marker", voffset+vertices[0]*vertices[1]+0, 1);CHKERRQ(ierr);
376         ierr    = DMSetLabelValue(dm, "marker", voffset+vertices[0]*vertices[1]+1, 1);CHKERRQ(ierr);
377         iface++;
378       }
379     }
380 
381     /* Side 3 (Back) */
382     for (vz = 0; vz < faces[2]; vz++) {
383       for (vx = 0; vx < faces[0]; vx++) {
384         voffset = numFaces + vz*vertices[0]*vertices[1] + vertices[0]*(vertices[1]-1) + vx;
385         cone[0] = voffset+vertices[0]*vertices[1]; cone[1] = voffset+vertices[0]*vertices[1]+1;
386         cone[2] = voffset+1; cone[3] = voffset;
387         ierr    = DMPlexSetCone(dm, iface, cone);CHKERRQ(ierr);
388         ierr    = DMSetLabelValue(dm, "marker", iface, 1);CHKERRQ(ierr);
389         ierr    = DMSetLabelValue(dm, "marker", voffset+0, 1);CHKERRQ(ierr);
390         ierr    = DMSetLabelValue(dm, "marker", voffset+1, 1);CHKERRQ(ierr);
391         ierr    = DMSetLabelValue(dm, "marker", voffset+vertices[0]*vertices[1]+0, 1);CHKERRQ(ierr);
392         ierr    = DMSetLabelValue(dm, "marker", voffset+vertices[0]*vertices[1]+1, 1);CHKERRQ(ierr);
393         iface++;
394       }
395     }
396 
397     /* Side 4 (Left) */
398     for (vz = 0; vz < faces[2]; vz++) {
399       for (vy = 0; vy < faces[1]; vy++) {
400         voffset = numFaces + vz*vertices[0]*vertices[1] + vy*vertices[0];
401         cone[0] = voffset; cone[1] = voffset+vertices[0]*vertices[1];
402         cone[2] = voffset+vertices[0]*vertices[1]+vertices[0]; cone[3] = voffset+vertices[0];
403         ierr    = DMPlexSetCone(dm, iface, cone);CHKERRQ(ierr);
404         ierr    = DMSetLabelValue(dm, "marker", iface, 1);CHKERRQ(ierr);
405         ierr    = DMSetLabelValue(dm, "marker", voffset+0, 1);CHKERRQ(ierr);
406         ierr    = DMSetLabelValue(dm, "marker", voffset+vertices[0]+0, 1);CHKERRQ(ierr);
407         ierr    = DMSetLabelValue(dm, "marker", voffset+vertices[1]+0, 1);CHKERRQ(ierr);
408         ierr    = DMSetLabelValue(dm, "marker", voffset+vertices[0]*vertices[1]+vertices[0], 1);CHKERRQ(ierr);
409         iface++;
410       }
411     }
412 
413     /* Side 5 (Right) */
414     for (vz = 0; vz < faces[2]; vz++) {
415       for (vy = 0; vy < faces[1]; vy++) {
416         voffset = numFaces + vz*vertices[0]*vertices[1] + vy*vertices[0] + faces[0];
417         cone[0] = voffset+vertices[0]*vertices[1]; cone[1] = voffset;
418         cone[2] = voffset+vertices[0]; cone[3] = voffset+vertices[0]*vertices[1]+vertices[0];
419         ierr    = DMPlexSetCone(dm, iface, cone);CHKERRQ(ierr);
420         ierr    = DMSetLabelValue(dm, "marker", iface, 1);CHKERRQ(ierr);
421         ierr    = DMSetLabelValue(dm, "marker", voffset+0, 1);CHKERRQ(ierr);
422         ierr    = DMSetLabelValue(dm, "marker", voffset+vertices[0]+0, 1);CHKERRQ(ierr);
423         ierr    = DMSetLabelValue(dm, "marker", voffset+vertices[0]*vertices[1]+0, 1);CHKERRQ(ierr);
424         ierr    = DMSetLabelValue(dm, "marker", voffset+vertices[0]*vertices[1]+vertices[0], 1);CHKERRQ(ierr);
425         iface++;
426       }
427     }
428   }
429   ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
430   ierr = DMPlexStratify(dm);CHKERRQ(ierr);
431   /* Build coordinates */
432   ierr = DMSetCoordinateDim(dm, 3);CHKERRQ(ierr);
433   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
434   ierr = PetscSectionSetChart(coordSection, numFaces, numFaces + numVertices);CHKERRQ(ierr);
435   for (v = numFaces; v < numFaces+numVertices; ++v) {
436     ierr = PetscSectionSetDof(coordSection, v, 3);CHKERRQ(ierr);
437   }
438   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
439   ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
440   ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
441   ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
442   ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
443   ierr = VecSetBlockSize(coordinates, 3);CHKERRQ(ierr);
444   ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
445   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
446   for (vz = 0; vz <= faces[2]; ++vz) {
447     for (vy = 0; vy <= faces[1]; ++vy) {
448       for (vx = 0; vx <= faces[0]; ++vx) {
449         coords[((vz*(faces[1]+1)+vy)*(faces[0]+1)+vx)*3+0] = lower[0] + ((upper[0] - lower[0])/faces[0])*vx;
450         coords[((vz*(faces[1]+1)+vy)*(faces[0]+1)+vx)*3+1] = lower[1] + ((upper[1] - lower[1])/faces[1])*vy;
451         coords[((vz*(faces[1]+1)+vy)*(faces[0]+1)+vx)*3+2] = lower[2] + ((upper[2] - lower[2])/faces[2])*vz;
452       }
453     }
454   }
455   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
456   ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
457   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
458   PetscFunctionReturn(0);
459 }
460 
461 static PetscErrorCode DMPlexCreateLineMesh_Internal(MPI_Comm comm,PetscInt segments,PetscReal lower,PetscReal upper,DMBoundaryType bd,DM *dm)
462 {
463   PetscInt       i,fStart,fEnd,numCells = 0,numVerts = 0;
464   PetscInt       numPoints[2],*coneSize,*cones,*coneOrientations;
465   PetscScalar    *vertexCoords;
466   PetscReal      L,maxCell;
467   PetscBool      markerSeparate = PETSC_FALSE;
468   PetscInt       markerLeft  = 1, faceMarkerLeft  = 1;
469   PetscInt       markerRight = 1, faceMarkerRight = 2;
470   PetscBool      wrap = (bd == DM_BOUNDARY_PERIODIC || bd == DM_BOUNDARY_TWIST) ? PETSC_TRUE : PETSC_FALSE;
471   PetscMPIInt    rank;
472   PetscErrorCode ierr;
473 
474   PetscFunctionBegin;
475   PetscValidPointer(dm,4);
476 
477   ierr = DMCreate(comm,dm);CHKERRQ(ierr);
478   ierr = DMSetType(*dm,DMPLEX);CHKERRQ(ierr);
479   ierr = DMSetDimension(*dm,1);CHKERRQ(ierr);
480   ierr = DMCreateLabel(*dm,"marker");CHKERRQ(ierr);
481   ierr = DMCreateLabel(*dm,"Face Sets");CHKERRQ(ierr);
482 
483   ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr);
484   if (!rank) numCells = segments;
485   if (!rank) numVerts = segments + (wrap ? 0 : 1);
486 
487   numPoints[0] = numVerts ; numPoints[1] = numCells;
488   ierr = PetscMalloc4(numCells+numVerts,&coneSize,numCells*2,&cones,numCells+numVerts,&coneOrientations,numVerts,&vertexCoords);CHKERRQ(ierr);
489   ierr = PetscArrayzero(coneOrientations,numCells+numVerts);CHKERRQ(ierr);
490   for (i = 0; i < numCells; ++i) { coneSize[i] = 2; }
491   for (i = 0; i < numVerts; ++i) { coneSize[numCells+i] = 0; }
492   for (i = 0; i < numCells; ++i) { cones[2*i] = numCells + i%numVerts; cones[2*i+1] = numCells + (i+1)%numVerts; }
493   for (i = 0; i < numVerts; ++i) { vertexCoords[i] = lower + (upper-lower)*((PetscReal)i/(PetscReal)numCells); }
494   ierr = DMPlexCreateFromDAG(*dm,1,numPoints,coneSize,cones,coneOrientations,vertexCoords);CHKERRQ(ierr);
495   ierr = PetscFree4(coneSize,cones,coneOrientations,vertexCoords);CHKERRQ(ierr);
496 
497   ierr = PetscOptionsGetBool(((PetscObject)*dm)->options,((PetscObject)*dm)->prefix,"-dm_plex_separate_marker",&markerSeparate,NULL);CHKERRQ(ierr);
498   if (markerSeparate) { markerLeft = faceMarkerLeft; markerRight = faceMarkerRight;}
499   if (!wrap && !rank) {
500     ierr = DMPlexGetHeightStratum(*dm,1,&fStart,&fEnd);CHKERRQ(ierr);
501     ierr = DMSetLabelValue(*dm,"marker",fStart,markerLeft);CHKERRQ(ierr);
502     ierr = DMSetLabelValue(*dm,"marker",fEnd-1,markerRight);CHKERRQ(ierr);
503     ierr = DMSetLabelValue(*dm,"Face Sets",fStart,faceMarkerLeft);CHKERRQ(ierr);
504     ierr = DMSetLabelValue(*dm,"Face Sets",fEnd-1,faceMarkerRight);CHKERRQ(ierr);
505   }
506   if (wrap) {
507     L       = upper - lower;
508     maxCell = (PetscReal)1.1*(L/(PetscReal)PetscMax(1,segments));
509     ierr = DMSetPeriodicity(*dm,PETSC_TRUE,&maxCell,&L,&bd);CHKERRQ(ierr);
510   }
511   PetscFunctionReturn(0);
512 }
513 
514 static PetscErrorCode DMPlexCreateBoxMesh_Simplex_Internal(MPI_Comm comm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate, DM *dm)
515 {
516   DM             boundary;
517   PetscInt       i;
518   PetscErrorCode ierr;
519 
520   PetscFunctionBegin;
521   PetscValidPointer(dm, 4);
522   for (i = 0; i < dim; ++i) if (periodicity[i] != DM_BOUNDARY_NONE) SETERRQ(comm, PETSC_ERR_SUP, "Periodicity is not supported for simplex meshes");
523   ierr = DMCreate(comm, &boundary);CHKERRQ(ierr);
524   PetscValidLogicalCollectiveInt(boundary,dim,2);
525   ierr = DMSetType(boundary, DMPLEX);CHKERRQ(ierr);
526   ierr = DMSetDimension(boundary, dim-1);CHKERRQ(ierr);
527   ierr = DMSetCoordinateDim(boundary, dim);CHKERRQ(ierr);
528   switch (dim) {
529   case 2: ierr = DMPlexCreateSquareBoundary(boundary, lower, upper, faces);CHKERRQ(ierr);break;
530   case 3: ierr = DMPlexCreateCubeBoundary(boundary, lower, upper, faces);CHKERRQ(ierr);break;
531   default: SETERRQ1(comm, PETSC_ERR_SUP, "Dimension not supported: %d", dim);
532   }
533   ierr = DMPlexGenerate(boundary, NULL, interpolate, dm);CHKERRQ(ierr);
534   ierr = DMDestroy(&boundary);CHKERRQ(ierr);
535   PetscFunctionReturn(0);
536 }
537 
538 static PetscErrorCode DMPlexCreateCubeMesh_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[], DMBoundaryType bdX, DMBoundaryType bdY, DMBoundaryType bdZ)
539 {
540   DMLabel        cutLabel = NULL;
541   PetscInt       markerTop      = 1, faceMarkerTop      = 1;
542   PetscInt       markerBottom   = 1, faceMarkerBottom   = 1;
543   PetscInt       markerFront    = 1, faceMarkerFront    = 1;
544   PetscInt       markerBack     = 1, faceMarkerBack     = 1;
545   PetscInt       markerRight    = 1, faceMarkerRight    = 1;
546   PetscInt       markerLeft     = 1, faceMarkerLeft     = 1;
547   PetscInt       dim;
548   PetscBool      markerSeparate = PETSC_FALSE, cutMarker = PETSC_FALSE;
549   PetscMPIInt    rank;
550   PetscErrorCode ierr;
551 
552   PetscFunctionBegin;
553   ierr = DMGetDimension(dm,&dim);CHKERRQ(ierr);
554   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRQ(ierr);
555   ierr = DMCreateLabel(dm,"marker");CHKERRQ(ierr);
556   ierr = DMCreateLabel(dm,"Face Sets");CHKERRQ(ierr);
557   ierr = PetscOptionsGetBool(((PetscObject) dm)->options,((PetscObject) dm)->prefix, "-dm_plex_periodic_cut", &cutMarker, NULL);CHKERRQ(ierr);
558   if (bdX == DM_BOUNDARY_PERIODIC || bdX == DM_BOUNDARY_TWIST ||
559       bdY == DM_BOUNDARY_PERIODIC || bdY == DM_BOUNDARY_TWIST ||
560       bdZ == DM_BOUNDARY_PERIODIC || bdZ == DM_BOUNDARY_TWIST) {
561 
562     if (cutMarker) {ierr = DMCreateLabel(dm, "periodic_cut");CHKERRQ(ierr); ierr = DMGetLabel(dm, "periodic_cut", &cutLabel);CHKERRQ(ierr);}
563   }
564   switch (dim) {
565   case 2:
566     faceMarkerTop    = 3;
567     faceMarkerBottom = 1;
568     faceMarkerRight  = 2;
569     faceMarkerLeft   = 4;
570     break;
571   case 3:
572     faceMarkerBottom = 1;
573     faceMarkerTop    = 2;
574     faceMarkerFront  = 3;
575     faceMarkerBack   = 4;
576     faceMarkerRight  = 5;
577     faceMarkerLeft   = 6;
578     break;
579   default:
580     SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SUP,"Dimension %d not supported",dim);
581     break;
582   }
583   ierr = PetscOptionsGetBool(((PetscObject) dm)->options,((PetscObject) dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL);CHKERRQ(ierr);
584   if (markerSeparate) {
585     markerBottom = faceMarkerBottom;
586     markerTop    = faceMarkerTop;
587     markerFront  = faceMarkerFront;
588     markerBack   = faceMarkerBack;
589     markerRight  = faceMarkerRight;
590     markerLeft   = faceMarkerLeft;
591   }
592   {
593     const PetscInt numXEdges    = !rank ? edges[0] : 0;
594     const PetscInt numYEdges    = !rank ? edges[1] : 0;
595     const PetscInt numZEdges    = !rank ? edges[2] : 0;
596     const PetscInt numXVertices = !rank ? (bdX == DM_BOUNDARY_PERIODIC || bdX == DM_BOUNDARY_TWIST ? edges[0] : edges[0]+1) : 0;
597     const PetscInt numYVertices = !rank ? (bdY == DM_BOUNDARY_PERIODIC || bdY == DM_BOUNDARY_TWIST ? edges[1] : edges[1]+1) : 0;
598     const PetscInt numZVertices = !rank ? (bdZ == DM_BOUNDARY_PERIODIC || bdZ == DM_BOUNDARY_TWIST ? edges[2] : edges[2]+1) : 0;
599     const PetscInt numCells     = numXEdges*numYEdges*numZEdges;
600     const PetscInt numXFaces    = numYEdges*numZEdges;
601     const PetscInt numYFaces    = numXEdges*numZEdges;
602     const PetscInt numZFaces    = numXEdges*numYEdges;
603     const PetscInt numTotXFaces = numXVertices*numXFaces;
604     const PetscInt numTotYFaces = numYVertices*numYFaces;
605     const PetscInt numTotZFaces = numZVertices*numZFaces;
606     const PetscInt numFaces     = numTotXFaces + numTotYFaces + numTotZFaces;
607     const PetscInt numTotXEdges = numXEdges*numYVertices*numZVertices;
608     const PetscInt numTotYEdges = numYEdges*numXVertices*numZVertices;
609     const PetscInt numTotZEdges = numZEdges*numXVertices*numYVertices;
610     const PetscInt numVertices  = numXVertices*numYVertices*numZVertices;
611     const PetscInt numEdges     = numTotXEdges + numTotYEdges + numTotZEdges;
612     const PetscInt firstVertex  = (dim == 2) ? numFaces : numCells;
613     const PetscInt firstXFace   = (dim == 2) ? 0 : numCells + numVertices;
614     const PetscInt firstYFace   = firstXFace + numTotXFaces;
615     const PetscInt firstZFace   = firstYFace + numTotYFaces;
616     const PetscInt firstXEdge   = numCells + numFaces + numVertices;
617     const PetscInt firstYEdge   = firstXEdge + numTotXEdges;
618     const PetscInt firstZEdge   = firstYEdge + numTotYEdges;
619     Vec            coordinates;
620     PetscSection   coordSection;
621     PetscScalar   *coords;
622     PetscInt       coordSize;
623     PetscInt       v, vx, vy, vz;
624     PetscInt       c, f, fx, fy, fz, e, ex, ey, ez;
625 
626     ierr = DMPlexSetChart(dm, 0, numCells+numFaces+numEdges+numVertices);CHKERRQ(ierr);
627     for (c = 0; c < numCells; c++) {
628       ierr = DMPlexSetConeSize(dm, c, 6);CHKERRQ(ierr);
629     }
630     for (f = firstXFace; f < firstXFace+numFaces; ++f) {
631       ierr = DMPlexSetConeSize(dm, f, 4);CHKERRQ(ierr);
632     }
633     for (e = firstXEdge; e < firstXEdge+numEdges; ++e) {
634       ierr = DMPlexSetConeSize(dm, e, 2);CHKERRQ(ierr);
635     }
636     ierr = DMSetUp(dm);CHKERRQ(ierr); /* Allocate space for cones */
637     /* Build cells */
638     for (fz = 0; fz < numZEdges; ++fz) {
639       for (fy = 0; fy < numYEdges; ++fy) {
640         for (fx = 0; fx < numXEdges; ++fx) {
641           PetscInt cell    = (fz*numYEdges + fy)*numXEdges + fx;
642           PetscInt faceB   = firstZFace + (fy*numXEdges+fx)*numZVertices +   fz;
643           PetscInt faceT   = firstZFace + (fy*numXEdges+fx)*numZVertices + ((fz+1)%numZVertices);
644           PetscInt faceF   = firstYFace + (fz*numXEdges+fx)*numYVertices +   fy;
645           PetscInt faceK   = firstYFace + (fz*numXEdges+fx)*numYVertices + ((fy+1)%numYVertices);
646           PetscInt faceL   = firstXFace + (fz*numYEdges+fy)*numXVertices +   fx;
647           PetscInt faceR   = firstXFace + (fz*numYEdges+fy)*numXVertices + ((fx+1)%numXVertices);
648                             /* B,  T,  F,  K,  R,  L */
649           PetscInt ornt[6] = {-4,  0,  0, -1,  0, -4}; /* ??? */
650           PetscInt cone[6];
651 
652           /* no boundary twisting in 3D */
653           cone[0] = faceB; cone[1] = faceT; cone[2] = faceF; cone[3] = faceK; cone[4] = faceR; cone[5] = faceL;
654           ierr    = DMPlexSetCone(dm, cell, cone);CHKERRQ(ierr);
655           ierr    = DMPlexSetConeOrientation(dm, cell, ornt);CHKERRQ(ierr);
656           if (bdX != DM_BOUNDARY_NONE && fx == numXEdges-1 && cutLabel) {ierr = DMLabelSetValue(cutLabel, cell, 2);CHKERRQ(ierr);}
657           if (bdY != DM_BOUNDARY_NONE && fy == numYEdges-1 && cutLabel) {ierr = DMLabelSetValue(cutLabel, cell, 2);CHKERRQ(ierr);}
658           if (bdZ != DM_BOUNDARY_NONE && fz == numZEdges-1 && cutLabel) {ierr = DMLabelSetValue(cutLabel, cell, 2);CHKERRQ(ierr);}
659         }
660       }
661     }
662     /* Build x faces */
663     for (fz = 0; fz < numZEdges; ++fz) {
664       for (fy = 0; fy < numYEdges; ++fy) {
665         for (fx = 0; fx < numXVertices; ++fx) {
666           PetscInt face    = firstXFace + (fz*numYEdges+fy)*numXVertices + fx;
667           PetscInt edgeL   = firstZEdge + (  fy*                 numXVertices+fx)*numZEdges + fz;
668           PetscInt edgeR   = firstZEdge + (((fy+1)%numYVertices)*numXVertices+fx)*numZEdges + fz;
669           PetscInt edgeB   = firstYEdge + (  fz*                 numXVertices+fx)*numYEdges + fy;
670           PetscInt edgeT   = firstYEdge + (((fz+1)%numZVertices)*numXVertices+fx)*numYEdges + fy;
671           PetscInt ornt[4] = {0, 0, -2, -2};
672           PetscInt cone[4];
673 
674           if (dim == 3) {
675             /* markers */
676             if (bdX != DM_BOUNDARY_PERIODIC) {
677               if (fx == numXVertices-1) {
678                 ierr = DMSetLabelValue(dm, "Face Sets", face, faceMarkerRight);CHKERRQ(ierr);
679                 ierr = DMSetLabelValue(dm, "marker", face, markerRight);CHKERRQ(ierr);
680               }
681               else if (fx == 0) {
682                 ierr = DMSetLabelValue(dm, "Face Sets", face, faceMarkerLeft);CHKERRQ(ierr);
683                 ierr = DMSetLabelValue(dm, "marker", face, markerLeft);CHKERRQ(ierr);
684               }
685             }
686           }
687           cone[0] = edgeB; cone[1] = edgeR; cone[2] = edgeT; cone[3] = edgeL;
688           ierr    = DMPlexSetCone(dm, face, cone);CHKERRQ(ierr);
689           ierr    = DMPlexSetConeOrientation(dm, face, ornt);CHKERRQ(ierr);
690         }
691       }
692     }
693     /* Build y faces */
694     for (fz = 0; fz < numZEdges; ++fz) {
695       for (fx = 0; fx < numXEdges; ++fx) {
696         for (fy = 0; fy < numYVertices; ++fy) {
697           PetscInt face    = firstYFace + (fz*numXEdges+fx)*numYVertices + fy;
698           PetscInt edgeL   = firstZEdge + (fy*numXVertices+  fx                 )*numZEdges + fz;
699           PetscInt edgeR   = firstZEdge + (fy*numXVertices+((fx+1)%numXVertices))*numZEdges + fz;
700           PetscInt edgeB   = firstXEdge + (  fz                 *numYVertices+fy)*numXEdges + fx;
701           PetscInt edgeT   = firstXEdge + (((fz+1)%numZVertices)*numYVertices+fy)*numXEdges + fx;
702           PetscInt ornt[4] = {0, 0, -2, -2};
703           PetscInt cone[4];
704 
705           if (dim == 3) {
706             /* markers */
707             if (bdY != DM_BOUNDARY_PERIODIC) {
708               if (fy == numYVertices-1) {
709                 ierr = DMSetLabelValue(dm, "Face Sets", face, faceMarkerBack);CHKERRQ(ierr);
710                 ierr = DMSetLabelValue(dm, "marker", face, markerBack);CHKERRQ(ierr);
711               }
712               else if (fy == 0) {
713                 ierr = DMSetLabelValue(dm, "Face Sets", face, faceMarkerFront);CHKERRQ(ierr);
714                 ierr = DMSetLabelValue(dm, "marker", face, markerFront);CHKERRQ(ierr);
715               }
716             }
717           }
718           cone[0] = edgeB; cone[1] = edgeR; cone[2] = edgeT; cone[3] = edgeL;
719           ierr    = DMPlexSetCone(dm, face, cone);CHKERRQ(ierr);
720           ierr    = DMPlexSetConeOrientation(dm, face, ornt);CHKERRQ(ierr);
721         }
722       }
723     }
724     /* Build z faces */
725     for (fy = 0; fy < numYEdges; ++fy) {
726       for (fx = 0; fx < numXEdges; ++fx) {
727         for (fz = 0; fz < numZVertices; fz++) {
728           PetscInt face    = firstZFace + (fy*numXEdges+fx)*numZVertices + fz;
729           PetscInt edgeL   = firstYEdge + (fz*numXVertices+  fx                 )*numYEdges + fy;
730           PetscInt edgeR   = firstYEdge + (fz*numXVertices+((fx+1)%numXVertices))*numYEdges + fy;
731           PetscInt edgeB   = firstXEdge + (fz*numYVertices+  fy                 )*numXEdges + fx;
732           PetscInt edgeT   = firstXEdge + (fz*numYVertices+((fy+1)%numYVertices))*numXEdges + fx;
733           PetscInt ornt[4] = {0, 0, -2, -2};
734           PetscInt cone[4];
735 
736           if (dim == 2) {
737             if (bdX == DM_BOUNDARY_TWIST && fx == numXEdges-1) {edgeR += numYEdges-1-2*fy; ornt[1] = -2;}
738             if (bdY == DM_BOUNDARY_TWIST && fy == numYEdges-1) {edgeT += numXEdges-1-2*fx; ornt[2] =  0;}
739             if (bdX != DM_BOUNDARY_NONE && fx == numXEdges-1 && cutLabel) {ierr = DMLabelSetValue(cutLabel, face, 2);CHKERRQ(ierr);}
740             if (bdY != DM_BOUNDARY_NONE && fy == numYEdges-1 && cutLabel) {ierr = DMLabelSetValue(cutLabel, face, 2);CHKERRQ(ierr);}
741           } else {
742             /* markers */
743             if (bdZ != DM_BOUNDARY_PERIODIC) {
744               if (fz == numZVertices-1) {
745                 ierr = DMSetLabelValue(dm, "Face Sets", face, faceMarkerTop);CHKERRQ(ierr);
746                 ierr = DMSetLabelValue(dm, "marker", face, markerTop);CHKERRQ(ierr);
747               }
748               else if (fz == 0) {
749                 ierr = DMSetLabelValue(dm, "Face Sets", face, faceMarkerBottom);CHKERRQ(ierr);
750                 ierr = DMSetLabelValue(dm, "marker", face, markerBottom);CHKERRQ(ierr);
751               }
752             }
753           }
754           cone[0] = edgeB; cone[1] = edgeR; cone[2] = edgeT; cone[3] = edgeL;
755           ierr    = DMPlexSetCone(dm, face, cone);CHKERRQ(ierr);
756           ierr    = DMPlexSetConeOrientation(dm, face, ornt);CHKERRQ(ierr);
757         }
758       }
759     }
760     /* Build Z edges*/
761     for (vy = 0; vy < numYVertices; vy++) {
762       for (vx = 0; vx < numXVertices; vx++) {
763         for (ez = 0; ez < numZEdges; ez++) {
764           const PetscInt edge    = firstZEdge  + (vy*numXVertices+vx)*numZEdges + ez;
765           const PetscInt vertexB = firstVertex + (  ez                 *numYVertices+vy)*numXVertices + vx;
766           const PetscInt vertexT = firstVertex + (((ez+1)%numZVertices)*numYVertices+vy)*numXVertices + vx;
767           PetscInt       cone[2];
768 
769           if (dim == 3) {
770             if (bdX != DM_BOUNDARY_PERIODIC) {
771               if (vx == numXVertices-1) {
772                 ierr = DMSetLabelValue(dm, "marker", edge, markerRight);CHKERRQ(ierr);
773               }
774               else if (vx == 0) {
775                 ierr = DMSetLabelValue(dm, "marker", edge, markerLeft);CHKERRQ(ierr);
776               }
777             }
778             if (bdY != DM_BOUNDARY_PERIODIC) {
779               if (vy == numYVertices-1) {
780                 ierr = DMSetLabelValue(dm, "marker", edge, markerBack);CHKERRQ(ierr);
781               }
782               else if (vy == 0) {
783                 ierr = DMSetLabelValue(dm, "marker", edge, markerFront);CHKERRQ(ierr);
784               }
785             }
786           }
787           cone[0] = vertexB; cone[1] = vertexT;
788           ierr = DMPlexSetCone(dm, edge, cone);CHKERRQ(ierr);
789         }
790       }
791     }
792     /* Build Y edges*/
793     for (vz = 0; vz < numZVertices; vz++) {
794       for (vx = 0; vx < numXVertices; vx++) {
795         for (ey = 0; ey < numYEdges; ey++) {
796           const PetscInt nextv   = (dim == 2 && bdY == DM_BOUNDARY_TWIST && ey == numYEdges-1) ? (numXVertices-vx-1) : (vz*numYVertices+((ey+1)%numYVertices))*numXVertices + vx;
797           const PetscInt edge    = firstYEdge  + (vz*numXVertices+vx)*numYEdges + ey;
798           const PetscInt vertexF = firstVertex + (vz*numYVertices+ey)*numXVertices + vx;
799           const PetscInt vertexK = firstVertex + nextv;
800           PetscInt       cone[2];
801 
802           cone[0] = vertexF; cone[1] = vertexK;
803           ierr = DMPlexSetCone(dm, edge, cone);CHKERRQ(ierr);
804           if (dim == 2) {
805             if ((bdX != DM_BOUNDARY_PERIODIC) && (bdX != DM_BOUNDARY_TWIST)) {
806               if (vx == numXVertices-1) {
807                 ierr = DMSetLabelValue(dm, "Face Sets", edge, faceMarkerRight);CHKERRQ(ierr);
808                 ierr = DMSetLabelValue(dm, "marker", edge,    markerRight);CHKERRQ(ierr);
809                 ierr = DMSetLabelValue(dm, "marker", cone[0], markerRight);CHKERRQ(ierr);
810                 if (ey == numYEdges-1) {
811                   ierr = DMSetLabelValue(dm, "marker", cone[1], markerRight);CHKERRQ(ierr);
812                 }
813               } else if (vx == 0) {
814                 ierr = DMSetLabelValue(dm, "Face Sets", edge, faceMarkerLeft);CHKERRQ(ierr);
815                 ierr = DMSetLabelValue(dm, "marker", edge,    markerLeft);CHKERRQ(ierr);
816                 ierr = DMSetLabelValue(dm, "marker", cone[0], markerLeft);CHKERRQ(ierr);
817                 if (ey == numYEdges-1) {
818                   ierr = DMSetLabelValue(dm, "marker", cone[1], markerLeft);CHKERRQ(ierr);
819                 }
820               }
821             } else {
822               if (vx == 0 && cutLabel) {
823                 ierr = DMLabelSetValue(cutLabel, edge,    1);CHKERRQ(ierr);
824                 ierr = DMLabelSetValue(cutLabel, cone[0], 1);CHKERRQ(ierr);
825                 if (ey == numYEdges-1) {
826                   ierr = DMLabelSetValue(cutLabel, cone[1], 1);CHKERRQ(ierr);
827                 }
828               }
829             }
830           } else {
831             if (bdX != DM_BOUNDARY_PERIODIC) {
832               if (vx == numXVertices-1) {
833                 ierr = DMSetLabelValue(dm, "marker", edge, markerRight);CHKERRQ(ierr);
834               } else if (vx == 0) {
835                 ierr = DMSetLabelValue(dm, "marker", edge, markerLeft);CHKERRQ(ierr);
836               }
837             }
838             if (bdZ != DM_BOUNDARY_PERIODIC) {
839               if (vz == numZVertices-1) {
840                 ierr = DMSetLabelValue(dm, "marker", edge, markerTop);CHKERRQ(ierr);
841               } else if (vz == 0) {
842                 ierr = DMSetLabelValue(dm, "marker", edge, markerBottom);CHKERRQ(ierr);
843               }
844             }
845           }
846         }
847       }
848     }
849     /* Build X edges*/
850     for (vz = 0; vz < numZVertices; vz++) {
851       for (vy = 0; vy < numYVertices; vy++) {
852         for (ex = 0; ex < numXEdges; ex++) {
853           const PetscInt nextv   = (dim == 2 && bdX == DM_BOUNDARY_TWIST && ex == numXEdges-1) ? (numYVertices-vy-1)*numXVertices : (vz*numYVertices+vy)*numXVertices + (ex+1)%numXVertices;
854           const PetscInt edge    = firstXEdge  + (vz*numYVertices+vy)*numXEdges + ex;
855           const PetscInt vertexL = firstVertex + (vz*numYVertices+vy)*numXVertices + ex;
856           const PetscInt vertexR = firstVertex + nextv;
857           PetscInt       cone[2];
858 
859           cone[0] = vertexL; cone[1] = vertexR;
860           ierr = DMPlexSetCone(dm, edge, cone);CHKERRQ(ierr);
861           if (dim == 2) {
862             if ((bdY != DM_BOUNDARY_PERIODIC) && (bdY != DM_BOUNDARY_TWIST)) {
863               if (vy == numYVertices-1) {
864                 ierr = DMSetLabelValue(dm, "Face Sets", edge, faceMarkerTop);CHKERRQ(ierr);
865                 ierr = DMSetLabelValue(dm, "marker", edge,    markerTop);CHKERRQ(ierr);
866                 ierr = DMSetLabelValue(dm, "marker", cone[0], markerTop);CHKERRQ(ierr);
867                 if (ex == numXEdges-1) {
868                   ierr = DMSetLabelValue(dm, "marker", cone[1], markerTop);CHKERRQ(ierr);
869                 }
870               } else if (vy == 0) {
871                 ierr = DMSetLabelValue(dm, "Face Sets", edge, faceMarkerBottom);CHKERRQ(ierr);
872                 ierr = DMSetLabelValue(dm, "marker", edge,    markerBottom);CHKERRQ(ierr);
873                 ierr = DMSetLabelValue(dm, "marker", cone[0], markerBottom);CHKERRQ(ierr);
874                 if (ex == numXEdges-1) {
875                   ierr = DMSetLabelValue(dm, "marker", cone[1], markerBottom);CHKERRQ(ierr);
876                 }
877               }
878             } else {
879               if (vy == 0 && cutLabel) {
880                 ierr = DMLabelSetValue(cutLabel, edge,    1);CHKERRQ(ierr);
881                 ierr = DMLabelSetValue(cutLabel, cone[0], 1);CHKERRQ(ierr);
882                 if (ex == numXEdges-1) {
883                   ierr = DMLabelSetValue(cutLabel, cone[1], 1);CHKERRQ(ierr);
884                 }
885               }
886             }
887           } else {
888             if (bdY != DM_BOUNDARY_PERIODIC) {
889               if (vy == numYVertices-1) {
890                 ierr = DMSetLabelValue(dm, "marker", edge, markerBack);CHKERRQ(ierr);
891               }
892               else if (vy == 0) {
893                 ierr = DMSetLabelValue(dm, "marker", edge, markerFront);CHKERRQ(ierr);
894               }
895             }
896             if (bdZ != DM_BOUNDARY_PERIODIC) {
897               if (vz == numZVertices-1) {
898                 ierr = DMSetLabelValue(dm, "marker", edge, markerTop);CHKERRQ(ierr);
899               }
900               else if (vz == 0) {
901                 ierr = DMSetLabelValue(dm, "marker", edge, markerBottom);CHKERRQ(ierr);
902               }
903             }
904           }
905         }
906       }
907     }
908     ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
909     ierr = DMPlexStratify(dm);CHKERRQ(ierr);
910     /* Build coordinates */
911     ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
912     ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
913     ierr = PetscSectionSetFieldComponents(coordSection, 0, dim);CHKERRQ(ierr);
914     ierr = PetscSectionSetChart(coordSection, firstVertex, firstVertex+numVertices);CHKERRQ(ierr);
915     for (v = firstVertex; v < firstVertex+numVertices; ++v) {
916       ierr = PetscSectionSetDof(coordSection, v, dim);CHKERRQ(ierr);
917       ierr = PetscSectionSetFieldDof(coordSection, v, 0, dim);CHKERRQ(ierr);
918     }
919     ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
920     ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
921     ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
922     ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
923     ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
924     ierr = VecSetBlockSize(coordinates, dim);CHKERRQ(ierr);
925     ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
926     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
927     for (vz = 0; vz < numZVertices; ++vz) {
928       for (vy = 0; vy < numYVertices; ++vy) {
929         for (vx = 0; vx < numXVertices; ++vx) {
930           coords[((vz*numYVertices+vy)*numXVertices+vx)*dim+0] = lower[0] + ((upper[0] - lower[0])/numXEdges)*vx;
931           coords[((vz*numYVertices+vy)*numXVertices+vx)*dim+1] = lower[1] + ((upper[1] - lower[1])/numYEdges)*vy;
932           if (dim == 3) {
933             coords[((vz*numYVertices+vy)*numXVertices+vx)*dim+2] = lower[2] + ((upper[2] - lower[2])/numZEdges)*vz;
934           }
935         }
936       }
937     }
938     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
939     ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
940     ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
941   }
942   PetscFunctionReturn(0);
943 }
944 
945 static PetscErrorCode DMPlexCreateBoxMesh_Tensor_Internal(MPI_Comm comm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate, DM *dm)
946 {
947   PetscInt       i;
948   PetscErrorCode ierr;
949 
950   PetscFunctionBegin;
951   PetscValidPointer(dm, 7);
952   ierr = DMCreate(comm, dm);CHKERRQ(ierr);
953   PetscValidLogicalCollectiveInt(*dm,dim,2);
954   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
955   ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
956   switch (dim) {
957   case 2: {ierr = DMPlexCreateCubeMesh_Internal(*dm, lower, upper, faces, periodicity[0], periodicity[1], DM_BOUNDARY_NONE);CHKERRQ(ierr);break;}
958   case 3: {ierr = DMPlexCreateCubeMesh_Internal(*dm, lower, upper, faces, periodicity[0], periodicity[1], periodicity[2]);CHKERRQ(ierr);break;}
959   default: SETERRQ1(comm, PETSC_ERR_SUP, "Dimension not supported: %d", dim);
960   }
961   if (periodicity[0] == DM_BOUNDARY_PERIODIC || periodicity[0] == DM_BOUNDARY_TWIST ||
962       periodicity[1] == DM_BOUNDARY_PERIODIC || periodicity[1] == DM_BOUNDARY_TWIST ||
963       (dim > 2 && (periodicity[2] == DM_BOUNDARY_PERIODIC || periodicity[2] == DM_BOUNDARY_TWIST))) {
964     PetscReal L[3];
965     PetscReal maxCell[3];
966 
967     for (i = 0; i < dim; i++) {
968       L[i]       = upper[i] - lower[i];
969       maxCell[i] = 1.1 * (L[i] / PetscMax(1,faces[i]));
970     }
971     ierr = DMSetPeriodicity(*dm,PETSC_TRUE,maxCell,L,periodicity);CHKERRQ(ierr);
972   }
973   if (!interpolate) {
974     DM udm;
975 
976     ierr = DMPlexUninterpolate(*dm, &udm);CHKERRQ(ierr);
977     ierr = DMPlexCopyCoordinates(*dm, udm);CHKERRQ(ierr);
978     ierr = DMDestroy(dm);CHKERRQ(ierr);
979     *dm  = udm;
980   }
981   PetscFunctionReturn(0);
982 }
983 
984 /*@C
985   DMPlexCreateBoxMesh - Creates a mesh on the tensor product of unit intervals (box) using simplices or tensor cells (hexahedra).
986 
987   Collective
988 
989   Input Parameters:
990 + comm        - The communicator for the DM object
991 . dim         - The spatial dimension
992 . simplex     - PETSC_TRUE for simplices, PETSC_FALSE for tensor cells
993 . faces       - Number of faces per dimension, or NULL for (1,) in 1D and (2, 2) in 2D and (1, 1, 1) in 3D
994 . lower       - The lower left corner, or NULL for (0, 0, 0)
995 . upper       - The upper right corner, or NULL for (1, 1, 1)
996 . periodicity - The boundary type for the X,Y,Z direction, or NULL for DM_BOUNDARY_NONE
997 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
998 
999   Output Parameter:
1000 . dm  - The DM object
1001 
1002   Options Database Keys:
1003 + -dm_plex_box_lower <x,y,z> - Specify lower-left-bottom coordinates for the box
1004 - -dm_plex_box_upper <x,y,z> - Specify upper-right-top coordinates for the box
1005 
1006   Note: Here is the numbering returned for 2 faces in each direction for tensor cells:
1007 $ 10---17---11---18----12
1008 $  |         |         |
1009 $  |         |         |
1010 $ 20    2   22    3    24
1011 $  |         |         |
1012 $  |         |         |
1013 $  7---15----8---16----9
1014 $  |         |         |
1015 $  |         |         |
1016 $ 19    0   21    1   23
1017 $  |         |         |
1018 $  |         |         |
1019 $  4---13----5---14----6
1020 
1021 and for simplicial cells
1022 
1023 $ 14----8---15----9----16
1024 $  |\     5  |\      7 |
1025 $  | \       | \       |
1026 $ 13   2    14    3    15
1027 $  | 4   \   | 6   \   |
1028 $  |       \ |       \ |
1029 $ 11----6---12----7----13
1030 $  |\        |\        |
1031 $  | \    1  | \     3 |
1032 $ 10   0    11    1    12
1033 $  | 0   \   | 2   \   |
1034 $  |       \ |       \ |
1035 $  8----4----9----5----10
1036 
1037   Level: beginner
1038 
1039 .seealso: DMPlexCreateFromFile(), DMPlexCreateHexCylinderMesh(), DMSetType(), DMCreate()
1040 @*/
1041 PetscErrorCode DMPlexCreateBoxMesh(MPI_Comm comm, PetscInt dim, PetscBool simplex, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate, DM *dm)
1042 {
1043   PetscInt       fac[3] = {0, 0, 0};
1044   PetscReal      low[3] = {0, 0, 0};
1045   PetscReal      upp[3] = {1, 1, 1};
1046   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1047   PetscInt       i, n;
1048   PetscBool      flg;
1049   PetscErrorCode ierr;
1050 
1051   PetscFunctionBegin;
1052   n    = 3;
1053   ierr = PetscOptionsGetIntArray(NULL, NULL, "-dm_plex_box_faces", fac, &n, &flg);CHKERRQ(ierr);
1054   for (i = 0; i < dim; ++i) fac[i] = faces ? faces[i] : (flg && i < n ? fac[i] : (dim == 1 ? 1 : 4-dim));
1055   if (lower) for (i = 0; i < dim; ++i) low[i] = lower[i];
1056   if (upper) for (i = 0; i < dim; ++i) upp[i] = upper[i];
1057   if (periodicity) for (i = 0; i < dim; ++i) bdt[i] = periodicity[i];
1058   /* Allow bounds to be specified from the command line */
1059   n    = 3;
1060   ierr = PetscOptionsGetRealArray(NULL, NULL, "-dm_plex_box_lower", low, &n, &flg);CHKERRQ(ierr);
1061   if (flg && (n != dim)) SETERRQ2(comm, PETSC_ERR_ARG_SIZ, "Lower box point had %D values, should have been %D", n, dim);
1062   n    = 3;
1063   ierr = PetscOptionsGetRealArray(NULL, NULL, "-dm_plex_box_upper", upp, &n, &flg);CHKERRQ(ierr);
1064   if (flg && (n != dim)) SETERRQ2(comm, PETSC_ERR_ARG_SIZ, "Upper box point had %D values, should have been %D", n, dim);
1065 
1066   if (dim == 1)      {ierr = DMPlexCreateLineMesh_Internal(comm, fac[0], low[0], upp[0], bdt[0], dm);CHKERRQ(ierr);}
1067   else if (simplex)  {ierr = DMPlexCreateBoxMesh_Simplex_Internal(comm, dim, fac, low, upp, bdt, interpolate, dm);CHKERRQ(ierr);}
1068   else               {ierr = DMPlexCreateBoxMesh_Tensor_Internal(comm, dim, fac, low, upp, bdt, interpolate, dm);CHKERRQ(ierr);}
1069   PetscFunctionReturn(0);
1070 }
1071 
1072 /*@
1073   DMPlexCreateWedgeBoxMesh - Creates a 3-D mesh tesselating the (x,y) plane and extruding in the third direction using wedge cells.
1074 
1075   Collective
1076 
1077   Input Parameters:
1078 + comm        - The communicator for the DM object
1079 . faces       - Number of faces per dimension, or NULL for (1, 1, 1)
1080 . lower       - The lower left corner, or NULL for (0, 0, 0)
1081 . upper       - The upper right corner, or NULL for (1, 1, 1)
1082 . periodicity - The boundary type for the X,Y,Z direction, or NULL for DM_BOUNDARY_NONE
1083 . ordExt      - If PETSC_TRUE, orders the extruded cells in the height first. Otherwise, orders the cell on the layers first
1084 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
1085 
1086   Output Parameter:
1087 . dm  - The DM object
1088 
1089   Level: beginner
1090 
1091 .seealso: DMPlexCreateHexCylinderMesh(), DMPlexCreateWedgeCylinderMesh(), DMPlexExtrude(), DMPlexCreateBoxMesh(), DMSetType(), DMCreate()
1092 @*/
1093 PetscErrorCode DMPlexCreateWedgeBoxMesh(MPI_Comm comm, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool ordExt, PetscBool interpolate, DM *dm)
1094 {
1095   DM             bdm, botdm;
1096   PetscInt       i;
1097   PetscInt       fac[3] = {0, 0, 0};
1098   PetscReal      low[3] = {0, 0, 0};
1099   PetscReal      upp[3] = {1, 1, 1};
1100   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1101   PetscErrorCode ierr;
1102 
1103   PetscFunctionBegin;
1104   for (i = 0; i < 3; ++i) fac[i] = faces ? (faces[i] > 0 ? faces[i] : 1) : 1;
1105   if (lower) for (i = 0; i < 3; ++i) low[i] = lower[i];
1106   if (upper) for (i = 0; i < 3; ++i) upp[i] = upper[i];
1107   if (periodicity) for (i = 0; i < 3; ++i) bdt[i] = periodicity[i];
1108   for (i = 0; i < 3; ++i) if (bdt[i] != DM_BOUNDARY_NONE) SETERRQ(comm, PETSC_ERR_SUP, "Periodicity not yet supported");
1109 
1110   ierr = DMCreate(comm, &bdm);CHKERRQ(ierr);
1111   ierr = DMSetType(bdm, DMPLEX);CHKERRQ(ierr);
1112   ierr = DMSetDimension(bdm, 1);CHKERRQ(ierr);
1113   ierr = DMSetCoordinateDim(bdm, 2);CHKERRQ(ierr);
1114   ierr = DMPlexCreateSquareBoundary(bdm, low, upp, fac);CHKERRQ(ierr);
1115   ierr = DMPlexGenerate(bdm, NULL, PETSC_FALSE, &botdm);CHKERRQ(ierr);
1116   ierr = DMDestroy(&bdm);CHKERRQ(ierr);
1117   ierr = DMPlexExtrude(botdm, fac[2], upp[2] - low[2], ordExt, interpolate, dm);CHKERRQ(ierr);
1118   if (low[2] != 0.0) {
1119     Vec         v;
1120     PetscScalar *x;
1121     PetscInt    cDim, n;
1122 
1123     ierr = DMGetCoordinatesLocal(*dm, &v);CHKERRQ(ierr);
1124     ierr = VecGetBlockSize(v, &cDim);CHKERRQ(ierr);
1125     ierr = VecGetLocalSize(v, &n);CHKERRQ(ierr);
1126     ierr = VecGetArray(v, &x);CHKERRQ(ierr);
1127     x   += cDim;
1128     for (i=0; i<n; i+=cDim) x[i] += low[2];
1129     ierr = VecRestoreArray(v,&x);CHKERRQ(ierr);
1130     ierr = DMSetCoordinatesLocal(*dm, v);CHKERRQ(ierr);
1131   }
1132   ierr = DMDestroy(&botdm);CHKERRQ(ierr);
1133   PetscFunctionReturn(0);
1134 }
1135 
1136 /*@
1137   DMPlexExtrude - Creates a (d+1)-D mesh by extruding a d-D mesh in the normal direction using prismatic cells.
1138 
1139   Collective on idm
1140 
1141   Input Parameters:
1142 + idm - The mesh to be extruted
1143 . layers - The number of layers
1144 . height - The height of the extruded layer
1145 . ordExt - If PETSC_TRUE, orders the extruded cells in the height first. Otherwise, orders the cell on the layers first
1146 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
1147 
1148   Output Parameter:
1149 . dm  - The DM object
1150 
1151   Notes: The object created is an hybrid mesh, the vertex ordering in the cone of the cell is that of the prismatic cells
1152 
1153   Level: advanced
1154 
1155 .seealso: DMPlexCreateWedgeCylinderMesh(), DMPlexCreateWedgeBoxMesh(), DMPlexSetHybridBounds(), DMSetType(), DMCreate()
1156 @*/
1157 PetscErrorCode DMPlexExtrude(DM idm, PetscInt layers, PetscReal height, PetscBool ordExt, PetscBool interpolate, DM* dm)
1158 {
1159   PetscScalar       *coordsB;
1160   const PetscScalar *coordsA;
1161   PetscReal         *normals = NULL;
1162   Vec               coordinatesA, coordinatesB;
1163   PetscSection      coordSectionA, coordSectionB;
1164   PetscInt          dim, cDim, cDimB, c, l, v, coordSize, *newCone;
1165   PetscInt          cStart, cEnd, vStart, vEnd, cellV, numCells, numVertices;
1166   PetscErrorCode    ierr;
1167 
1168   PetscFunctionBegin;
1169   PetscValidHeaderSpecific(idm, DM_CLASSID, 1);
1170   PetscValidLogicalCollectiveInt(idm, layers, 2);
1171   PetscValidLogicalCollectiveReal(idm, height, 3);
1172   PetscValidLogicalCollectiveBool(idm, interpolate, 4);
1173   ierr = DMGetDimension(idm, &dim);CHKERRQ(ierr);
1174   if (dim < 1 || dim > 3) SETERRQ1(PetscObjectComm((PetscObject)idm), PETSC_ERR_SUP, "Support for dimension %D not coded", dim);
1175 
1176   ierr = DMPlexGetHeightStratum(idm, 0, &cStart, &cEnd);CHKERRQ(ierr);
1177   ierr = DMPlexGetDepthStratum(idm, 0, &vStart, &vEnd);CHKERRQ(ierr);
1178   numCells = (cEnd - cStart)*layers;
1179   numVertices = (vEnd - vStart)*(layers+1);
1180   ierr = DMCreate(PetscObjectComm((PetscObject)idm), dm);CHKERRQ(ierr);
1181   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
1182   ierr = DMSetDimension(*dm, dim+1);CHKERRQ(ierr);
1183   ierr = DMPlexSetChart(*dm, 0, numCells+numVertices);CHKERRQ(ierr);
1184   for (c = cStart, cellV = 0; c < cEnd; ++c) {
1185     PetscInt *closure = NULL;
1186     PetscInt closureSize, numCorners = 0;
1187 
1188     ierr = DMPlexGetTransitiveClosure(idm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1189     for (v = 0; v < closureSize*2; v += 2) if ((closure[v] >= vStart) && (closure[v] < vEnd)) numCorners++;
1190     ierr = DMPlexRestoreTransitiveClosure(idm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1191     for (l = 0; l < layers; ++l) {
1192       ierr = DMPlexSetConeSize(*dm, ordExt ? layers*(c - cStart) + l : l*(cEnd - cStart) + c - cStart, 2*numCorners);CHKERRQ(ierr);
1193     }
1194     cellV = PetscMax(numCorners,cellV);
1195   }
1196   ierr = DMPlexSetHybridBounds(*dm, 0, PETSC_DETERMINE, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
1197   ierr = DMSetUp(*dm);CHKERRQ(ierr);
1198 
1199   ierr = DMGetCoordinateDim(idm, &cDim);CHKERRQ(ierr);
1200   if (dim != cDim) {
1201     ierr = PetscCalloc1(cDim*(vEnd - vStart), &normals);CHKERRQ(ierr);
1202   }
1203   ierr = PetscMalloc1(3*cellV,&newCone);CHKERRQ(ierr);
1204   for (c = cStart; c < cEnd; ++c) {
1205     PetscInt *closure = NULL;
1206     PetscInt closureSize, numCorners = 0, l;
1207     PetscReal normal[3] = {0, 0, 0};
1208 
1209     if (normals) {
1210       ierr = DMPlexComputeCellGeometryFVM(idm, c, NULL, NULL, normal);CHKERRQ(ierr);
1211     }
1212     ierr = DMPlexGetTransitiveClosure(idm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1213     for (v = 0; v < closureSize*2; v += 2) {
1214       if ((closure[v] >= vStart) && (closure[v] < vEnd)) {
1215         PetscInt d;
1216 
1217         newCone[numCorners++] = closure[v] - vStart;
1218         if (normals) { for (d = 0; d < cDim; ++d) normals[cDim*(closure[v]-vStart)+d] += normal[d]; }
1219       }
1220     }
1221     switch (numCorners) {
1222     case 4: /* do nothing */
1223     case 2: /* do nothing */
1224       break;
1225     case 3: /* from counter-clockwise to wedge ordering */
1226       l = newCone[1];
1227       newCone[1] = newCone[2];
1228       newCone[2] = l;
1229       break;
1230     default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_SUP, "Unsupported number of corners: %D", numCorners);
1231     }
1232     ierr = DMPlexRestoreTransitiveClosure(idm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1233     for (l = 0; l < layers; ++l) {
1234       PetscInt i;
1235 
1236       for (i = 0; i < numCorners; ++i) {
1237         newCone[  numCorners + i] = ordExt ? (layers+1)*newCone[i] + l     + numCells :     l*(vEnd - vStart) + newCone[i] + numCells;
1238         newCone[2*numCorners + i] = ordExt ? (layers+1)*newCone[i] + l + 1 + numCells : (l+1)*(vEnd - vStart) + newCone[i] + numCells;
1239       }
1240       ierr = DMPlexSetCone(*dm, ordExt ? layers*(c - cStart) + l : l*(cEnd - cStart) + c - cStart, newCone + numCorners);CHKERRQ(ierr);
1241     }
1242   }
1243   ierr = DMPlexSymmetrize(*dm);CHKERRQ(ierr);
1244   ierr = DMPlexStratify(*dm);CHKERRQ(ierr);
1245   ierr = PetscFree(newCone);CHKERRQ(ierr);
1246 
1247   cDimB = cDim == dim ? cDim+1 : cDim;
1248   ierr = DMGetCoordinateSection(*dm, &coordSectionB);CHKERRQ(ierr);
1249   ierr = PetscSectionSetNumFields(coordSectionB, 1);CHKERRQ(ierr);
1250   ierr = PetscSectionSetFieldComponents(coordSectionB, 0, cDimB);CHKERRQ(ierr);
1251   ierr = PetscSectionSetChart(coordSectionB, numCells, numCells+numVertices);CHKERRQ(ierr);
1252   for (v = numCells; v < numCells+numVertices; ++v) {
1253     ierr = PetscSectionSetDof(coordSectionB, v, cDimB);CHKERRQ(ierr);
1254     ierr = PetscSectionSetFieldDof(coordSectionB, v, 0, cDimB);CHKERRQ(ierr);
1255   }
1256   ierr = PetscSectionSetUp(coordSectionB);CHKERRQ(ierr);
1257   ierr = PetscSectionGetStorageSize(coordSectionB, &coordSize);CHKERRQ(ierr);
1258   ierr = VecCreate(PETSC_COMM_SELF, &coordinatesB);CHKERRQ(ierr);
1259   ierr = PetscObjectSetName((PetscObject) coordinatesB, "coordinates");CHKERRQ(ierr);
1260   ierr = VecSetSizes(coordinatesB, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
1261   ierr = VecSetBlockSize(coordinatesB, cDimB);CHKERRQ(ierr);
1262   ierr = VecSetType(coordinatesB,VECSTANDARD);CHKERRQ(ierr);
1263 
1264   ierr = DMGetCoordinateSection(idm, &coordSectionA);CHKERRQ(ierr);
1265   ierr = DMGetCoordinatesLocal(idm, &coordinatesA);CHKERRQ(ierr);
1266   ierr = VecGetArray(coordinatesB, &coordsB);CHKERRQ(ierr);
1267   ierr = VecGetArrayRead(coordinatesA, &coordsA);CHKERRQ(ierr);
1268   for (v = vStart; v < vEnd; ++v) {
1269     const PetscScalar *cptr;
1270     PetscReal         ones2[2] = { 0., 1.}, ones3[3] = { 0., 0., 1.};
1271     PetscReal         *normal, norm, h = height/layers;
1272     PetscInt          offA, d, cDimA = cDim;
1273 
1274     normal = normals ? normals + cDimB*(v - vStart) : (cDim > 1 ? ones3 : ones2);
1275     if (normals) {
1276       for (d = 0, norm = 0.0; d < cDimB; ++d) norm += normal[d]*normal[d];
1277       for (d = 0; d < cDimB; ++d) normal[d] *= 1./PetscSqrtReal(norm);
1278     }
1279 
1280     ierr = PetscSectionGetOffset(coordSectionA, v, &offA);CHKERRQ(ierr);
1281     cptr = coordsA + offA;
1282     for (l = 0; l < layers+1; ++l) {
1283       PetscInt offB, d, newV;
1284 
1285       newV = ordExt ? (layers+1)*(v -vStart) + l + numCells : (vEnd -vStart)*l + (v -vStart) + numCells;
1286       ierr = PetscSectionGetOffset(coordSectionB, newV, &offB);CHKERRQ(ierr);
1287       for (d = 0; d < cDimA; ++d) { coordsB[offB+d]  = cptr[d]; }
1288       for (d = 0; d < cDimB; ++d) { coordsB[offB+d] += l ? normal[d]*h : 0.0; }
1289       cptr    = coordsB + offB;
1290       cDimA   = cDimB;
1291     }
1292   }
1293   ierr = VecRestoreArrayRead(coordinatesA, &coordsA);CHKERRQ(ierr);
1294   ierr = VecRestoreArray(coordinatesB, &coordsB);CHKERRQ(ierr);
1295   ierr = DMSetCoordinatesLocal(*dm, coordinatesB);CHKERRQ(ierr);
1296   ierr = VecDestroy(&coordinatesB);CHKERRQ(ierr);
1297   ierr = PetscFree(normals);CHKERRQ(ierr);
1298   if (interpolate) {
1299     DM idm;
1300 
1301     ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
1302     ierr = DMPlexCopyCoordinates(*dm, idm);CHKERRQ(ierr);
1303     ierr = DMDestroy(dm);CHKERRQ(ierr);
1304     *dm  = idm;
1305   }
1306   PetscFunctionReturn(0);
1307 }
1308 
1309 /*@C
1310   DMPlexSetOptionsPrefix - Sets the prefix used for searching for all DM options in the database.
1311 
1312   Logically Collective on dm
1313 
1314   Input Parameters:
1315 + dm - the DM context
1316 - prefix - the prefix to prepend to all option names
1317 
1318   Notes:
1319   A hyphen (-) must NOT be given at the beginning of the prefix name.
1320   The first character of all runtime options is AUTOMATICALLY the hyphen.
1321 
1322   Level: advanced
1323 
1324 .seealso: SNESSetFromOptions()
1325 @*/
1326 PetscErrorCode DMPlexSetOptionsPrefix(DM dm, const char prefix[])
1327 {
1328   DM_Plex       *mesh = (DM_Plex *) dm->data;
1329   PetscErrorCode ierr;
1330 
1331   PetscFunctionBegin;
1332   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1333   ierr = PetscObjectSetOptionsPrefix((PetscObject) dm, prefix);CHKERRQ(ierr);
1334   ierr = PetscObjectSetOptionsPrefix((PetscObject) mesh->partitioner, prefix);CHKERRQ(ierr);
1335   PetscFunctionReturn(0);
1336 }
1337 
1338 /*@
1339   DMPlexCreateHexCylinderMesh - Creates a mesh on the tensor product of the unit interval with the circle (cylinder) using hexahedra.
1340 
1341   Collective
1342 
1343   Input Parameters:
1344 + comm      - The communicator for the DM object
1345 . numRefine - The number of regular refinements to the basic 5 cell structure
1346 - periodicZ - The boundary type for the Z direction
1347 
1348   Output Parameter:
1349 . dm  - The DM object
1350 
1351   Note: Here is the output numbering looking from the bottom of the cylinder:
1352 $       17-----14
1353 $        |     |
1354 $        |  2  |
1355 $        |     |
1356 $ 17-----8-----7-----14
1357 $  |     |     |     |
1358 $  |  3  |  0  |  1  |
1359 $  |     |     |     |
1360 $ 19-----5-----6-----13
1361 $        |     |
1362 $        |  4  |
1363 $        |     |
1364 $       19-----13
1365 $
1366 $ and up through the top
1367 $
1368 $       18-----16
1369 $        |     |
1370 $        |  2  |
1371 $        |     |
1372 $ 18----10----11-----16
1373 $  |     |     |     |
1374 $  |  3  |  0  |  1  |
1375 $  |     |     |     |
1376 $ 20-----9----12-----15
1377 $        |     |
1378 $        |  4  |
1379 $        |     |
1380 $       20-----15
1381 
1382   Level: beginner
1383 
1384 .seealso: DMPlexCreateBoxMesh(), DMSetType(), DMCreate()
1385 @*/
1386 PetscErrorCode DMPlexCreateHexCylinderMesh(MPI_Comm comm, PetscInt numRefine, DMBoundaryType periodicZ, DM *dm)
1387 {
1388   const PetscInt dim = 3;
1389   PetscInt       numCells, numVertices, r;
1390   PetscMPIInt    rank;
1391   PetscErrorCode ierr;
1392 
1393   PetscFunctionBegin;
1394   PetscValidPointer(dm, 4);
1395   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
1396   if (numRefine < 0) SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Number of refinements %D cannot be negative", numRefine);
1397   ierr = DMCreate(comm, dm);CHKERRQ(ierr);
1398   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
1399   ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
1400   /* Create topology */
1401   {
1402     PetscInt cone[8], c;
1403 
1404     numCells    = !rank ?  5 : 0;
1405     numVertices = !rank ? 16 : 0;
1406     if (periodicZ == DM_BOUNDARY_PERIODIC) {
1407       numCells   *= 3;
1408       numVertices = !rank ? 24 : 0;
1409     }
1410     ierr = DMPlexSetChart(*dm, 0, numCells+numVertices);CHKERRQ(ierr);
1411     for (c = 0; c < numCells; c++) {ierr = DMPlexSetConeSize(*dm, c, 8);CHKERRQ(ierr);}
1412     ierr = DMSetUp(*dm);CHKERRQ(ierr);
1413     if (!rank) {
1414       if (periodicZ == DM_BOUNDARY_PERIODIC) {
1415         cone[0] = 15; cone[1] = 18; cone[2] = 17; cone[3] = 16;
1416         cone[4] = 31; cone[5] = 32; cone[6] = 33; cone[7] = 34;
1417         ierr = DMPlexSetCone(*dm, 0, cone);CHKERRQ(ierr);
1418         cone[0] = 16; cone[1] = 17; cone[2] = 24; cone[3] = 23;
1419         cone[4] = 32; cone[5] = 36; cone[6] = 37; cone[7] = 33; /* 22 25 26 21 */
1420         ierr = DMPlexSetCone(*dm, 1, cone);CHKERRQ(ierr);
1421         cone[0] = 18; cone[1] = 27; cone[2] = 24; cone[3] = 17;
1422         cone[4] = 34; cone[5] = 33; cone[6] = 37; cone[7] = 38;
1423         ierr = DMPlexSetCone(*dm, 2, cone);CHKERRQ(ierr);
1424         cone[0] = 29; cone[1] = 27; cone[2] = 18; cone[3] = 15;
1425         cone[4] = 35; cone[5] = 31; cone[6] = 34; cone[7] = 38;
1426         ierr = DMPlexSetCone(*dm, 3, cone);CHKERRQ(ierr);
1427         cone[0] = 29; cone[1] = 15; cone[2] = 16; cone[3] = 23;
1428         cone[4] = 35; cone[5] = 36; cone[6] = 32; cone[7] = 31;
1429         ierr = DMPlexSetCone(*dm, 4, cone);CHKERRQ(ierr);
1430 
1431         cone[0] = 31; cone[1] = 34; cone[2] = 33; cone[3] = 32;
1432         cone[4] = 19; cone[5] = 22; cone[6] = 21; cone[7] = 20;
1433         ierr = DMPlexSetCone(*dm, 5, cone);CHKERRQ(ierr);
1434         cone[0] = 32; cone[1] = 33; cone[2] = 37; cone[3] = 36;
1435         cone[4] = 22; cone[5] = 25; cone[6] = 26; cone[7] = 21;
1436         ierr = DMPlexSetCone(*dm, 6, cone);CHKERRQ(ierr);
1437         cone[0] = 34; cone[1] = 38; cone[2] = 37; cone[3] = 33;
1438         cone[4] = 20; cone[5] = 21; cone[6] = 26; cone[7] = 28;
1439         ierr = DMPlexSetCone(*dm, 7, cone);CHKERRQ(ierr);
1440         cone[0] = 35; cone[1] = 38; cone[2] = 34; cone[3] = 31;
1441         cone[4] = 30; cone[5] = 19; cone[6] = 20; cone[7] = 28;
1442         ierr = DMPlexSetCone(*dm, 8, cone);CHKERRQ(ierr);
1443         cone[0] = 35; cone[1] = 31; cone[2] = 32; cone[3] = 36;
1444         cone[4] = 30; cone[5] = 25; cone[6] = 22; cone[7] = 19;
1445         ierr = DMPlexSetCone(*dm, 9, cone);CHKERRQ(ierr);
1446 
1447         cone[0] = 19; cone[1] = 20; cone[2] = 21; cone[3] = 22;
1448         cone[4] = 15; cone[5] = 16; cone[6] = 17; cone[7] = 18;
1449         ierr = DMPlexSetCone(*dm, 10, cone);CHKERRQ(ierr);
1450         cone[0] = 22; cone[1] = 21; cone[2] = 26; cone[3] = 25;
1451         cone[4] = 16; cone[5] = 23; cone[6] = 24; cone[7] = 17;
1452         ierr = DMPlexSetCone(*dm, 11, cone);CHKERRQ(ierr);
1453         cone[0] = 20; cone[1] = 28; cone[2] = 26; cone[3] = 21;
1454         cone[4] = 18; cone[5] = 17; cone[6] = 24; cone[7] = 27;
1455         ierr = DMPlexSetCone(*dm, 12, cone);CHKERRQ(ierr);
1456         cone[0] = 30; cone[1] = 28; cone[2] = 20; cone[3] = 19;
1457         cone[4] = 29; cone[5] = 15; cone[6] = 18; cone[7] = 27;
1458         ierr = DMPlexSetCone(*dm, 13, cone);CHKERRQ(ierr);
1459         cone[0] = 30; cone[1] = 19; cone[2] = 22; cone[3] = 25;
1460         cone[4] = 29; cone[5] = 23; cone[6] = 16; cone[7] = 15;
1461         ierr = DMPlexSetCone(*dm, 14, cone);CHKERRQ(ierr);
1462       } else {
1463         cone[0] =  5; cone[1] =  8; cone[2] =  7; cone[3] =  6;
1464         cone[4] =  9; cone[5] = 12; cone[6] = 11; cone[7] = 10;
1465         ierr = DMPlexSetCone(*dm, 0, cone);CHKERRQ(ierr);
1466         cone[0] =  6; cone[1] =  7; cone[2] = 14; cone[3] = 13;
1467         cone[4] = 12; cone[5] = 15; cone[6] = 16; cone[7] = 11;
1468         ierr = DMPlexSetCone(*dm, 1, cone);CHKERRQ(ierr);
1469         cone[0] =  8; cone[1] = 17; cone[2] = 14; cone[3] =  7;
1470         cone[4] = 10; cone[5] = 11; cone[6] = 16; cone[7] = 18;
1471         ierr = DMPlexSetCone(*dm, 2, cone);CHKERRQ(ierr);
1472         cone[0] = 19; cone[1] = 17; cone[2] =  8; cone[3] =  5;
1473         cone[4] = 20; cone[5] =  9; cone[6] = 10; cone[7] = 18;
1474         ierr = DMPlexSetCone(*dm, 3, cone);CHKERRQ(ierr);
1475         cone[0] = 19; cone[1] =  5; cone[2] =  6; cone[3] = 13;
1476         cone[4] = 20; cone[5] = 15; cone[6] = 12; cone[7] =  9;
1477         ierr = DMPlexSetCone(*dm, 4, cone);CHKERRQ(ierr);
1478       }
1479     }
1480     ierr = DMPlexSymmetrize(*dm);CHKERRQ(ierr);
1481     ierr = DMPlexStratify(*dm);CHKERRQ(ierr);
1482   }
1483   /* Interpolate */
1484   {
1485     DM idm;
1486 
1487     ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
1488     ierr = DMDestroy(dm);CHKERRQ(ierr);
1489     *dm  = idm;
1490   }
1491   /* Create cube geometry */
1492   {
1493     Vec             coordinates;
1494     PetscSection    coordSection;
1495     PetscScalar    *coords;
1496     PetscInt        coordSize, v;
1497     const PetscReal dis = 1.0/PetscSqrtReal(2.0);
1498     const PetscReal ds2 = dis/2.0;
1499 
1500     /* Build coordinates */
1501     ierr = DMGetCoordinateSection(*dm, &coordSection);CHKERRQ(ierr);
1502     ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
1503     ierr = PetscSectionSetFieldComponents(coordSection, 0, dim);CHKERRQ(ierr);
1504     ierr = PetscSectionSetChart(coordSection, numCells, numCells+numVertices);CHKERRQ(ierr);
1505     for (v = numCells; v < numCells+numVertices; ++v) {
1506       ierr = PetscSectionSetDof(coordSection, v, dim);CHKERRQ(ierr);
1507       ierr = PetscSectionSetFieldDof(coordSection, v, 0, dim);CHKERRQ(ierr);
1508     }
1509     ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
1510     ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
1511     ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
1512     ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
1513     ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
1514     ierr = VecSetBlockSize(coordinates, dim);CHKERRQ(ierr);
1515     ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
1516     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
1517     if (!rank) {
1518       coords[0*dim+0] = -ds2; coords[0*dim+1] = -ds2; coords[0*dim+2] = 0.0;
1519       coords[1*dim+0] =  ds2; coords[1*dim+1] = -ds2; coords[1*dim+2] = 0.0;
1520       coords[2*dim+0] =  ds2; coords[2*dim+1] =  ds2; coords[2*dim+2] = 0.0;
1521       coords[3*dim+0] = -ds2; coords[3*dim+1] =  ds2; coords[3*dim+2] = 0.0;
1522       coords[4*dim+0] = -ds2; coords[4*dim+1] = -ds2; coords[4*dim+2] = 1.0;
1523       coords[5*dim+0] = -ds2; coords[5*dim+1] =  ds2; coords[5*dim+2] = 1.0;
1524       coords[6*dim+0] =  ds2; coords[6*dim+1] =  ds2; coords[6*dim+2] = 1.0;
1525       coords[7*dim+0] =  ds2; coords[7*dim+1] = -ds2; coords[7*dim+2] = 1.0;
1526       coords[ 8*dim+0] =  dis; coords[ 8*dim+1] = -dis; coords[ 8*dim+2] = 0.0;
1527       coords[ 9*dim+0] =  dis; coords[ 9*dim+1] =  dis; coords[ 9*dim+2] = 0.0;
1528       coords[10*dim+0] =  dis; coords[10*dim+1] = -dis; coords[10*dim+2] = 1.0;
1529       coords[11*dim+0] =  dis; coords[11*dim+1] =  dis; coords[11*dim+2] = 1.0;
1530       coords[12*dim+0] = -dis; coords[12*dim+1] =  dis; coords[12*dim+2] = 0.0;
1531       coords[13*dim+0] = -dis; coords[13*dim+1] =  dis; coords[13*dim+2] = 1.0;
1532       coords[14*dim+0] = -dis; coords[14*dim+1] = -dis; coords[14*dim+2] = 0.0;
1533       coords[15*dim+0] = -dis; coords[15*dim+1] = -dis; coords[15*dim+2] = 1.0;
1534       if (periodicZ == DM_BOUNDARY_PERIODIC) {
1535         /* 15 31 19 */ coords[16*dim+0] = -ds2; coords[16*dim+1] = -ds2; coords[16*dim+2] = 0.5;
1536         /* 16 32 22 */ coords[17*dim+0] =  ds2; coords[17*dim+1] = -ds2; coords[17*dim+2] = 0.5;
1537         /* 17 33 21 */ coords[18*dim+0] =  ds2; coords[18*dim+1] =  ds2; coords[18*dim+2] = 0.5;
1538         /* 18 34 20 */ coords[19*dim+0] = -ds2; coords[19*dim+1] =  ds2; coords[19*dim+2] = 0.5;
1539         /* 29 35 30 */ coords[20*dim+0] = -dis; coords[20*dim+1] = -dis; coords[20*dim+2] = 0.5;
1540         /* 23 36 25 */ coords[21*dim+0] =  dis; coords[21*dim+1] = -dis; coords[21*dim+2] = 0.5;
1541         /* 24 37 26 */ coords[22*dim+0] =  dis; coords[22*dim+1] =  dis; coords[22*dim+2] = 0.5;
1542         /* 27 38 28 */ coords[23*dim+0] = -dis; coords[23*dim+1] =  dis; coords[23*dim+2] = 0.5;
1543       }
1544     }
1545     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
1546     ierr = DMSetCoordinatesLocal(*dm, coordinates);CHKERRQ(ierr);
1547     ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
1548   }
1549   /* Create periodicity */
1550   if (periodicZ == DM_BOUNDARY_PERIODIC || periodicZ == DM_BOUNDARY_TWIST) {
1551     PetscReal      L[3];
1552     PetscReal      maxCell[3];
1553     DMBoundaryType bdType[3];
1554     PetscReal      lower[3] = {0.0, 0.0, 0.0};
1555     PetscReal      upper[3] = {1.0, 1.0, 1.5};
1556     PetscInt       i, numZCells = 3;
1557 
1558     bdType[0] = DM_BOUNDARY_NONE;
1559     bdType[1] = DM_BOUNDARY_NONE;
1560     bdType[2] = periodicZ;
1561     for (i = 0; i < dim; i++) {
1562       L[i]       = upper[i] - lower[i];
1563       maxCell[i] = 1.1 * (L[i] / numZCells);
1564     }
1565     ierr = DMSetPeriodicity(*dm, PETSC_TRUE, maxCell, L, bdType);CHKERRQ(ierr);
1566   }
1567   /* Refine topology */
1568   for (r = 0; r < numRefine; ++r) {
1569     DM rdm = NULL;
1570 
1571     ierr = DMRefine(*dm, comm, &rdm);CHKERRQ(ierr);
1572     ierr = DMDestroy(dm);CHKERRQ(ierr);
1573     *dm  = rdm;
1574   }
1575   /* Remap geometry to cylinder
1576        Interior square: Linear interpolation is correct
1577        The other cells all have vertices on rays from the origin. We want to uniformly expand the spacing
1578        such that the last vertex is on the unit circle. So the closest and farthest vertices are at distance
1579 
1580          phi     = arctan(y/x)
1581          d_close = sqrt(1/8 + 1/4 sin^2(phi))
1582          d_far   = sqrt(1/2 + sin^2(phi))
1583 
1584        so we remap them using
1585 
1586          x_new = x_close + (x - x_close) (1 - d_close) / (d_far - d_close)
1587          y_new = y_close + (y - y_close) (1 - d_close) / (d_far - d_close)
1588 
1589        If pi/4 < phi < 3pi/4 or -3pi/4 < phi < -pi/4, then we switch x and y.
1590   */
1591   {
1592     Vec           coordinates;
1593     PetscSection  coordSection;
1594     PetscScalar  *coords;
1595     PetscInt      vStart, vEnd, v;
1596     const PetscReal dis = 1.0/PetscSqrtReal(2.0);
1597     const PetscReal ds2 = 0.5*dis;
1598 
1599     ierr = DMPlexGetDepthStratum(*dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
1600     ierr = DMGetCoordinateSection(*dm, &coordSection);CHKERRQ(ierr);
1601     ierr = DMGetCoordinatesLocal(*dm, &coordinates);CHKERRQ(ierr);
1602     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
1603     for (v = vStart; v < vEnd; ++v) {
1604       PetscReal phi, sinp, cosp, dc, df, x, y, xc, yc;
1605       PetscInt  off;
1606 
1607       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
1608       if ((PetscAbsScalar(coords[off+0]) <= ds2) && (PetscAbsScalar(coords[off+1]) <= ds2)) continue;
1609       x    = PetscRealPart(coords[off]);
1610       y    = PetscRealPart(coords[off+1]);
1611       phi  = PetscAtan2Real(y, x);
1612       sinp = PetscSinReal(phi);
1613       cosp = PetscCosReal(phi);
1614       if ((PetscAbsReal(phi) > PETSC_PI/4.0) && (PetscAbsReal(phi) < 3.0*PETSC_PI/4.0)) {
1615         dc = PetscAbsReal(ds2/sinp);
1616         df = PetscAbsReal(dis/sinp);
1617         xc = ds2*x/PetscAbsReal(y);
1618         yc = ds2*PetscSignReal(y);
1619       } else {
1620         dc = PetscAbsReal(ds2/cosp);
1621         df = PetscAbsReal(dis/cosp);
1622         xc = ds2*PetscSignReal(x);
1623         yc = ds2*y/PetscAbsReal(x);
1624       }
1625       coords[off+0] = xc + (coords[off+0] - xc)*(1.0 - dc)/(df - dc);
1626       coords[off+1] = yc + (coords[off+1] - yc)*(1.0 - dc)/(df - dc);
1627     }
1628     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
1629     if (periodicZ == DM_BOUNDARY_PERIODIC || periodicZ == DM_BOUNDARY_TWIST) {
1630       ierr = DMLocalizeCoordinates(*dm);CHKERRQ(ierr);
1631     }
1632   }
1633   PetscFunctionReturn(0);
1634 }
1635 
1636 /*@
1637   DMPlexCreateWedgeCylinderMesh - Creates a mesh on the tensor product of the unit interval with the circle (cylinder) using wedges.
1638 
1639   Collective
1640 
1641   Input Parameters:
1642 + comm - The communicator for the DM object
1643 . n    - The number of wedges around the origin
1644 - interpolate - Create edges and faces
1645 
1646   Output Parameter:
1647 . dm  - The DM object
1648 
1649   Level: beginner
1650 
1651 .seealso: DMPlexCreateHexCylinderMesh(), DMPlexCreateBoxMesh(), DMSetType(), DMCreate()
1652 @*/
1653 PetscErrorCode DMPlexCreateWedgeCylinderMesh(MPI_Comm comm, PetscInt n, PetscBool interpolate, DM *dm)
1654 {
1655   const PetscInt dim = 3;
1656   PetscInt       numCells, numVertices;
1657   PetscMPIInt    rank;
1658   PetscErrorCode ierr;
1659 
1660   PetscFunctionBegin;
1661   PetscValidPointer(dm, 3);
1662   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
1663   if (n < 0) SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Number of wedges %D cannot be negative", n);
1664   ierr = DMCreate(comm, dm);CHKERRQ(ierr);
1665   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
1666   ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
1667   /* Create topology */
1668   {
1669     PetscInt cone[6], c;
1670 
1671     numCells    = !rank ?        n : 0;
1672     numVertices = !rank ?  2*(n+1) : 0;
1673     ierr = DMPlexSetChart(*dm, 0, numCells+numVertices);CHKERRQ(ierr);
1674     ierr = DMPlexSetHybridBounds(*dm, 0, PETSC_DETERMINE, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
1675     for (c = 0; c < numCells; c++) {ierr = DMPlexSetConeSize(*dm, c, 6);CHKERRQ(ierr);}
1676     ierr = DMSetUp(*dm);CHKERRQ(ierr);
1677     for (c = 0; c < numCells; c++) {
1678       cone[0] =  c+n*1; cone[1] = (c+1)%n+n*1; cone[2] = 0+3*n;
1679       cone[3] =  c+n*2; cone[4] = (c+1)%n+n*2; cone[5] = 1+3*n;
1680       ierr = DMPlexSetCone(*dm, c, cone);CHKERRQ(ierr);
1681     }
1682     ierr = DMPlexSymmetrize(*dm);CHKERRQ(ierr);
1683     ierr = DMPlexStratify(*dm);CHKERRQ(ierr);
1684   }
1685   /* Interpolate */
1686   if (interpolate) {
1687     DM idm;
1688 
1689     ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
1690     ierr = DMDestroy(dm);CHKERRQ(ierr);
1691     *dm  = idm;
1692   }
1693   /* Create cylinder geometry */
1694   {
1695     Vec          coordinates;
1696     PetscSection coordSection;
1697     PetscScalar *coords;
1698     PetscInt     coordSize, v, c;
1699 
1700     /* Build coordinates */
1701     ierr = DMGetCoordinateSection(*dm, &coordSection);CHKERRQ(ierr);
1702     ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
1703     ierr = PetscSectionSetFieldComponents(coordSection, 0, dim);CHKERRQ(ierr);
1704     ierr = PetscSectionSetChart(coordSection, numCells, numCells+numVertices);CHKERRQ(ierr);
1705     for (v = numCells; v < numCells+numVertices; ++v) {
1706       ierr = PetscSectionSetDof(coordSection, v, dim);CHKERRQ(ierr);
1707       ierr = PetscSectionSetFieldDof(coordSection, v, 0, dim);CHKERRQ(ierr);
1708     }
1709     ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
1710     ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
1711     ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
1712     ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
1713     ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
1714     ierr = VecSetBlockSize(coordinates, dim);CHKERRQ(ierr);
1715     ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
1716     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
1717     for (c = 0; c < numCells; c++) {
1718       coords[(c+0*n)*dim+0] = PetscCosReal(2.0*c*PETSC_PI/n); coords[(c+0*n)*dim+1] = PetscSinReal(2.0*c*PETSC_PI/n); coords[(c+0*n)*dim+2] = 1.0;
1719       coords[(c+1*n)*dim+0] = PetscCosReal(2.0*c*PETSC_PI/n); coords[(c+1*n)*dim+1] = PetscSinReal(2.0*c*PETSC_PI/n); coords[(c+1*n)*dim+2] = 0.0;
1720     }
1721     if (!rank) {
1722       coords[(2*n+0)*dim+0] = 0.0; coords[(2*n+0)*dim+1] = 0.0; coords[(2*n+0)*dim+2] = 1.0;
1723       coords[(2*n+1)*dim+0] = 0.0; coords[(2*n+1)*dim+1] = 0.0; coords[(2*n+1)*dim+2] = 0.0;
1724     }
1725     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
1726     ierr = DMSetCoordinatesLocal(*dm, coordinates);CHKERRQ(ierr);
1727     ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
1728   }
1729   PetscFunctionReturn(0);
1730 }
1731 
1732 PETSC_STATIC_INLINE PetscReal DiffNormReal(PetscInt dim, const PetscReal x[], const PetscReal y[])
1733 {
1734   PetscReal prod = 0.0;
1735   PetscInt  i;
1736   for (i = 0; i < dim; ++i) prod += PetscSqr(x[i] - y[i]);
1737   return PetscSqrtReal(prod);
1738 }
1739 PETSC_STATIC_INLINE PetscReal DotReal(PetscInt dim, const PetscReal x[], const PetscReal y[])
1740 {
1741   PetscReal prod = 0.0;
1742   PetscInt  i;
1743   for (i = 0; i < dim; ++i) prod += x[i]*y[i];
1744   return prod;
1745 }
1746 
1747 /*@
1748   DMPlexCreateSphereMesh - Creates a mesh on the d-dimensional sphere, S^d.
1749 
1750   Collective
1751 
1752   Input Parameters:
1753 + comm  - The communicator for the DM object
1754 . dim   - The dimension
1755 - simplex - Use simplices, or tensor product cells
1756 
1757   Output Parameter:
1758 . dm  - The DM object
1759 
1760   Level: beginner
1761 
1762 .seealso: DMPlexCreateBoxMesh(), DMSetType(), DMCreate()
1763 @*/
1764 PetscErrorCode DMPlexCreateSphereMesh(MPI_Comm comm, PetscInt dim, PetscBool simplex, DM *dm)
1765 {
1766   const PetscInt  embedDim = dim+1;
1767   PetscSection    coordSection;
1768   Vec             coordinates;
1769   PetscScalar    *coords;
1770   PetscReal      *coordsIn;
1771   PetscInt        numCells, numEdges, numVerts, firstVertex, v, firstEdge, coordSize, d, c, e;
1772   PetscMPIInt     rank;
1773   PetscErrorCode  ierr;
1774 
1775   PetscFunctionBegin;
1776   PetscValidPointer(dm, 4);
1777   ierr = DMCreate(comm, dm);CHKERRQ(ierr);
1778   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
1779   ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
1780   ierr = DMSetCoordinateDim(*dm, dim+1);CHKERRQ(ierr);
1781   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) *dm), &rank);CHKERRQ(ierr);
1782   switch (dim) {
1783   case 2:
1784     if (simplex) {
1785       DM              idm;
1786       const PetscReal edgeLen     = 2.0/(1.0 + PETSC_PHI);
1787       const PetscReal vertex[3]   = {0.0, 1.0/(1.0 + PETSC_PHI), PETSC_PHI/(1.0 + PETSC_PHI)};
1788       const PetscInt  degree      = 5;
1789       PetscInt        s[3]        = {1, 1, 1};
1790       PetscInt        cone[3];
1791       PetscInt       *graph, p, i, j, k;
1792 
1793       numCells    = !rank ? 20 : 0;
1794       numVerts    = !rank ? 12 : 0;
1795       firstVertex = numCells;
1796       /* Use icosahedron, which for a unit sphere has coordinates which are all cyclic permutations of
1797 
1798            (0, \pm 1/\phi+1, \pm \phi/\phi+1)
1799 
1800          where \phi^2 - \phi - 1 = 0, meaning \phi is the golden ratio \frac{1 + \sqrt{5}}{2}. The edge
1801          length is then given by 2/\phi = 2 * 2.73606 = 5.47214.
1802        */
1803       /* Construct vertices */
1804       ierr = PetscCalloc1(numVerts * embedDim, &coordsIn);CHKERRQ(ierr);
1805       for (p = 0, i = 0; p < embedDim; ++p) {
1806         for (s[1] = -1; s[1] < 2; s[1] += 2) {
1807           for (s[2] = -1; s[2] < 2; s[2] += 2) {
1808             for (d = 0; d < embedDim; ++d) coordsIn[i*embedDim+d] = s[(d+p)%embedDim]*vertex[(d+p)%embedDim];
1809             ++i;
1810           }
1811         }
1812       }
1813       /* Construct graph */
1814       ierr = PetscCalloc1(numVerts * numVerts, &graph);CHKERRQ(ierr);
1815       for (i = 0; i < numVerts; ++i) {
1816         for (j = 0, k = 0; j < numVerts; ++j) {
1817           if (PetscAbsReal(DiffNormReal(embedDim, &coordsIn[i*embedDim], &coordsIn[j*embedDim]) - edgeLen) < PETSC_SMALL) {graph[i*numVerts+j] = 1; ++k;}
1818         }
1819         if (k != degree) SETERRQ3(comm, PETSC_ERR_PLIB, "Invalid icosahedron, vertex %D degree %D != %D", i, k, degree);
1820       }
1821       /* Build Topology */
1822       ierr = DMPlexSetChart(*dm, 0, numCells+numVerts);CHKERRQ(ierr);
1823       for (c = 0; c < numCells; c++) {
1824         ierr = DMPlexSetConeSize(*dm, c, embedDim);CHKERRQ(ierr);
1825       }
1826       ierr = DMSetUp(*dm);CHKERRQ(ierr); /* Allocate space for cones */
1827       /* Cells */
1828       for (i = 0, c = 0; i < numVerts; ++i) {
1829         for (j = 0; j < i; ++j) {
1830           for (k = 0; k < j; ++k) {
1831             if (graph[i*numVerts+j] && graph[j*numVerts+k] && graph[k*numVerts+i]) {
1832               cone[0] = firstVertex+i; cone[1] = firstVertex+j; cone[2] = firstVertex+k;
1833               /* Check orientation */
1834               {
1835                 const PetscInt epsilon[3][3][3] = {{{0, 0, 0}, {0, 0, 1}, {0, -1, 0}}, {{0, 0, -1}, {0, 0, 0}, {1, 0, 0}}, {{0, 1, 0}, {-1, 0, 0}, {0, 0, 0}}};
1836                 PetscReal normal[3];
1837                 PetscInt  e, f;
1838 
1839                 for (d = 0; d < embedDim; ++d) {
1840                   normal[d] = 0.0;
1841                   for (e = 0; e < embedDim; ++e) {
1842                     for (f = 0; f < embedDim; ++f) {
1843                       normal[d] += epsilon[d][e][f]*(coordsIn[j*embedDim+e] - coordsIn[i*embedDim+e])*(coordsIn[k*embedDim+f] - coordsIn[i*embedDim+f]);
1844                     }
1845                   }
1846                 }
1847                 if (DotReal(embedDim, normal, &coordsIn[i*embedDim]) < 0) {PetscInt tmp = cone[1]; cone[1] = cone[2]; cone[2] = tmp;}
1848               }
1849               ierr = DMPlexSetCone(*dm, c++, cone);CHKERRQ(ierr);
1850             }
1851           }
1852         }
1853       }
1854       ierr = DMPlexSymmetrize(*dm);CHKERRQ(ierr);
1855       ierr = DMPlexStratify(*dm);CHKERRQ(ierr);
1856       ierr = PetscFree(graph);CHKERRQ(ierr);
1857       /* Interpolate mesh */
1858       ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
1859       ierr = DMDestroy(dm);CHKERRQ(ierr);
1860       *dm  = idm;
1861     } else {
1862       /*
1863         12-21--13
1864          |     |
1865         25  4  24
1866          |     |
1867   12-25--9-16--8-24--13
1868    |     |     |     |
1869   23  5 17  0 15  3  22
1870    |     |     |     |
1871   10-20--6-14--7-19--11
1872          |     |
1873         20  1  19
1874          |     |
1875         10-18--11
1876          |     |
1877         23  2  22
1878          |     |
1879         12-21--13
1880        */
1881       const PetscReal dist = 1.0/PetscSqrtReal(3.0);
1882       PetscInt        cone[4], ornt[4];
1883 
1884       numCells    = !rank ?  6 : 0;
1885       numEdges    = !rank ? 12 : 0;
1886       numVerts    = !rank ?  8 : 0;
1887       firstVertex = numCells;
1888       firstEdge   = numCells + numVerts;
1889       /* Build Topology */
1890       ierr = DMPlexSetChart(*dm, 0, numCells+numEdges+numVerts);CHKERRQ(ierr);
1891       for (c = 0; c < numCells; c++) {
1892         ierr = DMPlexSetConeSize(*dm, c, 4);CHKERRQ(ierr);
1893       }
1894       for (e = firstEdge; e < firstEdge+numEdges; ++e) {
1895         ierr = DMPlexSetConeSize(*dm, e, 2);CHKERRQ(ierr);
1896       }
1897       ierr = DMSetUp(*dm);CHKERRQ(ierr); /* Allocate space for cones */
1898       /* Cell 0 */
1899       cone[0] = 14; cone[1] = 15; cone[2] = 16; cone[3] = 17;
1900       ierr = DMPlexSetCone(*dm, 0, cone);CHKERRQ(ierr);
1901       ornt[0] = 0; ornt[1] = 0; ornt[2] = 0; ornt[3] = 0;
1902       ierr = DMPlexSetConeOrientation(*dm, 0, ornt);CHKERRQ(ierr);
1903       /* Cell 1 */
1904       cone[0] = 18; cone[1] = 19; cone[2] = 14; cone[3] = 20;
1905       ierr = DMPlexSetCone(*dm, 1, cone);CHKERRQ(ierr);
1906       ornt[0] = 0; ornt[1] = 0; ornt[2] = -2; ornt[3] = 0;
1907       ierr = DMPlexSetConeOrientation(*dm, 1, ornt);CHKERRQ(ierr);
1908       /* Cell 2 */
1909       cone[0] = 21; cone[1] = 22; cone[2] = 18; cone[3] = 23;
1910       ierr = DMPlexSetCone(*dm, 2, cone);CHKERRQ(ierr);
1911       ornt[0] = 0; ornt[1] = 0; ornt[2] = -2; ornt[3] = 0;
1912       ierr = DMPlexSetConeOrientation(*dm, 2, ornt);CHKERRQ(ierr);
1913       /* Cell 3 */
1914       cone[0] = 19; cone[1] = 22; cone[2] = 24; cone[3] = 15;
1915       ierr = DMPlexSetCone(*dm, 3, cone);CHKERRQ(ierr);
1916       ornt[0] = -2; ornt[1] = -2; ornt[2] = 0; ornt[3] = -2;
1917       ierr = DMPlexSetConeOrientation(*dm, 3, ornt);CHKERRQ(ierr);
1918       /* Cell 4 */
1919       cone[0] = 16; cone[1] = 24; cone[2] = 21; cone[3] = 25;
1920       ierr = DMPlexSetCone(*dm, 4, cone);CHKERRQ(ierr);
1921       ornt[0] = -2; ornt[1] = -2; ornt[2] = -2; ornt[3] = 0;
1922       ierr = DMPlexSetConeOrientation(*dm, 4, ornt);CHKERRQ(ierr);
1923       /* Cell 5 */
1924       cone[0] = 20; cone[1] = 17; cone[2] = 25; cone[3] = 23;
1925       ierr = DMPlexSetCone(*dm, 5, cone);CHKERRQ(ierr);
1926       ornt[0] = -2; ornt[1] = -2; ornt[2] = -2; ornt[3] = -2;
1927       ierr = DMPlexSetConeOrientation(*dm, 5, ornt);CHKERRQ(ierr);
1928       /* Edges */
1929       cone[0] =  6; cone[1] =  7;
1930       ierr = DMPlexSetCone(*dm, 14, cone);CHKERRQ(ierr);
1931       cone[0] =  7; cone[1] =  8;
1932       ierr = DMPlexSetCone(*dm, 15, cone);CHKERRQ(ierr);
1933       cone[0] =  8; cone[1] =  9;
1934       ierr = DMPlexSetCone(*dm, 16, cone);CHKERRQ(ierr);
1935       cone[0] =  9; cone[1] =  6;
1936       ierr = DMPlexSetCone(*dm, 17, cone);CHKERRQ(ierr);
1937       cone[0] = 10; cone[1] = 11;
1938       ierr = DMPlexSetCone(*dm, 18, cone);CHKERRQ(ierr);
1939       cone[0] = 11; cone[1] =  7;
1940       ierr = DMPlexSetCone(*dm, 19, cone);CHKERRQ(ierr);
1941       cone[0] =  6; cone[1] = 10;
1942       ierr = DMPlexSetCone(*dm, 20, cone);CHKERRQ(ierr);
1943       cone[0] = 12; cone[1] = 13;
1944       ierr = DMPlexSetCone(*dm, 21, cone);CHKERRQ(ierr);
1945       cone[0] = 13; cone[1] = 11;
1946       ierr = DMPlexSetCone(*dm, 22, cone);CHKERRQ(ierr);
1947       cone[0] = 10; cone[1] = 12;
1948       ierr = DMPlexSetCone(*dm, 23, cone);CHKERRQ(ierr);
1949       cone[0] = 13; cone[1] =  8;
1950       ierr = DMPlexSetCone(*dm, 24, cone);CHKERRQ(ierr);
1951       cone[0] = 12; cone[1] =  9;
1952       ierr = DMPlexSetCone(*dm, 25, cone);CHKERRQ(ierr);
1953       ierr = DMPlexSymmetrize(*dm);CHKERRQ(ierr);
1954       ierr = DMPlexStratify(*dm);CHKERRQ(ierr);
1955       /* Build coordinates */
1956       ierr = PetscCalloc1(numVerts * embedDim, &coordsIn);CHKERRQ(ierr);
1957       coordsIn[0*embedDim+0] = -dist; coordsIn[0*embedDim+1] =  dist; coordsIn[0*embedDim+2] = -dist;
1958       coordsIn[1*embedDim+0] =  dist; coordsIn[1*embedDim+1] =  dist; coordsIn[1*embedDim+2] = -dist;
1959       coordsIn[2*embedDim+0] =  dist; coordsIn[2*embedDim+1] = -dist; coordsIn[2*embedDim+2] = -dist;
1960       coordsIn[3*embedDim+0] = -dist; coordsIn[3*embedDim+1] = -dist; coordsIn[3*embedDim+2] = -dist;
1961       coordsIn[4*embedDim+0] = -dist; coordsIn[4*embedDim+1] =  dist; coordsIn[4*embedDim+2] =  dist;
1962       coordsIn[5*embedDim+0] =  dist; coordsIn[5*embedDim+1] =  dist; coordsIn[5*embedDim+2] =  dist;
1963       coordsIn[6*embedDim+0] = -dist; coordsIn[6*embedDim+1] = -dist; coordsIn[6*embedDim+2] =  dist;
1964       coordsIn[7*embedDim+0] =  dist; coordsIn[7*embedDim+1] = -dist; coordsIn[7*embedDim+2] =  dist;
1965     }
1966     break;
1967   case 3:
1968     if (simplex) {
1969       DM              idm;
1970       const PetscReal edgeLen         = 1.0/PETSC_PHI;
1971       const PetscReal vertexA[4]      = {0.5, 0.5, 0.5, 0.5};
1972       const PetscReal vertexB[4]      = {1.0, 0.0, 0.0, 0.0};
1973       const PetscReal vertexC[4]      = {0.5, 0.5*PETSC_PHI, 0.5/PETSC_PHI, 0.0};
1974       const PetscInt  degree          = 12;
1975       PetscInt        s[4]            = {1, 1, 1};
1976       PetscInt        evenPerm[12][4] = {{0, 1, 2, 3}, {0, 2, 3, 1}, {0, 3, 1, 2}, {1, 0, 3, 2}, {1, 2, 0, 3}, {1, 3, 2, 0},
1977                                          {2, 0, 1, 3}, {2, 1, 3, 0}, {2, 3, 0, 1}, {3, 0, 2, 1}, {3, 1, 0, 2}, {3, 2, 1, 0}};
1978       PetscInt        cone[4];
1979       PetscInt       *graph, p, i, j, k, l;
1980 
1981       numCells    = !rank ? 600 : 0;
1982       numVerts    = !rank ? 120 : 0;
1983       firstVertex = numCells;
1984       /* Use the 600-cell, which for a unit sphere has coordinates which are
1985 
1986            1/2 (\pm 1, \pm 1,    \pm 1, \pm 1)                          16
1987                (\pm 1,    0,       0,      0)  all cyclic permutations   8
1988            1/2 (\pm 1, \pm phi, \pm 1/phi, 0)  all even permutations    96
1989 
1990          where \phi^2 - \phi - 1 = 0, meaning \phi is the golden ratio \frac{1 + \sqrt{5}}{2}. The edge
1991          length is then given by 1/\phi = 2.73606.
1992 
1993          http://buzzard.pugetsound.edu/sage-practice/ch03s03.html
1994          http://mathworld.wolfram.com/600-Cell.html
1995        */
1996       /* Construct vertices */
1997       ierr = PetscCalloc1(numVerts * embedDim, &coordsIn);CHKERRQ(ierr);
1998       i    = 0;
1999       for (s[0] = -1; s[0] < 2; s[0] += 2) {
2000         for (s[1] = -1; s[1] < 2; s[1] += 2) {
2001           for (s[2] = -1; s[2] < 2; s[2] += 2) {
2002             for (s[3] = -1; s[3] < 2; s[3] += 2) {
2003               for (d = 0; d < embedDim; ++d) coordsIn[i*embedDim+d] = s[d]*vertexA[d];
2004               ++i;
2005             }
2006           }
2007         }
2008       }
2009       for (p = 0; p < embedDim; ++p) {
2010         s[1] = s[2] = s[3] = 1;
2011         for (s[0] = -1; s[0] < 2; s[0] += 2) {
2012           for (d = 0; d < embedDim; ++d) coordsIn[i*embedDim+d] = s[(d+p)%embedDim]*vertexB[(d+p)%embedDim];
2013           ++i;
2014         }
2015       }
2016       for (p = 0; p < 12; ++p) {
2017         s[3] = 1;
2018         for (s[0] = -1; s[0] < 2; s[0] += 2) {
2019           for (s[1] = -1; s[1] < 2; s[1] += 2) {
2020             for (s[2] = -1; s[2] < 2; s[2] += 2) {
2021               for (d = 0; d < embedDim; ++d) coordsIn[i*embedDim+d] = s[evenPerm[p][d]]*vertexC[evenPerm[p][d]];
2022               ++i;
2023             }
2024           }
2025         }
2026       }
2027       if (i != numVerts) SETERRQ2(comm, PETSC_ERR_PLIB, "Invalid 600-cell, vertices %D != %D", i, numVerts);
2028       /* Construct graph */
2029       ierr = PetscCalloc1(numVerts * numVerts, &graph);CHKERRQ(ierr);
2030       for (i = 0; i < numVerts; ++i) {
2031         for (j = 0, k = 0; j < numVerts; ++j) {
2032           if (PetscAbsReal(DiffNormReal(embedDim, &coordsIn[i*embedDim], &coordsIn[j*embedDim]) - edgeLen) < PETSC_SMALL) {graph[i*numVerts+j] = 1; ++k;}
2033         }
2034         if (k != degree) SETERRQ3(comm, PETSC_ERR_PLIB, "Invalid 600-cell, vertex %D degree %D != %D", i, k, degree);
2035       }
2036       /* Build Topology */
2037       ierr = DMPlexSetChart(*dm, 0, numCells+numVerts);CHKERRQ(ierr);
2038       for (c = 0; c < numCells; c++) {
2039         ierr = DMPlexSetConeSize(*dm, c, embedDim);CHKERRQ(ierr);
2040       }
2041       ierr = DMSetUp(*dm);CHKERRQ(ierr); /* Allocate space for cones */
2042       /* Cells */
2043       for (i = 0, c = 0; i < numVerts; ++i) {
2044         for (j = 0; j < i; ++j) {
2045           for (k = 0; k < j; ++k) {
2046             for (l = 0; l < k; ++l) {
2047               if (graph[i*numVerts+j] && graph[j*numVerts+k] && graph[k*numVerts+i] &&
2048                   graph[l*numVerts+i] && graph[l*numVerts+j] && graph[l*numVerts+k]) {
2049                 cone[0] = firstVertex+i; cone[1] = firstVertex+j; cone[2] = firstVertex+k; cone[3] = firstVertex+l;
2050                 /* Check orientation: https://ef.gy/linear-algebra:normal-vectors-in-higher-dimensional-spaces */
2051                 {
2052                   const PetscInt epsilon[4][4][4][4] = {{{{0,  0,  0,  0}, { 0, 0,  0,  0}, { 0,  0, 0,  0}, { 0,  0,  0, 0}},
2053                                                          {{0,  0,  0,  0}, { 0, 0,  0,  0}, { 0,  0, 0,  1}, { 0,  0, -1, 0}},
2054                                                          {{0,  0,  0,  0}, { 0, 0,  0, -1}, { 0,  0, 0,  0}, { 0,  1,  0, 0}},
2055                                                          {{0,  0,  0,  0}, { 0, 0,  1,  0}, { 0, -1, 0,  0}, { 0,  0,  0, 0}}},
2056 
2057                                                         {{{0,  0,  0,  0}, { 0, 0,  0,  0}, { 0,  0, 0, -1}, { 0,  0,  1, 0}},
2058                                                          {{0,  0,  0,  0}, { 0, 0,  0,  0}, { 0,  0, 0,  0}, { 0,  0,  0, 0}},
2059                                                          {{0,  0,  0,  1}, { 0, 0,  0,  0}, { 0,  0, 0,  0}, {-1,  0,  0, 0}},
2060                                                          {{0,  0, -1,  0}, { 0, 0,  0,  0}, { 1,  0, 0,  0}, { 0,  0,  0, 0}}},
2061 
2062                                                         {{{0,  0,  0,  0}, { 0, 0,  0,  1}, { 0,  0, 0,  0}, { 0, -1,  0, 0}},
2063                                                          {{0,  0,  0, -1}, { 0, 0,  0,  0}, { 0,  0, 0,  0}, { 1,  0,  0, 0}},
2064                                                          {{0,  0,  0,  0}, { 0, 0,  0,  0}, { 0,  0, 0,  0}, { 0,  0,  0, 0}},
2065                                                          {{0,  1,  0,  0}, {-1, 0,  0,  0}, { 0,  0, 0,  0}, { 0,  0,  0, 0}}},
2066 
2067                                                         {{{0,  0,  0,  0}, { 0, 0, -1,  0}, { 0,  1, 0,  0}, { 0,  0,  0, 0}},
2068                                                          {{0,  0,  1,  0}, { 0, 0,  0,  0}, {-1,  0, 0,  0}, { 0,  0,  0, 0}},
2069                                                          {{0, -1,  0,  0}, { 1, 0,  0,  0}, { 0,  0, 0,  0}, { 0,  0,  0, 0}},
2070                                                          {{0,  0,  0,  0}, { 0, 0,  0,  0}, { 0,  0, 0,  0}, { 0,  0,  0, 0}}}};
2071                   PetscReal normal[4];
2072                   PetscInt  e, f, g;
2073 
2074                   for (d = 0; d < embedDim; ++d) {
2075                     normal[d] = 0.0;
2076                     for (e = 0; e < embedDim; ++e) {
2077                       for (f = 0; f < embedDim; ++f) {
2078                         for (g = 0; g < embedDim; ++g) {
2079                           normal[d] += epsilon[d][e][f][g]*(coordsIn[j*embedDim+e] - coordsIn[i*embedDim+e])*(coordsIn[k*embedDim+f] - coordsIn[i*embedDim+f])*(coordsIn[l*embedDim+f] - coordsIn[i*embedDim+f]);
2080                         }
2081                       }
2082                     }
2083                   }
2084                   if (DotReal(embedDim, normal, &coordsIn[i*embedDim]) < 0) {PetscInt tmp = cone[1]; cone[1] = cone[2]; cone[2] = tmp;}
2085                 }
2086                 ierr = DMPlexSetCone(*dm, c++, cone);CHKERRQ(ierr);
2087               }
2088             }
2089           }
2090         }
2091       }
2092       ierr = DMPlexSymmetrize(*dm);CHKERRQ(ierr);
2093       ierr = DMPlexStratify(*dm);CHKERRQ(ierr);
2094       ierr = PetscFree(graph);CHKERRQ(ierr);
2095       /* Interpolate mesh */
2096       ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
2097       ierr = DMDestroy(dm);CHKERRQ(ierr);
2098       *dm  = idm;
2099       break;
2100     }
2101   default: SETERRQ1(comm, PETSC_ERR_SUP, "Unsupported dimension for sphere: %D", dim);
2102   }
2103   /* Create coordinates */
2104   ierr = DMGetCoordinateSection(*dm, &coordSection);CHKERRQ(ierr);
2105   ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
2106   ierr = PetscSectionSetFieldComponents(coordSection, 0, embedDim);CHKERRQ(ierr);
2107   ierr = PetscSectionSetChart(coordSection, firstVertex, firstVertex+numVerts);CHKERRQ(ierr);
2108   for (v = firstVertex; v < firstVertex+numVerts; ++v) {
2109     ierr = PetscSectionSetDof(coordSection, v, embedDim);CHKERRQ(ierr);
2110     ierr = PetscSectionSetFieldDof(coordSection, v, 0, embedDim);CHKERRQ(ierr);
2111   }
2112   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
2113   ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
2114   ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
2115   ierr = VecSetBlockSize(coordinates, embedDim);CHKERRQ(ierr);
2116   ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
2117   ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
2118   ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
2119   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
2120   for (v = 0; v < numVerts; ++v) for (d = 0; d < embedDim; ++d) {coords[v*embedDim+d] = coordsIn[v*embedDim+d];}
2121   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
2122   ierr = DMSetCoordinatesLocal(*dm, coordinates);CHKERRQ(ierr);
2123   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
2124   ierr = PetscFree(coordsIn);CHKERRQ(ierr);
2125   PetscFunctionReturn(0);
2126 }
2127 
2128 /* External function declarations here */
2129 extern PetscErrorCode DMCreateInterpolation_Plex(DM dmCoarse, DM dmFine, Mat *interpolation, Vec *scaling);
2130 extern PetscErrorCode DMCreateInjection_Plex(DM dmCoarse, DM dmFine, Mat *mat);
2131 extern PetscErrorCode DMCreateMassMatrix_Plex(DM dmCoarse, DM dmFine, Mat *mat);
2132 extern PetscErrorCode DMCreateLocalSection_Plex(DM dm);
2133 extern PetscErrorCode DMCreateDefaultConstraints_Plex(DM dm);
2134 extern PetscErrorCode DMCreateMatrix_Plex(DM dm,  Mat *J);
2135 extern PetscErrorCode DMCreateCoordinateDM_Plex(DM dm, DM *cdm);
2136 extern PetscErrorCode DMCreateCoordinateField_Plex(DM dm, DMField *field);
2137 PETSC_INTERN PetscErrorCode DMClone_Plex(DM dm, DM *newdm);
2138 extern PetscErrorCode DMSetUp_Plex(DM dm);
2139 extern PetscErrorCode DMDestroy_Plex(DM dm);
2140 extern PetscErrorCode DMView_Plex(DM dm, PetscViewer viewer);
2141 extern PetscErrorCode DMLoad_Plex(DM dm, PetscViewer viewer);
2142 extern PetscErrorCode DMCreateSubDM_Plex(DM dm, PetscInt numFields, const PetscInt fields[], IS *is, DM *subdm);
2143 extern PetscErrorCode DMCreateSuperDM_Plex(DM dms[], PetscInt len, IS **is, DM *superdm);
2144 static PetscErrorCode DMInitialize_Plex(DM dm);
2145 
2146 /* Replace dm with the contents of dmNew
2147    - Share the DM_Plex structure
2148    - Share the coordinates
2149    - Share the SF
2150 */
2151 static PetscErrorCode DMPlexReplace_Static(DM dm, DM dmNew)
2152 {
2153   PetscSF               sf;
2154   DM                    coordDM, coarseDM;
2155   Vec                   coords;
2156   PetscBool             isper;
2157   const PetscReal      *maxCell, *L;
2158   const DMBoundaryType *bd;
2159   PetscErrorCode        ierr;
2160 
2161   PetscFunctionBegin;
2162   ierr = DMGetPointSF(dmNew, &sf);CHKERRQ(ierr);
2163   ierr = DMSetPointSF(dm, sf);CHKERRQ(ierr);
2164   ierr = DMGetCoordinateDM(dmNew, &coordDM);CHKERRQ(ierr);
2165   ierr = DMGetCoordinatesLocal(dmNew, &coords);CHKERRQ(ierr);
2166   ierr = DMSetCoordinateDM(dm, coordDM);CHKERRQ(ierr);
2167   ierr = DMSetCoordinatesLocal(dm, coords);CHKERRQ(ierr);
2168   ierr = DMGetPeriodicity(dm, &isper, &maxCell, &L, &bd);CHKERRQ(ierr);
2169   ierr = DMSetPeriodicity(dmNew, isper, maxCell, L, bd);CHKERRQ(ierr);
2170   ierr = DMDestroy_Plex(dm);CHKERRQ(ierr);
2171   ierr = DMInitialize_Plex(dm);CHKERRQ(ierr);
2172   dm->data = dmNew->data;
2173   ((DM_Plex *) dmNew->data)->refct++;
2174   dmNew->labels->refct++;
2175   if (!--(dm->labels->refct)) {
2176     DMLabelLink next = dm->labels->next;
2177 
2178     /* destroy the labels */
2179     while (next) {
2180       DMLabelLink tmp = next->next;
2181 
2182       ierr = DMLabelDestroy(&next->label);CHKERRQ(ierr);
2183       ierr = PetscFree(next);CHKERRQ(ierr);
2184       next = tmp;
2185     }
2186     ierr = PetscFree(dm->labels);CHKERRQ(ierr);
2187   }
2188   dm->labels = dmNew->labels;
2189   dm->depthLabel = dmNew->depthLabel;
2190   ierr = DMGetCoarseDM(dmNew,&coarseDM);CHKERRQ(ierr);
2191   ierr = DMSetCoarseDM(dm,coarseDM);CHKERRQ(ierr);
2192   PetscFunctionReturn(0);
2193 }
2194 
2195 /* Swap dm with the contents of dmNew
2196    - Swap the DM_Plex structure
2197    - Swap the coordinates
2198    - Swap the point PetscSF
2199 */
2200 static PetscErrorCode DMPlexSwap_Static(DM dmA, DM dmB)
2201 {
2202   DM              coordDMA, coordDMB;
2203   Vec             coordsA,  coordsB;
2204   PetscSF         sfA,      sfB;
2205   void            *tmp;
2206   DMLabelLinkList listTmp;
2207   DMLabel         depthTmp;
2208   PetscInt        tmpI;
2209   PetscErrorCode  ierr;
2210 
2211   PetscFunctionBegin;
2212   ierr = DMGetPointSF(dmA, &sfA);CHKERRQ(ierr);
2213   ierr = DMGetPointSF(dmB, &sfB);CHKERRQ(ierr);
2214   ierr = PetscObjectReference((PetscObject) sfA);CHKERRQ(ierr);
2215   ierr = DMSetPointSF(dmA, sfB);CHKERRQ(ierr);
2216   ierr = DMSetPointSF(dmB, sfA);CHKERRQ(ierr);
2217   ierr = PetscObjectDereference((PetscObject) sfA);CHKERRQ(ierr);
2218 
2219   ierr = DMGetCoordinateDM(dmA, &coordDMA);CHKERRQ(ierr);
2220   ierr = DMGetCoordinateDM(dmB, &coordDMB);CHKERRQ(ierr);
2221   ierr = PetscObjectReference((PetscObject) coordDMA);CHKERRQ(ierr);
2222   ierr = DMSetCoordinateDM(dmA, coordDMB);CHKERRQ(ierr);
2223   ierr = DMSetCoordinateDM(dmB, coordDMA);CHKERRQ(ierr);
2224   ierr = PetscObjectDereference((PetscObject) coordDMA);CHKERRQ(ierr);
2225 
2226   ierr = DMGetCoordinatesLocal(dmA, &coordsA);CHKERRQ(ierr);
2227   ierr = DMGetCoordinatesLocal(dmB, &coordsB);CHKERRQ(ierr);
2228   ierr = PetscObjectReference((PetscObject) coordsA);CHKERRQ(ierr);
2229   ierr = DMSetCoordinatesLocal(dmA, coordsB);CHKERRQ(ierr);
2230   ierr = DMSetCoordinatesLocal(dmB, coordsA);CHKERRQ(ierr);
2231   ierr = PetscObjectDereference((PetscObject) coordsA);CHKERRQ(ierr);
2232 
2233   tmp       = dmA->data;
2234   dmA->data = dmB->data;
2235   dmB->data = tmp;
2236   listTmp   = dmA->labels;
2237   dmA->labels = dmB->labels;
2238   dmB->labels = listTmp;
2239   depthTmp  = dmA->depthLabel;
2240   dmA->depthLabel = dmB->depthLabel;
2241   dmB->depthLabel = depthTmp;
2242   tmpI         = dmA->levelup;
2243   dmA->levelup = dmB->levelup;
2244   dmB->levelup = tmpI;
2245   PetscFunctionReturn(0);
2246 }
2247 
2248 PetscErrorCode DMSetFromOptions_NonRefinement_Plex(PetscOptionItems *PetscOptionsObject,DM dm)
2249 {
2250   DM_Plex       *mesh = (DM_Plex*) dm->data;
2251   PetscErrorCode ierr;
2252 
2253   PetscFunctionBegin;
2254   /* Handle viewing */
2255   ierr = PetscOptionsBool("-dm_plex_print_set_values", "Output all set values info", "DMPlexMatSetClosure", PETSC_FALSE, &mesh->printSetValues, NULL);CHKERRQ(ierr);
2256   ierr = PetscOptionsBoundedInt("-dm_plex_print_fem", "Debug output level all fem computations", "DMPlexSNESComputeResidualFEM", 0, &mesh->printFEM, NULL,0);CHKERRQ(ierr);
2257   ierr = PetscOptionsReal("-dm_plex_print_tol", "Tolerance for FEM output", "DMPlexSNESComputeResidualFEM", mesh->printTol, &mesh->printTol, NULL);CHKERRQ(ierr);
2258   ierr = PetscOptionsBoundedInt("-dm_plex_print_l2", "Debug output level all L2 diff computations", "DMComputeL2Diff", 0, &mesh->printL2, NULL,0);CHKERRQ(ierr);
2259   /* Point Location */
2260   ierr = PetscOptionsBool("-dm_plex_hash_location", "Use grid hashing for point location", "DMInterpolate", PETSC_FALSE, &mesh->useHashLocation, NULL);CHKERRQ(ierr);
2261   /* Partitioning and distribution */
2262   ierr = PetscOptionsBool("-dm_plex_partition_balance", "Attempt to evenly divide points on partition boundary between processes", "DMPlexSetPartitionBalance", PETSC_FALSE, &mesh->partitionBalance, NULL);CHKERRQ(ierr);
2263   /* Generation and remeshing */
2264   ierr = PetscOptionsBool("-dm_plex_remesh_bd", "Allow changes to the boundary on remeshing", "DMAdapt", PETSC_FALSE, &mesh->remeshBd, NULL);CHKERRQ(ierr);
2265   /* Projection behavior */
2266   ierr = PetscOptionsBoundedInt("-dm_plex_max_projection_height", "Maxmimum mesh point height used to project locally", "DMPlexSetMaxProjectionHeight", 0, &mesh->maxProjectionHeight, NULL,0);CHKERRQ(ierr);
2267   ierr = PetscOptionsBool("-dm_plex_regular_refinement", "Use special nested projection algorithm for regular refinement", "DMPlexSetRegularRefinement", mesh->regularRefinement, &mesh->regularRefinement, NULL);CHKERRQ(ierr);
2268   /* Checking structure */
2269   {
2270     PetscBool   flg = PETSC_FALSE, flg2 = PETSC_FALSE;
2271 
2272     ierr = PetscOptionsBool("-dm_plex_check_symmetry", "Check that the adjacency information in the mesh is symmetric", "DMPlexCheckSymmetry", PETSC_FALSE, &flg, &flg2);CHKERRQ(ierr);
2273     if (flg && flg2) {ierr = DMPlexCheckSymmetry(dm);CHKERRQ(ierr);}
2274     ierr = PetscOptionsBool("-dm_plex_check_skeleton", "Check that each cell has the correct number of vertices (only for homogeneous simplex or tensor meshes)", "DMPlexCheckSkeleton", PETSC_FALSE, &flg, &flg2);CHKERRQ(ierr);
2275     if (flg && flg2) {ierr = DMPlexCheckSkeleton(dm, 0);CHKERRQ(ierr);}
2276     ierr = PetscOptionsBool("-dm_plex_check_faces", "Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type", "DMPlexCheckFaces", PETSC_FALSE, &flg, &flg2);CHKERRQ(ierr);
2277     if (flg && flg2) {ierr = DMPlexCheckFaces(dm, 0);CHKERRQ(ierr);}
2278     ierr = PetscOptionsBool("-dm_plex_check_geometry", "Check that cells have positive volume", "DMPlexCheckGeometry", PETSC_FALSE, &flg, &flg2);CHKERRQ(ierr);
2279     if (flg && flg2) {ierr = DMPlexCheckGeometry(dm);CHKERRQ(ierr);}
2280   }
2281 
2282   ierr = PetscPartitionerSetFromOptions(mesh->partitioner);CHKERRQ(ierr);
2283   PetscFunctionReturn(0);
2284 }
2285 
2286 static PetscErrorCode DMSetFromOptions_Plex(PetscOptionItems *PetscOptionsObject,DM dm)
2287 {
2288   PetscInt       refine = 0, coarsen = 0, r;
2289   PetscBool      isHierarchy;
2290   PetscErrorCode ierr;
2291 
2292   PetscFunctionBegin;
2293   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2294   ierr = PetscOptionsHead(PetscOptionsObject,"DMPlex Options");CHKERRQ(ierr);
2295   /* Handle DMPlex refinement */
2296   ierr = PetscOptionsBoundedInt("-dm_refine", "The number of uniform refinements", "DMCreate", refine, &refine, NULL,0);CHKERRQ(ierr);
2297   ierr = PetscOptionsBoundedInt("-dm_refine_hierarchy", "The number of uniform refinements", "DMCreate", refine, &refine, &isHierarchy,0);CHKERRQ(ierr);
2298   if (refine) {ierr = DMPlexSetRefinementUniform(dm, PETSC_TRUE);CHKERRQ(ierr);}
2299   if (refine && isHierarchy) {
2300     DM *dms, coarseDM;
2301 
2302     ierr = DMGetCoarseDM(dm, &coarseDM);CHKERRQ(ierr);
2303     ierr = PetscObjectReference((PetscObject)coarseDM);CHKERRQ(ierr);
2304     ierr = PetscMalloc1(refine,&dms);CHKERRQ(ierr);
2305     ierr = DMRefineHierarchy(dm, refine, dms);CHKERRQ(ierr);
2306     /* Total hack since we do not pass in a pointer */
2307     ierr = DMPlexSwap_Static(dm, dms[refine-1]);CHKERRQ(ierr);
2308     if (refine == 1) {
2309       ierr = DMSetCoarseDM(dm, dms[0]);CHKERRQ(ierr);
2310       ierr = DMPlexSetRegularRefinement(dm, PETSC_TRUE);CHKERRQ(ierr);
2311     } else {
2312       ierr = DMSetCoarseDM(dm, dms[refine-2]);CHKERRQ(ierr);
2313       ierr = DMPlexSetRegularRefinement(dm, PETSC_TRUE);CHKERRQ(ierr);
2314       ierr = DMSetCoarseDM(dms[0], dms[refine-1]);CHKERRQ(ierr);
2315       ierr = DMPlexSetRegularRefinement(dms[0], PETSC_TRUE);CHKERRQ(ierr);
2316     }
2317     ierr = DMSetCoarseDM(dms[refine-1], coarseDM);CHKERRQ(ierr);
2318     ierr = PetscObjectDereference((PetscObject)coarseDM);CHKERRQ(ierr);
2319     /* Free DMs */
2320     for (r = 0; r < refine; ++r) {
2321       ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dms[r]);CHKERRQ(ierr);
2322       ierr = DMDestroy(&dms[r]);CHKERRQ(ierr);
2323     }
2324     ierr = PetscFree(dms);CHKERRQ(ierr);
2325   } else {
2326     for (r = 0; r < refine; ++r) {
2327       DM refinedMesh;
2328 
2329       ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dm);CHKERRQ(ierr);
2330       ierr = DMRefine(dm, PetscObjectComm((PetscObject) dm), &refinedMesh);CHKERRQ(ierr);
2331       /* Total hack since we do not pass in a pointer */
2332       ierr = DMPlexReplace_Static(dm, refinedMesh);CHKERRQ(ierr);
2333       ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dm);CHKERRQ(ierr);
2334       ierr = DMDestroy(&refinedMesh);CHKERRQ(ierr);
2335     }
2336   }
2337   /* Handle DMPlex coarsening */
2338   ierr = PetscOptionsBoundedInt("-dm_coarsen", "Coarsen the mesh", "DMCreate", coarsen, &coarsen, NULL,0);CHKERRQ(ierr);
2339   ierr = PetscOptionsBoundedInt("-dm_coarsen_hierarchy", "The number of coarsenings", "DMCreate", coarsen, &coarsen, &isHierarchy,0);CHKERRQ(ierr);
2340   if (coarsen && isHierarchy) {
2341     DM *dms;
2342 
2343     ierr = PetscMalloc1(coarsen, &dms);CHKERRQ(ierr);
2344     ierr = DMCoarsenHierarchy(dm, coarsen, dms);CHKERRQ(ierr);
2345     /* Free DMs */
2346     for (r = 0; r < coarsen; ++r) {
2347       ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dms[r]);CHKERRQ(ierr);
2348       ierr = DMDestroy(&dms[r]);CHKERRQ(ierr);
2349     }
2350     ierr = PetscFree(dms);CHKERRQ(ierr);
2351   } else {
2352     for (r = 0; r < coarsen; ++r) {
2353       DM coarseMesh;
2354 
2355       ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dm);CHKERRQ(ierr);
2356       ierr = DMCoarsen(dm, PetscObjectComm((PetscObject) dm), &coarseMesh);CHKERRQ(ierr);
2357       /* Total hack since we do not pass in a pointer */
2358       ierr = DMPlexReplace_Static(dm, coarseMesh);CHKERRQ(ierr);
2359       ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dm);CHKERRQ(ierr);
2360       ierr = DMDestroy(&coarseMesh);CHKERRQ(ierr);
2361     }
2362   }
2363   /* Handle */
2364   ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dm);CHKERRQ(ierr);
2365   ierr = PetscOptionsTail();CHKERRQ(ierr);
2366   PetscFunctionReturn(0);
2367 }
2368 
2369 static PetscErrorCode DMCreateGlobalVector_Plex(DM dm,Vec *vec)
2370 {
2371   PetscErrorCode ierr;
2372 
2373   PetscFunctionBegin;
2374   ierr = DMCreateGlobalVector_Section_Private(dm,vec);CHKERRQ(ierr);
2375   /* ierr = VecSetOperation(*vec, VECOP_DUPLICATE, (void(*)(void)) VecDuplicate_MPI_DM);CHKERRQ(ierr); */
2376   ierr = VecSetOperation(*vec, VECOP_VIEW, (void (*)(void)) VecView_Plex);CHKERRQ(ierr);
2377   ierr = VecSetOperation(*vec, VECOP_VIEWNATIVE, (void (*)(void)) VecView_Plex_Native);CHKERRQ(ierr);
2378   ierr = VecSetOperation(*vec, VECOP_LOAD, (void (*)(void)) VecLoad_Plex);CHKERRQ(ierr);
2379   ierr = VecSetOperation(*vec, VECOP_LOADNATIVE, (void (*)(void)) VecLoad_Plex_Native);CHKERRQ(ierr);
2380   PetscFunctionReturn(0);
2381 }
2382 
2383 static PetscErrorCode DMCreateLocalVector_Plex(DM dm,Vec *vec)
2384 {
2385   PetscErrorCode ierr;
2386 
2387   PetscFunctionBegin;
2388   ierr = DMCreateLocalVector_Section_Private(dm,vec);CHKERRQ(ierr);
2389   ierr = VecSetOperation(*vec, VECOP_VIEW, (void (*)(void)) VecView_Plex_Local);CHKERRQ(ierr);
2390   ierr = VecSetOperation(*vec, VECOP_LOAD, (void (*)(void)) VecLoad_Plex_Local);CHKERRQ(ierr);
2391   PetscFunctionReturn(0);
2392 }
2393 
2394 static PetscErrorCode DMGetDimPoints_Plex(DM dm, PetscInt dim, PetscInt *pStart, PetscInt *pEnd)
2395 {
2396   PetscInt       depth, d;
2397   PetscErrorCode ierr;
2398 
2399   PetscFunctionBegin;
2400   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
2401   if (depth == 1) {
2402     ierr = DMGetDimension(dm, &d);CHKERRQ(ierr);
2403     if (dim == 0)      {ierr = DMPlexGetDepthStratum(dm, dim, pStart, pEnd);CHKERRQ(ierr);}
2404     else if (dim == d) {ierr = DMPlexGetDepthStratum(dm, 1, pStart, pEnd);CHKERRQ(ierr);}
2405     else               {*pStart = 0; *pEnd = 0;}
2406   } else {
2407     ierr = DMPlexGetDepthStratum(dm, dim, pStart, pEnd);CHKERRQ(ierr);
2408   }
2409   PetscFunctionReturn(0);
2410 }
2411 
2412 static PetscErrorCode DMGetNeighbors_Plex(DM dm, PetscInt *nranks, const PetscMPIInt *ranks[])
2413 {
2414   PetscSF        sf;
2415   PetscErrorCode ierr;
2416 
2417   PetscFunctionBegin;
2418   ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr);
2419   ierr = PetscSFGetRootRanks(sf, nranks, ranks, NULL, NULL, NULL);CHKERRQ(ierr);
2420   PetscFunctionReturn(0);
2421 }
2422 
2423 static PetscErrorCode DMInitialize_Plex(DM dm)
2424 {
2425   PetscErrorCode ierr;
2426 
2427   PetscFunctionBegin;
2428   dm->ops->view                            = DMView_Plex;
2429   dm->ops->load                            = DMLoad_Plex;
2430   dm->ops->setfromoptions                  = DMSetFromOptions_Plex;
2431   dm->ops->clone                           = DMClone_Plex;
2432   dm->ops->setup                           = DMSetUp_Plex;
2433   dm->ops->createlocalsection              = DMCreateLocalSection_Plex;
2434   dm->ops->createdefaultconstraints        = DMCreateDefaultConstraints_Plex;
2435   dm->ops->createglobalvector              = DMCreateGlobalVector_Plex;
2436   dm->ops->createlocalvector               = DMCreateLocalVector_Plex;
2437   dm->ops->getlocaltoglobalmapping         = NULL;
2438   dm->ops->createfieldis                   = NULL;
2439   dm->ops->createcoordinatedm              = DMCreateCoordinateDM_Plex;
2440   dm->ops->createcoordinatefield           = DMCreateCoordinateField_Plex;
2441   dm->ops->getcoloring                     = NULL;
2442   dm->ops->creatematrix                    = DMCreateMatrix_Plex;
2443   dm->ops->createinterpolation             = DMCreateInterpolation_Plex;
2444   dm->ops->createmassmatrix                = DMCreateMassMatrix_Plex;
2445   dm->ops->createinjection                 = DMCreateInjection_Plex;
2446   dm->ops->refine                          = DMRefine_Plex;
2447   dm->ops->coarsen                         = DMCoarsen_Plex;
2448   dm->ops->refinehierarchy                 = DMRefineHierarchy_Plex;
2449   dm->ops->coarsenhierarchy                = DMCoarsenHierarchy_Plex;
2450   dm->ops->adaptlabel                      = DMAdaptLabel_Plex;
2451   dm->ops->adaptmetric                     = DMAdaptMetric_Plex;
2452   dm->ops->globaltolocalbegin              = NULL;
2453   dm->ops->globaltolocalend                = NULL;
2454   dm->ops->localtoglobalbegin              = NULL;
2455   dm->ops->localtoglobalend                = NULL;
2456   dm->ops->destroy                         = DMDestroy_Plex;
2457   dm->ops->createsubdm                     = DMCreateSubDM_Plex;
2458   dm->ops->createsuperdm                   = DMCreateSuperDM_Plex;
2459   dm->ops->getdimpoints                    = DMGetDimPoints_Plex;
2460   dm->ops->locatepoints                    = DMLocatePoints_Plex;
2461   dm->ops->projectfunctionlocal            = DMProjectFunctionLocal_Plex;
2462   dm->ops->projectfunctionlabellocal       = DMProjectFunctionLabelLocal_Plex;
2463   dm->ops->projectfieldlocal               = DMProjectFieldLocal_Plex;
2464   dm->ops->projectfieldlabellocal          = DMProjectFieldLabelLocal_Plex;
2465   dm->ops->computel2diff                   = DMComputeL2Diff_Plex;
2466   dm->ops->computel2gradientdiff           = DMComputeL2GradientDiff_Plex;
2467   dm->ops->computel2fielddiff              = DMComputeL2FieldDiff_Plex;
2468   dm->ops->getneighbors                    = DMGetNeighbors_Plex;
2469   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMPlexInsertBoundaryValues_C",DMPlexInsertBoundaryValues_Plex);CHKERRQ(ierr);
2470   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMSetUpGLVisViewer_C",DMSetUpGLVisViewer_Plex);CHKERRQ(ierr);
2471   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMCreateNeumannOverlap_C",DMCreateNeumannOverlap_Plex);CHKERRQ(ierr);
2472   PetscFunctionReturn(0);
2473 }
2474 
2475 PETSC_INTERN PetscErrorCode DMClone_Plex(DM dm, DM *newdm)
2476 {
2477   DM_Plex        *mesh = (DM_Plex *) dm->data;
2478   PetscErrorCode ierr;
2479 
2480   PetscFunctionBegin;
2481   mesh->refct++;
2482   (*newdm)->data = mesh;
2483   ierr = PetscObjectChangeTypeName((PetscObject) *newdm, DMPLEX);CHKERRQ(ierr);
2484   ierr = DMInitialize_Plex(*newdm);CHKERRQ(ierr);
2485   PetscFunctionReturn(0);
2486 }
2487 
2488 /*MC
2489   DMPLEX = "plex" - A DM object that encapsulates an unstructured mesh, or CW Complex, which can be expressed using a Hasse Diagram.
2490                     In the local representation, Vecs contain all unknowns in the interior and shared boundary. This is
2491                     specified by a PetscSection object. Ownership in the global representation is determined by
2492                     ownership of the underlying DMPlex points. This is specified by another PetscSection object.
2493 
2494   Options Database Keys:
2495 + -dm_plex_hash_location             - Use grid hashing for point location
2496 . -dm_plex_partition_balance         - Attempt to evenly divide points on partition boundary between processes
2497 . -dm_plex_remesh_bd                 - Allow changes to the boundary on remeshing
2498 . -dm_plex_max_projection_height     - Maxmimum mesh point height used to project locally
2499 . -dm_plex_regular_refinement        - Use special nested projection algorithm for regular refinement
2500 . -dm_plex_check_symmetry            - Check that the adjacency information in the mesh is symmetric
2501 . -dm_plex_check_skeleton <celltype> - Check that each cell has the correct number of vertices
2502 . -dm_plex_check_faces <celltype>    - Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type
2503 . -dm_plex_check_geometry            - Check that cells have positive volume
2504 . -dm_view :mesh.tex:ascii_latex     - View the mesh in LaTeX/TikZ
2505 . -dm_plex_view_scale <num>          - Scale the TikZ
2506 - -dm_plex_print_fem <num>           - View FEM assembly information, such as element vectors and matrices
2507 
2508 
2509   Level: intermediate
2510 
2511 .seealso: DMType, DMPlexCreate(), DMCreate(), DMSetType()
2512 M*/
2513 
2514 PETSC_EXTERN PetscErrorCode DMCreate_Plex(DM dm)
2515 {
2516   DM_Plex        *mesh;
2517   PetscInt       unit, d;
2518   PetscErrorCode ierr;
2519 
2520   PetscFunctionBegin;
2521   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2522   ierr     = PetscNewLog(dm,&mesh);CHKERRQ(ierr);
2523   dm->dim  = 0;
2524   dm->data = mesh;
2525 
2526   mesh->refct             = 1;
2527   ierr                    = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &mesh->coneSection);CHKERRQ(ierr);
2528   mesh->maxConeSize       = 0;
2529   mesh->cones             = NULL;
2530   mesh->coneOrientations  = NULL;
2531   ierr                    = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &mesh->supportSection);CHKERRQ(ierr);
2532   mesh->maxSupportSize    = 0;
2533   mesh->supports          = NULL;
2534   mesh->refinementUniform = PETSC_TRUE;
2535   mesh->refinementLimit   = -1.0;
2536 
2537   mesh->facesTmp = NULL;
2538 
2539   mesh->tetgenOpts   = NULL;
2540   mesh->triangleOpts = NULL;
2541   ierr = PetscPartitionerCreate(PetscObjectComm((PetscObject)dm), &mesh->partitioner);CHKERRQ(ierr);
2542   mesh->remeshBd     = PETSC_FALSE;
2543 
2544   mesh->subpointMap = NULL;
2545 
2546   for (unit = 0; unit < NUM_PETSC_UNITS; ++unit) mesh->scale[unit] = 1.0;
2547 
2548   mesh->regularRefinement   = PETSC_FALSE;
2549   mesh->depthState          = -1;
2550   mesh->globalVertexNumbers = NULL;
2551   mesh->globalCellNumbers   = NULL;
2552   mesh->anchorSection       = NULL;
2553   mesh->anchorIS            = NULL;
2554   mesh->createanchors       = NULL;
2555   mesh->computeanchormatrix = NULL;
2556   mesh->parentSection       = NULL;
2557   mesh->parents             = NULL;
2558   mesh->childIDs            = NULL;
2559   mesh->childSection        = NULL;
2560   mesh->children            = NULL;
2561   mesh->referenceTree       = NULL;
2562   mesh->getchildsymmetry    = NULL;
2563   for (d = 0; d < 8; ++d) mesh->hybridPointMax[d] = PETSC_DETERMINE;
2564   mesh->vtkCellHeight       = 0;
2565   mesh->useAnchors          = PETSC_FALSE;
2566 
2567   mesh->maxProjectionHeight = 0;
2568 
2569   mesh->printSetValues = PETSC_FALSE;
2570   mesh->printFEM       = 0;
2571   mesh->printTol       = 1.0e-10;
2572 
2573   ierr = DMInitialize_Plex(dm);CHKERRQ(ierr);
2574   PetscFunctionReturn(0);
2575 }
2576 
2577 /*@
2578   DMPlexCreate - Creates a DMPlex object, which encapsulates an unstructured mesh, or CW complex, which can be expressed using a Hasse Diagram.
2579 
2580   Collective
2581 
2582   Input Parameter:
2583 . comm - The communicator for the DMPlex object
2584 
2585   Output Parameter:
2586 . mesh  - The DMPlex object
2587 
2588   Level: beginner
2589 
2590 @*/
2591 PetscErrorCode DMPlexCreate(MPI_Comm comm, DM *mesh)
2592 {
2593   PetscErrorCode ierr;
2594 
2595   PetscFunctionBegin;
2596   PetscValidPointer(mesh,2);
2597   ierr = DMCreate(comm, mesh);CHKERRQ(ierr);
2598   ierr = DMSetType(*mesh, DMPLEX);CHKERRQ(ierr);
2599   PetscFunctionReturn(0);
2600 }
2601 
2602 /*
2603   This takes as input the common mesh generator output, a list of the vertices for each cell, but vertex numbers are global and an SF is built for them
2604 */
2605 /* TODO: invertCells and spaceDim arguments could be added also to to DMPlexCreateFromCellListParallel(), DMPlexBuildFromCellList_Internal() and DMPlexCreateFromCellList() */
2606 PetscErrorCode DMPlexBuildFromCellList_Parallel_Internal(DM dm, PetscInt spaceDim, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, const int cells[], PetscBool invertCells, PetscSF *sfVert)
2607 {
2608   PetscSF         sfPoint;
2609   PetscLayout     vLayout;
2610   PetscHSetI      vhash;
2611   PetscSFNode    *remoteVerticesAdj, *vertexLocal, *vertexOwner, *remoteVertex;
2612   const PetscInt *vrange;
2613   PetscInt        numVerticesAdj, off = 0, *verticesAdj, numVerticesGhost = 0, *localVertex, *cone, c, p, v, g;
2614   PetscMPIInt     rank, size;
2615   PetscErrorCode  ierr;
2616 
2617   PetscFunctionBegin;
2618   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRQ(ierr);
2619   ierr = MPI_Comm_size(PetscObjectComm((PetscObject) dm), &size);CHKERRQ(ierr);
2620   /* Partition vertices */
2621   ierr = PetscLayoutCreate(PetscObjectComm((PetscObject) dm), &vLayout);CHKERRQ(ierr);
2622   ierr = PetscLayoutSetLocalSize(vLayout, numVertices);CHKERRQ(ierr);
2623   ierr = PetscLayoutSetBlockSize(vLayout, 1);CHKERRQ(ierr);
2624   ierr = PetscLayoutSetUp(vLayout);CHKERRQ(ierr);
2625   ierr = PetscLayoutGetRanges(vLayout, &vrange);CHKERRQ(ierr);
2626   /* Count vertices and map them to procs */
2627   ierr = PetscHSetICreate(&vhash);CHKERRQ(ierr);
2628   for (c = 0; c < numCells; ++c) {
2629     for (p = 0; p < numCorners; ++p) {
2630       ierr = PetscHSetIAdd(vhash, cells[c*numCorners+p]);CHKERRQ(ierr);
2631     }
2632   }
2633   ierr = PetscHSetIGetSize(vhash, &numVerticesAdj);CHKERRQ(ierr);
2634   ierr = PetscMalloc1(numVerticesAdj, &verticesAdj);CHKERRQ(ierr);
2635   ierr = PetscHSetIGetElems(vhash, &off, verticesAdj);CHKERRQ(ierr);
2636   ierr = PetscHSetIDestroy(&vhash);CHKERRQ(ierr);
2637   if (off != numVerticesAdj) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid number of local vertices %D should be %D", off, numVerticesAdj);
2638   ierr = PetscSortInt(numVerticesAdj, verticesAdj);CHKERRQ(ierr);
2639   ierr = PetscMalloc1(numVerticesAdj, &remoteVerticesAdj);CHKERRQ(ierr);
2640   for (v = 0; v < numVerticesAdj; ++v) {
2641     const PetscInt gv = verticesAdj[v];
2642     PetscInt       vrank;
2643 
2644     ierr = PetscFindInt(gv, size+1, vrange, &vrank);CHKERRQ(ierr);
2645     vrank = vrank < 0 ? -(vrank+2) : vrank;
2646     remoteVerticesAdj[v].index = gv - vrange[vrank];
2647     remoteVerticesAdj[v].rank  = vrank;
2648   }
2649   /* Create cones */
2650   ierr = DMPlexSetChart(dm, 0, numCells+numVerticesAdj);CHKERRQ(ierr);
2651   for (c = 0; c < numCells; ++c) {ierr = DMPlexSetConeSize(dm, c, numCorners);CHKERRQ(ierr);}
2652   ierr = DMSetUp(dm);CHKERRQ(ierr);
2653   ierr = DMGetWorkArray(dm, numCorners, MPIU_INT, &cone);CHKERRQ(ierr);
2654   for (c = 0; c < numCells; ++c) {
2655     for (p = 0; p < numCorners; ++p) {
2656       const PetscInt gv = cells[c*numCorners+p];
2657       PetscInt       lv;
2658 
2659       ierr = PetscFindInt(gv, numVerticesAdj, verticesAdj, &lv);CHKERRQ(ierr);
2660       if (lv < 0) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Could not find global vertex %D in local connectivity", gv);
2661       cone[p] = lv+numCells;
2662     }
2663     if (invertCells) { ierr = DMPlexInvertCell_Internal(spaceDim, numCorners, cone);CHKERRQ(ierr); }
2664     ierr = DMPlexSetCone(dm, c, cone);CHKERRQ(ierr);
2665   }
2666   ierr = DMRestoreWorkArray(dm, numCorners, MPIU_INT, &cone);CHKERRQ(ierr);
2667   /* Create SF for vertices */
2668   ierr = PetscSFCreate(PetscObjectComm((PetscObject)dm), sfVert);CHKERRQ(ierr);
2669   ierr = PetscObjectSetName((PetscObject) *sfVert, "Vertex Ownership SF");CHKERRQ(ierr);
2670   ierr = PetscSFSetFromOptions(*sfVert);CHKERRQ(ierr);
2671   ierr = PetscSFSetGraph(*sfVert, numVertices, numVerticesAdj, NULL, PETSC_OWN_POINTER, remoteVerticesAdj, PETSC_OWN_POINTER);CHKERRQ(ierr);
2672   ierr = PetscFree(verticesAdj);CHKERRQ(ierr);
2673   /* Build pointSF */
2674   ierr = PetscMalloc2(numVerticesAdj, &vertexLocal, numVertices, &vertexOwner);CHKERRQ(ierr);
2675   for (v = 0; v < numVerticesAdj; ++v) {vertexLocal[v].index = v+numCells; vertexLocal[v].rank = rank;}
2676   for (v = 0; v < numVertices;    ++v) {vertexOwner[v].index = -1;         vertexOwner[v].rank = -1;}
2677   ierr = PetscSFReduceBegin(*sfVert, MPIU_2INT, vertexLocal, vertexOwner, MPI_MAXLOC);CHKERRQ(ierr);
2678   ierr = PetscSFReduceEnd(*sfVert, MPIU_2INT, vertexLocal, vertexOwner, MPI_MAXLOC);CHKERRQ(ierr);
2679   for (v = 0; v < numVertices;    ++v) if (vertexOwner[v].rank < 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Global vertex %d on rank %d was unclaimed", v + vrange[rank], rank);
2680   ierr = PetscSFBcastBegin(*sfVert, MPIU_2INT, vertexOwner, vertexLocal);CHKERRQ(ierr);
2681   ierr = PetscSFBcastEnd(*sfVert, MPIU_2INT, vertexOwner, vertexLocal);CHKERRQ(ierr);
2682   for (v = 0; v < numVerticesAdj; ++v) if (vertexLocal[v].rank != rank) ++numVerticesGhost;
2683   ierr = PetscMalloc1(numVerticesGhost, &localVertex);CHKERRQ(ierr);
2684   ierr = PetscMalloc1(numVerticesGhost, &remoteVertex);CHKERRQ(ierr);
2685   for (v = 0, g = 0; v < numVerticesAdj; ++v) {
2686     if (vertexLocal[v].rank != rank) {
2687       localVertex[g]        = v+numCells;
2688       remoteVertex[g].index = vertexLocal[v].index;
2689       remoteVertex[g].rank  = vertexLocal[v].rank;
2690       ++g;
2691     }
2692   }
2693   ierr = PetscFree2(vertexLocal, vertexOwner);CHKERRQ(ierr);
2694   if (g != numVerticesGhost) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid number of vertex ghosts %D should be %D", g, numVerticesGhost);
2695   ierr = DMGetPointSF(dm, &sfPoint);CHKERRQ(ierr);
2696   ierr = PetscObjectSetName((PetscObject) sfPoint, "point SF");CHKERRQ(ierr);
2697   ierr = PetscSFSetGraph(sfPoint, numCells+numVerticesAdj, numVerticesGhost, localVertex, PETSC_OWN_POINTER, remoteVertex, PETSC_OWN_POINTER);CHKERRQ(ierr);
2698   ierr = PetscLayoutDestroy(&vLayout);CHKERRQ(ierr);
2699   /* Fill in the rest of the topology structure */
2700   ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
2701   ierr = DMPlexStratify(dm);CHKERRQ(ierr);
2702   PetscFunctionReturn(0);
2703 }
2704 
2705 /*
2706   This takes as input the coordinates for each owned vertex
2707 */
2708 PetscErrorCode DMPlexBuildCoordinates_Parallel_Internal(DM dm, PetscInt spaceDim, PetscInt numCells, PetscInt numV, PetscSF sfVert, const PetscReal vertexCoords[])
2709 {
2710   PetscSection   coordSection;
2711   Vec            coordinates;
2712   PetscScalar   *coords;
2713   PetscInt       numVertices, numVerticesAdj, coordSize, v;
2714   PetscErrorCode ierr;
2715 
2716   PetscFunctionBegin;
2717   ierr = DMSetCoordinateDim(dm, spaceDim);CHKERRQ(ierr);
2718   ierr = PetscSFGetGraph(sfVert, &numVertices, &numVerticesAdj, NULL, NULL);CHKERRQ(ierr);
2719   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
2720   ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
2721   ierr = PetscSectionSetFieldComponents(coordSection, 0, spaceDim);CHKERRQ(ierr);
2722   ierr = PetscSectionSetChart(coordSection, numCells, numCells + numVerticesAdj);CHKERRQ(ierr);
2723   for (v = numCells; v < numCells+numVerticesAdj; ++v) {
2724     ierr = PetscSectionSetDof(coordSection, v, spaceDim);CHKERRQ(ierr);
2725     ierr = PetscSectionSetFieldDof(coordSection, v, 0, spaceDim);CHKERRQ(ierr);
2726   }
2727   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
2728   ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
2729   ierr = VecCreate(PetscObjectComm((PetscObject)dm), &coordinates);CHKERRQ(ierr);
2730   ierr = VecSetBlockSize(coordinates, spaceDim);CHKERRQ(ierr);
2731   ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
2732   ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
2733   ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
2734   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
2735   {
2736     MPI_Datatype coordtype;
2737 
2738     /* Need a temp buffer for coords if we have complex/single */
2739     ierr = MPI_Type_contiguous(spaceDim, MPIU_SCALAR, &coordtype);CHKERRQ(ierr);
2740     ierr = MPI_Type_commit(&coordtype);CHKERRQ(ierr);
2741 #if defined(PETSC_USE_COMPLEX)
2742     {
2743     PetscScalar *svertexCoords;
2744     PetscInt    i;
2745     ierr = PetscMalloc1(numV*spaceDim,&svertexCoords);CHKERRQ(ierr);
2746     for (i=0; i<numV*spaceDim; i++) svertexCoords[i] = vertexCoords[i];
2747     ierr = PetscSFBcastBegin(sfVert, coordtype, svertexCoords, coords);CHKERRQ(ierr);
2748     ierr = PetscSFBcastEnd(sfVert, coordtype, svertexCoords, coords);CHKERRQ(ierr);
2749     ierr = PetscFree(svertexCoords);CHKERRQ(ierr);
2750     }
2751 #else
2752     ierr = PetscSFBcastBegin(sfVert, coordtype, vertexCoords, coords);CHKERRQ(ierr);
2753     ierr = PetscSFBcastEnd(sfVert, coordtype, vertexCoords, coords);CHKERRQ(ierr);
2754 #endif
2755     ierr = MPI_Type_free(&coordtype);CHKERRQ(ierr);
2756   }
2757   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
2758   ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
2759   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
2760   PetscFunctionReturn(0);
2761 }
2762 
2763 /*@
2764   DMPlexCreateFromCellListParallel - This takes as input common mesh generator output, a list of the vertices for each cell, and produces a DM
2765 
2766   Input Parameters:
2767 + comm - The communicator
2768 . dim - The topological dimension of the mesh
2769 . numCells - The number of cells owned by this process
2770 . numVertices - The number of vertices owned by this process
2771 . numCorners - The number of vertices for each cell
2772 . interpolate - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
2773 . cells - An array of numCells*numCorners numbers, the global vertex numbers for each cell
2774 . spaceDim - The spatial dimension used for coordinates
2775 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
2776 
2777   Output Parameter:
2778 + dm - The DM
2779 - vertexSF - Optional, SF describing complete vertex ownership
2780 
2781   Note: Two triangles sharing a face
2782 $
2783 $        2
2784 $      / | \
2785 $     /  |  \
2786 $    /   |   \
2787 $   0  0 | 1  3
2788 $    \   |   /
2789 $     \  |  /
2790 $      \ | /
2791 $        1
2792 would have input
2793 $  numCells = 2, numVertices = 4
2794 $  cells = [0 1 2  1 3 2]
2795 $
2796 which would result in the DMPlex
2797 $
2798 $        4
2799 $      / | \
2800 $     /  |  \
2801 $    /   |   \
2802 $   2  0 | 1  5
2803 $    \   |   /
2804 $     \  |  /
2805 $      \ | /
2806 $        3
2807 
2808   Level: beginner
2809 
2810 .seealso: DMPlexCreateFromCellList(), DMPlexCreateFromDAG(), DMPlexCreate()
2811 @*/
2812 PetscErrorCode DMPlexCreateFromCellListParallel(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, PetscBool interpolate, const int cells[], PetscInt spaceDim, const PetscReal vertexCoords[], PetscSF *vertexSF, DM *dm)
2813 {
2814   PetscSF        sfVert;
2815   PetscErrorCode ierr;
2816 
2817   PetscFunctionBegin;
2818   ierr = DMCreate(comm, dm);CHKERRQ(ierr);
2819   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
2820   PetscValidLogicalCollectiveInt(*dm, dim, 2);
2821   PetscValidLogicalCollectiveInt(*dm, spaceDim, 8);
2822   ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
2823   ierr = DMPlexBuildFromCellList_Parallel_Internal(*dm, spaceDim, numCells, numVertices, numCorners, cells, PETSC_FALSE, &sfVert);CHKERRQ(ierr);
2824   if (interpolate) {
2825     DM idm;
2826 
2827     ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
2828     ierr = DMDestroy(dm);CHKERRQ(ierr);
2829     *dm  = idm;
2830   }
2831   ierr = DMPlexBuildCoordinates_Parallel_Internal(*dm, spaceDim, numCells, numVertices, sfVert, vertexCoords);CHKERRQ(ierr);
2832   if (vertexSF) *vertexSF = sfVert;
2833   else {ierr = PetscSFDestroy(&sfVert);CHKERRQ(ierr);}
2834   PetscFunctionReturn(0);
2835 }
2836 
2837 /*
2838   This takes as input the common mesh generator output, a list of the vertices for each cell
2839 */
2840 PetscErrorCode DMPlexBuildFromCellList_Internal(DM dm, PetscInt spaceDim, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, const int cells[], PetscBool invertCells)
2841 {
2842   PetscInt      *cone, c, p;
2843   PetscErrorCode ierr;
2844 
2845   PetscFunctionBegin;
2846   ierr = DMPlexSetChart(dm, 0, numCells+numVertices);CHKERRQ(ierr);
2847   for (c = 0; c < numCells; ++c) {
2848     ierr = DMPlexSetConeSize(dm, c, numCorners);CHKERRQ(ierr);
2849   }
2850   ierr = DMSetUp(dm);CHKERRQ(ierr);
2851   ierr = DMGetWorkArray(dm, numCorners, MPIU_INT, &cone);CHKERRQ(ierr);
2852   for (c = 0; c < numCells; ++c) {
2853     for (p = 0; p < numCorners; ++p) {
2854       cone[p] = cells[c*numCorners+p]+numCells;
2855     }
2856     if (invertCells) { ierr = DMPlexInvertCell_Internal(spaceDim, numCorners, cone);CHKERRQ(ierr); }
2857     ierr = DMPlexSetCone(dm, c, cone);CHKERRQ(ierr);
2858   }
2859   ierr = DMRestoreWorkArray(dm, numCorners, MPIU_INT, &cone);CHKERRQ(ierr);
2860   ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
2861   ierr = DMPlexStratify(dm);CHKERRQ(ierr);
2862   PetscFunctionReturn(0);
2863 }
2864 
2865 /*
2866   This takes as input the coordinates for each vertex
2867 */
2868 PetscErrorCode DMPlexBuildCoordinates_Internal(DM dm, PetscInt spaceDim, PetscInt numCells, PetscInt numVertices, const double vertexCoords[])
2869 {
2870   PetscSection   coordSection;
2871   Vec            coordinates;
2872   DM             cdm;
2873   PetscScalar   *coords;
2874   PetscInt       v, d;
2875   PetscErrorCode ierr;
2876 
2877   PetscFunctionBegin;
2878   ierr = DMSetCoordinateDim(dm, spaceDim);CHKERRQ(ierr);
2879   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
2880   ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
2881   ierr = PetscSectionSetFieldComponents(coordSection, 0, spaceDim);CHKERRQ(ierr);
2882   ierr = PetscSectionSetChart(coordSection, numCells, numCells + numVertices);CHKERRQ(ierr);
2883   for (v = numCells; v < numCells+numVertices; ++v) {
2884     ierr = PetscSectionSetDof(coordSection, v, spaceDim);CHKERRQ(ierr);
2885     ierr = PetscSectionSetFieldDof(coordSection, v, 0, spaceDim);CHKERRQ(ierr);
2886   }
2887   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
2888 
2889   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
2890   ierr = DMCreateLocalVector(cdm, &coordinates);CHKERRQ(ierr);
2891   ierr = VecSetBlockSize(coordinates, spaceDim);CHKERRQ(ierr);
2892   ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
2893   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
2894   for (v = 0; v < numVertices; ++v) {
2895     for (d = 0; d < spaceDim; ++d) {
2896       coords[v*spaceDim+d] = vertexCoords[v*spaceDim+d];
2897     }
2898   }
2899   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
2900   ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
2901   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
2902   PetscFunctionReturn(0);
2903 }
2904 
2905 /*@
2906   DMPlexCreateFromCellList - This takes as input common mesh generator output, a list of the vertices for each cell, and produces a DM
2907 
2908   Input Parameters:
2909 + comm - The communicator
2910 . dim - The topological dimension of the mesh
2911 . numCells - The number of cells
2912 . numVertices - The number of vertices
2913 . numCorners - The number of vertices for each cell
2914 . interpolate - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
2915 . cells - An array of numCells*numCorners numbers, the vertices for each cell
2916 . spaceDim - The spatial dimension used for coordinates
2917 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
2918 
2919   Output Parameter:
2920 . dm - The DM
2921 
2922   Note: Two triangles sharing a face
2923 $
2924 $        2
2925 $      / | \
2926 $     /  |  \
2927 $    /   |   \
2928 $   0  0 | 1  3
2929 $    \   |   /
2930 $     \  |  /
2931 $      \ | /
2932 $        1
2933 would have input
2934 $  numCells = 2, numVertices = 4
2935 $  cells = [0 1 2  1 3 2]
2936 $
2937 which would result in the DMPlex
2938 $
2939 $        4
2940 $      / | \
2941 $     /  |  \
2942 $    /   |   \
2943 $   2  0 | 1  5
2944 $    \   |   /
2945 $     \  |  /
2946 $      \ | /
2947 $        3
2948 
2949   Level: beginner
2950 
2951 .seealso: DMPlexCreateFromDAG(), DMPlexCreate()
2952 @*/
2953 PetscErrorCode DMPlexCreateFromCellList(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, PetscBool interpolate, const int cells[], PetscInt spaceDim, const double vertexCoords[], DM *dm)
2954 {
2955   PetscErrorCode ierr;
2956 
2957   PetscFunctionBegin;
2958   ierr = DMCreate(comm, dm);CHKERRQ(ierr);
2959   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
2960   ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
2961   ierr = DMPlexBuildFromCellList_Internal(*dm, spaceDim, numCells, numVertices, numCorners, cells, PETSC_FALSE);CHKERRQ(ierr);
2962   if (interpolate) {
2963     DM idm;
2964 
2965     ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
2966     ierr = DMDestroy(dm);CHKERRQ(ierr);
2967     *dm  = idm;
2968   }
2969   ierr = DMPlexBuildCoordinates_Internal(*dm, spaceDim, numCells, numVertices, vertexCoords);CHKERRQ(ierr);
2970   PetscFunctionReturn(0);
2971 }
2972 
2973 /*@
2974   DMPlexCreateFromDAG - This takes as input the adjacency-list representation of the Directed Acyclic Graph (Hasse Diagram) encoding a mesh, and produces a DM
2975 
2976   Input Parameters:
2977 + dm - The empty DM object, usually from DMCreate() and DMSetDimension()
2978 . depth - The depth of the DAG
2979 . numPoints - Array of size depth + 1 containing the number of points at each depth
2980 . coneSize - The cone size of each point
2981 . cones - The concatenation of the cone points for each point, the cone list must be oriented correctly for each point
2982 . coneOrientations - The orientation of each cone point
2983 - vertexCoords - An array of numPoints[0]*spacedim numbers representing the coordinates of each vertex, with spacedim the value set via DMSetCoordinateDim()
2984 
2985   Output Parameter:
2986 . dm - The DM
2987 
2988   Note: Two triangles sharing a face would have input
2989 $  depth = 1, numPoints = [4 2], coneSize = [3 3 0 0 0 0]
2990 $  cones = [2 3 4  3 5 4], coneOrientations = [0 0 0  0 0 0]
2991 $ vertexCoords = [-1.0 0.0  0.0 -1.0  0.0 1.0  1.0 0.0]
2992 $
2993 which would result in the DMPlex
2994 $
2995 $        4
2996 $      / | \
2997 $     /  |  \
2998 $    /   |   \
2999 $   2  0 | 1  5
3000 $    \   |   /
3001 $     \  |  /
3002 $      \ | /
3003 $        3
3004 $
3005 $ Notice that all points are numbered consecutively, unlikely DMPlexCreateFromCellList()
3006 
3007   Level: advanced
3008 
3009 .seealso: DMPlexCreateFromCellList(), DMPlexCreate()
3010 @*/
3011 PetscErrorCode DMPlexCreateFromDAG(DM dm, PetscInt depth, const PetscInt numPoints[], const PetscInt coneSize[], const PetscInt cones[], const PetscInt coneOrientations[], const PetscScalar vertexCoords[])
3012 {
3013   Vec            coordinates;
3014   PetscSection   coordSection;
3015   PetscScalar    *coords;
3016   PetscInt       coordSize, firstVertex = -1, pStart = 0, pEnd = 0, p, v, dim, dimEmbed, d, off;
3017   PetscErrorCode ierr;
3018 
3019   PetscFunctionBegin;
3020   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
3021   ierr = DMGetCoordinateDim(dm, &dimEmbed);CHKERRQ(ierr);
3022   if (dimEmbed < dim) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Embedding dimension %d cannot be less than intrinsic dimension %d",dimEmbed,dim);
3023   for (d = 0; d <= depth; ++d) pEnd += numPoints[d];
3024   ierr = DMPlexSetChart(dm, pStart, pEnd);CHKERRQ(ierr);
3025   for (p = pStart; p < pEnd; ++p) {
3026     ierr = DMPlexSetConeSize(dm, p, coneSize[p-pStart]);CHKERRQ(ierr);
3027     if (firstVertex < 0 && !coneSize[p - pStart]) {
3028       firstVertex = p - pStart;
3029     }
3030   }
3031   if (firstVertex < 0 && numPoints[0]) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONG,"Expected %d vertices but could not find any", numPoints[0]);
3032   ierr = DMSetUp(dm);CHKERRQ(ierr); /* Allocate space for cones */
3033   for (p = pStart, off = 0; p < pEnd; off += coneSize[p-pStart], ++p) {
3034     ierr = DMPlexSetCone(dm, p, &cones[off]);CHKERRQ(ierr);
3035     ierr = DMPlexSetConeOrientation(dm, p, &coneOrientations[off]);CHKERRQ(ierr);
3036   }
3037   ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
3038   ierr = DMPlexStratify(dm);CHKERRQ(ierr);
3039   /* Build coordinates */
3040   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
3041   ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
3042   ierr = PetscSectionSetFieldComponents(coordSection, 0, dimEmbed);CHKERRQ(ierr);
3043   ierr = PetscSectionSetChart(coordSection, firstVertex, firstVertex+numPoints[0]);CHKERRQ(ierr);
3044   for (v = firstVertex; v < firstVertex+numPoints[0]; ++v) {
3045     ierr = PetscSectionSetDof(coordSection, v, dimEmbed);CHKERRQ(ierr);
3046     ierr = PetscSectionSetFieldDof(coordSection, v, 0, dimEmbed);CHKERRQ(ierr);
3047   }
3048   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
3049   ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
3050   ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
3051   ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
3052   ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
3053   ierr = VecSetBlockSize(coordinates, dimEmbed);CHKERRQ(ierr);
3054   ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
3055   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
3056   for (v = 0; v < numPoints[0]; ++v) {
3057     PetscInt off;
3058 
3059     ierr = PetscSectionGetOffset(coordSection, v+firstVertex, &off);CHKERRQ(ierr);
3060     for (d = 0; d < dimEmbed; ++d) {
3061       coords[off+d] = vertexCoords[v*dimEmbed+d];
3062     }
3063   }
3064   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
3065   ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
3066   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
3067   PetscFunctionReturn(0);
3068 }
3069 
3070 /*@C
3071   DMPlexCreateCellVertexFromFile - Create a DMPlex mesh from a simple cell-vertex file.
3072 
3073 + comm        - The MPI communicator
3074 . filename    - Name of the .dat file
3075 - interpolate - Create faces and edges in the mesh
3076 
3077   Output Parameter:
3078 . dm  - The DM object representing the mesh
3079 
3080   Note: The format is the simplest possible:
3081 $ Ne
3082 $ v0 v1 ... vk
3083 $ Nv
3084 $ x y z marker
3085 
3086   Level: beginner
3087 
3088 .seealso: DMPlexCreateFromFile(), DMPlexCreateMedFromFile(), DMPlexCreateGmsh(), DMPlexCreate()
3089 @*/
3090 PetscErrorCode DMPlexCreateCellVertexFromFile(MPI_Comm comm, const char filename[], PetscBool interpolate, DM *dm)
3091 {
3092   DMLabel         marker;
3093   PetscViewer     viewer;
3094   Vec             coordinates;
3095   PetscSection    coordSection;
3096   PetscScalar    *coords;
3097   char            line[PETSC_MAX_PATH_LEN];
3098   PetscInt        dim = 3, cdim = 3, coordSize, v, c, d;
3099   PetscMPIInt     rank;
3100   int             snum, Nv, Nc;
3101   PetscErrorCode  ierr;
3102 
3103   PetscFunctionBegin;
3104   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
3105   ierr = PetscViewerCreate(comm, &viewer);CHKERRQ(ierr);
3106   ierr = PetscViewerSetType(viewer, PETSCVIEWERASCII);CHKERRQ(ierr);
3107   ierr = PetscViewerFileSetMode(viewer, FILE_MODE_READ);CHKERRQ(ierr);
3108   ierr = PetscViewerFileSetName(viewer, filename);CHKERRQ(ierr);
3109   if (!rank) {
3110     ierr = PetscViewerRead(viewer, line, 2, NULL, PETSC_STRING);CHKERRQ(ierr);
3111     snum = sscanf(line, "%d %d", &Nc, &Nv);
3112     if (snum != 2) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
3113   } else {
3114     Nc = Nv = 0;
3115   }
3116   ierr = DMCreate(comm, dm);CHKERRQ(ierr);
3117   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
3118   ierr = DMPlexSetChart(*dm, 0, Nc+Nv);CHKERRQ(ierr);
3119   ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
3120   ierr = DMSetCoordinateDim(*dm, cdim);CHKERRQ(ierr);
3121   /* Read topology */
3122   if (!rank) {
3123     PetscInt cone[8], corners = 8;
3124     int      vbuf[8], v;
3125 
3126     for (c = 0; c < Nc; ++c) {ierr = DMPlexSetConeSize(*dm, c, corners);CHKERRQ(ierr);}
3127     ierr = DMSetUp(*dm);CHKERRQ(ierr);
3128     for (c = 0; c < Nc; ++c) {
3129       ierr = PetscViewerRead(viewer, line, corners, NULL, PETSC_STRING);CHKERRQ(ierr);
3130       snum = sscanf(line, "%d %d %d %d %d %d %d %d", &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3], &vbuf[4], &vbuf[5], &vbuf[6], &vbuf[7]);
3131       if (snum != corners) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
3132       for (v = 0; v < corners; ++v) cone[v] = vbuf[v] + Nc;
3133       /* Hexahedra are inverted */
3134       {
3135         PetscInt tmp = cone[1];
3136         cone[1] = cone[3];
3137         cone[3] = tmp;
3138       }
3139       ierr = DMPlexSetCone(*dm, c, cone);CHKERRQ(ierr);
3140     }
3141   }
3142   ierr = DMPlexSymmetrize(*dm);CHKERRQ(ierr);
3143   ierr = DMPlexStratify(*dm);CHKERRQ(ierr);
3144   /* Read coordinates */
3145   ierr = DMGetCoordinateSection(*dm, &coordSection);CHKERRQ(ierr);
3146   ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
3147   ierr = PetscSectionSetFieldComponents(coordSection, 0, cdim);CHKERRQ(ierr);
3148   ierr = PetscSectionSetChart(coordSection, Nc, Nc + Nv);CHKERRQ(ierr);
3149   for (v = Nc; v < Nc+Nv; ++v) {
3150     ierr = PetscSectionSetDof(coordSection, v, cdim);CHKERRQ(ierr);
3151     ierr = PetscSectionSetFieldDof(coordSection, v, 0, cdim);CHKERRQ(ierr);
3152   }
3153   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
3154   ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
3155   ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
3156   ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
3157   ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
3158   ierr = VecSetBlockSize(coordinates, cdim);CHKERRQ(ierr);
3159   ierr = VecSetType(coordinates, VECSTANDARD);CHKERRQ(ierr);
3160   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
3161   if (!rank) {
3162     double x[3];
3163     int    val;
3164 
3165     ierr = DMCreateLabel(*dm, "marker");CHKERRQ(ierr);
3166     ierr = DMGetLabel(*dm, "marker", &marker);CHKERRQ(ierr);
3167     for (v = 0; v < Nv; ++v) {
3168       ierr = PetscViewerRead(viewer, line, 4, NULL, PETSC_STRING);CHKERRQ(ierr);
3169       snum = sscanf(line, "%lg %lg %lg %d", &x[0], &x[1], &x[2], &val);
3170       if (snum != 4) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
3171       for (d = 0; d < cdim; ++d) coords[v*cdim+d] = x[d];
3172       if (val) {ierr = DMLabelSetValue(marker, v+Nc, val);CHKERRQ(ierr);}
3173     }
3174   }
3175   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
3176   ierr = DMSetCoordinatesLocal(*dm, coordinates);CHKERRQ(ierr);
3177   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
3178   ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
3179   if (interpolate) {
3180     DM      idm;
3181     DMLabel bdlabel;
3182 
3183     ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
3184     ierr = DMDestroy(dm);CHKERRQ(ierr);
3185     *dm  = idm;
3186 
3187     ierr = DMGetLabel(*dm, "marker", &bdlabel);CHKERRQ(ierr);
3188     ierr = DMPlexMarkBoundaryFaces(*dm, PETSC_DETERMINE, bdlabel);CHKERRQ(ierr);
3189     ierr = DMPlexLabelComplete(*dm, bdlabel);CHKERRQ(ierr);
3190   }
3191   PetscFunctionReturn(0);
3192 }
3193 
3194 /*@C
3195   DMPlexCreateFromFile - This takes a filename and produces a DM
3196 
3197   Input Parameters:
3198 + comm - The communicator
3199 . filename - A file name
3200 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
3201 
3202   Output Parameter:
3203 . dm - The DM
3204 
3205   Options Database Keys:
3206 . -dm_plex_create_from_hdf5_xdmf - use the PETSC_VIEWER_HDF5_XDMF format for reading HDF5
3207 
3208   Level: beginner
3209 
3210 .seealso: DMPlexCreateFromDAG(), DMPlexCreateFromCellList(), DMPlexCreate()
3211 @*/
3212 PetscErrorCode DMPlexCreateFromFile(MPI_Comm comm, const char filename[], PetscBool interpolate, DM *dm)
3213 {
3214   const char    *extGmsh    = ".msh";
3215   const char    *extGmsh2   = ".msh2";
3216   const char    *extGmsh4   = ".msh4";
3217   const char    *extCGNS    = ".cgns";
3218   const char    *extExodus  = ".exo";
3219   const char    *extGenesis = ".gen";
3220   const char    *extFluent  = ".cas";
3221   const char    *extHDF5    = ".h5";
3222   const char    *extMed     = ".med";
3223   const char    *extPLY     = ".ply";
3224   const char    *extCV      = ".dat";
3225   size_t         len;
3226   PetscBool      isGmsh, isGmsh2, isGmsh4, isCGNS, isExodus, isGenesis, isFluent, isHDF5, isMed, isPLY, isCV;
3227   PetscMPIInt    rank;
3228   PetscErrorCode ierr;
3229 
3230   PetscFunctionBegin;
3231   PetscValidPointer(filename, 2);
3232   PetscValidPointer(dm, 4);
3233   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
3234   ierr = PetscStrlen(filename, &len);CHKERRQ(ierr);
3235   if (!len) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Filename must be a valid path");
3236   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)], extGmsh,    4, &isGmsh);CHKERRQ(ierr);
3237   ierr = PetscStrncmp(&filename[PetscMax(0,len-5)], extGmsh2,   5, &isGmsh2);CHKERRQ(ierr);
3238   ierr = PetscStrncmp(&filename[PetscMax(0,len-5)], extGmsh4,   5, &isGmsh4);CHKERRQ(ierr);
3239   ierr = PetscStrncmp(&filename[PetscMax(0,len-5)], extCGNS,    5, &isCGNS);CHKERRQ(ierr);
3240   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)], extExodus,  4, &isExodus);CHKERRQ(ierr);
3241   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)], extGenesis, 4, &isGenesis);CHKERRQ(ierr);
3242   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)], extFluent,  4, &isFluent);CHKERRQ(ierr);
3243   ierr = PetscStrncmp(&filename[PetscMax(0,len-3)], extHDF5,    3, &isHDF5);CHKERRQ(ierr);
3244   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)], extMed,     4, &isMed);CHKERRQ(ierr);
3245   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)], extPLY,     4, &isPLY);CHKERRQ(ierr);
3246   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)], extCV,      4, &isCV);CHKERRQ(ierr);
3247   if (isGmsh || isGmsh2 || isGmsh4) {
3248     ierr = DMPlexCreateGmshFromFile(comm, filename, interpolate, dm);CHKERRQ(ierr);
3249   } else if (isCGNS) {
3250     ierr = DMPlexCreateCGNSFromFile(comm, filename, interpolate, dm);CHKERRQ(ierr);
3251   } else if (isExodus || isGenesis) {
3252     ierr = DMPlexCreateExodusFromFile(comm, filename, interpolate, dm);CHKERRQ(ierr);
3253   } else if (isFluent) {
3254     ierr = DMPlexCreateFluentFromFile(comm, filename, interpolate, dm);CHKERRQ(ierr);
3255   } else if (isHDF5) {
3256     PetscBool      load_hdf5_xdmf = PETSC_FALSE;
3257     PetscViewer viewer;
3258 
3259     /* PETSC_VIEWER_HDF5_XDMF is used if the filename ends with .xdmf.h5, or if -dm_plex_create_from_hdf5_xdmf option is present */
3260     ierr = PetscStrncmp(&filename[PetscMax(0,len-8)], ".xdmf",  5, &load_hdf5_xdmf);CHKERRQ(ierr);
3261     ierr = PetscOptionsGetBool(NULL, NULL, "-dm_plex_create_from_hdf5_xdmf", &load_hdf5_xdmf, NULL);CHKERRQ(ierr);
3262     ierr = PetscViewerCreate(comm, &viewer);CHKERRQ(ierr);
3263     ierr = PetscViewerSetType(viewer, PETSCVIEWERHDF5);CHKERRQ(ierr);
3264     ierr = PetscViewerFileSetMode(viewer, FILE_MODE_READ);CHKERRQ(ierr);
3265     ierr = PetscViewerFileSetName(viewer, filename);CHKERRQ(ierr);
3266     ierr = DMCreate(comm, dm);CHKERRQ(ierr);
3267     ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
3268     if (load_hdf5_xdmf) {ierr = PetscViewerPushFormat(viewer, PETSC_VIEWER_HDF5_XDMF);CHKERRQ(ierr);}
3269     ierr = DMLoad(*dm, viewer);CHKERRQ(ierr);
3270     if (load_hdf5_xdmf) {ierr = PetscViewerPopFormat(viewer);CHKERRQ(ierr);}
3271     ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
3272 
3273     if (interpolate) {
3274       DM idm;
3275 
3276       ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
3277       ierr = DMDestroy(dm);CHKERRQ(ierr);
3278       *dm  = idm;
3279     }
3280   } else if (isMed) {
3281     ierr = DMPlexCreateMedFromFile(comm, filename, interpolate, dm);CHKERRQ(ierr);
3282   } else if (isPLY) {
3283     ierr = DMPlexCreatePLYFromFile(comm, filename, interpolate, dm);CHKERRQ(ierr);
3284   } else if (isCV) {
3285     ierr = DMPlexCreateCellVertexFromFile(comm, filename, interpolate, dm);CHKERRQ(ierr);
3286   } else SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cannot load file %s: unrecognized extension", filename);
3287   PetscFunctionReturn(0);
3288 }
3289 
3290 /*@
3291   DMPlexCreateReferenceCell - Create a DMPLEX with the appropriate FEM reference cell
3292 
3293   Collective
3294 
3295   Input Parameters:
3296 + comm    - The communicator
3297 . dim     - The spatial dimension
3298 - simplex - Flag for simplex, otherwise use a tensor-product cell
3299 
3300   Output Parameter:
3301 . refdm - The reference cell
3302 
3303   Level: intermediate
3304 
3305 .seealso:
3306 @*/
3307 PetscErrorCode DMPlexCreateReferenceCell(MPI_Comm comm, PetscInt dim, PetscBool simplex, DM *refdm)
3308 {
3309   DM             rdm;
3310   Vec            coords;
3311   PetscErrorCode ierr;
3312 
3313   PetscFunctionBeginUser;
3314   ierr = DMCreate(comm, &rdm);CHKERRQ(ierr);
3315   ierr = DMSetType(rdm, DMPLEX);CHKERRQ(ierr);
3316   ierr = DMSetDimension(rdm, dim);CHKERRQ(ierr);
3317   switch (dim) {
3318   case 0:
3319   {
3320     PetscInt    numPoints[1]        = {1};
3321     PetscInt    coneSize[1]         = {0};
3322     PetscInt    cones[1]            = {0};
3323     PetscInt    coneOrientations[1] = {0};
3324     PetscScalar vertexCoords[1]     = {0.0};
3325 
3326     ierr = DMPlexCreateFromDAG(rdm, 0, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
3327   }
3328   break;
3329   case 1:
3330   {
3331     PetscInt    numPoints[2]        = {2, 1};
3332     PetscInt    coneSize[3]         = {2, 0, 0};
3333     PetscInt    cones[2]            = {1, 2};
3334     PetscInt    coneOrientations[2] = {0, 0};
3335     PetscScalar vertexCoords[2]     = {-1.0,  1.0};
3336 
3337     ierr = DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
3338   }
3339   break;
3340   case 2:
3341     if (simplex) {
3342       PetscInt    numPoints[2]        = {3, 1};
3343       PetscInt    coneSize[4]         = {3, 0, 0, 0};
3344       PetscInt    cones[3]            = {1, 2, 3};
3345       PetscInt    coneOrientations[3] = {0, 0, 0};
3346       PetscScalar vertexCoords[6]     = {-1.0, -1.0,  1.0, -1.0,  -1.0, 1.0};
3347 
3348       ierr = DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
3349     } else {
3350       PetscInt    numPoints[2]        = {4, 1};
3351       PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
3352       PetscInt    cones[4]            = {1, 2, 3, 4};
3353       PetscInt    coneOrientations[4] = {0, 0, 0, 0};
3354       PetscScalar vertexCoords[8]     = {-1.0, -1.0,  1.0, -1.0,  1.0, 1.0,  -1.0, 1.0};
3355 
3356       ierr = DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
3357     }
3358   break;
3359   case 3:
3360     if (simplex) {
3361       PetscInt    numPoints[2]        = {4, 1};
3362       PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
3363       PetscInt    cones[4]            = {1, 3, 2, 4};
3364       PetscInt    coneOrientations[4] = {0, 0, 0, 0};
3365       PetscScalar vertexCoords[12]    = {-1.0, -1.0, -1.0,  1.0, -1.0, -1.0,  -1.0, 1.0, -1.0,  -1.0, -1.0, 1.0};
3366 
3367       ierr = DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
3368     } else {
3369       PetscInt    numPoints[2]        = {8, 1};
3370       PetscInt    coneSize[9]         = {8, 0, 0, 0, 0, 0, 0, 0, 0};
3371       PetscInt    cones[8]            = {1, 4, 3, 2, 5, 6, 7, 8};
3372       PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
3373       PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0,  1.0, -1.0, -1.0,  1.0, 1.0, -1.0,  -1.0, 1.0, -1.0,
3374                                          -1.0, -1.0,  1.0,  1.0, -1.0,  1.0,  1.0, 1.0,  1.0,  -1.0, 1.0,  1.0};
3375 
3376       ierr = DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
3377     }
3378   break;
3379   default:
3380     SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Cannot create reference cell for dimension %d", dim);
3381   }
3382   ierr = DMPlexInterpolate(rdm, refdm);CHKERRQ(ierr);
3383   if (rdm->coordinateDM) {
3384     DM           ncdm;
3385     PetscSection cs;
3386     PetscInt     pEnd = -1;
3387 
3388     ierr = DMGetLocalSection(rdm->coordinateDM, &cs);CHKERRQ(ierr);
3389     if (cs) {ierr = PetscSectionGetChart(cs, NULL, &pEnd);CHKERRQ(ierr);}
3390     if (pEnd >= 0) {
3391       ierr = DMClone(*refdm, &ncdm);CHKERRQ(ierr);
3392       ierr = DMCopyDisc(rdm->coordinateDM, ncdm);CHKERRQ(ierr);
3393       ierr = DMSetLocalSection(ncdm, cs);CHKERRQ(ierr);
3394       ierr = DMSetCoordinateDM(*refdm, ncdm);CHKERRQ(ierr);
3395       ierr = DMDestroy(&ncdm);CHKERRQ(ierr);
3396     }
3397   }
3398   ierr = DMGetCoordinatesLocal(rdm, &coords);CHKERRQ(ierr);
3399   if (coords) {
3400     ierr = DMSetCoordinatesLocal(*refdm, coords);CHKERRQ(ierr);
3401   } else {
3402     ierr = DMGetCoordinates(rdm, &coords);CHKERRQ(ierr);
3403     if (coords) {ierr = DMSetCoordinates(*refdm, coords);CHKERRQ(ierr);}
3404   }
3405   ierr = DMDestroy(&rdm);CHKERRQ(ierr);
3406   PetscFunctionReturn(0);
3407 }
3408