xref: /petsc/src/dm/impls/plex/plexcreate.c (revision d6685f554fbda8d96c6a5d73ab0e7a4e21a05c51)
1 #define PETSCDM_DLL
2 #include <petsc/private/dmpleximpl.h>    /*I   "petscdmplex.h"   I*/
3 #include <petsc/private/hashseti.h>          /*I   "petscdmplex.h"   I*/
4 #include <petscsf.h>
5 #include <petsc/private/kernels/blockmatmult.h>
6 #include <petsc/private/kernels/blockinvert.h>
7 
8 PetscLogEvent DMPLEX_CreateFromFile, DMPLEX_BuildFromCellList, DMPLEX_BuildCoordinatesFromCellList;
9 
10 /* External function declarations here */
11 static PetscErrorCode DMInitialize_Plex(DM dm);
12 
13 /* This copies internal things in the Plex structure that we generally want when making a new, related Plex */
14 PetscErrorCode DMPlexCopy_Internal(DM dmin, PetscBool copyPeriodicity, DM dmout)
15 {
16   const DMBoundaryType *bd;
17   const PetscReal      *maxCell, *L;
18   PetscBool             isper, dist;
19   PetscErrorCode        ierr;
20 
21   PetscFunctionBegin;
22   if (copyPeriodicity) {
23     ierr = DMGetPeriodicity(dmin, &isper, &maxCell, &L, &bd);CHKERRQ(ierr);
24     ierr = DMSetPeriodicity(dmout, isper,  maxCell,  L,  bd);CHKERRQ(ierr);
25   }
26   ierr = DMPlexDistributeGetDefault(dmin, &dist);CHKERRQ(ierr);
27   ierr = DMPlexDistributeSetDefault(dmout, dist);CHKERRQ(ierr);
28   ((DM_Plex *) dmout->data)->useHashLocation = ((DM_Plex *) dmin->data)->useHashLocation;
29   PetscFunctionReturn(0);
30 }
31 
32 /* Replace dm with the contents of ndm, and then destroy ndm
33    - Share the DM_Plex structure
34    - Share the coordinates
35    - Share the SF
36 */
37 static PetscErrorCode DMPlexReplace_Static(DM dm, DM *ndm)
38 {
39   PetscSF               sf;
40   DM                    dmNew = *ndm, coordDM, coarseDM;
41   Vec                   coords;
42   PetscBool             isper;
43   const PetscReal      *maxCell, *L;
44   const DMBoundaryType *bd;
45   PetscInt              dim, cdim;
46   PetscErrorCode        ierr;
47 
48   PetscFunctionBegin;
49   if (dm == dmNew) {
50     ierr = DMDestroy(ndm);CHKERRQ(ierr);
51     PetscFunctionReturn(0);
52   }
53   dm->setupcalled = dmNew->setupcalled;
54   ierr = DMGetDimension(dmNew, &dim);CHKERRQ(ierr);
55   ierr = DMSetDimension(dm, dim);CHKERRQ(ierr);
56   ierr = DMGetCoordinateDim(dmNew, &cdim);CHKERRQ(ierr);
57   ierr = DMSetCoordinateDim(dm, cdim);CHKERRQ(ierr);
58   ierr = DMGetPointSF(dmNew, &sf);CHKERRQ(ierr);
59   ierr = DMSetPointSF(dm, sf);CHKERRQ(ierr);
60   ierr = DMGetCoordinateDM(dmNew, &coordDM);CHKERRQ(ierr);
61   ierr = DMGetCoordinatesLocal(dmNew, &coords);CHKERRQ(ierr);
62   ierr = DMSetCoordinateDM(dm, coordDM);CHKERRQ(ierr);
63   ierr = DMSetCoordinatesLocal(dm, coords);CHKERRQ(ierr);
64   /* Do not want to create the coordinate field if it does not already exist, so do not call DMGetCoordinateField() */
65   ierr = DMFieldDestroy(&dm->coordinateField);CHKERRQ(ierr);
66   dm->coordinateField = dmNew->coordinateField;
67   ((DM_Plex *) dmNew->data)->coordFunc = ((DM_Plex *) dm->data)->coordFunc;
68   ierr = DMGetPeriodicity(dmNew, &isper, &maxCell, &L, &bd);CHKERRQ(ierr);
69   ierr = DMSetPeriodicity(dm, isper, maxCell, L, bd);CHKERRQ(ierr);
70   ierr = DMDestroy_Plex(dm);CHKERRQ(ierr);
71   ierr = DMInitialize_Plex(dm);CHKERRQ(ierr);
72   dm->data = dmNew->data;
73   ((DM_Plex *) dmNew->data)->refct++;
74   ierr = DMDestroyLabelLinkList_Internal(dm);CHKERRQ(ierr);
75   ierr = DMCopyLabels(dmNew, dm, PETSC_OWN_POINTER, PETSC_TRUE, DM_COPY_LABELS_FAIL);CHKERRQ(ierr);
76   ierr = DMGetCoarseDM(dmNew,&coarseDM);CHKERRQ(ierr);
77   ierr = DMSetCoarseDM(dm,coarseDM);CHKERRQ(ierr);
78   ierr = DMDestroy(ndm);CHKERRQ(ierr);
79   PetscFunctionReturn(0);
80 }
81 
82 /* Swap dm with the contents of dmNew
83    - Swap the DM_Plex structure
84    - Swap the coordinates
85    - Swap the point PetscSF
86 */
87 static PetscErrorCode DMPlexSwap_Static(DM dmA, DM dmB)
88 {
89   DM              coordDMA, coordDMB;
90   Vec             coordsA,  coordsB;
91   PetscSF         sfA,      sfB;
92   DMField         fieldTmp;
93   void            *tmp;
94   DMLabelLink     listTmp;
95   DMLabel         depthTmp;
96   PetscInt        tmpI;
97   PetscErrorCode  ierr;
98 
99   PetscFunctionBegin;
100   if (dmA == dmB) PetscFunctionReturn(0);
101   ierr = DMGetPointSF(dmA, &sfA);CHKERRQ(ierr);
102   ierr = DMGetPointSF(dmB, &sfB);CHKERRQ(ierr);
103   ierr = PetscObjectReference((PetscObject) sfA);CHKERRQ(ierr);
104   ierr = DMSetPointSF(dmA, sfB);CHKERRQ(ierr);
105   ierr = DMSetPointSF(dmB, sfA);CHKERRQ(ierr);
106   ierr = PetscObjectDereference((PetscObject) sfA);CHKERRQ(ierr);
107 
108   ierr = DMGetCoordinateDM(dmA, &coordDMA);CHKERRQ(ierr);
109   ierr = DMGetCoordinateDM(dmB, &coordDMB);CHKERRQ(ierr);
110   ierr = PetscObjectReference((PetscObject) coordDMA);CHKERRQ(ierr);
111   ierr = DMSetCoordinateDM(dmA, coordDMB);CHKERRQ(ierr);
112   ierr = DMSetCoordinateDM(dmB, coordDMA);CHKERRQ(ierr);
113   ierr = PetscObjectDereference((PetscObject) coordDMA);CHKERRQ(ierr);
114 
115   ierr = DMGetCoordinatesLocal(dmA, &coordsA);CHKERRQ(ierr);
116   ierr = DMGetCoordinatesLocal(dmB, &coordsB);CHKERRQ(ierr);
117   ierr = PetscObjectReference((PetscObject) coordsA);CHKERRQ(ierr);
118   ierr = DMSetCoordinatesLocal(dmA, coordsB);CHKERRQ(ierr);
119   ierr = DMSetCoordinatesLocal(dmB, coordsA);CHKERRQ(ierr);
120   ierr = PetscObjectDereference((PetscObject) coordsA);CHKERRQ(ierr);
121 
122   fieldTmp             = dmA->coordinateField;
123   dmA->coordinateField = dmB->coordinateField;
124   dmB->coordinateField = fieldTmp;
125   tmp       = dmA->data;
126   dmA->data = dmB->data;
127   dmB->data = tmp;
128   listTmp   = dmA->labels;
129   dmA->labels = dmB->labels;
130   dmB->labels = listTmp;
131   depthTmp  = dmA->depthLabel;
132   dmA->depthLabel = dmB->depthLabel;
133   dmB->depthLabel = depthTmp;
134   depthTmp  = dmA->celltypeLabel;
135   dmA->celltypeLabel = dmB->celltypeLabel;
136   dmB->celltypeLabel = depthTmp;
137   tmpI         = dmA->levelup;
138   dmA->levelup = dmB->levelup;
139   dmB->levelup = tmpI;
140   PetscFunctionReturn(0);
141 }
142 
143 static PetscErrorCode DMPlexInterpolateInPlace_Internal(DM dm)
144 {
145   DM             idm;
146   PetscErrorCode ierr;
147 
148   PetscFunctionBegin;
149   ierr = DMPlexInterpolate(dm, &idm);CHKERRQ(ierr);
150   ierr = DMPlexCopyCoordinates(dm, idm);CHKERRQ(ierr);
151   ierr = DMPlexReplace_Static(dm, &idm);CHKERRQ(ierr);
152   PetscFunctionReturn(0);
153 }
154 
155 /*@C
156   DMPlexCreateCoordinateSpace - Creates a finite element space for the coordinates
157 
158   Collective
159 
160   Input Parameters:
161 + DM        - The DM
162 . degree    - The degree of the finite element
163 - coordFunc - An optional function to map new points from refinement to the surface
164 
165   Level: advanced
166 
167 .seealso: PetscFECreateLagrange(), DMGetCoordinateDM()
168 @*/
169 PetscErrorCode DMPlexCreateCoordinateSpace(DM dm, PetscInt degree, PetscPointFunc coordFunc)
170 {
171   DM_Plex       *mesh = (DM_Plex *) dm->data;
172   DM             cdm;
173   PetscDS        cds;
174   PetscFE        fe;
175   PetscClassId   id;
176   PetscErrorCode ierr;
177 
178   PetscFunctionBegin;
179   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
180   ierr = DMGetDS(cdm, &cds);CHKERRQ(ierr);
181   ierr = PetscDSGetDiscretization(cds, 0, (PetscObject *) &fe);CHKERRQ(ierr);
182   ierr = PetscObjectGetClassId((PetscObject) fe, &id);CHKERRQ(ierr);
183   if (id != PETSCFE_CLASSID) {
184     PetscBool simplex;
185     PetscInt  dim, dE, qorder;
186 
187     ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
188     ierr = DMGetCoordinateDim(dm, &dE);CHKERRQ(ierr);
189     ierr = DMPlexIsSimplex(dm, &simplex);CHKERRQ(ierr);
190     qorder = degree;
191     ierr = PetscObjectOptionsBegin((PetscObject) cdm);CHKERRQ(ierr);
192     ierr = PetscOptionsBoundedInt("-coord_dm_default_quadrature_order", "Quadrature order is one less than quadrature points per edge", "DMPlexCreateCoordinateSpace", qorder, &qorder, NULL, 0);CHKERRQ(ierr);
193     ierr = PetscOptionsEnd();CHKERRQ(ierr);
194     ierr = PetscFECreateLagrange(PETSC_COMM_SELF, dim, dE, simplex, degree, qorder, &fe);CHKERRQ(ierr);
195     ierr = DMSetField(cdm, 0, NULL, (PetscObject) fe);CHKERRQ(ierr);
196     ierr = DMCreateDS(cdm);CHKERRQ(ierr);
197     ierr = DMProjectCoordinates(dm, fe);CHKERRQ(ierr);
198     ierr = PetscFEDestroy(&fe);CHKERRQ(ierr);
199   }
200   mesh->coordFunc = coordFunc;
201   PetscFunctionReturn(0);
202 }
203 
204 /*@
205   DMPlexCreateDoublet - Creates a mesh of two cells of the specified type, optionally with later refinement.
206 
207   Collective
208 
209   Input Parameters:
210 + comm - The communicator for the DM object
211 . dim - The spatial dimension
212 . simplex - Flag for simplicial cells, otherwise they are tensor product cells
213 . interpolate - Flag to create intermediate mesh pieces (edges, faces)
214 - refinementLimit - A nonzero number indicates the largest admissible volume for a refined cell
215 
216   Output Parameter:
217 . dm - The DM object
218 
219   Level: beginner
220 
221 .seealso: DMSetType(), DMCreate()
222 @*/
223 PetscErrorCode DMPlexCreateDoublet(MPI_Comm comm, PetscInt dim, PetscBool simplex, PetscBool interpolate, PetscReal refinementLimit, DM *newdm)
224 {
225   DM             dm;
226   PetscMPIInt    rank;
227   PetscErrorCode ierr;
228 
229   PetscFunctionBegin;
230   ierr = DMCreate(comm, &dm);CHKERRQ(ierr);
231   ierr = DMSetType(dm, DMPLEX);CHKERRQ(ierr);
232   ierr = DMSetDimension(dm, dim);CHKERRQ(ierr);
233   ierr = MPI_Comm_rank(comm, &rank);CHKERRMPI(ierr);
234   switch (dim) {
235   case 2:
236     if (simplex) {ierr = PetscObjectSetName((PetscObject) dm, "triangular");CHKERRQ(ierr);}
237     else         {ierr = PetscObjectSetName((PetscObject) dm, "quadrilateral");CHKERRQ(ierr);}
238     break;
239   case 3:
240     if (simplex) {ierr = PetscObjectSetName((PetscObject) dm, "tetrahedral");CHKERRQ(ierr);}
241     else         {ierr = PetscObjectSetName((PetscObject) dm, "hexahedral");CHKERRQ(ierr);}
242     break;
243   default:
244     SETERRQ(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make meshes for dimension %D", dim);
245   }
246   if (rank) {
247     PetscInt numPoints[2] = {0, 0};
248     ierr = DMPlexCreateFromDAG(dm, 1, numPoints, NULL, NULL, NULL, NULL);CHKERRQ(ierr);
249   } else {
250     switch (dim) {
251     case 2:
252       if (simplex) {
253         PetscInt    numPoints[2]        = {4, 2};
254         PetscInt    coneSize[6]         = {3, 3, 0, 0, 0, 0};
255         PetscInt    cones[6]            = {2, 3, 4,  5, 4, 3};
256         PetscInt    coneOrientations[6] = {0, 0, 0,  0, 0, 0};
257         PetscScalar vertexCoords[8]     = {-0.5, 0.5, 0.0, 0.0, 0.0, 1.0, 0.5, 0.5};
258 
259         ierr = DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
260       } else {
261         PetscInt    numPoints[2]        = {6, 2};
262         PetscInt    coneSize[8]         = {4, 4, 0, 0, 0, 0, 0, 0};
263         PetscInt    cones[8]            = {2, 3, 4, 5,  3, 6, 7, 4};
264         PetscInt    coneOrientations[8] = {0, 0, 0, 0,  0, 0, 0, 0};
265         PetscScalar vertexCoords[12]    = {-1.0, -0.5,  0.0, -0.5,  0.0, 0.5,  -1.0, 0.5,  1.0, -0.5,  1.0, 0.5};
266 
267         ierr = DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
268       }
269       break;
270     case 3:
271       if (simplex) {
272         PetscInt    numPoints[2]        = {5, 2};
273         PetscInt    coneSize[7]         = {4, 4, 0, 0, 0, 0, 0};
274         PetscInt    cones[8]            = {4, 3, 5, 2,  5, 3, 4, 6};
275         PetscInt    coneOrientations[8] = {0, 0, 0, 0,  0, 0, 0, 0};
276         PetscScalar vertexCoords[15]    = {-1.0, 0.0, 0.0,  0.0, -1.0, 0.0,  0.0, 0.0, 1.0,  0.0, 1.0, 0.0,  1.0, 0.0, 0.0};
277 
278         ierr = DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
279       } else {
280         PetscInt    numPoints[2]         = {12, 2};
281         PetscInt    coneSize[14]         = {8, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
282         PetscInt    cones[16]            = {2, 3, 4, 5, 6, 7, 8, 9,  5, 4, 10, 11, 7, 12, 13, 8};
283         PetscInt    coneOrientations[16] = {0, 0, 0, 0, 0, 0, 0, 0,  0, 0,  0,  0, 0,  0,  0, 0};
284         PetscScalar vertexCoords[36]     = {-1.0, -0.5, -0.5,  -1.0,  0.5, -0.5,  0.0,  0.5, -0.5,   0.0, -0.5, -0.5,
285                                             -1.0, -0.5,  0.5,   0.0, -0.5,  0.5,  0.0,  0.5,  0.5,  -1.0,  0.5,  0.5,
286                                              1.0,  0.5, -0.5,   1.0, -0.5, -0.5,  1.0, -0.5,  0.5,   1.0,  0.5,  0.5};
287 
288         ierr = DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
289       }
290       break;
291     default:
292       SETERRQ(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make meshes for dimension %D", dim);
293     }
294   }
295   *newdm = dm;
296   if (refinementLimit > 0.0) {
297     DM rdm;
298     const char *name;
299 
300     ierr = DMPlexSetRefinementUniform(*newdm, PETSC_FALSE);CHKERRQ(ierr);
301     ierr = DMPlexSetRefinementLimit(*newdm, refinementLimit);CHKERRQ(ierr);
302     ierr = DMRefine(*newdm, comm, &rdm);CHKERRQ(ierr);
303     ierr = PetscObjectGetName((PetscObject) *newdm, &name);CHKERRQ(ierr);
304     ierr = PetscObjectSetName((PetscObject)    rdm,  name);CHKERRQ(ierr);
305     ierr = DMDestroy(newdm);CHKERRQ(ierr);
306     *newdm = rdm;
307   }
308   if (interpolate) {
309     DM idm;
310 
311     ierr = DMPlexInterpolate(*newdm, &idm);CHKERRQ(ierr);
312     ierr = DMDestroy(newdm);CHKERRQ(ierr);
313     *newdm = idm;
314   }
315   PetscFunctionReturn(0);
316 }
317 
318 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_1D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[])
319 {
320   const PetscInt numVertices    = 2;
321   PetscInt       markerRight    = 1;
322   PetscInt       markerLeft     = 1;
323   PetscBool      markerSeparate = PETSC_FALSE;
324   Vec            coordinates;
325   PetscSection   coordSection;
326   PetscScalar   *coords;
327   PetscInt       coordSize;
328   PetscMPIInt    rank;
329   PetscInt       cdim = 1, v;
330   PetscErrorCode ierr;
331 
332   PetscFunctionBegin;
333   ierr = PetscOptionsGetBool(((PetscObject) dm)->options,((PetscObject) dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL);CHKERRQ(ierr);
334   if (markerSeparate) {
335     markerRight  = 2;
336     markerLeft   = 1;
337   }
338   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRMPI(ierr);
339   if (!rank) {
340     ierr = DMPlexSetChart(dm, 0, numVertices);CHKERRQ(ierr);
341     ierr = DMSetUp(dm);CHKERRQ(ierr); /* Allocate space for cones */
342     ierr = DMSetLabelValue(dm, "marker", 0, markerLeft);CHKERRQ(ierr);
343     ierr = DMSetLabelValue(dm, "marker", 1, markerRight);CHKERRQ(ierr);
344   }
345   ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
346   ierr = DMPlexStratify(dm);CHKERRQ(ierr);
347   /* Build coordinates */
348   ierr = DMSetCoordinateDim(dm, cdim);CHKERRQ(ierr);
349   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
350   ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
351   ierr = PetscSectionSetChart(coordSection, 0, numVertices);CHKERRQ(ierr);
352   ierr = PetscSectionSetFieldComponents(coordSection, 0, cdim);CHKERRQ(ierr);
353   for (v = 0; v < numVertices; ++v) {
354     ierr = PetscSectionSetDof(coordSection, v, cdim);CHKERRQ(ierr);
355     ierr = PetscSectionSetFieldDof(coordSection, v, 0, cdim);CHKERRQ(ierr);
356   }
357   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
358   ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
359   ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
360   ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
361   ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
362   ierr = VecSetBlockSize(coordinates, cdim);CHKERRQ(ierr);
363   ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
364   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
365   coords[0] = lower[0];
366   coords[1] = upper[0];
367   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
368   ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
369   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
370   PetscFunctionReturn(0);
371 }
372 
373 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_2D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[])
374 {
375   const PetscInt numVertices    = (edges[0]+1)*(edges[1]+1);
376   const PetscInt numEdges       = edges[0]*(edges[1]+1) + (edges[0]+1)*edges[1];
377   PetscInt       markerTop      = 1;
378   PetscInt       markerBottom   = 1;
379   PetscInt       markerRight    = 1;
380   PetscInt       markerLeft     = 1;
381   PetscBool      markerSeparate = PETSC_FALSE;
382   Vec            coordinates;
383   PetscSection   coordSection;
384   PetscScalar    *coords;
385   PetscInt       coordSize;
386   PetscMPIInt    rank;
387   PetscInt       v, vx, vy;
388   PetscErrorCode ierr;
389 
390   PetscFunctionBegin;
391   ierr = PetscOptionsGetBool(((PetscObject) dm)->options,((PetscObject) dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL);CHKERRQ(ierr);
392   if (markerSeparate) {
393     markerTop    = 3;
394     markerBottom = 1;
395     markerRight  = 2;
396     markerLeft   = 4;
397   }
398   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRMPI(ierr);
399   if (rank == 0) {
400     PetscInt e, ex, ey;
401 
402     ierr = DMPlexSetChart(dm, 0, numEdges+numVertices);CHKERRQ(ierr);
403     for (e = 0; e < numEdges; ++e) {
404       ierr = DMPlexSetConeSize(dm, e, 2);CHKERRQ(ierr);
405     }
406     ierr = DMSetUp(dm);CHKERRQ(ierr); /* Allocate space for cones */
407     for (vx = 0; vx <= edges[0]; vx++) {
408       for (ey = 0; ey < edges[1]; ey++) {
409         PetscInt edge   = vx*edges[1] + ey + edges[0]*(edges[1]+1);
410         PetscInt vertex = ey*(edges[0]+1) + vx + numEdges;
411         PetscInt cone[2];
412 
413         cone[0] = vertex; cone[1] = vertex+edges[0]+1;
414         ierr    = DMPlexSetCone(dm, edge, cone);CHKERRQ(ierr);
415         if (vx == edges[0]) {
416           ierr = DMSetLabelValue(dm, "marker", edge,    markerRight);CHKERRQ(ierr);
417           ierr = DMSetLabelValue(dm, "marker", cone[0], markerRight);CHKERRQ(ierr);
418           if (ey == edges[1]-1) {
419             ierr = DMSetLabelValue(dm, "marker", cone[1], markerRight);CHKERRQ(ierr);
420             ierr = DMSetLabelValue(dm, "Face Sets", cone[1], markerRight);CHKERRQ(ierr);
421           }
422         } else if (vx == 0) {
423           ierr = DMSetLabelValue(dm, "marker", edge,    markerLeft);CHKERRQ(ierr);
424           ierr = DMSetLabelValue(dm, "marker", cone[0], markerLeft);CHKERRQ(ierr);
425           if (ey == edges[1]-1) {
426             ierr = DMSetLabelValue(dm, "marker", cone[1], markerLeft);CHKERRQ(ierr);
427             ierr = DMSetLabelValue(dm, "Face Sets", cone[1], markerLeft);CHKERRQ(ierr);
428           }
429         }
430       }
431     }
432     for (vy = 0; vy <= edges[1]; vy++) {
433       for (ex = 0; ex < edges[0]; ex++) {
434         PetscInt edge   = vy*edges[0]     + ex;
435         PetscInt vertex = vy*(edges[0]+1) + ex + numEdges;
436         PetscInt cone[2];
437 
438         cone[0] = vertex; cone[1] = vertex+1;
439         ierr    = DMPlexSetCone(dm, edge, cone);CHKERRQ(ierr);
440         if (vy == edges[1]) {
441           ierr = DMSetLabelValue(dm, "marker", edge,    markerTop);CHKERRQ(ierr);
442           ierr = DMSetLabelValue(dm, "marker", cone[0], markerTop);CHKERRQ(ierr);
443           if (ex == edges[0]-1) {
444             ierr = DMSetLabelValue(dm, "marker", cone[1], markerTop);CHKERRQ(ierr);
445             ierr = DMSetLabelValue(dm, "Face Sets", cone[1], markerTop);CHKERRQ(ierr);
446           }
447         } else if (vy == 0) {
448           ierr = DMSetLabelValue(dm, "marker", edge,    markerBottom);CHKERRQ(ierr);
449           ierr = DMSetLabelValue(dm, "marker", cone[0], markerBottom);CHKERRQ(ierr);
450           if (ex == edges[0]-1) {
451             ierr = DMSetLabelValue(dm, "marker", cone[1], markerBottom);CHKERRQ(ierr);
452             ierr = DMSetLabelValue(dm, "Face Sets", cone[1], markerBottom);CHKERRQ(ierr);
453           }
454         }
455       }
456     }
457   }
458   ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
459   ierr = DMPlexStratify(dm);CHKERRQ(ierr);
460   /* Build coordinates */
461   ierr = DMSetCoordinateDim(dm, 2);CHKERRQ(ierr);
462   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
463   ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
464   ierr = PetscSectionSetChart(coordSection, numEdges, numEdges + numVertices);CHKERRQ(ierr);
465   ierr = PetscSectionSetFieldComponents(coordSection, 0, 2);CHKERRQ(ierr);
466   for (v = numEdges; v < numEdges+numVertices; ++v) {
467     ierr = PetscSectionSetDof(coordSection, v, 2);CHKERRQ(ierr);
468     ierr = PetscSectionSetFieldDof(coordSection, v, 0, 2);CHKERRQ(ierr);
469   }
470   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
471   ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
472   ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
473   ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
474   ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
475   ierr = VecSetBlockSize(coordinates, 2);CHKERRQ(ierr);
476   ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
477   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
478   for (vy = 0; vy <= edges[1]; ++vy) {
479     for (vx = 0; vx <= edges[0]; ++vx) {
480       coords[(vy*(edges[0]+1)+vx)*2+0] = lower[0] + ((upper[0] - lower[0])/edges[0])*vx;
481       coords[(vy*(edges[0]+1)+vx)*2+1] = lower[1] + ((upper[1] - lower[1])/edges[1])*vy;
482     }
483   }
484   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
485   ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
486   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
487   PetscFunctionReturn(0);
488 }
489 
490 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_3D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt faces[])
491 {
492   PetscInt       vertices[3], numVertices;
493   PetscInt       numFaces    = 2*faces[0]*faces[1] + 2*faces[1]*faces[2] + 2*faces[0]*faces[2];
494   Vec            coordinates;
495   PetscSection   coordSection;
496   PetscScalar    *coords;
497   PetscInt       coordSize;
498   PetscMPIInt    rank;
499   PetscInt       v, vx, vy, vz;
500   PetscInt       voffset, iface=0, cone[4];
501   PetscErrorCode ierr;
502 
503   PetscFunctionBegin;
504   PetscCheckFalse((faces[0] < 1) || (faces[1] < 1) || (faces[2] < 1),PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Must have at least 1 face per side");
505   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRMPI(ierr);
506   vertices[0] = faces[0]+1; vertices[1] = faces[1]+1; vertices[2] = faces[2]+1;
507   numVertices = vertices[0]*vertices[1]*vertices[2];
508   if (rank == 0) {
509     PetscInt f;
510 
511     ierr = DMPlexSetChart(dm, 0, numFaces+numVertices);CHKERRQ(ierr);
512     for (f = 0; f < numFaces; ++f) {
513       ierr = DMPlexSetConeSize(dm, f, 4);CHKERRQ(ierr);
514     }
515     ierr = DMSetUp(dm);CHKERRQ(ierr); /* Allocate space for cones */
516 
517     /* Side 0 (Top) */
518     for (vy = 0; vy < faces[1]; vy++) {
519       for (vx = 0; vx < faces[0]; vx++) {
520         voffset = numFaces + vertices[0]*vertices[1]*(vertices[2]-1) + vy*vertices[0] + vx;
521         cone[0] = voffset; cone[1] = voffset+1; cone[2] = voffset+vertices[0]+1; cone[3] = voffset+vertices[0];
522         ierr    = DMPlexSetCone(dm, iface, cone);CHKERRQ(ierr);
523         ierr    = DMSetLabelValue(dm, "marker", iface, 1);CHKERRQ(ierr);
524         ierr    = DMSetLabelValue(dm, "marker", voffset+0, 1);CHKERRQ(ierr);
525         ierr    = DMSetLabelValue(dm, "marker", voffset+1, 1);CHKERRQ(ierr);
526         ierr    = DMSetLabelValue(dm, "marker", voffset+vertices[0]+0, 1);CHKERRQ(ierr);
527         ierr    = DMSetLabelValue(dm, "marker", voffset+vertices[0]+1, 1);CHKERRQ(ierr);
528         iface++;
529       }
530     }
531 
532     /* Side 1 (Bottom) */
533     for (vy = 0; vy < faces[1]; vy++) {
534       for (vx = 0; vx < faces[0]; vx++) {
535         voffset = numFaces + vy*(faces[0]+1) + vx;
536         cone[0] = voffset+1; cone[1] = voffset; cone[2] = voffset+vertices[0]; cone[3] = voffset+vertices[0]+1;
537         ierr    = DMPlexSetCone(dm, iface, cone);CHKERRQ(ierr);
538         ierr    = DMSetLabelValue(dm, "marker", iface, 1);CHKERRQ(ierr);
539         ierr    = DMSetLabelValue(dm, "marker", voffset+0, 1);CHKERRQ(ierr);
540         ierr    = DMSetLabelValue(dm, "marker", voffset+1, 1);CHKERRQ(ierr);
541         ierr    = DMSetLabelValue(dm, "marker", voffset+vertices[0]+0, 1);CHKERRQ(ierr);
542         ierr    = DMSetLabelValue(dm, "marker", voffset+vertices[0]+1, 1);CHKERRQ(ierr);
543         iface++;
544       }
545     }
546 
547     /* Side 2 (Front) */
548     for (vz = 0; vz < faces[2]; vz++) {
549       for (vx = 0; vx < faces[0]; vx++) {
550         voffset = numFaces + vz*vertices[0]*vertices[1] + vx;
551         cone[0] = voffset; cone[1] = voffset+1; cone[2] = voffset+vertices[0]*vertices[1]+1; cone[3] = voffset+vertices[0]*vertices[1];
552         ierr    = DMPlexSetCone(dm, iface, cone);CHKERRQ(ierr);
553         ierr    = DMSetLabelValue(dm, "marker", iface, 1);CHKERRQ(ierr);
554         ierr    = DMSetLabelValue(dm, "marker", voffset+0, 1);CHKERRQ(ierr);
555         ierr    = DMSetLabelValue(dm, "marker", voffset+1, 1);CHKERRQ(ierr);
556         ierr    = DMSetLabelValue(dm, "marker", voffset+vertices[0]*vertices[1]+0, 1);CHKERRQ(ierr);
557         ierr    = DMSetLabelValue(dm, "marker", voffset+vertices[0]*vertices[1]+1, 1);CHKERRQ(ierr);
558         iface++;
559       }
560     }
561 
562     /* Side 3 (Back) */
563     for (vz = 0; vz < faces[2]; vz++) {
564       for (vx = 0; vx < faces[0]; vx++) {
565         voffset = numFaces + vz*vertices[0]*vertices[1] + vertices[0]*(vertices[1]-1) + vx;
566         cone[0] = voffset+vertices[0]*vertices[1]; cone[1] = voffset+vertices[0]*vertices[1]+1;
567         cone[2] = voffset+1; cone[3] = voffset;
568         ierr    = DMPlexSetCone(dm, iface, cone);CHKERRQ(ierr);
569         ierr    = DMSetLabelValue(dm, "marker", iface, 1);CHKERRQ(ierr);
570         ierr    = DMSetLabelValue(dm, "marker", voffset+0, 1);CHKERRQ(ierr);
571         ierr    = DMSetLabelValue(dm, "marker", voffset+1, 1);CHKERRQ(ierr);
572         ierr    = DMSetLabelValue(dm, "marker", voffset+vertices[0]*vertices[1]+0, 1);CHKERRQ(ierr);
573         ierr    = DMSetLabelValue(dm, "marker", voffset+vertices[0]*vertices[1]+1, 1);CHKERRQ(ierr);
574         iface++;
575       }
576     }
577 
578     /* Side 4 (Left) */
579     for (vz = 0; vz < faces[2]; vz++) {
580       for (vy = 0; vy < faces[1]; vy++) {
581         voffset = numFaces + vz*vertices[0]*vertices[1] + vy*vertices[0];
582         cone[0] = voffset; cone[1] = voffset+vertices[0]*vertices[1];
583         cone[2] = voffset+vertices[0]*vertices[1]+vertices[0]; cone[3] = voffset+vertices[0];
584         ierr    = DMPlexSetCone(dm, iface, cone);CHKERRQ(ierr);
585         ierr    = DMSetLabelValue(dm, "marker", iface, 1);CHKERRQ(ierr);
586         ierr    = DMSetLabelValue(dm, "marker", voffset+0, 1);CHKERRQ(ierr);
587         ierr    = DMSetLabelValue(dm, "marker", voffset+vertices[0]+0, 1);CHKERRQ(ierr);
588         ierr    = DMSetLabelValue(dm, "marker", voffset+vertices[1]+0, 1);CHKERRQ(ierr);
589         ierr    = DMSetLabelValue(dm, "marker", voffset+vertices[0]*vertices[1]+vertices[0], 1);CHKERRQ(ierr);
590         iface++;
591       }
592     }
593 
594     /* Side 5 (Right) */
595     for (vz = 0; vz < faces[2]; vz++) {
596       for (vy = 0; vy < faces[1]; vy++) {
597         voffset = numFaces + vz*vertices[0]*vertices[1] + vy*vertices[0] + faces[0];
598         cone[0] = voffset+vertices[0]*vertices[1]; cone[1] = voffset;
599         cone[2] = voffset+vertices[0]; cone[3] = voffset+vertices[0]*vertices[1]+vertices[0];
600         ierr    = DMPlexSetCone(dm, iface, cone);CHKERRQ(ierr);
601         ierr    = DMSetLabelValue(dm, "marker", iface, 1);CHKERRQ(ierr);
602         ierr    = DMSetLabelValue(dm, "marker", voffset+0, 1);CHKERRQ(ierr);
603         ierr    = DMSetLabelValue(dm, "marker", voffset+vertices[0]+0, 1);CHKERRQ(ierr);
604         ierr    = DMSetLabelValue(dm, "marker", voffset+vertices[0]*vertices[1]+0, 1);CHKERRQ(ierr);
605         ierr    = DMSetLabelValue(dm, "marker", voffset+vertices[0]*vertices[1]+vertices[0], 1);CHKERRQ(ierr);
606         iface++;
607       }
608     }
609   }
610   ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
611   ierr = DMPlexStratify(dm);CHKERRQ(ierr);
612   /* Build coordinates */
613   ierr = DMSetCoordinateDim(dm, 3);CHKERRQ(ierr);
614   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
615   ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
616   ierr = PetscSectionSetChart(coordSection, numFaces, numFaces + numVertices);CHKERRQ(ierr);
617   ierr = PetscSectionSetFieldComponents(coordSection, 0, 3);CHKERRQ(ierr);
618   for (v = numFaces; v < numFaces+numVertices; ++v) {
619     ierr = PetscSectionSetDof(coordSection, v, 3);CHKERRQ(ierr);
620     ierr = PetscSectionSetFieldDof(coordSection, v, 0, 3);CHKERRQ(ierr);
621   }
622   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
623   ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
624   ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
625   ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
626   ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
627   ierr = VecSetBlockSize(coordinates, 3);CHKERRQ(ierr);
628   ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
629   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
630   for (vz = 0; vz <= faces[2]; ++vz) {
631     for (vy = 0; vy <= faces[1]; ++vy) {
632       for (vx = 0; vx <= faces[0]; ++vx) {
633         coords[((vz*(faces[1]+1)+vy)*(faces[0]+1)+vx)*3+0] = lower[0] + ((upper[0] - lower[0])/faces[0])*vx;
634         coords[((vz*(faces[1]+1)+vy)*(faces[0]+1)+vx)*3+1] = lower[1] + ((upper[1] - lower[1])/faces[1])*vy;
635         coords[((vz*(faces[1]+1)+vy)*(faces[0]+1)+vx)*3+2] = lower[2] + ((upper[2] - lower[2])/faces[2])*vz;
636       }
637     }
638   }
639   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
640   ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
641   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
642   PetscFunctionReturn(0);
643 }
644 
645 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], PetscBool interpolate)
646 {
647   PetscErrorCode ierr;
648 
649   PetscFunctionBegin;
650   PetscValidLogicalCollectiveInt(dm, dim, 2);
651   ierr = DMSetDimension(dm, dim-1);CHKERRQ(ierr);
652   ierr = DMSetCoordinateDim(dm, dim);CHKERRQ(ierr);
653   switch (dim) {
654     case 1: ierr = DMPlexCreateBoxSurfaceMesh_Tensor_1D_Internal(dm, lower, upper, faces);CHKERRQ(ierr);break;
655     case 2: ierr = DMPlexCreateBoxSurfaceMesh_Tensor_2D_Internal(dm, lower, upper, faces);CHKERRQ(ierr);break;
656     case 3: ierr = DMPlexCreateBoxSurfaceMesh_Tensor_3D_Internal(dm, lower, upper, faces);CHKERRQ(ierr);break;
657     default: SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Dimension not supported: %D", dim);
658   }
659   if (interpolate) {ierr = DMPlexInterpolateInPlace_Internal(dm);CHKERRQ(ierr);}
660   PetscFunctionReturn(0);
661 }
662 
663 /*@C
664   DMPlexCreateBoxSurfaceMesh - Creates a mesh on the surface of the tensor product of unit intervals (box) using tensor cells (hexahedra).
665 
666   Collective
667 
668   Input Parameters:
669 + comm        - The communicator for the DM object
670 . dim         - The spatial dimension of the box, so the resulting mesh is has dimension dim-1
671 . faces       - Number of faces per dimension, or NULL for (1,) in 1D and (2, 2) in 2D and (1, 1, 1) in 3D
672 . lower       - The lower left corner, or NULL for (0, 0, 0)
673 . upper       - The upper right corner, or NULL for (1, 1, 1)
674 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
675 
676   Output Parameter:
677 . dm  - The DM object
678 
679   Level: beginner
680 
681 .seealso: DMSetFromOptions(), DMPlexCreateBoxMesh(), DMPlexCreateFromFile(), DMSetType(), DMCreate()
682 @*/
683 PetscErrorCode DMPlexCreateBoxSurfaceMesh(MPI_Comm comm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], PetscBool interpolate, DM *dm)
684 {
685   PetscInt       fac[3] = {1, 1, 1};
686   PetscReal      low[3] = {0, 0, 0};
687   PetscReal      upp[3] = {1, 1, 1};
688   PetscErrorCode ierr;
689 
690   PetscFunctionBegin;
691   ierr = DMCreate(comm,dm);CHKERRQ(ierr);
692   ierr = DMSetType(*dm,DMPLEX);CHKERRQ(ierr);
693   ierr = DMPlexCreateBoxSurfaceMesh_Internal(*dm, dim, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, interpolate);CHKERRQ(ierr);
694   PetscFunctionReturn(0);
695 }
696 
697 static PetscErrorCode DMPlexCreateLineMesh_Internal(DM dm,PetscInt segments,PetscReal lower,PetscReal upper,DMBoundaryType bd)
698 {
699   PetscInt       i,fStart,fEnd,numCells = 0,numVerts = 0;
700   PetscInt       numPoints[2],*coneSize,*cones,*coneOrientations;
701   PetscScalar    *vertexCoords;
702   PetscReal      L,maxCell;
703   PetscBool      markerSeparate = PETSC_FALSE;
704   PetscInt       markerLeft  = 1, faceMarkerLeft  = 1;
705   PetscInt       markerRight = 1, faceMarkerRight = 2;
706   PetscBool      wrap = (bd == DM_BOUNDARY_PERIODIC || bd == DM_BOUNDARY_TWIST) ? PETSC_TRUE : PETSC_FALSE;
707   PetscMPIInt    rank;
708   PetscErrorCode ierr;
709 
710   PetscFunctionBegin;
711   PetscValidPointer(dm,1);
712 
713   ierr = DMSetDimension(dm,1);CHKERRQ(ierr);
714   ierr = DMCreateLabel(dm,"marker");CHKERRQ(ierr);
715   ierr = DMCreateLabel(dm,"Face Sets");CHKERRQ(ierr);
716 
717   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm),&rank);CHKERRMPI(ierr);
718   if (rank == 0) numCells = segments;
719   if (rank == 0) numVerts = segments + (wrap ? 0 : 1);
720 
721   numPoints[0] = numVerts ; numPoints[1] = numCells;
722   ierr = PetscMalloc4(numCells+numVerts,&coneSize,numCells*2,&cones,numCells+numVerts,&coneOrientations,numVerts,&vertexCoords);CHKERRQ(ierr);
723   ierr = PetscArrayzero(coneOrientations,numCells+numVerts);CHKERRQ(ierr);
724   for (i = 0; i < numCells; ++i) { coneSize[i] = 2; }
725   for (i = 0; i < numVerts; ++i) { coneSize[numCells+i] = 0; }
726   for (i = 0; i < numCells; ++i) { cones[2*i] = numCells + i%numVerts; cones[2*i+1] = numCells + (i+1)%numVerts; }
727   for (i = 0; i < numVerts; ++i) { vertexCoords[i] = lower + (upper-lower)*((PetscReal)i/(PetscReal)numCells); }
728   ierr = DMPlexCreateFromDAG(dm,1,numPoints,coneSize,cones,coneOrientations,vertexCoords);CHKERRQ(ierr);
729   ierr = PetscFree4(coneSize,cones,coneOrientations,vertexCoords);CHKERRQ(ierr);
730 
731   ierr = PetscOptionsGetBool(((PetscObject)dm)->options,((PetscObject)dm)->prefix,"-dm_plex_separate_marker",&markerSeparate,NULL);CHKERRQ(ierr);
732   if (markerSeparate) { markerLeft = faceMarkerLeft; markerRight = faceMarkerRight;}
733   if (!wrap && rank == 0) {
734     ierr = DMPlexGetHeightStratum(dm,1,&fStart,&fEnd);CHKERRQ(ierr);
735     ierr = DMSetLabelValue(dm,"marker",fStart,markerLeft);CHKERRQ(ierr);
736     ierr = DMSetLabelValue(dm,"marker",fEnd-1,markerRight);CHKERRQ(ierr);
737     ierr = DMSetLabelValue(dm,"Face Sets",fStart,faceMarkerLeft);CHKERRQ(ierr);
738     ierr = DMSetLabelValue(dm,"Face Sets",fEnd-1,faceMarkerRight);CHKERRQ(ierr);
739   }
740   if (wrap) {
741     L       = upper - lower;
742     maxCell = (PetscReal)1.1*(L/(PetscReal)PetscMax(1,segments));
743     ierr = DMSetPeriodicity(dm,PETSC_TRUE,&maxCell,&L,&bd);CHKERRQ(ierr);
744   }
745   ierr = DMPlexSetRefinementUniform(dm, PETSC_TRUE);CHKERRQ(ierr);
746   PetscFunctionReturn(0);
747 }
748 
749 static PetscErrorCode DMPlexCreateBoxMesh_Simplex_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate)
750 {
751   DM             boundary, vol;
752   PetscInt       i;
753   PetscErrorCode ierr;
754 
755   PetscFunctionBegin;
756   PetscValidPointer(dm, 1);
757   for (i = 0; i < dim; ++i) PetscCheckFalse(periodicity[i] != DM_BOUNDARY_NONE,PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Periodicity is not supported for simplex meshes");
758   ierr = DMCreate(PetscObjectComm((PetscObject) dm), &boundary);CHKERRQ(ierr);
759   ierr = DMSetType(boundary, DMPLEX);CHKERRQ(ierr);
760   ierr = DMPlexCreateBoxSurfaceMesh_Internal(boundary, dim, faces, lower, upper, PETSC_FALSE);CHKERRQ(ierr);
761   ierr = DMPlexGenerate(boundary, NULL, interpolate, &vol);CHKERRQ(ierr);
762   ierr = DMPlexCopy_Internal(dm, PETSC_TRUE, vol);CHKERRQ(ierr);
763   ierr = DMPlexReplace_Static(dm, &vol);CHKERRQ(ierr);
764   ierr = DMDestroy(&boundary);CHKERRQ(ierr);
765   PetscFunctionReturn(0);
766 }
767 
768 static PetscErrorCode DMPlexCreateCubeMesh_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[], DMBoundaryType bdX, DMBoundaryType bdY, DMBoundaryType bdZ)
769 {
770   DMLabel        cutLabel = NULL;
771   PetscInt       markerTop      = 1, faceMarkerTop      = 1;
772   PetscInt       markerBottom   = 1, faceMarkerBottom   = 1;
773   PetscInt       markerFront    = 1, faceMarkerFront    = 1;
774   PetscInt       markerBack     = 1, faceMarkerBack     = 1;
775   PetscInt       markerRight    = 1, faceMarkerRight    = 1;
776   PetscInt       markerLeft     = 1, faceMarkerLeft     = 1;
777   PetscInt       dim;
778   PetscBool      markerSeparate = PETSC_FALSE, cutMarker = PETSC_FALSE;
779   PetscMPIInt    rank;
780   PetscErrorCode ierr;
781 
782   PetscFunctionBegin;
783   ierr = DMGetDimension(dm,&dim);CHKERRQ(ierr);
784   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRMPI(ierr);
785   ierr = DMCreateLabel(dm,"marker");CHKERRQ(ierr);
786   ierr = DMCreateLabel(dm,"Face Sets");CHKERRQ(ierr);
787   ierr = PetscOptionsGetBool(((PetscObject) dm)->options,((PetscObject) dm)->prefix, "-dm_plex_periodic_cut", &cutMarker, NULL);CHKERRQ(ierr);
788   if (bdX == DM_BOUNDARY_PERIODIC || bdX == DM_BOUNDARY_TWIST ||
789       bdY == DM_BOUNDARY_PERIODIC || bdY == DM_BOUNDARY_TWIST ||
790       bdZ == DM_BOUNDARY_PERIODIC || bdZ == DM_BOUNDARY_TWIST) {
791 
792     if (cutMarker) {ierr = DMCreateLabel(dm, "periodic_cut");CHKERRQ(ierr); ierr = DMGetLabel(dm, "periodic_cut", &cutLabel);CHKERRQ(ierr);}
793   }
794   switch (dim) {
795   case 2:
796     faceMarkerTop    = 3;
797     faceMarkerBottom = 1;
798     faceMarkerRight  = 2;
799     faceMarkerLeft   = 4;
800     break;
801   case 3:
802     faceMarkerBottom = 1;
803     faceMarkerTop    = 2;
804     faceMarkerFront  = 3;
805     faceMarkerBack   = 4;
806     faceMarkerRight  = 5;
807     faceMarkerLeft   = 6;
808     break;
809   default:
810     SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"Dimension %D not supported",dim);
811   }
812   ierr = PetscOptionsGetBool(((PetscObject) dm)->options,((PetscObject) dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL);CHKERRQ(ierr);
813   if (markerSeparate) {
814     markerBottom = faceMarkerBottom;
815     markerTop    = faceMarkerTop;
816     markerFront  = faceMarkerFront;
817     markerBack   = faceMarkerBack;
818     markerRight  = faceMarkerRight;
819     markerLeft   = faceMarkerLeft;
820   }
821   {
822     const PetscInt numXEdges    = rank == 0 ? edges[0] : 0;
823     const PetscInt numYEdges    = rank == 0 ? edges[1] : 0;
824     const PetscInt numZEdges    = rank == 0 ? edges[2] : 0;
825     const PetscInt numXVertices = rank == 0 ? (bdX == DM_BOUNDARY_PERIODIC || bdX == DM_BOUNDARY_TWIST ? edges[0] : edges[0]+1) : 0;
826     const PetscInt numYVertices = rank == 0 ? (bdY == DM_BOUNDARY_PERIODIC || bdY == DM_BOUNDARY_TWIST ? edges[1] : edges[1]+1) : 0;
827     const PetscInt numZVertices = rank == 0 ? (bdZ == DM_BOUNDARY_PERIODIC || bdZ == DM_BOUNDARY_TWIST ? edges[2] : edges[2]+1) : 0;
828     const PetscInt numCells     = numXEdges*numYEdges*numZEdges;
829     const PetscInt numXFaces    = numYEdges*numZEdges;
830     const PetscInt numYFaces    = numXEdges*numZEdges;
831     const PetscInt numZFaces    = numXEdges*numYEdges;
832     const PetscInt numTotXFaces = numXVertices*numXFaces;
833     const PetscInt numTotYFaces = numYVertices*numYFaces;
834     const PetscInt numTotZFaces = numZVertices*numZFaces;
835     const PetscInt numFaces     = numTotXFaces + numTotYFaces + numTotZFaces;
836     const PetscInt numTotXEdges = numXEdges*numYVertices*numZVertices;
837     const PetscInt numTotYEdges = numYEdges*numXVertices*numZVertices;
838     const PetscInt numTotZEdges = numZEdges*numXVertices*numYVertices;
839     const PetscInt numVertices  = numXVertices*numYVertices*numZVertices;
840     const PetscInt numEdges     = numTotXEdges + numTotYEdges + numTotZEdges;
841     const PetscInt firstVertex  = (dim == 2) ? numFaces : numCells;
842     const PetscInt firstXFace   = (dim == 2) ? 0 : numCells + numVertices;
843     const PetscInt firstYFace   = firstXFace + numTotXFaces;
844     const PetscInt firstZFace   = firstYFace + numTotYFaces;
845     const PetscInt firstXEdge   = numCells + numFaces + numVertices;
846     const PetscInt firstYEdge   = firstXEdge + numTotXEdges;
847     const PetscInt firstZEdge   = firstYEdge + numTotYEdges;
848     Vec            coordinates;
849     PetscSection   coordSection;
850     PetscScalar   *coords;
851     PetscInt       coordSize;
852     PetscInt       v, vx, vy, vz;
853     PetscInt       c, f, fx, fy, fz, e, ex, ey, ez;
854 
855     ierr = DMPlexSetChart(dm, 0, numCells+numFaces+numEdges+numVertices);CHKERRQ(ierr);
856     for (c = 0; c < numCells; c++) {
857       ierr = DMPlexSetConeSize(dm, c, 6);CHKERRQ(ierr);
858     }
859     for (f = firstXFace; f < firstXFace+numFaces; ++f) {
860       ierr = DMPlexSetConeSize(dm, f, 4);CHKERRQ(ierr);
861     }
862     for (e = firstXEdge; e < firstXEdge+numEdges; ++e) {
863       ierr = DMPlexSetConeSize(dm, e, 2);CHKERRQ(ierr);
864     }
865     ierr = DMSetUp(dm);CHKERRQ(ierr); /* Allocate space for cones */
866     /* Build cells */
867     for (fz = 0; fz < numZEdges; ++fz) {
868       for (fy = 0; fy < numYEdges; ++fy) {
869         for (fx = 0; fx < numXEdges; ++fx) {
870           PetscInt cell    = (fz*numYEdges + fy)*numXEdges + fx;
871           PetscInt faceB   = firstZFace + (fy*numXEdges+fx)*numZVertices +   fz;
872           PetscInt faceT   = firstZFace + (fy*numXEdges+fx)*numZVertices + ((fz+1)%numZVertices);
873           PetscInt faceF   = firstYFace + (fz*numXEdges+fx)*numYVertices +   fy;
874           PetscInt faceK   = firstYFace + (fz*numXEdges+fx)*numYVertices + ((fy+1)%numYVertices);
875           PetscInt faceL   = firstXFace + (fz*numYEdges+fy)*numXVertices +   fx;
876           PetscInt faceR   = firstXFace + (fz*numYEdges+fy)*numXVertices + ((fx+1)%numXVertices);
877                             /* B,  T,  F,  K,  R,  L */
878           PetscInt ornt[6] = {-2,  0,  0, -3,  0, -2}; /* ??? */
879           PetscInt cone[6];
880 
881           /* no boundary twisting in 3D */
882           cone[0] = faceB; cone[1] = faceT; cone[2] = faceF; cone[3] = faceK; cone[4] = faceR; cone[5] = faceL;
883           ierr    = DMPlexSetCone(dm, cell, cone);CHKERRQ(ierr);
884           ierr    = DMPlexSetConeOrientation(dm, cell, ornt);CHKERRQ(ierr);
885           if (bdX != DM_BOUNDARY_NONE && fx == numXEdges-1 && cutLabel) {ierr = DMLabelSetValue(cutLabel, cell, 2);CHKERRQ(ierr);}
886           if (bdY != DM_BOUNDARY_NONE && fy == numYEdges-1 && cutLabel) {ierr = DMLabelSetValue(cutLabel, cell, 2);CHKERRQ(ierr);}
887           if (bdZ != DM_BOUNDARY_NONE && fz == numZEdges-1 && cutLabel) {ierr = DMLabelSetValue(cutLabel, cell, 2);CHKERRQ(ierr);}
888         }
889       }
890     }
891     /* Build x faces */
892     for (fz = 0; fz < numZEdges; ++fz) {
893       for (fy = 0; fy < numYEdges; ++fy) {
894         for (fx = 0; fx < numXVertices; ++fx) {
895           PetscInt face    = firstXFace + (fz*numYEdges+fy)     *numXVertices+fx;
896           PetscInt edgeL   = firstZEdge + (fy                   *numXVertices+fx)*numZEdges + fz;
897           PetscInt edgeR   = firstZEdge + (((fy+1)%numYVertices)*numXVertices+fx)*numZEdges + fz;
898           PetscInt edgeB   = firstYEdge + (fz                   *numXVertices+fx)*numYEdges + fy;
899           PetscInt edgeT   = firstYEdge + (((fz+1)%numZVertices)*numXVertices+fx)*numYEdges + fy;
900           PetscInt ornt[4] = {0, 0, -1, -1};
901           PetscInt cone[4];
902 
903           if (dim == 3) {
904             /* markers */
905             if (bdX != DM_BOUNDARY_PERIODIC) {
906               if (fx == numXVertices-1) {
907                 ierr = DMSetLabelValue(dm, "Face Sets", face, faceMarkerRight);CHKERRQ(ierr);
908                 ierr = DMSetLabelValue(dm, "marker", face, markerRight);CHKERRQ(ierr);
909               }
910               else if (fx == 0) {
911                 ierr = DMSetLabelValue(dm, "Face Sets", face, faceMarkerLeft);CHKERRQ(ierr);
912                 ierr = DMSetLabelValue(dm, "marker", face, markerLeft);CHKERRQ(ierr);
913               }
914             }
915           }
916           cone[0] = edgeB; cone[1] = edgeR; cone[2] = edgeT; cone[3] = edgeL;
917           ierr    = DMPlexSetCone(dm, face, cone);CHKERRQ(ierr);
918           ierr    = DMPlexSetConeOrientation(dm, face, ornt);CHKERRQ(ierr);
919         }
920       }
921     }
922     /* Build y faces */
923     for (fz = 0; fz < numZEdges; ++fz) {
924       for (fx = 0; fx < numXEdges; ++fx) {
925         for (fy = 0; fy < numYVertices; ++fy) {
926           PetscInt face    = firstYFace + (fz*numXEdges+fx)*numYVertices + fy;
927           PetscInt edgeL   = firstZEdge + (fy*numXVertices+  fx)*numZEdges + fz;
928           PetscInt edgeR   = firstZEdge + (fy*numXVertices+((fx+1)%numXVertices))*numZEdges + fz;
929           PetscInt edgeB   = firstXEdge + (fz                   *numYVertices+fy)*numXEdges + fx;
930           PetscInt edgeT   = firstXEdge + (((fz+1)%numZVertices)*numYVertices+fy)*numXEdges + fx;
931           PetscInt ornt[4] = {0, 0, -1, -1};
932           PetscInt cone[4];
933 
934           if (dim == 3) {
935             /* markers */
936             if (bdY != DM_BOUNDARY_PERIODIC) {
937               if (fy == numYVertices-1) {
938                 ierr = DMSetLabelValue(dm, "Face Sets", face, faceMarkerBack);CHKERRQ(ierr);
939                 ierr = DMSetLabelValue(dm, "marker", face, markerBack);CHKERRQ(ierr);
940               }
941               else if (fy == 0) {
942                 ierr = DMSetLabelValue(dm, "Face Sets", face, faceMarkerFront);CHKERRQ(ierr);
943                 ierr = DMSetLabelValue(dm, "marker", face, markerFront);CHKERRQ(ierr);
944               }
945             }
946           }
947           cone[0] = edgeB; cone[1] = edgeR; cone[2] = edgeT; cone[3] = edgeL;
948           ierr    = DMPlexSetCone(dm, face, cone);CHKERRQ(ierr);
949           ierr    = DMPlexSetConeOrientation(dm, face, ornt);CHKERRQ(ierr);
950         }
951       }
952     }
953     /* Build z faces */
954     for (fy = 0; fy < numYEdges; ++fy) {
955       for (fx = 0; fx < numXEdges; ++fx) {
956         for (fz = 0; fz < numZVertices; fz++) {
957           PetscInt face    = firstZFace + (fy*numXEdges+fx)*numZVertices + fz;
958           PetscInt edgeL   = firstYEdge + (fz*numXVertices+  fx)*numYEdges + fy;
959           PetscInt edgeR   = firstYEdge + (fz*numXVertices+((fx+1)%numXVertices))*numYEdges + fy;
960           PetscInt edgeB   = firstXEdge + (fz*numYVertices+  fy)*numXEdges + fx;
961           PetscInt edgeT   = firstXEdge + (fz*numYVertices+((fy+1)%numYVertices))*numXEdges + fx;
962           PetscInt ornt[4] = {0, 0, -1, -1};
963           PetscInt cone[4];
964 
965           if (dim == 2) {
966             if (bdX == DM_BOUNDARY_TWIST && fx == numXEdges-1) {edgeR += numYEdges-1-2*fy; ornt[1] = -1;}
967             if (bdY == DM_BOUNDARY_TWIST && fy == numYEdges-1) {edgeT += numXEdges-1-2*fx; ornt[2] =  0;}
968             if (bdX != DM_BOUNDARY_NONE && fx == numXEdges-1 && cutLabel) {ierr = DMLabelSetValue(cutLabel, face, 2);CHKERRQ(ierr);}
969             if (bdY != DM_BOUNDARY_NONE && fy == numYEdges-1 && cutLabel) {ierr = DMLabelSetValue(cutLabel, face, 2);CHKERRQ(ierr);}
970           } else {
971             /* markers */
972             if (bdZ != DM_BOUNDARY_PERIODIC) {
973               if (fz == numZVertices-1) {
974                 ierr = DMSetLabelValue(dm, "Face Sets", face, faceMarkerTop);CHKERRQ(ierr);
975                 ierr = DMSetLabelValue(dm, "marker", face, markerTop);CHKERRQ(ierr);
976               }
977               else if (fz == 0) {
978                 ierr = DMSetLabelValue(dm, "Face Sets", face, faceMarkerBottom);CHKERRQ(ierr);
979                 ierr = DMSetLabelValue(dm, "marker", face, markerBottom);CHKERRQ(ierr);
980               }
981             }
982           }
983           cone[0] = edgeB; cone[1] = edgeR; cone[2] = edgeT; cone[3] = edgeL;
984           ierr    = DMPlexSetCone(dm, face, cone);CHKERRQ(ierr);
985           ierr    = DMPlexSetConeOrientation(dm, face, ornt);CHKERRQ(ierr);
986         }
987       }
988     }
989     /* Build Z edges*/
990     for (vy = 0; vy < numYVertices; vy++) {
991       for (vx = 0; vx < numXVertices; vx++) {
992         for (ez = 0; ez < numZEdges; ez++) {
993           const PetscInt edge    = firstZEdge  + (vy*numXVertices+vx)*numZEdges + ez;
994           const PetscInt vertexB = firstVertex + (ez                   *numYVertices+vy)*numXVertices + vx;
995           const PetscInt vertexT = firstVertex + (((ez+1)%numZVertices)*numYVertices+vy)*numXVertices + vx;
996           PetscInt       cone[2];
997 
998           if (dim == 3) {
999             if (bdX != DM_BOUNDARY_PERIODIC) {
1000               if (vx == numXVertices-1) {
1001                 ierr = DMSetLabelValue(dm, "marker", edge, markerRight);CHKERRQ(ierr);
1002               }
1003               else if (vx == 0) {
1004                 ierr = DMSetLabelValue(dm, "marker", edge, markerLeft);CHKERRQ(ierr);
1005               }
1006             }
1007             if (bdY != DM_BOUNDARY_PERIODIC) {
1008               if (vy == numYVertices-1) {
1009                 ierr = DMSetLabelValue(dm, "marker", edge, markerBack);CHKERRQ(ierr);
1010               }
1011               else if (vy == 0) {
1012                 ierr = DMSetLabelValue(dm, "marker", edge, markerFront);CHKERRQ(ierr);
1013               }
1014             }
1015           }
1016           cone[0] = vertexB; cone[1] = vertexT;
1017           ierr = DMPlexSetCone(dm, edge, cone);CHKERRQ(ierr);
1018         }
1019       }
1020     }
1021     /* Build Y edges*/
1022     for (vz = 0; vz < numZVertices; vz++) {
1023       for (vx = 0; vx < numXVertices; vx++) {
1024         for (ey = 0; ey < numYEdges; ey++) {
1025           const PetscInt nextv   = (dim == 2 && bdY == DM_BOUNDARY_TWIST && ey == numYEdges-1) ? (numXVertices-vx-1) : (vz*numYVertices+((ey+1)%numYVertices))*numXVertices + vx;
1026           const PetscInt edge    = firstYEdge  + (vz*numXVertices+vx)*numYEdges + ey;
1027           const PetscInt vertexF = firstVertex + (vz*numYVertices+ey)*numXVertices + vx;
1028           const PetscInt vertexK = firstVertex + nextv;
1029           PetscInt       cone[2];
1030 
1031           cone[0] = vertexF; cone[1] = vertexK;
1032           ierr = DMPlexSetCone(dm, edge, cone);CHKERRQ(ierr);
1033           if (dim == 2) {
1034             if ((bdX != DM_BOUNDARY_PERIODIC) && (bdX != DM_BOUNDARY_TWIST)) {
1035               if (vx == numXVertices-1) {
1036                 ierr = DMSetLabelValue(dm, "Face Sets", edge, faceMarkerRight);CHKERRQ(ierr);
1037                 ierr = DMSetLabelValue(dm, "marker", edge,    markerRight);CHKERRQ(ierr);
1038                 ierr = DMSetLabelValue(dm, "marker", cone[0], markerRight);CHKERRQ(ierr);
1039                 if (ey == numYEdges-1) {
1040                   ierr = DMSetLabelValue(dm, "marker", cone[1], markerRight);CHKERRQ(ierr);
1041                 }
1042               } else if (vx == 0) {
1043                 ierr = DMSetLabelValue(dm, "Face Sets", edge, faceMarkerLeft);CHKERRQ(ierr);
1044                 ierr = DMSetLabelValue(dm, "marker", edge,    markerLeft);CHKERRQ(ierr);
1045                 ierr = DMSetLabelValue(dm, "marker", cone[0], markerLeft);CHKERRQ(ierr);
1046                 if (ey == numYEdges-1) {
1047                   ierr = DMSetLabelValue(dm, "marker", cone[1], markerLeft);CHKERRQ(ierr);
1048                 }
1049               }
1050             } else {
1051               if (vx == 0 && cutLabel) {
1052                 ierr = DMLabelSetValue(cutLabel, edge,    1);CHKERRQ(ierr);
1053                 ierr = DMLabelSetValue(cutLabel, cone[0], 1);CHKERRQ(ierr);
1054                 if (ey == numYEdges-1) {
1055                   ierr = DMLabelSetValue(cutLabel, cone[1], 1);CHKERRQ(ierr);
1056                 }
1057               }
1058             }
1059           } else {
1060             if (bdX != DM_BOUNDARY_PERIODIC) {
1061               if (vx == numXVertices-1) {
1062                 ierr = DMSetLabelValue(dm, "marker", edge, markerRight);CHKERRQ(ierr);
1063               } else if (vx == 0) {
1064                 ierr = DMSetLabelValue(dm, "marker", edge, markerLeft);CHKERRQ(ierr);
1065               }
1066             }
1067             if (bdZ != DM_BOUNDARY_PERIODIC) {
1068               if (vz == numZVertices-1) {
1069                 ierr = DMSetLabelValue(dm, "marker", edge, markerTop);CHKERRQ(ierr);
1070               } else if (vz == 0) {
1071                 ierr = DMSetLabelValue(dm, "marker", edge, markerBottom);CHKERRQ(ierr);
1072               }
1073             }
1074           }
1075         }
1076       }
1077     }
1078     /* Build X edges*/
1079     for (vz = 0; vz < numZVertices; vz++) {
1080       for (vy = 0; vy < numYVertices; vy++) {
1081         for (ex = 0; ex < numXEdges; ex++) {
1082           const PetscInt nextv   = (dim == 2 && bdX == DM_BOUNDARY_TWIST && ex == numXEdges-1) ? (numYVertices-vy-1)*numXVertices : (vz*numYVertices+vy)*numXVertices + (ex+1)%numXVertices;
1083           const PetscInt edge    = firstXEdge  + (vz*numYVertices+vy)*numXEdges + ex;
1084           const PetscInt vertexL = firstVertex + (vz*numYVertices+vy)*numXVertices + ex;
1085           const PetscInt vertexR = firstVertex + nextv;
1086           PetscInt       cone[2];
1087 
1088           cone[0] = vertexL; cone[1] = vertexR;
1089           ierr = DMPlexSetCone(dm, edge, cone);CHKERRQ(ierr);
1090           if (dim == 2) {
1091             if ((bdY != DM_BOUNDARY_PERIODIC) && (bdY != DM_BOUNDARY_TWIST)) {
1092               if (vy == numYVertices-1) {
1093                 ierr = DMSetLabelValue(dm, "Face Sets", edge, faceMarkerTop);CHKERRQ(ierr);
1094                 ierr = DMSetLabelValue(dm, "marker", edge,    markerTop);CHKERRQ(ierr);
1095                 ierr = DMSetLabelValue(dm, "marker", cone[0], markerTop);CHKERRQ(ierr);
1096                 if (ex == numXEdges-1) {
1097                   ierr = DMSetLabelValue(dm, "marker", cone[1], markerTop);CHKERRQ(ierr);
1098                 }
1099               } else if (vy == 0) {
1100                 ierr = DMSetLabelValue(dm, "Face Sets", edge, faceMarkerBottom);CHKERRQ(ierr);
1101                 ierr = DMSetLabelValue(dm, "marker", edge,    markerBottom);CHKERRQ(ierr);
1102                 ierr = DMSetLabelValue(dm, "marker", cone[0], markerBottom);CHKERRQ(ierr);
1103                 if (ex == numXEdges-1) {
1104                   ierr = DMSetLabelValue(dm, "marker", cone[1], markerBottom);CHKERRQ(ierr);
1105                 }
1106               }
1107             } else {
1108               if (vy == 0 && cutLabel) {
1109                 ierr = DMLabelSetValue(cutLabel, edge,    1);CHKERRQ(ierr);
1110                 ierr = DMLabelSetValue(cutLabel, cone[0], 1);CHKERRQ(ierr);
1111                 if (ex == numXEdges-1) {
1112                   ierr = DMLabelSetValue(cutLabel, cone[1], 1);CHKERRQ(ierr);
1113                 }
1114               }
1115             }
1116           } else {
1117             if (bdY != DM_BOUNDARY_PERIODIC) {
1118               if (vy == numYVertices-1) {
1119                 ierr = DMSetLabelValue(dm, "marker", edge, markerBack);CHKERRQ(ierr);
1120               }
1121               else if (vy == 0) {
1122                 ierr = DMSetLabelValue(dm, "marker", edge, markerFront);CHKERRQ(ierr);
1123               }
1124             }
1125             if (bdZ != DM_BOUNDARY_PERIODIC) {
1126               if (vz == numZVertices-1) {
1127                 ierr = DMSetLabelValue(dm, "marker", edge, markerTop);CHKERRQ(ierr);
1128               }
1129               else if (vz == 0) {
1130                 ierr = DMSetLabelValue(dm, "marker", edge, markerBottom);CHKERRQ(ierr);
1131               }
1132             }
1133           }
1134         }
1135       }
1136     }
1137     ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
1138     ierr = DMPlexStratify(dm);CHKERRQ(ierr);
1139     /* Build coordinates */
1140     ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
1141     ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
1142     ierr = PetscSectionSetFieldComponents(coordSection, 0, dim);CHKERRQ(ierr);
1143     ierr = PetscSectionSetChart(coordSection, firstVertex, firstVertex+numVertices);CHKERRQ(ierr);
1144     for (v = firstVertex; v < firstVertex+numVertices; ++v) {
1145       ierr = PetscSectionSetDof(coordSection, v, dim);CHKERRQ(ierr);
1146       ierr = PetscSectionSetFieldDof(coordSection, v, 0, dim);CHKERRQ(ierr);
1147     }
1148     ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
1149     ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
1150     ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
1151     ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
1152     ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
1153     ierr = VecSetBlockSize(coordinates, dim);CHKERRQ(ierr);
1154     ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
1155     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
1156     for (vz = 0; vz < numZVertices; ++vz) {
1157       for (vy = 0; vy < numYVertices; ++vy) {
1158         for (vx = 0; vx < numXVertices; ++vx) {
1159           coords[((vz*numYVertices+vy)*numXVertices+vx)*dim+0] = lower[0] + ((upper[0] - lower[0])/numXEdges)*vx;
1160           coords[((vz*numYVertices+vy)*numXVertices+vx)*dim+1] = lower[1] + ((upper[1] - lower[1])/numYEdges)*vy;
1161           if (dim == 3) {
1162             coords[((vz*numYVertices+vy)*numXVertices+vx)*dim+2] = lower[2] + ((upper[2] - lower[2])/numZEdges)*vz;
1163           }
1164         }
1165       }
1166     }
1167     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
1168     ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
1169     ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
1170   }
1171   PetscFunctionReturn(0);
1172 }
1173 
1174 static PetscErrorCode DMPlexCreateBoxMesh_Tensor_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[])
1175 {
1176   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1177   PetscInt       fac[3] = {0, 0, 0}, d;
1178   PetscErrorCode ierr;
1179 
1180   PetscFunctionBegin;
1181   PetscValidPointer(dm, 1);
1182   PetscValidLogicalCollectiveInt(dm, dim, 2);
1183   ierr = DMSetDimension(dm, dim);CHKERRQ(ierr);
1184   for (d = 0; d < dim; ++d) {fac[d] = faces[d]; bdt[d] = periodicity[d];}
1185   ierr = DMPlexCreateCubeMesh_Internal(dm, lower, upper, fac, bdt[0], bdt[1], bdt[2]);CHKERRQ(ierr);
1186   if (periodicity[0] == DM_BOUNDARY_PERIODIC || periodicity[0] == DM_BOUNDARY_TWIST ||
1187       periodicity[1] == DM_BOUNDARY_PERIODIC || periodicity[1] == DM_BOUNDARY_TWIST ||
1188       (dim > 2 && (periodicity[2] == DM_BOUNDARY_PERIODIC || periodicity[2] == DM_BOUNDARY_TWIST))) {
1189     PetscReal L[3];
1190     PetscReal maxCell[3];
1191 
1192     for (d = 0; d < dim; ++d) {
1193       L[d]       = upper[d] - lower[d];
1194       maxCell[d] = 1.1 * (L[d] / PetscMax(1, faces[d]));
1195     }
1196     ierr = DMSetPeriodicity(dm, PETSC_TRUE, maxCell, L, periodicity);CHKERRQ(ierr);
1197   }
1198   ierr = DMPlexSetRefinementUniform(dm, PETSC_TRUE);CHKERRQ(ierr);
1199   PetscFunctionReturn(0);
1200 }
1201 
1202 static PetscErrorCode DMPlexCreateBoxMesh_Internal(DM dm, PetscInt dim, PetscBool simplex, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate)
1203 {
1204   PetscErrorCode ierr;
1205 
1206   PetscFunctionBegin;
1207   if (dim == 1)      {ierr = DMPlexCreateLineMesh_Internal(dm, faces[0], lower[0], upper[0], periodicity[0]);CHKERRQ(ierr);}
1208   else if (simplex)  {ierr = DMPlexCreateBoxMesh_Simplex_Internal(dm, dim, faces, lower, upper, periodicity, interpolate);CHKERRQ(ierr);}
1209   else               {ierr = DMPlexCreateBoxMesh_Tensor_Internal(dm, dim, faces, lower, upper, periodicity);CHKERRQ(ierr);}
1210   if (!interpolate && dim > 1 && !simplex) {
1211     DM udm;
1212 
1213     ierr = DMPlexUninterpolate(dm, &udm);CHKERRQ(ierr);
1214     ierr = DMPlexCopyCoordinates(dm, udm);CHKERRQ(ierr);
1215     ierr = DMPlexReplace_Static(dm, &udm);CHKERRQ(ierr);
1216   }
1217   PetscFunctionReturn(0);
1218 }
1219 
1220 /*@C
1221   DMPlexCreateBoxMesh - Creates a mesh on the tensor product of unit intervals (box) using simplices or tensor cells (hexahedra).
1222 
1223   Collective
1224 
1225   Input Parameters:
1226 + comm        - The communicator for the DM object
1227 . dim         - The spatial dimension
1228 . simplex     - PETSC_TRUE for simplices, PETSC_FALSE for tensor cells
1229 . faces       - Number of faces per dimension, or NULL for (1,) in 1D and (2, 2) in 2D and (1, 1, 1) in 3D
1230 . lower       - The lower left corner, or NULL for (0, 0, 0)
1231 . upper       - The upper right corner, or NULL for (1, 1, 1)
1232 . periodicity - The boundary type for the X,Y,Z direction, or NULL for DM_BOUNDARY_NONE
1233 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
1234 
1235   Output Parameter:
1236 . dm  - The DM object
1237 
1238   Note: If you want to customize this mesh using options, you just need to
1239 $  DMCreate(comm, &dm);
1240 $  DMSetType(dm, DMPLEX);
1241 $  DMSetFromOptions(dm);
1242 and use the options on the DMSetFromOptions() page.
1243 
1244   Here is the numbering returned for 2 faces in each direction for tensor cells:
1245 $ 10---17---11---18----12
1246 $  |         |         |
1247 $  |         |         |
1248 $ 20    2   22    3    24
1249 $  |         |         |
1250 $  |         |         |
1251 $  7---15----8---16----9
1252 $  |         |         |
1253 $  |         |         |
1254 $ 19    0   21    1   23
1255 $  |         |         |
1256 $  |         |         |
1257 $  4---13----5---14----6
1258 
1259 and for simplicial cells
1260 
1261 $ 14----8---15----9----16
1262 $  |\     5  |\      7 |
1263 $  | \       | \       |
1264 $ 13   2    14    3    15
1265 $  | 4   \   | 6   \   |
1266 $  |       \ |       \ |
1267 $ 11----6---12----7----13
1268 $  |\        |\        |
1269 $  | \    1  | \     3 |
1270 $ 10   0    11    1    12
1271 $  | 0   \   | 2   \   |
1272 $  |       \ |       \ |
1273 $  8----4----9----5----10
1274 
1275   Level: beginner
1276 
1277 .seealso: DMSetFromOptions(), DMPlexCreateFromFile(), DMPlexCreateHexCylinderMesh(), DMSetType(), DMCreate()
1278 @*/
1279 PetscErrorCode DMPlexCreateBoxMesh(MPI_Comm comm, PetscInt dim, PetscBool simplex, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate, DM *dm)
1280 {
1281   PetscInt       fac[3] = {1, 1, 1};
1282   PetscReal      low[3] = {0, 0, 0};
1283   PetscReal      upp[3] = {1, 1, 1};
1284   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1285   PetscErrorCode ierr;
1286 
1287   PetscFunctionBegin;
1288   ierr = DMCreate(comm,dm);CHKERRQ(ierr);
1289   ierr = DMSetType(*dm,DMPLEX);CHKERRQ(ierr);
1290   ierr = DMPlexCreateBoxMesh_Internal(*dm, dim, simplex, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, periodicity ? periodicity : bdt, interpolate);CHKERRQ(ierr);
1291   PetscFunctionReturn(0);
1292 }
1293 
1294 static PetscErrorCode DMPlexCreateWedgeBoxMesh_Internal(DM dm, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[])
1295 {
1296   DM             bdm, vol;
1297   PetscInt       i;
1298   PetscErrorCode ierr;
1299 
1300   PetscFunctionBegin;
1301   for (i = 0; i < 3; ++i) PetscCheckFalse(periodicity[i] != DM_BOUNDARY_NONE,PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Periodicity not yet supported");
1302   ierr = DMCreate(PetscObjectComm((PetscObject) dm), &bdm);CHKERRQ(ierr);
1303   ierr = DMSetType(bdm, DMPLEX);CHKERRQ(ierr);
1304   ierr = DMSetDimension(bdm, 2);CHKERRQ(ierr);
1305   ierr = DMPlexCreateBoxMesh_Simplex_Internal(bdm, 2, faces, lower, upper, periodicity, PETSC_TRUE);CHKERRQ(ierr);
1306   ierr = DMPlexExtrude(bdm, faces[2], upper[2] - lower[2], PETSC_TRUE, PETSC_FALSE, NULL, NULL, &vol);CHKERRQ(ierr);
1307   ierr = DMDestroy(&bdm);CHKERRQ(ierr);
1308   ierr = DMPlexReplace_Static(dm, &vol);CHKERRQ(ierr);
1309   if (lower[2] != 0.0) {
1310     Vec          v;
1311     PetscScalar *x;
1312     PetscInt     cDim, n;
1313 
1314     ierr = DMGetCoordinatesLocal(dm, &v);CHKERRQ(ierr);
1315     ierr = VecGetBlockSize(v, &cDim);CHKERRQ(ierr);
1316     ierr = VecGetLocalSize(v, &n);CHKERRQ(ierr);
1317     ierr = VecGetArray(v, &x);CHKERRQ(ierr);
1318     x   += cDim;
1319     for (i = 0; i < n; i += cDim) x[i] += lower[2];
1320     ierr = VecRestoreArray(v,&x);CHKERRQ(ierr);
1321     ierr = DMSetCoordinatesLocal(dm, v);CHKERRQ(ierr);
1322   }
1323   PetscFunctionReturn(0);
1324 }
1325 
1326 /*@
1327   DMPlexCreateWedgeBoxMesh - Creates a 3-D mesh tesselating the (x,y) plane and extruding in the third direction using wedge cells.
1328 
1329   Collective
1330 
1331   Input Parameters:
1332 + comm        - The communicator for the DM object
1333 . faces       - Number of faces per dimension, or NULL for (1, 1, 1)
1334 . lower       - The lower left corner, or NULL for (0, 0, 0)
1335 . upper       - The upper right corner, or NULL for (1, 1, 1)
1336 . periodicity - The boundary type for the X,Y,Z direction, or NULL for DM_BOUNDARY_NONE
1337 . orderHeight - If PETSC_TRUE, orders the extruded cells in the height first. Otherwise, orders the cell on the layers first
1338 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
1339 
1340   Output Parameter:
1341 . dm  - The DM object
1342 
1343   Level: beginner
1344 
1345 .seealso: DMPlexCreateHexCylinderMesh(), DMPlexCreateWedgeCylinderMesh(), DMExtrude(), DMPlexCreateBoxMesh(), DMSetType(), DMCreate()
1346 @*/
1347 PetscErrorCode DMPlexCreateWedgeBoxMesh(MPI_Comm comm, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool orderHeight, PetscBool interpolate, DM *dm)
1348 {
1349   PetscInt       fac[3] = {1, 1, 1};
1350   PetscReal      low[3] = {0, 0, 0};
1351   PetscReal      upp[3] = {1, 1, 1};
1352   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1353   PetscErrorCode ierr;
1354 
1355   PetscFunctionBegin;
1356   ierr = DMCreate(comm,dm);CHKERRQ(ierr);
1357   ierr = DMSetType(*dm,DMPLEX);CHKERRQ(ierr);
1358   ierr = DMPlexCreateWedgeBoxMesh_Internal(*dm, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, periodicity ? periodicity : bdt);CHKERRQ(ierr);
1359   if (!interpolate) {
1360     DM udm;
1361 
1362     ierr = DMPlexUninterpolate(*dm, &udm);CHKERRQ(ierr);
1363     ierr = DMPlexReplace_Static(*dm, &udm);CHKERRQ(ierr);
1364   }
1365   PetscFunctionReturn(0);
1366 }
1367 
1368 /*@C
1369   DMPlexSetOptionsPrefix - Sets the prefix used for searching for all DM options in the database.
1370 
1371   Logically Collective on dm
1372 
1373   Input Parameters:
1374 + dm - the DM context
1375 - prefix - the prefix to prepend to all option names
1376 
1377   Notes:
1378   A hyphen (-) must NOT be given at the beginning of the prefix name.
1379   The first character of all runtime options is AUTOMATICALLY the hyphen.
1380 
1381   Level: advanced
1382 
1383 .seealso: SNESSetFromOptions()
1384 @*/
1385 PetscErrorCode DMPlexSetOptionsPrefix(DM dm, const char prefix[])
1386 {
1387   DM_Plex       *mesh = (DM_Plex *) dm->data;
1388   PetscErrorCode ierr;
1389 
1390   PetscFunctionBegin;
1391   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1392   ierr = PetscObjectSetOptionsPrefix((PetscObject) dm, prefix);CHKERRQ(ierr);
1393   ierr = PetscObjectSetOptionsPrefix((PetscObject) mesh->partitioner, prefix);CHKERRQ(ierr);
1394   PetscFunctionReturn(0);
1395 }
1396 
1397 /* Remap geometry to cylinder
1398    TODO: This only works for a single refinement, then it is broken
1399 
1400      Interior square: Linear interpolation is correct
1401      The other cells all have vertices on rays from the origin. We want to uniformly expand the spacing
1402      such that the last vertex is on the unit circle. So the closest and farthest vertices are at distance
1403 
1404        phi     = arctan(y/x)
1405        d_close = sqrt(1/8 + 1/4 sin^2(phi))
1406        d_far   = sqrt(1/2 + sin^2(phi))
1407 
1408      so we remap them using
1409 
1410        x_new = x_close + (x - x_close) (1 - d_close) / (d_far - d_close)
1411        y_new = y_close + (y - y_close) (1 - d_close) / (d_far - d_close)
1412 
1413      If pi/4 < phi < 3pi/4 or -3pi/4 < phi < -pi/4, then we switch x and y.
1414 */
1415 static void snapToCylinder(PetscInt dim, PetscInt Nf, PetscInt NfAux,
1416                            const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[],
1417                            const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[],
1418                            PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
1419 {
1420   const PetscReal dis = 1.0/PetscSqrtReal(2.0);
1421   const PetscReal ds2 = 0.5*dis;
1422 
1423   if ((PetscAbsScalar(u[0]) <= ds2) && (PetscAbsScalar(u[1]) <= ds2)) {
1424     f0[0] = u[0];
1425     f0[1] = u[1];
1426   } else {
1427     PetscReal phi, sinp, cosp, dc, df, x, y, xc, yc;
1428 
1429     x    = PetscRealPart(u[0]);
1430     y    = PetscRealPart(u[1]);
1431     phi  = PetscAtan2Real(y, x);
1432     sinp = PetscSinReal(phi);
1433     cosp = PetscCosReal(phi);
1434     if ((PetscAbsReal(phi) > PETSC_PI/4.0) && (PetscAbsReal(phi) < 3.0*PETSC_PI/4.0)) {
1435       dc = PetscAbsReal(ds2/sinp);
1436       df = PetscAbsReal(dis/sinp);
1437       xc = ds2*x/PetscAbsReal(y);
1438       yc = ds2*PetscSignReal(y);
1439     } else {
1440       dc = PetscAbsReal(ds2/cosp);
1441       df = PetscAbsReal(dis/cosp);
1442       xc = ds2*PetscSignReal(x);
1443       yc = ds2*y/PetscAbsReal(x);
1444     }
1445     f0[0] = xc + (u[0] - xc)*(1.0 - dc)/(df - dc);
1446     f0[1] = yc + (u[1] - yc)*(1.0 - dc)/(df - dc);
1447   }
1448   f0[2] = u[2];
1449 }
1450 
1451 static PetscErrorCode DMPlexCreateHexCylinderMesh_Internal(DM dm, DMBoundaryType periodicZ)
1452 {
1453   const PetscInt dim = 3;
1454   PetscInt       numCells, numVertices;
1455   PetscMPIInt    rank;
1456   PetscErrorCode ierr;
1457 
1458   PetscFunctionBegin;
1459   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRMPI(ierr);
1460   ierr = DMSetDimension(dm, dim);CHKERRQ(ierr);
1461   /* Create topology */
1462   {
1463     PetscInt cone[8], c;
1464 
1465     numCells    = rank == 0 ?  5 : 0;
1466     numVertices = rank == 0 ? 16 : 0;
1467     if (periodicZ == DM_BOUNDARY_PERIODIC) {
1468       numCells   *= 3;
1469       numVertices = rank == 0 ? 24 : 0;
1470     }
1471     ierr = DMPlexSetChart(dm, 0, numCells+numVertices);CHKERRQ(ierr);
1472     for (c = 0; c < numCells; c++) {ierr = DMPlexSetConeSize(dm, c, 8);CHKERRQ(ierr);}
1473     ierr = DMSetUp(dm);CHKERRQ(ierr);
1474     if (rank == 0) {
1475       if (periodicZ == DM_BOUNDARY_PERIODIC) {
1476         cone[0] = 15; cone[1] = 18; cone[2] = 17; cone[3] = 16;
1477         cone[4] = 31; cone[5] = 32; cone[6] = 33; cone[7] = 34;
1478         ierr = DMPlexSetCone(dm, 0, cone);CHKERRQ(ierr);
1479         cone[0] = 16; cone[1] = 17; cone[2] = 24; cone[3] = 23;
1480         cone[4] = 32; cone[5] = 36; cone[6] = 37; cone[7] = 33; /* 22 25 26 21 */
1481         ierr = DMPlexSetCone(dm, 1, cone);CHKERRQ(ierr);
1482         cone[0] = 18; cone[1] = 27; cone[2] = 24; cone[3] = 17;
1483         cone[4] = 34; cone[5] = 33; cone[6] = 37; cone[7] = 38;
1484         ierr = DMPlexSetCone(dm, 2, cone);CHKERRQ(ierr);
1485         cone[0] = 29; cone[1] = 27; cone[2] = 18; cone[3] = 15;
1486         cone[4] = 35; cone[5] = 31; cone[6] = 34; cone[7] = 38;
1487         ierr = DMPlexSetCone(dm, 3, cone);CHKERRQ(ierr);
1488         cone[0] = 29; cone[1] = 15; cone[2] = 16; cone[3] = 23;
1489         cone[4] = 35; cone[5] = 36; cone[6] = 32; cone[7] = 31;
1490         ierr = DMPlexSetCone(dm, 4, cone);CHKERRQ(ierr);
1491 
1492         cone[0] = 31; cone[1] = 34; cone[2] = 33; cone[3] = 32;
1493         cone[4] = 19; cone[5] = 22; cone[6] = 21; cone[7] = 20;
1494         ierr = DMPlexSetCone(dm, 5, cone);CHKERRQ(ierr);
1495         cone[0] = 32; cone[1] = 33; cone[2] = 37; cone[3] = 36;
1496         cone[4] = 22; cone[5] = 25; cone[6] = 26; cone[7] = 21;
1497         ierr = DMPlexSetCone(dm, 6, cone);CHKERRQ(ierr);
1498         cone[0] = 34; cone[1] = 38; cone[2] = 37; cone[3] = 33;
1499         cone[4] = 20; cone[5] = 21; cone[6] = 26; cone[7] = 28;
1500         ierr = DMPlexSetCone(dm, 7, cone);CHKERRQ(ierr);
1501         cone[0] = 35; cone[1] = 38; cone[2] = 34; cone[3] = 31;
1502         cone[4] = 30; cone[5] = 19; cone[6] = 20; cone[7] = 28;
1503         ierr = DMPlexSetCone(dm, 8, cone);CHKERRQ(ierr);
1504         cone[0] = 35; cone[1] = 31; cone[2] = 32; cone[3] = 36;
1505         cone[4] = 30; cone[5] = 25; cone[6] = 22; cone[7] = 19;
1506         ierr = DMPlexSetCone(dm, 9, cone);CHKERRQ(ierr);
1507 
1508         cone[0] = 19; cone[1] = 20; cone[2] = 21; cone[3] = 22;
1509         cone[4] = 15; cone[5] = 16; cone[6] = 17; cone[7] = 18;
1510         ierr = DMPlexSetCone(dm, 10, cone);CHKERRQ(ierr);
1511         cone[0] = 22; cone[1] = 21; cone[2] = 26; cone[3] = 25;
1512         cone[4] = 16; cone[5] = 23; cone[6] = 24; cone[7] = 17;
1513         ierr = DMPlexSetCone(dm, 11, cone);CHKERRQ(ierr);
1514         cone[0] = 20; cone[1] = 28; cone[2] = 26; cone[3] = 21;
1515         cone[4] = 18; cone[5] = 17; cone[6] = 24; cone[7] = 27;
1516         ierr = DMPlexSetCone(dm, 12, cone);CHKERRQ(ierr);
1517         cone[0] = 30; cone[1] = 28; cone[2] = 20; cone[3] = 19;
1518         cone[4] = 29; cone[5] = 15; cone[6] = 18; cone[7] = 27;
1519         ierr = DMPlexSetCone(dm, 13, cone);CHKERRQ(ierr);
1520         cone[0] = 30; cone[1] = 19; cone[2] = 22; cone[3] = 25;
1521         cone[4] = 29; cone[5] = 23; cone[6] = 16; cone[7] = 15;
1522         ierr = DMPlexSetCone(dm, 14, cone);CHKERRQ(ierr);
1523       } else {
1524         cone[0] =  5; cone[1] =  8; cone[2] =  7; cone[3] =  6;
1525         cone[4] =  9; cone[5] = 12; cone[6] = 11; cone[7] = 10;
1526         ierr = DMPlexSetCone(dm, 0, cone);CHKERRQ(ierr);
1527         cone[0] =  6; cone[1] =  7; cone[2] = 14; cone[3] = 13;
1528         cone[4] = 12; cone[5] = 15; cone[6] = 16; cone[7] = 11;
1529         ierr = DMPlexSetCone(dm, 1, cone);CHKERRQ(ierr);
1530         cone[0] =  8; cone[1] = 17; cone[2] = 14; cone[3] =  7;
1531         cone[4] = 10; cone[5] = 11; cone[6] = 16; cone[7] = 18;
1532         ierr = DMPlexSetCone(dm, 2, cone);CHKERRQ(ierr);
1533         cone[0] = 19; cone[1] = 17; cone[2] =  8; cone[3] =  5;
1534         cone[4] = 20; cone[5] =  9; cone[6] = 10; cone[7] = 18;
1535         ierr = DMPlexSetCone(dm, 3, cone);CHKERRQ(ierr);
1536         cone[0] = 19; cone[1] =  5; cone[2] =  6; cone[3] = 13;
1537         cone[4] = 20; cone[5] = 15; cone[6] = 12; cone[7] =  9;
1538         ierr = DMPlexSetCone(dm, 4, cone);CHKERRQ(ierr);
1539       }
1540     }
1541     ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
1542     ierr = DMPlexStratify(dm);CHKERRQ(ierr);
1543   }
1544   /* Create cube geometry */
1545   {
1546     Vec             coordinates;
1547     PetscSection    coordSection;
1548     PetscScalar    *coords;
1549     PetscInt        coordSize, v;
1550     const PetscReal dis = 1.0/PetscSqrtReal(2.0);
1551     const PetscReal ds2 = dis/2.0;
1552 
1553     /* Build coordinates */
1554     ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
1555     ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
1556     ierr = PetscSectionSetFieldComponents(coordSection, 0, dim);CHKERRQ(ierr);
1557     ierr = PetscSectionSetChart(coordSection, numCells, numCells+numVertices);CHKERRQ(ierr);
1558     for (v = numCells; v < numCells+numVertices; ++v) {
1559       ierr = PetscSectionSetDof(coordSection, v, dim);CHKERRQ(ierr);
1560       ierr = PetscSectionSetFieldDof(coordSection, v, 0, dim);CHKERRQ(ierr);
1561     }
1562     ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
1563     ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
1564     ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
1565     ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
1566     ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
1567     ierr = VecSetBlockSize(coordinates, dim);CHKERRQ(ierr);
1568     ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
1569     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
1570     if (rank == 0) {
1571       coords[0*dim+0] = -ds2; coords[0*dim+1] = -ds2; coords[0*dim+2] = 0.0;
1572       coords[1*dim+0] =  ds2; coords[1*dim+1] = -ds2; coords[1*dim+2] = 0.0;
1573       coords[2*dim+0] =  ds2; coords[2*dim+1] =  ds2; coords[2*dim+2] = 0.0;
1574       coords[3*dim+0] = -ds2; coords[3*dim+1] =  ds2; coords[3*dim+2] = 0.0;
1575       coords[4*dim+0] = -ds2; coords[4*dim+1] = -ds2; coords[4*dim+2] = 1.0;
1576       coords[5*dim+0] = -ds2; coords[5*dim+1] =  ds2; coords[5*dim+2] = 1.0;
1577       coords[6*dim+0] =  ds2; coords[6*dim+1] =  ds2; coords[6*dim+2] = 1.0;
1578       coords[7*dim+0] =  ds2; coords[7*dim+1] = -ds2; coords[7*dim+2] = 1.0;
1579       coords[ 8*dim+0] =  dis; coords[ 8*dim+1] = -dis; coords[ 8*dim+2] = 0.0;
1580       coords[ 9*dim+0] =  dis; coords[ 9*dim+1] =  dis; coords[ 9*dim+2] = 0.0;
1581       coords[10*dim+0] =  dis; coords[10*dim+1] = -dis; coords[10*dim+2] = 1.0;
1582       coords[11*dim+0] =  dis; coords[11*dim+1] =  dis; coords[11*dim+2] = 1.0;
1583       coords[12*dim+0] = -dis; coords[12*dim+1] =  dis; coords[12*dim+2] = 0.0;
1584       coords[13*dim+0] = -dis; coords[13*dim+1] =  dis; coords[13*dim+2] = 1.0;
1585       coords[14*dim+0] = -dis; coords[14*dim+1] = -dis; coords[14*dim+2] = 0.0;
1586       coords[15*dim+0] = -dis; coords[15*dim+1] = -dis; coords[15*dim+2] = 1.0;
1587       if (periodicZ == DM_BOUNDARY_PERIODIC) {
1588         /* 15 31 19 */ coords[16*dim+0] = -ds2; coords[16*dim+1] = -ds2; coords[16*dim+2] = 0.5;
1589         /* 16 32 22 */ coords[17*dim+0] =  ds2; coords[17*dim+1] = -ds2; coords[17*dim+2] = 0.5;
1590         /* 17 33 21 */ coords[18*dim+0] =  ds2; coords[18*dim+1] =  ds2; coords[18*dim+2] = 0.5;
1591         /* 18 34 20 */ coords[19*dim+0] = -ds2; coords[19*dim+1] =  ds2; coords[19*dim+2] = 0.5;
1592         /* 29 35 30 */ coords[20*dim+0] = -dis; coords[20*dim+1] = -dis; coords[20*dim+2] = 0.5;
1593         /* 23 36 25 */ coords[21*dim+0] =  dis; coords[21*dim+1] = -dis; coords[21*dim+2] = 0.5;
1594         /* 24 37 26 */ coords[22*dim+0] =  dis; coords[22*dim+1] =  dis; coords[22*dim+2] = 0.5;
1595         /* 27 38 28 */ coords[23*dim+0] = -dis; coords[23*dim+1] =  dis; coords[23*dim+2] = 0.5;
1596       }
1597     }
1598     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
1599     ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
1600     ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
1601   }
1602   /* Create periodicity */
1603   if (periodicZ == DM_BOUNDARY_PERIODIC || periodicZ == DM_BOUNDARY_TWIST) {
1604     PetscReal      L[3];
1605     PetscReal      maxCell[3];
1606     DMBoundaryType bdType[3];
1607     PetscReal      lower[3] = {0.0, 0.0, 0.0};
1608     PetscReal      upper[3] = {1.0, 1.0, 1.5};
1609     PetscInt       i, numZCells = 3;
1610 
1611     bdType[0] = DM_BOUNDARY_NONE;
1612     bdType[1] = DM_BOUNDARY_NONE;
1613     bdType[2] = periodicZ;
1614     for (i = 0; i < dim; i++) {
1615       L[i]       = upper[i] - lower[i];
1616       maxCell[i] = 1.1 * (L[i] / numZCells);
1617     }
1618     ierr = DMSetPeriodicity(dm, PETSC_TRUE, maxCell, L, bdType);CHKERRQ(ierr);
1619   }
1620   {
1621     DM          cdm;
1622     PetscDS     cds;
1623     PetscScalar c[2] = {1.0, 1.0};
1624 
1625     ierr = DMPlexCreateCoordinateSpace(dm, 1, snapToCylinder);CHKERRQ(ierr);
1626     ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
1627     ierr = DMGetDS(cdm, &cds);CHKERRQ(ierr);
1628     ierr = PetscDSSetConstants(cds, 2, c);CHKERRQ(ierr);
1629   }
1630   /* Wait for coordinate creation before doing in-place modification */
1631   ierr = DMPlexInterpolateInPlace_Internal(dm);CHKERRQ(ierr);
1632   PetscFunctionReturn(0);
1633 }
1634 
1635 /*@
1636   DMPlexCreateHexCylinderMesh - Creates a mesh on the tensor product of the unit interval with the circle (cylinder) using hexahedra.
1637 
1638   Collective
1639 
1640   Input Parameters:
1641 + comm      - The communicator for the DM object
1642 - periodicZ - The boundary type for the Z direction
1643 
1644   Output Parameter:
1645 . dm  - The DM object
1646 
1647   Note:
1648   Here is the output numbering looking from the bottom of the cylinder:
1649 $       17-----14
1650 $        |     |
1651 $        |  2  |
1652 $        |     |
1653 $ 17-----8-----7-----14
1654 $  |     |     |     |
1655 $  |  3  |  0  |  1  |
1656 $  |     |     |     |
1657 $ 19-----5-----6-----13
1658 $        |     |
1659 $        |  4  |
1660 $        |     |
1661 $       19-----13
1662 $
1663 $ and up through the top
1664 $
1665 $       18-----16
1666 $        |     |
1667 $        |  2  |
1668 $        |     |
1669 $ 18----10----11-----16
1670 $  |     |     |     |
1671 $  |  3  |  0  |  1  |
1672 $  |     |     |     |
1673 $ 20-----9----12-----15
1674 $        |     |
1675 $        |  4  |
1676 $        |     |
1677 $       20-----15
1678 
1679   Level: beginner
1680 
1681 .seealso: DMPlexCreateBoxMesh(), DMSetType(), DMCreate()
1682 @*/
1683 PetscErrorCode DMPlexCreateHexCylinderMesh(MPI_Comm comm, DMBoundaryType periodicZ, DM *dm)
1684 {
1685   PetscErrorCode ierr;
1686 
1687   PetscFunctionBegin;
1688   PetscValidPointer(dm, 3);
1689   ierr = DMCreate(comm, dm);CHKERRQ(ierr);
1690   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
1691   ierr = DMPlexCreateHexCylinderMesh_Internal(*dm, periodicZ);CHKERRQ(ierr);
1692   PetscFunctionReturn(0);
1693 }
1694 
1695 static PetscErrorCode DMPlexCreateWedgeCylinderMesh_Internal(DM dm, PetscInt n, PetscBool interpolate)
1696 {
1697   const PetscInt dim = 3;
1698   PetscInt       numCells, numVertices, v;
1699   PetscMPIInt    rank;
1700   PetscErrorCode ierr;
1701 
1702   PetscFunctionBegin;
1703   PetscCheckFalse(n < 0,PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of wedges %D cannot be negative", n);
1704   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRMPI(ierr);
1705   ierr = DMSetDimension(dm, dim);CHKERRQ(ierr);
1706   /* Must create the celltype label here so that we do not automatically try to compute the types */
1707   ierr = DMCreateLabel(dm, "celltype");CHKERRQ(ierr);
1708   /* Create topology */
1709   {
1710     PetscInt cone[6], c;
1711 
1712     numCells    = rank == 0 ?        n : 0;
1713     numVertices = rank == 0 ?  2*(n+1) : 0;
1714     ierr = DMPlexSetChart(dm, 0, numCells+numVertices);CHKERRQ(ierr);
1715     for (c = 0; c < numCells; c++) {ierr = DMPlexSetConeSize(dm, c, 6);CHKERRQ(ierr);}
1716     ierr = DMSetUp(dm);CHKERRQ(ierr);
1717     for (c = 0; c < numCells; c++) {
1718       cone[0] =  c+n*1; cone[1] = (c+1)%n+n*1; cone[2] = 0+3*n;
1719       cone[3] =  c+n*2; cone[4] = (c+1)%n+n*2; cone[5] = 1+3*n;
1720       ierr = DMPlexSetCone(dm, c, cone);CHKERRQ(ierr);
1721       ierr = DMPlexSetCellType(dm, c, DM_POLYTOPE_TRI_PRISM_TENSOR);CHKERRQ(ierr);
1722     }
1723     ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
1724     ierr = DMPlexStratify(dm);CHKERRQ(ierr);
1725   }
1726   for (v = numCells; v < numCells+numVertices; ++v) {
1727     ierr = DMPlexSetCellType(dm, v, DM_POLYTOPE_POINT);CHKERRQ(ierr);
1728   }
1729   /* Create cylinder geometry */
1730   {
1731     Vec          coordinates;
1732     PetscSection coordSection;
1733     PetscScalar *coords;
1734     PetscInt     coordSize, c;
1735 
1736     /* Build coordinates */
1737     ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
1738     ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
1739     ierr = PetscSectionSetFieldComponents(coordSection, 0, dim);CHKERRQ(ierr);
1740     ierr = PetscSectionSetChart(coordSection, numCells, numCells+numVertices);CHKERRQ(ierr);
1741     for (v = numCells; v < numCells+numVertices; ++v) {
1742       ierr = PetscSectionSetDof(coordSection, v, dim);CHKERRQ(ierr);
1743       ierr = PetscSectionSetFieldDof(coordSection, v, 0, dim);CHKERRQ(ierr);
1744     }
1745     ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
1746     ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
1747     ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
1748     ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
1749     ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
1750     ierr = VecSetBlockSize(coordinates, dim);CHKERRQ(ierr);
1751     ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
1752     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
1753     for (c = 0; c < numCells; c++) {
1754       coords[(c+0*n)*dim+0] = PetscCosReal(2.0*c*PETSC_PI/n); coords[(c+0*n)*dim+1] = PetscSinReal(2.0*c*PETSC_PI/n); coords[(c+0*n)*dim+2] = 1.0;
1755       coords[(c+1*n)*dim+0] = PetscCosReal(2.0*c*PETSC_PI/n); coords[(c+1*n)*dim+1] = PetscSinReal(2.0*c*PETSC_PI/n); coords[(c+1*n)*dim+2] = 0.0;
1756     }
1757     if (rank == 0) {
1758       coords[(2*n+0)*dim+0] = 0.0; coords[(2*n+0)*dim+1] = 0.0; coords[(2*n+0)*dim+2] = 1.0;
1759       coords[(2*n+1)*dim+0] = 0.0; coords[(2*n+1)*dim+1] = 0.0; coords[(2*n+1)*dim+2] = 0.0;
1760     }
1761     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
1762     ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
1763     ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
1764   }
1765   /* Interpolate */
1766   if (interpolate) {ierr = DMPlexInterpolateInPlace_Internal(dm);CHKERRQ(ierr);}
1767   PetscFunctionReturn(0);
1768 }
1769 
1770 /*@
1771   DMPlexCreateWedgeCylinderMesh - Creates a mesh on the tensor product of the unit interval with the circle (cylinder) using wedges.
1772 
1773   Collective
1774 
1775   Input Parameters:
1776 + comm - The communicator for the DM object
1777 . n    - The number of wedges around the origin
1778 - interpolate - Create edges and faces
1779 
1780   Output Parameter:
1781 . dm  - The DM object
1782 
1783   Level: beginner
1784 
1785 .seealso: DMPlexCreateHexCylinderMesh(), DMPlexCreateBoxMesh(), DMSetType(), DMCreate()
1786 @*/
1787 PetscErrorCode DMPlexCreateWedgeCylinderMesh(MPI_Comm comm, PetscInt n, PetscBool interpolate, DM *dm)
1788 {
1789   PetscErrorCode ierr;
1790 
1791   PetscFunctionBegin;
1792   PetscValidPointer(dm, 4);
1793   ierr = DMCreate(comm, dm);CHKERRQ(ierr);
1794   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
1795   ierr = DMPlexCreateWedgeCylinderMesh_Internal(*dm, n, interpolate);CHKERRQ(ierr);
1796   PetscFunctionReturn(0);
1797 }
1798 
1799 static inline PetscReal DiffNormReal(PetscInt dim, const PetscReal x[], const PetscReal y[])
1800 {
1801   PetscReal prod = 0.0;
1802   PetscInt  i;
1803   for (i = 0; i < dim; ++i) prod += PetscSqr(x[i] - y[i]);
1804   return PetscSqrtReal(prod);
1805 }
1806 static inline PetscReal DotReal(PetscInt dim, const PetscReal x[], const PetscReal y[])
1807 {
1808   PetscReal prod = 0.0;
1809   PetscInt  i;
1810   for (i = 0; i < dim; ++i) prod += x[i]*y[i];
1811   return prod;
1812 }
1813 
1814 /* The first constant is the sphere radius */
1815 static void snapToSphere(PetscInt dim, PetscInt Nf, PetscInt NfAux,
1816                          const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[],
1817                          const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[],
1818                          PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
1819 {
1820   PetscReal r = PetscRealPart(constants[0]);
1821   PetscReal norm2 = 0.0, fac;
1822   PetscInt  n = uOff[1] - uOff[0], d;
1823 
1824   for (d = 0; d < n; ++d) norm2 += PetscSqr(PetscRealPart(u[d]));
1825   fac = r/PetscSqrtReal(norm2);
1826   for (d = 0; d < n; ++d) f0[d] = u[d]*fac;
1827 }
1828 
1829 static PetscErrorCode DMPlexCreateSphereMesh_Internal(DM dm, PetscInt dim, PetscBool simplex, PetscReal R)
1830 {
1831   const PetscInt  embedDim = dim+1;
1832   PetscSection    coordSection;
1833   Vec             coordinates;
1834   PetscScalar    *coords;
1835   PetscReal      *coordsIn;
1836   PetscInt        numCells, numEdges, numVerts, firstVertex, v, firstEdge, coordSize, d, c, e;
1837   PetscMPIInt     rank;
1838   PetscErrorCode  ierr;
1839 
1840   PetscFunctionBegin;
1841   PetscValidLogicalCollectiveBool(dm, simplex, 3);
1842   ierr = DMSetDimension(dm, dim);CHKERRQ(ierr);
1843   ierr = DMSetCoordinateDim(dm, dim+1);CHKERRQ(ierr);
1844   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRMPI(ierr);
1845   switch (dim) {
1846   case 2:
1847     if (simplex) {
1848       const PetscReal radius    = PetscSqrtReal(1 + PETSC_PHI*PETSC_PHI)/(1.0 + PETSC_PHI);
1849       const PetscReal edgeLen   = 2.0/(1.0 + PETSC_PHI) * (R/radius);
1850       const PetscInt  degree    = 5;
1851       PetscReal       vertex[3] = {0.0, 1.0/(1.0 + PETSC_PHI), PETSC_PHI/(1.0 + PETSC_PHI)};
1852       PetscInt        s[3]      = {1, 1, 1};
1853       PetscInt        cone[3];
1854       PetscInt       *graph, p, i, j, k;
1855 
1856       vertex[0] *= R/radius; vertex[1] *= R/radius; vertex[2] *= R/radius;
1857       numCells    = rank == 0 ? 20 : 0;
1858       numVerts    = rank == 0 ? 12 : 0;
1859       firstVertex = numCells;
1860       /* Use icosahedron, which for a R-sphere has coordinates which are all cyclic permutations of
1861 
1862            (0, \pm 1/\phi+1, \pm \phi/\phi+1)
1863 
1864          where \phi^2 - \phi - 1 = 0, meaning \phi is the golden ratio \frac{1 + \sqrt{5}}{2}. The edge
1865          length is then given by 2/(1+\phi) = 2 * 0.38197 = 0.76393.
1866       */
1867       /* Construct vertices */
1868       ierr = PetscCalloc1(numVerts * embedDim, &coordsIn);CHKERRQ(ierr);
1869       if (rank == 0) {
1870         for (p = 0, i = 0; p < embedDim; ++p) {
1871           for (s[1] = -1; s[1] < 2; s[1] += 2) {
1872             for (s[2] = -1; s[2] < 2; s[2] += 2) {
1873               for (d = 0; d < embedDim; ++d) coordsIn[i*embedDim+d] = s[(d+p)%embedDim]*vertex[(d+p)%embedDim];
1874               ++i;
1875             }
1876           }
1877         }
1878       }
1879       /* Construct graph */
1880       ierr = PetscCalloc1(numVerts * numVerts, &graph);CHKERRQ(ierr);
1881       for (i = 0; i < numVerts; ++i) {
1882         for (j = 0, k = 0; j < numVerts; ++j) {
1883           if (PetscAbsReal(DiffNormReal(embedDim, &coordsIn[i*embedDim], &coordsIn[j*embedDim]) - edgeLen) < PETSC_SMALL) {graph[i*numVerts+j] = 1; ++k;}
1884         }
1885         PetscCheckFalse(k != degree,PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Invalid icosahedron, vertex %D degree %D != %D", i, k, degree);
1886       }
1887       /* Build Topology */
1888       ierr = DMPlexSetChart(dm, 0, numCells+numVerts);CHKERRQ(ierr);
1889       for (c = 0; c < numCells; c++) {
1890         ierr = DMPlexSetConeSize(dm, c, embedDim);CHKERRQ(ierr);
1891       }
1892       ierr = DMSetUp(dm);CHKERRQ(ierr); /* Allocate space for cones */
1893       /* Cells */
1894       for (i = 0, c = 0; i < numVerts; ++i) {
1895         for (j = 0; j < i; ++j) {
1896           for (k = 0; k < j; ++k) {
1897             if (graph[i*numVerts+j] && graph[j*numVerts+k] && graph[k*numVerts+i]) {
1898               cone[0] = firstVertex+i; cone[1] = firstVertex+j; cone[2] = firstVertex+k;
1899               /* Check orientation */
1900               {
1901                 const PetscInt epsilon[3][3][3] = {{{0, 0, 0}, {0, 0, 1}, {0, -1, 0}}, {{0, 0, -1}, {0, 0, 0}, {1, 0, 0}}, {{0, 1, 0}, {-1, 0, 0}, {0, 0, 0}}};
1902                 PetscReal normal[3];
1903                 PetscInt  e, f;
1904 
1905                 for (d = 0; d < embedDim; ++d) {
1906                   normal[d] = 0.0;
1907                   for (e = 0; e < embedDim; ++e) {
1908                     for (f = 0; f < embedDim; ++f) {
1909                       normal[d] += epsilon[d][e][f]*(coordsIn[j*embedDim+e] - coordsIn[i*embedDim+e])*(coordsIn[k*embedDim+f] - coordsIn[i*embedDim+f]);
1910                     }
1911                   }
1912                 }
1913                 if (DotReal(embedDim, normal, &coordsIn[i*embedDim]) < 0) {PetscInt tmp = cone[1]; cone[1] = cone[2]; cone[2] = tmp;}
1914               }
1915               ierr = DMPlexSetCone(dm, c++, cone);CHKERRQ(ierr);
1916             }
1917           }
1918         }
1919       }
1920       ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
1921       ierr = DMPlexStratify(dm);CHKERRQ(ierr);
1922       ierr = PetscFree(graph);CHKERRQ(ierr);
1923     } else {
1924       /*
1925         12-21--13
1926          |     |
1927         25  4  24
1928          |     |
1929   12-25--9-16--8-24--13
1930    |     |     |     |
1931   23  5 17  0 15  3  22
1932    |     |     |     |
1933   10-20--6-14--7-19--11
1934          |     |
1935         20  1  19
1936          |     |
1937         10-18--11
1938          |     |
1939         23  2  22
1940          |     |
1941         12-21--13
1942        */
1943       PetscInt cone[4], ornt[4];
1944 
1945       numCells    = rank == 0 ?  6 : 0;
1946       numEdges    = rank == 0 ? 12 : 0;
1947       numVerts    = rank == 0 ?  8 : 0;
1948       firstVertex = numCells;
1949       firstEdge   = numCells + numVerts;
1950       /* Build Topology */
1951       ierr = DMPlexSetChart(dm, 0, numCells+numEdges+numVerts);CHKERRQ(ierr);
1952       for (c = 0; c < numCells; c++) {
1953         ierr = DMPlexSetConeSize(dm, c, 4);CHKERRQ(ierr);
1954       }
1955       for (e = firstEdge; e < firstEdge+numEdges; ++e) {
1956         ierr = DMPlexSetConeSize(dm, e, 2);CHKERRQ(ierr);
1957       }
1958       ierr = DMSetUp(dm);CHKERRQ(ierr); /* Allocate space for cones */
1959       if (rank == 0) {
1960         /* Cell 0 */
1961         cone[0] = 14; cone[1] = 15; cone[2] = 16; cone[3] = 17;
1962         ierr = DMPlexSetCone(dm, 0, cone);CHKERRQ(ierr);
1963         ornt[0] = 0; ornt[1] = 0; ornt[2] = 0; ornt[3] = 0;
1964         ierr = DMPlexSetConeOrientation(dm, 0, ornt);CHKERRQ(ierr);
1965         /* Cell 1 */
1966         cone[0] = 18; cone[1] = 19; cone[2] = 14; cone[3] = 20;
1967         ierr = DMPlexSetCone(dm, 1, cone);CHKERRQ(ierr);
1968         ornt[0] = 0; ornt[1] = 0; ornt[2] = -1; ornt[3] = 0;
1969         ierr = DMPlexSetConeOrientation(dm, 1, ornt);CHKERRQ(ierr);
1970         /* Cell 2 */
1971         cone[0] = 21; cone[1] = 22; cone[2] = 18; cone[3] = 23;
1972         ierr = DMPlexSetCone(dm, 2, cone);CHKERRQ(ierr);
1973         ornt[0] = 0; ornt[1] = 0; ornt[2] = -1; ornt[3] = 0;
1974         ierr = DMPlexSetConeOrientation(dm, 2, ornt);CHKERRQ(ierr);
1975         /* Cell 3 */
1976         cone[0] = 19; cone[1] = 22; cone[2] = 24; cone[3] = 15;
1977         ierr = DMPlexSetCone(dm, 3, cone);CHKERRQ(ierr);
1978         ornt[0] = -1; ornt[1] = -1; ornt[2] = 0; ornt[3] = -1;
1979         ierr = DMPlexSetConeOrientation(dm, 3, ornt);CHKERRQ(ierr);
1980         /* Cell 4 */
1981         cone[0] = 16; cone[1] = 24; cone[2] = 21; cone[3] = 25;
1982         ierr = DMPlexSetCone(dm, 4, cone);CHKERRQ(ierr);
1983         ornt[0] = -1; ornt[1] = -1; ornt[2] = -1; ornt[3] = 0;
1984         ierr = DMPlexSetConeOrientation(dm, 4, ornt);CHKERRQ(ierr);
1985         /* Cell 5 */
1986         cone[0] = 20; cone[1] = 17; cone[2] = 25; cone[3] = 23;
1987         ierr = DMPlexSetCone(dm, 5, cone);CHKERRQ(ierr);
1988         ornt[0] = -1; ornt[1] = -1; ornt[2] = -1; ornt[3] = -1;
1989         ierr = DMPlexSetConeOrientation(dm, 5, ornt);CHKERRQ(ierr);
1990         /* Edges */
1991         cone[0] =  6; cone[1] =  7;
1992         ierr = DMPlexSetCone(dm, 14, cone);CHKERRQ(ierr);
1993         cone[0] =  7; cone[1] =  8;
1994         ierr = DMPlexSetCone(dm, 15, cone);CHKERRQ(ierr);
1995         cone[0] =  8; cone[1] =  9;
1996         ierr = DMPlexSetCone(dm, 16, cone);CHKERRQ(ierr);
1997         cone[0] =  9; cone[1] =  6;
1998         ierr = DMPlexSetCone(dm, 17, cone);CHKERRQ(ierr);
1999         cone[0] = 10; cone[1] = 11;
2000         ierr = DMPlexSetCone(dm, 18, cone);CHKERRQ(ierr);
2001         cone[0] = 11; cone[1] =  7;
2002         ierr = DMPlexSetCone(dm, 19, cone);CHKERRQ(ierr);
2003         cone[0] =  6; cone[1] = 10;
2004         ierr = DMPlexSetCone(dm, 20, cone);CHKERRQ(ierr);
2005         cone[0] = 12; cone[1] = 13;
2006         ierr = DMPlexSetCone(dm, 21, cone);CHKERRQ(ierr);
2007         cone[0] = 13; cone[1] = 11;
2008         ierr = DMPlexSetCone(dm, 22, cone);CHKERRQ(ierr);
2009         cone[0] = 10; cone[1] = 12;
2010         ierr = DMPlexSetCone(dm, 23, cone);CHKERRQ(ierr);
2011         cone[0] = 13; cone[1] =  8;
2012         ierr = DMPlexSetCone(dm, 24, cone);CHKERRQ(ierr);
2013         cone[0] = 12; cone[1] =  9;
2014         ierr = DMPlexSetCone(dm, 25, cone);CHKERRQ(ierr);
2015       }
2016       ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
2017       ierr = DMPlexStratify(dm);CHKERRQ(ierr);
2018       /* Build coordinates */
2019       ierr = PetscCalloc1(numVerts * embedDim, &coordsIn);CHKERRQ(ierr);
2020       if (rank == 0) {
2021         coordsIn[0*embedDim+0] = -R; coordsIn[0*embedDim+1] =  R; coordsIn[0*embedDim+2] = -R;
2022         coordsIn[1*embedDim+0] =  R; coordsIn[1*embedDim+1] =  R; coordsIn[1*embedDim+2] = -R;
2023         coordsIn[2*embedDim+0] =  R; coordsIn[2*embedDim+1] = -R; coordsIn[2*embedDim+2] = -R;
2024         coordsIn[3*embedDim+0] = -R; coordsIn[3*embedDim+1] = -R; coordsIn[3*embedDim+2] = -R;
2025         coordsIn[4*embedDim+0] = -R; coordsIn[4*embedDim+1] =  R; coordsIn[4*embedDim+2] =  R;
2026         coordsIn[5*embedDim+0] =  R; coordsIn[5*embedDim+1] =  R; coordsIn[5*embedDim+2] =  R;
2027         coordsIn[6*embedDim+0] = -R; coordsIn[6*embedDim+1] = -R; coordsIn[6*embedDim+2] =  R;
2028         coordsIn[7*embedDim+0] =  R; coordsIn[7*embedDim+1] = -R; coordsIn[7*embedDim+2] =  R;
2029       }
2030     }
2031     break;
2032   case 3:
2033     if (simplex) {
2034       const PetscReal edgeLen         = 1.0/PETSC_PHI;
2035       PetscReal       vertexA[4]      = {0.5, 0.5, 0.5, 0.5};
2036       PetscReal       vertexB[4]      = {1.0, 0.0, 0.0, 0.0};
2037       PetscReal       vertexC[4]      = {0.5, 0.5*PETSC_PHI, 0.5/PETSC_PHI, 0.0};
2038       const PetscInt  degree          = 12;
2039       PetscInt        s[4]            = {1, 1, 1};
2040       PetscInt        evenPerm[12][4] = {{0, 1, 2, 3}, {0, 2, 3, 1}, {0, 3, 1, 2}, {1, 0, 3, 2}, {1, 2, 0, 3}, {1, 3, 2, 0},
2041                                          {2, 0, 1, 3}, {2, 1, 3, 0}, {2, 3, 0, 1}, {3, 0, 2, 1}, {3, 1, 0, 2}, {3, 2, 1, 0}};
2042       PetscInt        cone[4];
2043       PetscInt       *graph, p, i, j, k, l;
2044 
2045       vertexA[0] *= R; vertexA[1] *= R; vertexA[2] *= R; vertexA[3] *= R;
2046       vertexB[0] *= R; vertexB[1] *= R; vertexB[2] *= R; vertexB[3] *= R;
2047       vertexC[0] *= R; vertexC[1] *= R; vertexC[2] *= R; vertexC[3] *= R;
2048       numCells    = rank == 0 ? 600 : 0;
2049       numVerts    = rank == 0 ? 120 : 0;
2050       firstVertex = numCells;
2051       /* Use the 600-cell, which for a unit sphere has coordinates which are
2052 
2053            1/2 (\pm 1, \pm 1,    \pm 1, \pm 1)                          16
2054                (\pm 1,    0,       0,      0)  all cyclic permutations   8
2055            1/2 (\pm 1, \pm phi, \pm 1/phi, 0)  all even permutations    96
2056 
2057          where \phi^2 - \phi - 1 = 0, meaning \phi is the golden ratio \frac{1 + \sqrt{5}}{2}. The edge
2058          length is then given by 1/\phi = 0.61803.
2059 
2060          http://buzzard.pugetsound.edu/sage-practice/ch03s03.html
2061          http://mathworld.wolfram.com/600-Cell.html
2062       */
2063       /* Construct vertices */
2064       ierr = PetscCalloc1(numVerts * embedDim, &coordsIn);CHKERRQ(ierr);
2065       i    = 0;
2066       if (rank == 0) {
2067         for (s[0] = -1; s[0] < 2; s[0] += 2) {
2068           for (s[1] = -1; s[1] < 2; s[1] += 2) {
2069             for (s[2] = -1; s[2] < 2; s[2] += 2) {
2070               for (s[3] = -1; s[3] < 2; s[3] += 2) {
2071                 for (d = 0; d < embedDim; ++d) coordsIn[i*embedDim+d] = s[d]*vertexA[d];
2072                 ++i;
2073               }
2074             }
2075           }
2076         }
2077         for (p = 0; p < embedDim; ++p) {
2078           s[1] = s[2] = s[3] = 1;
2079           for (s[0] = -1; s[0] < 2; s[0] += 2) {
2080             for (d = 0; d < embedDim; ++d) coordsIn[i*embedDim+d] = s[(d+p)%embedDim]*vertexB[(d+p)%embedDim];
2081             ++i;
2082           }
2083         }
2084         for (p = 0; p < 12; ++p) {
2085           s[3] = 1;
2086           for (s[0] = -1; s[0] < 2; s[0] += 2) {
2087             for (s[1] = -1; s[1] < 2; s[1] += 2) {
2088               for (s[2] = -1; s[2] < 2; s[2] += 2) {
2089                 for (d = 0; d < embedDim; ++d) coordsIn[i*embedDim+d] = s[evenPerm[p][d]]*vertexC[evenPerm[p][d]];
2090                 ++i;
2091               }
2092             }
2093           }
2094         }
2095       }
2096       PetscCheckFalse(i != numVerts,PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Invalid 600-cell, vertices %D != %D", i, numVerts);
2097       /* Construct graph */
2098       ierr = PetscCalloc1(numVerts * numVerts, &graph);CHKERRQ(ierr);
2099       for (i = 0; i < numVerts; ++i) {
2100         for (j = 0, k = 0; j < numVerts; ++j) {
2101           if (PetscAbsReal(DiffNormReal(embedDim, &coordsIn[i*embedDim], &coordsIn[j*embedDim]) - edgeLen) < PETSC_SMALL) {graph[i*numVerts+j] = 1; ++k;}
2102         }
2103         PetscCheckFalse(k != degree,PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Invalid 600-cell, vertex %D degree %D != %D", i, k, degree);
2104       }
2105       /* Build Topology */
2106       ierr = DMPlexSetChart(dm, 0, numCells+numVerts);CHKERRQ(ierr);
2107       for (c = 0; c < numCells; c++) {
2108         ierr = DMPlexSetConeSize(dm, c, embedDim);CHKERRQ(ierr);
2109       }
2110       ierr = DMSetUp(dm);CHKERRQ(ierr); /* Allocate space for cones */
2111       /* Cells */
2112       if (rank == 0) {
2113         for (i = 0, c = 0; i < numVerts; ++i) {
2114           for (j = 0; j < i; ++j) {
2115             for (k = 0; k < j; ++k) {
2116               for (l = 0; l < k; ++l) {
2117                 if (graph[i*numVerts+j] && graph[j*numVerts+k] && graph[k*numVerts+i] &&
2118                     graph[l*numVerts+i] && graph[l*numVerts+j] && graph[l*numVerts+k]) {
2119                   cone[0] = firstVertex+i; cone[1] = firstVertex+j; cone[2] = firstVertex+k; cone[3] = firstVertex+l;
2120                   /* Check orientation: https://ef.gy/linear-algebra:normal-vectors-in-higher-dimensional-spaces */
2121                   {
2122                     const PetscInt epsilon[4][4][4][4] = {{{{0,  0,  0,  0}, { 0, 0,  0,  0}, { 0,  0, 0,  0}, { 0,  0,  0, 0}},
2123                                                            {{0,  0,  0,  0}, { 0, 0,  0,  0}, { 0,  0, 0,  1}, { 0,  0, -1, 0}},
2124                                                            {{0,  0,  0,  0}, { 0, 0,  0, -1}, { 0,  0, 0,  0}, { 0,  1,  0, 0}},
2125                                                            {{0,  0,  0,  0}, { 0, 0,  1,  0}, { 0, -1, 0,  0}, { 0,  0,  0, 0}}},
2126 
2127                                                           {{{0,  0,  0,  0}, { 0, 0,  0,  0}, { 0,  0, 0, -1}, { 0,  0,  1, 0}},
2128                                                            {{0,  0,  0,  0}, { 0, 0,  0,  0}, { 0,  0, 0,  0}, { 0,  0,  0, 0}},
2129                                                            {{0,  0,  0,  1}, { 0, 0,  0,  0}, { 0,  0, 0,  0}, {-1,  0,  0, 0}},
2130                                                            {{0,  0, -1,  0}, { 0, 0,  0,  0}, { 1,  0, 0,  0}, { 0,  0,  0, 0}}},
2131 
2132                                                           {{{0,  0,  0,  0}, { 0, 0,  0,  1}, { 0,  0, 0,  0}, { 0, -1,  0, 0}},
2133                                                            {{0,  0,  0, -1}, { 0, 0,  0,  0}, { 0,  0, 0,  0}, { 1,  0,  0, 0}},
2134                                                            {{0,  0,  0,  0}, { 0, 0,  0,  0}, { 0,  0, 0,  0}, { 0,  0,  0, 0}},
2135                                                            {{0,  1,  0,  0}, {-1, 0,  0,  0}, { 0,  0, 0,  0}, { 0,  0,  0, 0}}},
2136 
2137                                                           {{{0,  0,  0,  0}, { 0, 0, -1,  0}, { 0,  1, 0,  0}, { 0,  0,  0, 0}},
2138                                                            {{0,  0,  1,  0}, { 0, 0,  0,  0}, {-1,  0, 0,  0}, { 0,  0,  0, 0}},
2139                                                            {{0, -1,  0,  0}, { 1, 0,  0,  0}, { 0,  0, 0,  0}, { 0,  0,  0, 0}},
2140                                                            {{0,  0,  0,  0}, { 0, 0,  0,  0}, { 0,  0, 0,  0}, { 0,  0,  0, 0}}}};
2141                     PetscReal normal[4];
2142                     PetscInt  e, f, g;
2143 
2144                     for (d = 0; d < embedDim; ++d) {
2145                       normal[d] = 0.0;
2146                       for (e = 0; e < embedDim; ++e) {
2147                         for (f = 0; f < embedDim; ++f) {
2148                           for (g = 0; g < embedDim; ++g) {
2149                             normal[d] += epsilon[d][e][f][g]*(coordsIn[j*embedDim+e] - coordsIn[i*embedDim+e])*(coordsIn[k*embedDim+f] - coordsIn[i*embedDim+f])*(coordsIn[l*embedDim+f] - coordsIn[i*embedDim+f]);
2150                           }
2151                         }
2152                       }
2153                     }
2154                     if (DotReal(embedDim, normal, &coordsIn[i*embedDim]) < 0) {PetscInt tmp = cone[1]; cone[1] = cone[2]; cone[2] = tmp;}
2155                   }
2156                   ierr = DMPlexSetCone(dm, c++, cone);CHKERRQ(ierr);
2157                 }
2158               }
2159             }
2160           }
2161         }
2162       }
2163       ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
2164       ierr = DMPlexStratify(dm);CHKERRQ(ierr);
2165       ierr = PetscFree(graph);CHKERRQ(ierr);
2166       break;
2167     }
2168   default: SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Unsupported dimension for sphere: %D", dim);
2169   }
2170   /* Create coordinates */
2171   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
2172   ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
2173   ierr = PetscSectionSetFieldComponents(coordSection, 0, embedDim);CHKERRQ(ierr);
2174   ierr = PetscSectionSetChart(coordSection, firstVertex, firstVertex+numVerts);CHKERRQ(ierr);
2175   for (v = firstVertex; v < firstVertex+numVerts; ++v) {
2176     ierr = PetscSectionSetDof(coordSection, v, embedDim);CHKERRQ(ierr);
2177     ierr = PetscSectionSetFieldDof(coordSection, v, 0, embedDim);CHKERRQ(ierr);
2178   }
2179   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
2180   ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
2181   ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
2182   ierr = VecSetBlockSize(coordinates, embedDim);CHKERRQ(ierr);
2183   ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
2184   ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
2185   ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
2186   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
2187   for (v = 0; v < numVerts; ++v) for (d = 0; d < embedDim; ++d) {coords[v*embedDim+d] = coordsIn[v*embedDim+d];}
2188   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
2189   ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
2190   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
2191   ierr = PetscFree(coordsIn);CHKERRQ(ierr);
2192   {
2193     DM          cdm;
2194     PetscDS     cds;
2195     PetscScalar c = R;
2196 
2197     ierr = DMPlexCreateCoordinateSpace(dm, 1, snapToSphere);CHKERRQ(ierr);
2198     ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
2199     ierr = DMGetDS(cdm, &cds);CHKERRQ(ierr);
2200     ierr = PetscDSSetConstants(cds, 1, &c);CHKERRQ(ierr);
2201   }
2202   /* Wait for coordinate creation before doing in-place modification */
2203   if (simplex) {ierr = DMPlexInterpolateInPlace_Internal(dm);CHKERRQ(ierr);}
2204   PetscFunctionReturn(0);
2205 }
2206 
2207 typedef void (*TPSEvaluateFunc)(const PetscReal[], PetscReal*, PetscReal[], PetscReal(*)[3]);
2208 
2209 /*
2210  The Schwarz P implicit surface is
2211 
2212      f(x) = cos(x0) + cos(x1) + cos(x2) = 0
2213 */
2214 static void TPSEvaluate_SchwarzP(const PetscReal y[3], PetscReal *f, PetscReal grad[], PetscReal (*hess)[3])
2215 {
2216   PetscReal c[3] = {PetscCosReal(y[0] * PETSC_PI), PetscCosReal(y[1] * PETSC_PI), PetscCosReal(y[2] * PETSC_PI)};
2217   PetscReal g[3] = {-PetscSinReal(y[0] * PETSC_PI), -PetscSinReal(y[1] * PETSC_PI), -PetscSinReal(y[2] * PETSC_PI)};
2218   f[0] = c[0] + c[1] + c[2];
2219   for (PetscInt i=0; i<3; i++) {
2220     grad[i] = PETSC_PI * g[i];
2221     for (PetscInt j=0; j<3; j++) {
2222       hess[i][j] = (i == j) ? -PetscSqr(PETSC_PI) * c[i] : 0.;
2223     }
2224   }
2225 }
2226 
2227 /*
2228  The Gyroid implicit surface is
2229 
2230  f(x,y,z) = sin(pi * x) * cos (pi * (y + 1/2))  + sin(pi * (y + 1/2)) * cos(pi * (z + 1/4)) + sin(pi * (z + 1/4)) * cos(pi * x)
2231 
2232 */
2233 static void TPSEvaluate_Gyroid(const PetscReal y[3], PetscReal *f, PetscReal grad[], PetscReal (*hess)[3])
2234 {
2235   PetscReal s[3] = {PetscSinReal(PETSC_PI * y[0]), PetscSinReal(PETSC_PI * (y[1] + .5)), PetscSinReal(PETSC_PI * (y[2] + .25))};
2236   PetscReal c[3] = {PetscCosReal(PETSC_PI * y[0]), PetscCosReal(PETSC_PI * (y[1] + .5)), PetscCosReal(PETSC_PI * (y[2] + .25))};
2237   f[0] = s[0] * c[1] + s[1] * c[2] + s[2] * c[0];
2238   grad[0] = PETSC_PI * (c[0] * c[1] - s[2] * s[0]);
2239   grad[1] = PETSC_PI * (c[1] * c[2] - s[0] * s[1]);
2240   grad[2] = PETSC_PI * (c[2] * c[0] - s[1] * s[2]);
2241   hess[0][0] = -PetscSqr(PETSC_PI) * (s[0] * c[1] + s[2] * c[0]);
2242   hess[0][1] = -PetscSqr(PETSC_PI) * (c[0] * s[1]);
2243   hess[0][2] = -PetscSqr(PETSC_PI) * (c[2] * s[0]);
2244   hess[1][0] = -PetscSqr(PETSC_PI) * (s[1] * c[2] + s[0] * c[1]);
2245   hess[1][1] = -PetscSqr(PETSC_PI) * (c[1] * s[2]);
2246   hess[2][2] = -PetscSqr(PETSC_PI) * (c[0] * s[1]);
2247   hess[2][0] = -PetscSqr(PETSC_PI) * (s[2] * c[0] + s[1] * c[2]);
2248   hess[2][1] = -PetscSqr(PETSC_PI) * (c[2] * s[0]);
2249   hess[2][2] = -PetscSqr(PETSC_PI) * (c[1] * s[2]);
2250 }
2251 
2252 /*
2253    We wish to solve
2254 
2255          min_y || y - x ||^2  subject to f(y) = 0
2256 
2257    Let g(y) = grad(f).  The minimization problem is equivalent to asking to satisfy
2258    f(y) = 0 and (y-x) is parallel to g(y).  We do this by using Householder QR to obtain a basis for the
2259    tangent space and ask for both components in the tangent space to be zero.
2260 
2261    Take g to be a column vector and compute the "full QR" factorization Q R = g,
2262    where Q = I - 2 n n^T is a symmetric orthogonal matrix.
2263    The first column of Q is parallel to g so the remaining two columns span the null space.
2264    Let Qn = Q[:,1:] be those remaining columns.  Then Qn Qn^T is an orthogonal projector into the tangent space.
2265    Since Q is symmetric, this is equivalent to multipyling by Q and taking the last two entries.
2266    In total, we have a system of 3 equations in 3 unknowns:
2267 
2268      f(y) = 0                       1 equation
2269      Qn^T (y - x) = 0               2 equations
2270 
2271    Here, we compute the residual and Jacobian of this system.
2272 */
2273 static void TPSNearestPointResJac(TPSEvaluateFunc feval, const PetscScalar x[], const PetscScalar y[], PetscScalar res[], PetscScalar J[])
2274 {
2275   PetscReal yreal[3] = {PetscRealPart(y[0]), PetscRealPart(y[1]), PetscRealPart(y[2])};
2276   PetscReal d[3] = {PetscRealPart(y[0] - x[0]), PetscRealPart(y[1] - x[1]), PetscRealPart(y[2] - x[2])};
2277   PetscReal f, grad[3], n[3], n_y[3][3], norm, norm_y[3], nd, nd_y[3], sign;
2278 
2279   feval(yreal, &f, grad, n_y);
2280 
2281   for (PetscInt i=0; i<3; i++) n[i] = grad[i];
2282   norm = PetscSqrtReal(PetscSqr(n[0]) + PetscSqr(n[1]) + PetscSqr(n[2]));
2283   for (PetscInt i=0; i<3; i++) {
2284     norm_y[i] = 1. / norm * n[i] * n_y[i][i];
2285   }
2286 
2287   // Define the Householder reflector
2288   sign = n[0] >= 0 ? 1. : -1.;
2289   n[0] += norm * sign;
2290   for (PetscInt i=0; i<3; i++) n_y[0][i] += norm_y[i] * sign;
2291 
2292   norm = PetscSqrtReal(PetscSqr(n[0]) + PetscSqr(n[1]) + PetscSqr(n[2]));
2293   norm_y[0] = 1. / norm * (n[0] * n_y[0][0]);
2294   norm_y[1] = 1. / norm * (n[0] * n_y[0][1] + n[1] * n_y[1][1]);
2295   norm_y[2] = 1. / norm * (n[0] * n_y[0][2] + n[2] * n_y[2][2]);
2296 
2297   for (PetscInt i=0; i<3; i++) {
2298     n[i] /= norm;
2299     for (PetscInt j=0; j<3; j++) {
2300       // note that n[i] is n_old[i]/norm when executing the code below
2301       n_y[i][j] = n_y[i][j] / norm - n[i] / norm * norm_y[j];
2302     }
2303   }
2304 
2305   nd = n[0] * d[0] + n[1] * d[1] + n[2] * d[2];
2306   for (PetscInt i=0; i<3; i++) nd_y[i] = n[i] + n_y[0][i] * d[0] + n_y[1][i] * d[1] + n_y[2][i] * d[2];
2307 
2308   res[0] = f;
2309   res[1] = d[1] - 2 * n[1] * nd;
2310   res[2] = d[2] - 2 * n[2] * nd;
2311   // J[j][i] is J_{ij} (column major)
2312   for (PetscInt j=0; j<3; j++) {
2313     J[0 + j*3] = grad[j];
2314     J[1 + j*3] = (j == 1)*1. - 2 * (n_y[1][j] * nd + n[1] * nd_y[j]);
2315     J[2 + j*3] = (j == 2)*1. - 2 * (n_y[2][j] * nd + n[2] * nd_y[j]);
2316   }
2317 }
2318 
2319 /*
2320    Project x to the nearest point on the implicit surface using Newton's method.
2321 */
2322 static PetscErrorCode TPSNearestPoint(TPSEvaluateFunc feval, PetscScalar x[])
2323 {
2324   PetscScalar y[3] = {x[0], x[1], x[2]}; // Initial guess
2325   PetscErrorCode ierr;
2326 
2327   PetscFunctionBegin;
2328   for (PetscInt iter=0; iter<10; iter++) {
2329     PetscScalar res[3], J[9];
2330     PetscReal resnorm;
2331     TPSNearestPointResJac(feval, x, y, res, J);
2332     resnorm = PetscSqrtReal(PetscSqr(PetscRealPart(res[0])) + PetscSqr(PetscRealPart(res[1])) + PetscSqr(PetscRealPart(res[2])));
2333     if (0) { // Turn on this monitor if you need to confirm quadratic convergence
2334       ierr = PetscPrintf(PETSC_COMM_SELF, "[%D] res [%g %g %g]\n", iter, PetscRealPart(res[0]), PetscRealPart(res[1]), PetscRealPart(res[2]));CHKERRQ(ierr);
2335     }
2336     if (resnorm < PETSC_SMALL) break;
2337 
2338     // Take the Newton step
2339     ierr = PetscKernel_A_gets_inverse_A_3(J, 0., PETSC_FALSE, NULL);CHKERRQ(ierr);
2340     PetscKernel_v_gets_v_minus_A_times_w_3(y, J, res);
2341   }
2342   for (PetscInt i=0; i<3; i++) x[i] = y[i];
2343   PetscFunctionReturn(0);
2344 }
2345 
2346 const char *const DMPlexTPSTypes[] = {"SCHWARZ_P", "GYROID", "DMPlexTPSType", "DMPLEX_TPS_", NULL};
2347 
2348 static PetscErrorCode DMPlexCreateTPSMesh_Internal(DM dm, DMPlexTPSType tpstype, const PetscInt extent[], const DMBoundaryType periodic[], PetscInt refinements, PetscInt layers, PetscReal thickness)
2349 {
2350   PetscErrorCode ierr;
2351   PetscMPIInt rank;
2352   PetscInt topoDim = 2, spaceDim = 3, numFaces = 0, numVertices = 0, numEdges = 0;
2353   PetscInt (*edges)[2] = NULL, *edgeSets = NULL;
2354   PetscInt *cells_flat = NULL;
2355   PetscReal *vtxCoords = NULL;
2356   TPSEvaluateFunc evalFunc = NULL;
2357   DMLabel label;
2358 
2359   PetscFunctionBegin;
2360   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRMPI(ierr);
2361   PetscCheck((layers != 0) ^ (thickness == 0.), PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_INCOMP, "Layers %D must be nonzero iff thickness %g is nonzero", layers, (double)thickness);
2362   switch (tpstype) {
2363   case DMPLEX_TPS_SCHWARZ_P:
2364     PetscCheck(!periodic || (periodic[0] == DM_BOUNDARY_NONE && periodic[1] == DM_BOUNDARY_NONE && periodic[2] == DM_BOUNDARY_NONE), PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Schwarz P does not support periodic meshes");
2365     if (!rank) {
2366       PetscInt (*cells)[6][4][4] = NULL; // [junction, junction-face, cell, conn]
2367       PetscInt Njunctions = 0, Ncuts = 0, Npipes[3], vcount;
2368       PetscReal L = 1;
2369 
2370       Npipes[0] = (extent[0] + 1) * extent[1] * extent[2];
2371       Npipes[1] = extent[0] * (extent[1] + 1) * extent[2];
2372       Npipes[2] = extent[0] * extent[1] * (extent[2] + 1);
2373       Njunctions = extent[0] * extent[1] * extent[2];
2374       Ncuts = 2 * (extent[0] * extent[1] + extent[1] * extent[2] + extent[2] * extent[0]);
2375       numVertices = 4 * (Npipes[0] + Npipes[1] + Npipes[2]) + 8 * Njunctions;
2376       ierr = PetscMalloc1(3*numVertices, &vtxCoords);CHKERRQ(ierr);
2377       ierr = PetscMalloc1(Njunctions, &cells);CHKERRQ(ierr);
2378       ierr = PetscMalloc1(Ncuts*4, &edges);CHKERRQ(ierr);
2379       ierr = PetscMalloc1(Ncuts*4, &edgeSets);CHKERRQ(ierr);
2380       // x-normal pipes
2381       vcount = 0;
2382       for (PetscInt i=0; i<extent[0]+1; i++) {
2383         for (PetscInt j=0; j<extent[1]; j++) {
2384           for (PetscInt k=0; k<extent[2]; k++) {
2385             for (PetscInt l=0; l<4; l++) {
2386               vtxCoords[vcount++] = (2*i - 1) * L;
2387               vtxCoords[vcount++] = 2 * j * L + PetscCosReal((2*l + 1) * PETSC_PI / 4) * L / 2;
2388               vtxCoords[vcount++] = 2 * k * L + PetscSinReal((2*l + 1) * PETSC_PI / 4) * L / 2;
2389             }
2390           }
2391         }
2392       }
2393       // y-normal pipes
2394       for (PetscInt i=0; i<extent[0]; i++) {
2395         for (PetscInt j=0; j<extent[1]+1; j++) {
2396           for (PetscInt k=0; k<extent[2]; k++) {
2397             for (PetscInt l=0; l<4; l++) {
2398               vtxCoords[vcount++] = 2 * i * L + PetscSinReal((2*l + 1) * PETSC_PI / 4) * L / 2;
2399               vtxCoords[vcount++] = (2*j - 1) * L;
2400               vtxCoords[vcount++] = 2 * k * L + PetscCosReal((2*l + 1) * PETSC_PI / 4) * L / 2;
2401             }
2402           }
2403         }
2404       }
2405       // z-normal pipes
2406       for (PetscInt i=0; i<extent[0]; i++) {
2407         for (PetscInt j=0; j<extent[1]; j++) {
2408           for (PetscInt k=0; k<extent[2]+1; k++) {
2409             for (PetscInt l=0; l<4; l++) {
2410               vtxCoords[vcount++] = 2 * i * L + PetscCosReal((2*l + 1) * PETSC_PI / 4) * L / 2;
2411               vtxCoords[vcount++] = 2 * j * L + PetscSinReal((2*l + 1) * PETSC_PI / 4) * L / 2;
2412               vtxCoords[vcount++] = (2*k - 1) * L;
2413             }
2414           }
2415         }
2416       }
2417       // junctions
2418       for (PetscInt i=0; i<extent[0]; i++) {
2419         for (PetscInt j=0; j<extent[1]; j++) {
2420           for (PetscInt k=0; k<extent[2]; k++) {
2421             const PetscInt J = (i*extent[1] + j)*extent[2] + k, Jvoff = (Npipes[0] + Npipes[1] + Npipes[2])*4 + J*8;
2422             PetscCheck(vcount / 3 == Jvoff, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Unexpected vertex count");
2423             for (PetscInt ii=0; ii<2; ii++) {
2424               for (PetscInt jj=0; jj<2; jj++) {
2425                 for (PetscInt kk=0; kk<2; kk++) {
2426                   double Ls = (1 - sqrt(2) / 4) * L;
2427                   vtxCoords[vcount++] = 2*i*L + (2*ii-1) * Ls;
2428                   vtxCoords[vcount++] = 2*j*L + (2*jj-1) * Ls;
2429                   vtxCoords[vcount++] = 2*k*L + (2*kk-1) * Ls;
2430                 }
2431               }
2432             }
2433             const PetscInt jfaces[3][2][4] = {
2434               {{3,1,0,2}, {7,5,4,6}}, // x-aligned
2435               {{5,4,0,1}, {7,6,2,3}}, // y-aligned
2436               {{6,2,0,4}, {7,3,1,5}}  // z-aligned
2437             };
2438             const PetscInt pipe_lo[3] = { // vertex numbers of pipes
2439               ((i * extent[1] + j) * extent[2] + k)*4,
2440               ((i * (extent[1] + 1) + j) * extent[2] + k + Npipes[0])*4,
2441               ((i * extent[1] + j) * (extent[2]+1) + k + Npipes[0] + Npipes[1])*4
2442             };
2443             const PetscInt pipe_hi[3] = { // vertex numbers of pipes
2444               (((i + 1) * extent[1] + j) * extent[2] + k)*4,
2445               ((i * (extent[1] + 1) + j + 1) * extent[2] + k + Npipes[0])*4,
2446               ((i * extent[1] + j) * (extent[2]+1) + k + 1 + Npipes[0] + Npipes[1])*4
2447             };
2448             for (PetscInt dir=0; dir<3; dir++) { // x,y,z
2449               const PetscInt ijk[3] = {i, j, k};
2450               for (PetscInt l=0; l<4; l++) { // rotations
2451                 cells[J][dir*2+0][l][0] = pipe_lo[dir] + l;
2452                 cells[J][dir*2+0][l][1] = Jvoff + jfaces[dir][0][l];
2453                 cells[J][dir*2+0][l][2] = Jvoff + jfaces[dir][0][(l-1+4)%4];
2454                 cells[J][dir*2+0][l][3] = pipe_lo[dir] + (l-1+4)%4;
2455                 cells[J][dir*2+1][l][0] = Jvoff + jfaces[dir][1][l];
2456                 cells[J][dir*2+1][l][1] = pipe_hi[dir] + l;
2457                 cells[J][dir*2+1][l][2] = pipe_hi[dir] + (l-1+4)%4;
2458                 cells[J][dir*2+1][l][3] = Jvoff + jfaces[dir][1][(l-1+4)%4];
2459                 if (ijk[dir] == 0) {
2460                   edges[numEdges][0] = pipe_lo[dir] + l;
2461                   edges[numEdges][1] = pipe_lo[dir] + (l+1) % 4;
2462                   edgeSets[numEdges] = dir*2 + 1;
2463                   numEdges++;
2464                 }
2465                 if (ijk[dir] + 1 == extent[dir]) {
2466                   edges[numEdges][0] = pipe_hi[dir] + l;
2467                   edges[numEdges][1] = pipe_hi[dir] + (l+1) % 4;
2468                   edgeSets[numEdges] = dir*2 + 2;
2469                   numEdges++;
2470                 }
2471               }
2472             }
2473           }
2474         }
2475       }
2476       PetscCheck(numEdges == Ncuts * 4, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Edge count %D incompatible with number of cuts %D", numEdges, Ncuts);
2477       numFaces = 24 * Njunctions;
2478       cells_flat = cells[0][0][0];
2479     }
2480     evalFunc = TPSEvaluate_SchwarzP;
2481     break;
2482   case DMPLEX_TPS_GYROID:
2483     if (!rank) {
2484       // This is a coarse mesh approximation of the gyroid shifted to being the zero of the level set
2485       //
2486       //     sin(pi*x)*cos(pi*(y+1/2)) + sin(pi*(y+1/2))*cos(pi*(z+1/4)) + sin(pi*(z+1/4))*cos(x)
2487       //
2488       // on the cell [0,2]^3.
2489       //
2490       // Think about dividing that cell into four columns, and focus on the column [0,1]x[0,1]x[0,2].
2491       // If you looked at the gyroid in that column at different slices of z you would see that it kind of spins
2492       // like a boomerang:
2493       //
2494       //     z = 0          z = 1/4        z = 1/2        z = 3/4     //
2495       //     -----          -------        -------        -------     //
2496       //                                                              //
2497       //     +       +      +       +      +       +      +   \   +   //
2498       //      \                                   /            \      //
2499       //       \            `-_   _-'            /              }     //
2500       //        *-_            `-'            _-'              /      //
2501       //     +     `-+      +       +      +-'     +      +   /   +   //
2502       //                                                              //
2503       //                                                              //
2504       //     z = 1          z = 5/4        z = 3/2        z = 7/4     //
2505       //     -----          -------        -------        -------     //
2506       //                                                              //
2507       //     +-_     +      +       +      +     _-+      +   /   +   //
2508       //        `-_            _-_            _-`            /        //
2509       //           \        _-'   `-_        /              {         //
2510       //            \                       /                \        //
2511       //     +       +      +       +      +       +      +   \   +   //
2512       //
2513       //
2514       // This course mesh approximates each of these slices by two line segments,
2515       // and then connects the segments in consecutive layers with quadrilateral faces.
2516       // All of the end points of the segments are multiples of 1/4 except for the
2517       // point * in the picture for z = 0 above and the similar points in other layers.
2518       // That point is at (gamma, gamma, 0), where gamma is calculated below.
2519       //
2520       // The column  [1,2]x[1,2]x[0,2] looks the same as this column;
2521       // The columns [1,2]x[0,1]x[0,2] and [0,1]x[1,2]x[0,2] are mirror images.
2522       //
2523       // As for how this method turned into the names given to the vertices:
2524       // that was not systematic, it was just the way it worked out in my handwritten notes.
2525 
2526       PetscInt facesPerBlock = 64;
2527       PetscInt vertsPerBlock = 56;
2528       PetscInt extentPlus[3];
2529       PetscInt numBlocks, numBlocksPlus;
2530       const PetscInt A =  0,   B =  1,   C =  2,   D =  3,   E =  4,   F =  5,   G =  6,   H =  7,
2531         II =  8,   J =  9,   K = 10,   L = 11,   M = 12,   N = 13,   O = 14,   P = 15,
2532         Q = 16,   R = 17,   S = 18,   T = 19,   U = 20,   V = 21,   W = 22,   X = 23,
2533         Y = 24,   Z = 25,  Ap = 26,  Bp = 27,  Cp = 28,  Dp = 29,  Ep = 30,  Fp = 31,
2534         Gp = 32,  Hp = 33,  Ip = 34,  Jp = 35,  Kp = 36,  Lp = 37,  Mp = 38,  Np = 39,
2535         Op = 40,  Pp = 41,  Qp = 42,  Rp = 43,  Sp = 44,  Tp = 45,  Up = 46,  Vp = 47,
2536         Wp = 48,  Xp = 49,  Yp = 50,  Zp = 51,  Aq = 52,  Bq = 53,  Cq = 54,  Dq = 55;
2537       const PetscInt pattern[64][4] =
2538         { /* face to vertex within the coarse discretization of a single gyroid block */
2539           /* layer 0 */
2540           {A,C,K,G},{C,B,II,K},{D,A,H,L},{B+56*1,D,L,J},{E,B+56*1,J,N},{A+56*2,E,N,H+56*2},{F,A+56*2,G+56*2,M},{B,F,M,II},
2541           /* layer 1 */
2542           {G,K,Q,O},{K,II,P,Q},{L,H,O+56*1,R},{J,L,R,P},{N,J,P,S},{H+56*2,N,S,O+56*3},{M,G+56*2,O+56*2,T},{II,M,T,P},
2543           /* layer 2 */
2544           {O,Q,Y,U},{Q,P,W,Y},{R,O+56*1,U+56*1,Ap},{P,R,Ap,W},{S,P,X,Bp},{O+56*3,S,Bp,V+56*1},{T,O+56*2,V,Z},{P,T,Z,X},
2545           /* layer 3 */
2546           {U,Y,Ep,Dp},{Y,W,Cp,Ep},{Ap,U+56*1,Dp+56*1,Gp},{W,Ap,Gp,Cp},{Bp,X,Cp+56*2,Fp},{V+56*1,Bp,Fp,Dp+56*1},{Z,V,Dp,Hp},{X,Z,Hp,Cp+56*2},
2547           /* layer 4 */
2548           {Dp,Ep,Mp,Kp},{Ep,Cp,Ip,Mp},{Gp,Dp+56*1,Lp,Np},{Cp,Gp,Np,Jp},{Fp,Cp+56*2,Jp+56*2,Pp},{Dp+56*1,Fp,Pp,Lp},{Hp,Dp,Kp,Op},{Cp+56*2,Hp,Op,Ip+56*2},
2549           /* layer 5 */
2550           {Kp,Mp,Sp,Rp},{Mp,Ip,Qp,Sp},{Np,Lp,Rp,Tp},{Jp,Np,Tp,Qp+56*1},{Pp,Jp+56*2,Qp+56*3,Up},{Lp,Pp,Up,Rp},{Op,Kp,Rp,Vp},{Ip+56*2,Op,Vp,Qp+56*2},
2551           /* layer 6 */
2552           {Rp,Sp,Aq,Yp},{Sp,Qp,Wp,Aq},{Tp,Rp,Yp,Cq},{Qp+56*1,Tp,Cq,Wp+56*1},{Up,Qp+56*3,Xp+56*1,Dq},{Rp,Up,Dq,Zp},{Vp,Rp,Zp,Bq},{Qp+56*2,Vp,Bq,Xp},
2553           /* layer 7 (the top is the periodic image of the bottom of layer 0) */
2554           {Yp,Aq,C+56*4,A+56*4},{Aq,Wp,B+56*4,C+56*4},{Cq,Yp,A+56*4,D+56*4},{Wp+56*1,Cq,D+56*4,B+56*5},{Dq,Xp+56*1,B+56*5,E+56*4},{Zp,Dq,E+56*4,A+56*6},{Bq,Zp,A+56*6,F+56*4},{Xp,Bq,F+56*4,B+56*4}
2555         };
2556       const PetscReal gamma = PetscAcosReal((PetscSqrtReal(3.)-1.) / PetscSqrtReal(2.)) / PETSC_PI;
2557       const PetscReal patternCoords[56][3] =
2558         {
2559           /* A  */ {1.,0.,0.},
2560           /* B  */ {0.,1.,0.},
2561           /* C  */ {gamma,gamma,0.},
2562           /* D  */ {1+gamma,1-gamma,0.},
2563           /* E  */ {2-gamma,2-gamma,0.},
2564           /* F  */ {1-gamma,1+gamma,0.},
2565 
2566           /* G  */ {.5,0,.25},
2567           /* H  */ {1.5,0.,.25},
2568           /* II */ {.5,1.,.25},
2569           /* J  */ {1.5,1.,.25},
2570           /* K  */ {.25,.5,.25},
2571           /* L  */ {1.25,.5,.25},
2572           /* M  */ {.75,1.5,.25},
2573           /* N  */ {1.75,1.5,.25},
2574 
2575           /* O  */ {0.,0.,.5},
2576           /* P  */ {1.,1.,.5},
2577           /* Q  */ {gamma,1-gamma,.5},
2578           /* R  */ {1+gamma,gamma,.5},
2579           /* S  */ {2-gamma,1+gamma,.5},
2580           /* T  */ {1-gamma,2-gamma,.5},
2581 
2582           /* U  */ {0.,.5,.75},
2583           /* V  */ {0.,1.5,.75},
2584           /* W  */ {1.,.5,.75},
2585           /* X  */ {1.,1.5,.75},
2586           /* Y  */ {.5,.75,.75},
2587           /* Z  */ {.5,1.75,.75},
2588           /* Ap */ {1.5,.25,.75},
2589           /* Bp */ {1.5,1.25,.75},
2590 
2591           /* Cp */ {1.,0.,1.},
2592           /* Dp */ {0.,1.,1.},
2593           /* Ep */ {1-gamma,1-gamma,1.},
2594           /* Fp */ {1+gamma,1+gamma,1.},
2595           /* Gp */ {2-gamma,gamma,1.},
2596           /* Hp */ {gamma,2-gamma,1.},
2597 
2598           /* Ip */ {.5,0.,1.25},
2599           /* Jp */ {1.5,0.,1.25},
2600           /* Kp */ {.5,1.,1.25},
2601           /* Lp */ {1.5,1.,1.25},
2602           /* Mp */ {.75,.5,1.25},
2603           /* Np */ {1.75,.5,1.25},
2604           /* Op */ {.25,1.5,1.25},
2605           /* Pp */ {1.25,1.5,1.25},
2606 
2607           /* Qp */ {0.,0.,1.5},
2608           /* Rp */ {1.,1.,1.5},
2609           /* Sp */ {1-gamma,gamma,1.5},
2610           /* Tp */ {2-gamma,1-gamma,1.5},
2611           /* Up */ {1+gamma,2-gamma,1.5},
2612           /* Vp */ {gamma,1+gamma,1.5},
2613 
2614           /* Wp */ {0.,.5,1.75},
2615           /* Xp */ {0.,1.5,1.75},
2616           /* Yp */ {1.,.5,1.75},
2617           /* Zp */ {1.,1.5,1.75},
2618           /* Aq */ {.5,.25,1.75},
2619           /* Bq */ {.5,1.25,1.75},
2620           /* Cq */ {1.5,.75,1.75},
2621           /* Dq */ {1.5,1.75,1.75},
2622         };
2623       PetscInt  (*cells)[64][4] = NULL;
2624       PetscBool *seen;
2625       PetscInt  *vertToTrueVert;
2626       PetscInt  count;
2627 
2628       for (PetscInt i = 0; i < 3; i++) extentPlus[i]  = extent[i] + 1;
2629       numBlocks = 1;
2630       for (PetscInt i = 0; i < 3; i++)     numBlocks *= extent[i];
2631       numBlocksPlus = 1;
2632       for (PetscInt i = 0; i < 3; i++) numBlocksPlus *= extentPlus[i];
2633       numFaces = numBlocks * facesPerBlock;
2634       ierr = PetscMalloc1(numBlocks, &cells);CHKERRQ(ierr);
2635       ierr = PetscCalloc1(numBlocksPlus * vertsPerBlock,&seen);CHKERRQ(ierr);
2636       for (PetscInt k = 0; k < extent[2]; k++) {
2637         for (PetscInt j = 0; j < extent[1]; j++) {
2638           for (PetscInt i = 0; i < extent[0]; i++) {
2639             for (PetscInt f = 0; f < facesPerBlock; f++) {
2640               for (PetscInt v = 0; v < 4; v++) {
2641                 PetscInt vertRaw = pattern[f][v];
2642                 PetscInt blockidx = vertRaw / 56;
2643                 PetscInt patternvert = vertRaw % 56;
2644                 PetscInt xplus = (blockidx & 1);
2645                 PetscInt yplus = (blockidx & 2) >> 1;
2646                 PetscInt zplus = (blockidx & 4) >> 2;
2647                 PetscInt zcoord = (periodic && periodic[2] == DM_BOUNDARY_PERIODIC) ? ((k + zplus) % extent[2]) : (k + zplus);
2648                 PetscInt ycoord = (periodic && periodic[1] == DM_BOUNDARY_PERIODIC) ? ((j + yplus) % extent[1]) : (j + yplus);
2649                 PetscInt xcoord = (periodic && periodic[0] == DM_BOUNDARY_PERIODIC) ? ((i + xplus) % extent[0]) : (i + xplus);
2650                 PetscInt vert = ((zcoord * extentPlus[1] + ycoord) * extentPlus[0] + xcoord) * 56 + patternvert;
2651 
2652                 cells[(k * extent[1] + j) * extent[0] + i][f][v] = vert;
2653                 seen[vert] = PETSC_TRUE;
2654               }
2655             }
2656           }
2657         }
2658       }
2659       for (PetscInt i = 0; i < numBlocksPlus * vertsPerBlock; i++) if (seen[i]) numVertices++;
2660       count = 0;
2661       ierr = PetscMalloc1(numBlocksPlus * vertsPerBlock, &vertToTrueVert);CHKERRQ(ierr);
2662       ierr = PetscMalloc1(numVertices * 3, &vtxCoords);CHKERRQ(ierr);
2663       for (PetscInt i = 0; i < numBlocksPlus * vertsPerBlock; i++) vertToTrueVert[i] = -1;
2664       for (PetscInt k = 0; k < extentPlus[2]; k++) {
2665         for (PetscInt j = 0; j < extentPlus[1]; j++) {
2666           for (PetscInt i = 0; i < extentPlus[0]; i++) {
2667             for (PetscInt v = 0; v < vertsPerBlock; v++) {
2668               PetscInt vIdx = ((k * extentPlus[1] + j) * extentPlus[0] + i) * vertsPerBlock + v;
2669 
2670               if (seen[vIdx]) {
2671                 PetscInt thisVert;
2672 
2673                 vertToTrueVert[vIdx] = thisVert = count++;
2674 
2675                 for (PetscInt d = 0; d < 3; d++) vtxCoords[3 * thisVert + d] = patternCoords[v][d];
2676                 vtxCoords[3 * thisVert + 0] += i * 2;
2677                 vtxCoords[3 * thisVert + 1] += j * 2;
2678                 vtxCoords[3 * thisVert + 2] += k * 2;
2679               }
2680             }
2681           }
2682         }
2683       }
2684       for (PetscInt i = 0; i < numBlocks; i++) {
2685         for (PetscInt f = 0; f < facesPerBlock; f++) {
2686           for (PetscInt v = 0; v < 4; v++) {
2687             cells[i][f][v] = vertToTrueVert[cells[i][f][v]];
2688           }
2689         }
2690       }
2691       ierr = PetscFree(vertToTrueVert);CHKERRQ(ierr);
2692       ierr = PetscFree(seen);CHKERRQ(ierr);
2693       cells_flat = cells[0][0];
2694       numEdges = 0;
2695       for (PetscInt i = 0; i < numFaces; i++) {
2696         for (PetscInt e = 0; e < 4; e++) {
2697           PetscInt ev[] = {cells_flat[i*4 + e], cells_flat[i*4 + ((e+1)%4)]};
2698           const PetscReal *evCoords[] = {&vtxCoords[3*ev[0]], &vtxCoords[3*ev[1]]};
2699 
2700           for (PetscInt d = 0; d < 3; d++) {
2701             if (!periodic || periodic[0] != DM_BOUNDARY_PERIODIC) {
2702               if (evCoords[0][d] == 0. && evCoords[1][d] == 0.) numEdges++;
2703               if (evCoords[0][d] == 2.*extent[d] && evCoords[1][d] == 2.*extent[d]) numEdges++;
2704             }
2705           }
2706         }
2707       }
2708       ierr = PetscMalloc1(numEdges, &edges);CHKERRQ(ierr);
2709       ierr = PetscMalloc1(numEdges, &edgeSets);CHKERRQ(ierr);
2710       for (PetscInt edge = 0, i = 0; i < numFaces; i++) {
2711         for (PetscInt e = 0; e < 4; e++) {
2712           PetscInt ev[] = {cells_flat[i*4 + e], cells_flat[i*4 + ((e+1)%4)]};
2713           const PetscReal *evCoords[] = {&vtxCoords[3*ev[0]], &vtxCoords[3*ev[1]]};
2714 
2715           for (PetscInt d = 0; d < 3; d++) {
2716             if (!periodic || periodic[d] != DM_BOUNDARY_PERIODIC) {
2717               if (evCoords[0][d] == 0. && evCoords[1][d] == 0.) {
2718                 edges[edge][0] = ev[0];
2719                 edges[edge][1] = ev[1];
2720                 edgeSets[edge++] = 2 * d;
2721               }
2722               if (evCoords[0][d] == 2.*extent[d] && evCoords[1][d] == 2.*extent[d]) {
2723                 edges[edge][0] = ev[0];
2724                 edges[edge][1] = ev[1];
2725                 edgeSets[edge++] = 2 * d + 1;
2726               }
2727             }
2728           }
2729         }
2730       }
2731     }
2732     evalFunc = TPSEvaluate_Gyroid;
2733     break;
2734   }
2735 
2736   ierr = DMSetDimension(dm, topoDim);CHKERRQ(ierr);
2737   if (!rank) {ierr = DMPlexBuildFromCellList(dm, numFaces, numVertices, 4, cells_flat);CHKERRQ(ierr);}
2738   else       {ierr = DMPlexBuildFromCellList(dm, 0, 0, 0, NULL);CHKERRQ(ierr);}
2739   ierr = PetscFree(cells_flat);CHKERRQ(ierr);
2740   {
2741     DM idm;
2742     ierr = DMPlexInterpolate(dm, &idm);CHKERRQ(ierr);
2743     ierr = DMPlexReplace_Static(dm, &idm);CHKERRQ(ierr);
2744   }
2745   if (!rank) {ierr = DMPlexBuildCoordinatesFromCellList(dm, spaceDim, vtxCoords);CHKERRQ(ierr);}
2746   else       {ierr = DMPlexBuildCoordinatesFromCellList(dm, spaceDim, NULL);CHKERRQ(ierr);}
2747   ierr = PetscFree(vtxCoords);CHKERRQ(ierr);
2748 
2749   ierr = DMCreateLabel(dm, "Face Sets");CHKERRQ(ierr);
2750   ierr = DMGetLabel(dm, "Face Sets", &label);CHKERRQ(ierr);
2751   for (PetscInt e=0; e<numEdges; e++) {
2752     PetscInt njoin;
2753     const PetscInt *join, verts[] = {numFaces + edges[e][0], numFaces + edges[e][1]};
2754     ierr = DMPlexGetJoin(dm, 2, verts, &njoin, &join);CHKERRQ(ierr);
2755     PetscCheck(njoin == 1, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Expected unique join of vertices %D and %D", edges[e][0], edges[e][1]);
2756     ierr = DMLabelSetValue(label, join[0], edgeSets[e]);CHKERRQ(ierr);
2757     ierr = DMPlexRestoreJoin(dm, 2, verts, &njoin, &join);CHKERRQ(ierr);
2758   }
2759   ierr = PetscFree(edges);CHKERRQ(ierr);
2760   ierr = PetscFree(edgeSets);CHKERRQ(ierr);
2761 
2762   ierr = DMPlexSetRefinementUniform(dm, PETSC_TRUE);CHKERRQ(ierr);
2763   for (PetscInt refine=0; refine<refinements; refine++) {
2764     PetscInt m;
2765     DM dmf;
2766     Vec X;
2767     PetscScalar *x;
2768     ierr = DMRefine(dm, MPI_COMM_NULL, &dmf);CHKERRQ(ierr);
2769     ierr = DMPlexReplace_Static(dm, &dmf);CHKERRQ(ierr);
2770 
2771     ierr = DMGetCoordinatesLocal(dm, &X);CHKERRQ(ierr);
2772     ierr = VecGetLocalSize(X, &m);CHKERRQ(ierr);
2773     ierr = VecGetArray(X, &x);CHKERRQ(ierr);
2774     for (PetscInt i=0; i<m; i+=3) {
2775       ierr = TPSNearestPoint(evalFunc, &x[i]);CHKERRQ(ierr);
2776     }
2777     ierr = VecRestoreArray(X, &x);CHKERRQ(ierr);
2778   }
2779 
2780   // Face Sets has already been propagated to new vertices during refinement; this propagates to the initial vertices.
2781   ierr = DMGetLabel(dm, "Face Sets", &label);CHKERRQ(ierr);
2782   ierr = DMPlexLabelComplete(dm, label);CHKERRQ(ierr);
2783 
2784   if (thickness > 0) {
2785     DM dm3;
2786     ierr = DMPlexExtrude(dm, layers, thickness, PETSC_FALSE, PETSC_TRUE, NULL, NULL, &dm3);CHKERRQ(ierr);
2787     ierr = DMPlexReplace_Static(dm, &dm3);CHKERRQ(ierr);
2788   }
2789   PetscFunctionReturn(0);
2790 }
2791 
2792 /*@
2793   DMPlexCreateTPSMesh - Create a distributed, interpolated mesh of a triply-periodic surface
2794 
2795   Collective
2796 
2797   Input Parameters:
2798 + comm   - The communicator for the DM object
2799 . tpstype - Type of triply-periodic surface
2800 . extent - Array of length 3 containing number of periods in each direction
2801 . periodic - array of length 3 with periodicity, or NULL for non-periodic
2802 . thickness - Thickness in normal direction
2803 - refinements - Number of factor-of-2 refinements
2804 
2805   Output Parameter:
2806 . dm  - The DM object
2807 
2808   Notes:
2809   This meshes the surface of the Schwarz P or Gyroid surfaces.  Schwarz P is is the simplest member of the triply-periodic minimal surfaces.
2810   https://en.wikipedia.org/wiki/Schwarz_minimal_surface#Schwarz_P_(%22Primitive%22) and can be cut with "clean" boundaries.
2811   The Gyroid (https://en.wikipedia.org/wiki/Gyroid) is another triply-periodic minimal surface with applications in additive manufacturing; it is much more difficult to "cut" since there are no planes of symmetry.
2812   Our implementation creates a very coarse mesh of the surface and refines (by 4-way splitting) as many times as requested.
2813   On each refinement, all vertices are projected to their nearest point on the surface.
2814   This projection could readily be extended to related surfaces.
2815 
2816   The face (edge) sets for the Schwarz P surface are numbered 1(-x), 2(+x), 3(-y), 4(+y), 5(-z), 6(+z).
2817   When the mesh is refined, "Face Sets" contain the new vertices (created during refinement).  Use DMPlexLabelComplete() to propagate to coarse-level vertices.
2818 
2819   References:
2820   Maskery et al, Insights into the mechanical properties of several triply periodic minimal surface lattice structures made by polymer additive manufacturing, 2017. https://doi.org/10.1016/j.polymer.2017.11.049
2821 
2822   Developer Notes:
2823   The Gyroid mesh does not currently mark boundary sets.
2824 
2825   Level: beginner
2826 
2827 .seealso: DMPlexCreateSphereMesh(), DMSetType(), DMCreate()
2828 @*/
2829 PetscErrorCode DMPlexCreateTPSMesh(MPI_Comm comm, DMPlexTPSType tpstype, const PetscInt extent[], const DMBoundaryType periodic[], PetscInt refinements, PetscInt layers, PetscReal thickness, DM *dm)
2830 {
2831   PetscErrorCode ierr;
2832 
2833   PetscFunctionBegin;
2834   ierr = DMCreate(comm, dm);CHKERRQ(ierr);
2835   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
2836   ierr = DMPlexCreateTPSMesh_Internal(*dm, tpstype, extent, periodic, refinements, layers, thickness);CHKERRQ(ierr);
2837   PetscFunctionReturn(0);
2838 }
2839 
2840 /*@
2841   DMPlexCreateSphereMesh - Creates a mesh on the d-dimensional sphere, S^d.
2842 
2843   Collective
2844 
2845   Input Parameters:
2846 + comm    - The communicator for the DM object
2847 . dim     - The dimension
2848 . simplex - Use simplices, or tensor product cells
2849 - R       - The radius
2850 
2851   Output Parameter:
2852 . dm  - The DM object
2853 
2854   Level: beginner
2855 
2856 .seealso: DMPlexCreateBallMesh(), DMPlexCreateBoxMesh(), DMSetType(), DMCreate()
2857 @*/
2858 PetscErrorCode DMPlexCreateSphereMesh(MPI_Comm comm, PetscInt dim, PetscBool simplex, PetscReal R, DM *dm)
2859 {
2860   PetscErrorCode ierr;
2861 
2862   PetscFunctionBegin;
2863   PetscValidPointer(dm, 5);
2864   ierr = DMCreate(comm, dm);CHKERRQ(ierr);
2865   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
2866   ierr = DMPlexCreateSphereMesh_Internal(*dm, dim, simplex, R);CHKERRQ(ierr);
2867   PetscFunctionReturn(0);
2868 }
2869 
2870 static PetscErrorCode DMPlexCreateBallMesh_Internal(DM dm, PetscInt dim, PetscReal R)
2871 {
2872   DM             sdm, vol;
2873   DMLabel        bdlabel;
2874   PetscErrorCode ierr;
2875 
2876   PetscFunctionBegin;
2877   ierr = DMCreate(PetscObjectComm((PetscObject) dm), &sdm);CHKERRQ(ierr);
2878   ierr = DMSetType(sdm, DMPLEX);CHKERRQ(ierr);
2879   ierr = PetscObjectSetOptionsPrefix((PetscObject) sdm, "bd_");CHKERRQ(ierr);
2880   ierr = DMPlexCreateSphereMesh_Internal(sdm, dim-1, PETSC_TRUE, R);CHKERRQ(ierr);
2881   ierr = DMSetFromOptions(sdm);CHKERRQ(ierr);
2882   ierr = DMViewFromOptions(sdm, NULL, "-dm_view");CHKERRQ(ierr);
2883   ierr = DMPlexGenerate(sdm, NULL, PETSC_TRUE, &vol);CHKERRQ(ierr);
2884   ierr = DMDestroy(&sdm);CHKERRQ(ierr);
2885   ierr = DMPlexReplace_Static(dm, &vol);CHKERRQ(ierr);
2886   ierr = DMCreateLabel(dm, "marker");CHKERRQ(ierr);
2887   ierr = DMGetLabel(dm, "marker", &bdlabel);CHKERRQ(ierr);
2888   ierr = DMPlexMarkBoundaryFaces(dm, PETSC_DETERMINE, bdlabel);CHKERRQ(ierr);
2889   ierr = DMPlexLabelComplete(dm, bdlabel);CHKERRQ(ierr);
2890   PetscFunctionReturn(0);
2891 }
2892 
2893 /*@
2894   DMPlexCreateBallMesh - Creates a simplex mesh on the d-dimensional ball, B^d.
2895 
2896   Collective
2897 
2898   Input Parameters:
2899 + comm  - The communicator for the DM object
2900 . dim   - The dimension
2901 - R     - The radius
2902 
2903   Output Parameter:
2904 . dm  - The DM object
2905 
2906   Options Database Keys:
2907 - bd_dm_refine - This will refine the surface mesh preserving the sphere geometry
2908 
2909   Level: beginner
2910 
2911 .seealso: DMPlexCreateSphereMesh(), DMPlexCreateBoxMesh(), DMSetType(), DMCreate()
2912 @*/
2913 PetscErrorCode DMPlexCreateBallMesh(MPI_Comm comm, PetscInt dim, PetscReal R, DM *dm)
2914 {
2915   PetscErrorCode ierr;
2916 
2917   PetscFunctionBegin;
2918   ierr = DMCreate(comm, dm);CHKERRQ(ierr);
2919   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
2920   ierr = DMPlexCreateBallMesh_Internal(*dm, dim, R);CHKERRQ(ierr);
2921   PetscFunctionReturn(0);
2922 }
2923 
2924 static PetscErrorCode DMPlexCreateReferenceCell_Internal(DM rdm, DMPolytopeType ct)
2925 {
2926   PetscErrorCode ierr;
2927 
2928   PetscFunctionBegin;
2929   switch (ct) {
2930     case DM_POLYTOPE_POINT:
2931     {
2932       PetscInt    numPoints[1]        = {1};
2933       PetscInt    coneSize[1]         = {0};
2934       PetscInt    cones[1]            = {0};
2935       PetscInt    coneOrientations[1] = {0};
2936       PetscScalar vertexCoords[1]     = {0.0};
2937 
2938       ierr = DMSetDimension(rdm, 0);CHKERRQ(ierr);
2939       ierr = DMPlexCreateFromDAG(rdm, 0, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
2940     }
2941     break;
2942     case DM_POLYTOPE_SEGMENT:
2943     {
2944       PetscInt    numPoints[2]        = {2, 1};
2945       PetscInt    coneSize[3]         = {2, 0, 0};
2946       PetscInt    cones[2]            = {1, 2};
2947       PetscInt    coneOrientations[2] = {0, 0};
2948       PetscScalar vertexCoords[2]     = {-1.0,  1.0};
2949 
2950       ierr = DMSetDimension(rdm, 1);CHKERRQ(ierr);
2951       ierr = DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
2952     }
2953     break;
2954     case DM_POLYTOPE_POINT_PRISM_TENSOR:
2955     {
2956       PetscInt    numPoints[2]        = {2, 1};
2957       PetscInt    coneSize[3]         = {2, 0, 0};
2958       PetscInt    cones[2]            = {1, 2};
2959       PetscInt    coneOrientations[2] = {0, 0};
2960       PetscScalar vertexCoords[2]     = {-1.0,  1.0};
2961 
2962       ierr = DMSetDimension(rdm, 1);CHKERRQ(ierr);
2963       ierr = DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
2964     }
2965     break;
2966     case DM_POLYTOPE_TRIANGLE:
2967     {
2968       PetscInt    numPoints[2]        = {3, 1};
2969       PetscInt    coneSize[4]         = {3, 0, 0, 0};
2970       PetscInt    cones[3]            = {1, 2, 3};
2971       PetscInt    coneOrientations[3] = {0, 0, 0};
2972       PetscScalar vertexCoords[6]     = {-1.0, -1.0,  1.0, -1.0,  -1.0, 1.0};
2973 
2974       ierr = DMSetDimension(rdm, 2);CHKERRQ(ierr);
2975       ierr = DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
2976     }
2977     break;
2978     case DM_POLYTOPE_QUADRILATERAL:
2979     {
2980       PetscInt    numPoints[2]        = {4, 1};
2981       PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
2982       PetscInt    cones[4]            = {1, 2, 3, 4};
2983       PetscInt    coneOrientations[4] = {0, 0, 0, 0};
2984       PetscScalar vertexCoords[8]     = {-1.0, -1.0,  1.0, -1.0,  1.0, 1.0,  -1.0, 1.0};
2985 
2986       ierr = DMSetDimension(rdm, 2);CHKERRQ(ierr);
2987       ierr = DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
2988     }
2989     break;
2990     case DM_POLYTOPE_SEG_PRISM_TENSOR:
2991     {
2992       PetscInt    numPoints[2]        = {4, 1};
2993       PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
2994       PetscInt    cones[4]            = {1, 2, 3, 4};
2995       PetscInt    coneOrientations[4] = {0, 0, 0, 0};
2996       PetscScalar vertexCoords[8]     = {-1.0, -1.0,  1.0, -1.0,  -1.0, 1.0,  1.0, 1.0};
2997 
2998       ierr = DMSetDimension(rdm, 2);CHKERRQ(ierr);
2999       ierr = DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
3000     }
3001     break;
3002     case DM_POLYTOPE_TETRAHEDRON:
3003     {
3004       PetscInt    numPoints[2]        = {4, 1};
3005       PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
3006       PetscInt    cones[4]            = {1, 2, 3, 4};
3007       PetscInt    coneOrientations[4] = {0, 0, 0, 0};
3008       PetscScalar vertexCoords[12]    = {-1.0, -1.0, -1.0,  -1.0, 1.0, -1.0,  1.0, -1.0, -1.0,  -1.0, -1.0, 1.0};
3009 
3010       ierr = DMSetDimension(rdm, 3);CHKERRQ(ierr);
3011       ierr = DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
3012     }
3013     break;
3014     case DM_POLYTOPE_HEXAHEDRON:
3015     {
3016       PetscInt    numPoints[2]        = {8, 1};
3017       PetscInt    coneSize[9]         = {8, 0, 0, 0, 0, 0, 0, 0, 0};
3018       PetscInt    cones[8]            = {1, 2, 3, 4, 5, 6, 7, 8};
3019       PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
3020       PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0,  -1.0,  1.0, -1.0,  1.0, 1.0, -1.0,   1.0, -1.0, -1.0,
3021                                          -1.0, -1.0,  1.0,   1.0, -1.0,  1.0,  1.0, 1.0,  1.0,  -1.0,  1.0,  1.0};
3022 
3023       ierr = DMSetDimension(rdm, 3);CHKERRQ(ierr);
3024       ierr = DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
3025     }
3026     break;
3027     case DM_POLYTOPE_TRI_PRISM:
3028     {
3029       PetscInt    numPoints[2]        = {6, 1};
3030       PetscInt    coneSize[7]         = {6, 0, 0, 0, 0, 0, 0};
3031       PetscInt    cones[6]            = {1, 2, 3, 4, 5, 6};
3032       PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
3033       PetscScalar vertexCoords[18]    = {-1.0, -1.0, -1.0, -1.0,  1.0, -1.0,   1.0, -1.0, -1.0,
3034                                          -1.0, -1.0,  1.0,  1.0, -1.0,  1.0,  -1.0,  1.0,  1.0};
3035 
3036       ierr = DMSetDimension(rdm, 3);CHKERRQ(ierr);
3037       ierr = DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
3038     }
3039     break;
3040     case DM_POLYTOPE_TRI_PRISM_TENSOR:
3041     {
3042       PetscInt    numPoints[2]        = {6, 1};
3043       PetscInt    coneSize[7]         = {6, 0, 0, 0, 0, 0, 0};
3044       PetscInt    cones[6]            = {1, 2, 3, 4, 5, 6};
3045       PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
3046       PetscScalar vertexCoords[18]    = {-1.0, -1.0, -1.0,  1.0, -1.0, -1.0,  -1.0, 1.0, -1.0,
3047                                          -1.0, -1.0,  1.0,  1.0, -1.0,  1.0,  -1.0, 1.0,  1.0};
3048 
3049       ierr = DMSetDimension(rdm, 3);CHKERRQ(ierr);
3050       ierr = DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
3051     }
3052     break;
3053     case DM_POLYTOPE_QUAD_PRISM_TENSOR:
3054     {
3055       PetscInt    numPoints[2]        = {8, 1};
3056       PetscInt    coneSize[9]         = {8, 0, 0, 0, 0, 0, 0, 0, 0};
3057       PetscInt    cones[8]            = {1, 2, 3, 4, 5, 6, 7, 8};
3058       PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
3059       PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0,  1.0, -1.0, -1.0,  1.0, 1.0, -1.0,  -1.0, 1.0, -1.0,
3060                                          -1.0, -1.0,  1.0,  1.0, -1.0,  1.0,  1.0, 1.0,  1.0,  -1.0, 1.0,  1.0};
3061 
3062       ierr = DMSetDimension(rdm, 3);CHKERRQ(ierr);
3063       ierr = DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
3064     }
3065     break;
3066     case DM_POLYTOPE_PYRAMID:
3067     {
3068       PetscInt    numPoints[2]        = {5, 1};
3069       PetscInt    coneSize[6]         = {5, 0, 0, 0, 0, 0};
3070       PetscInt    cones[5]            = {1, 2, 3, 4, 5};
3071       PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
3072       PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0,  -1.0, 1.0, -1.0,  1.0, 1.0, -1.0,  1.0, -1.0, -1.0,
3073                                           0.0,  0.0,  1.0};
3074 
3075       ierr = DMSetDimension(rdm, 3);CHKERRQ(ierr);
3076       ierr = DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
3077     }
3078     break;
3079     default: SETERRQ(PetscObjectComm((PetscObject) rdm), PETSC_ERR_ARG_WRONG, "Cannot create reference cell for cell type %s", DMPolytopeTypes[ct]);
3080   }
3081   {
3082     PetscInt Nv, v;
3083 
3084     /* Must create the celltype label here so that we do not automatically try to compute the types */
3085     ierr = DMCreateLabel(rdm, "celltype");CHKERRQ(ierr);
3086     ierr = DMPlexSetCellType(rdm, 0, ct);CHKERRQ(ierr);
3087     ierr = DMPlexGetChart(rdm, NULL, &Nv);CHKERRQ(ierr);
3088     for (v = 1; v < Nv; ++v) {ierr = DMPlexSetCellType(rdm, v, DM_POLYTOPE_POINT);CHKERRQ(ierr);}
3089   }
3090   ierr = DMPlexInterpolateInPlace_Internal(rdm);CHKERRQ(ierr);
3091   ierr = PetscObjectSetName((PetscObject) rdm, DMPolytopeTypes[ct]);CHKERRQ(ierr);
3092   PetscFunctionReturn(0);
3093 }
3094 
3095 /*@
3096   DMPlexCreateReferenceCell - Create a DMPLEX with the appropriate FEM reference cell
3097 
3098   Collective
3099 
3100   Input Parameters:
3101 + comm - The communicator
3102 - ct   - The cell type of the reference cell
3103 
3104   Output Parameter:
3105 . refdm - The reference cell
3106 
3107   Level: intermediate
3108 
3109 .seealso: DMPlexCreateReferenceCell(), DMPlexCreateBoxMesh()
3110 @*/
3111 PetscErrorCode DMPlexCreateReferenceCell(MPI_Comm comm, DMPolytopeType ct, DM *refdm)
3112 {
3113   PetscErrorCode ierr;
3114 
3115   PetscFunctionBegin;
3116   ierr = DMCreate(comm, refdm);CHKERRQ(ierr);
3117   ierr = DMSetType(*refdm, DMPLEX);CHKERRQ(ierr);
3118   ierr = DMPlexCreateReferenceCell_Internal(*refdm, ct);CHKERRQ(ierr);
3119   PetscFunctionReturn(0);
3120 }
3121 
3122 static PetscErrorCode DMPlexCreateBoundaryLabel_Private(DM dm, const char name[])
3123 {
3124   DM             plex;
3125   DMLabel        label;
3126   PetscBool      hasLabel;
3127   PetscErrorCode ierr;
3128 
3129   PetscFunctionBeginUser;
3130   ierr = DMHasLabel(dm, name, &hasLabel);CHKERRQ(ierr);
3131   if (hasLabel) PetscFunctionReturn(0);
3132   ierr = DMCreateLabel(dm, name);CHKERRQ(ierr);
3133   ierr = DMGetLabel(dm, name, &label);CHKERRQ(ierr);
3134   ierr = DMConvert(dm, DMPLEX, &plex);CHKERRQ(ierr);
3135   ierr = DMPlexMarkBoundaryFaces(plex, 1, label);CHKERRQ(ierr);
3136   ierr = DMDestroy(&plex);CHKERRQ(ierr);
3137   PetscFunctionReturn(0);
3138 }
3139 
3140 const char * const DMPlexShapes[] = {"box", "box_surface", "ball", "sphere", "cylinder", "schwarz_p", "gyroid", "unknown", "DMPlexShape", "DM_SHAPE_", NULL};
3141 
3142 static PetscErrorCode DMPlexCreateFromOptions_Internal(PetscOptionItems *PetscOptionsObject, PetscBool *useCoordSpace, DM dm)
3143 {
3144   DMPlexShape    shape = DM_SHAPE_BOX;
3145   DMPolytopeType cell  = DM_POLYTOPE_TRIANGLE;
3146   PetscInt       dim   = 2;
3147   PetscBool      simplex = PETSC_TRUE, interpolate = PETSC_TRUE, adjCone = PETSC_FALSE, adjClosure = PETSC_TRUE, refDomain = PETSC_FALSE;
3148   PetscBool      flg, flg2, fflg, bdfflg, nameflg;
3149   MPI_Comm       comm;
3150   char           filename[PETSC_MAX_PATH_LEN]   = "<unspecified>";
3151   char           bdFilename[PETSC_MAX_PATH_LEN] = "<unspecified>";
3152   char           plexname[PETSC_MAX_PATH_LEN]   = "";
3153   PetscErrorCode ierr;
3154 
3155   PetscFunctionBegin;
3156   ierr = PetscObjectGetComm((PetscObject) dm, &comm);CHKERRQ(ierr);
3157   /* TODO Turn this into a registration interface */
3158   ierr = PetscOptionsString("-dm_plex_filename", "File containing a mesh", "DMPlexCreateFromFile", filename, filename, sizeof(filename), &fflg);CHKERRQ(ierr);
3159   ierr = PetscOptionsString("-dm_plex_boundary_filename", "File containing a mesh boundary", "DMPlexCreateFromFile", bdFilename, bdFilename, sizeof(bdFilename), &bdfflg);CHKERRQ(ierr);
3160   ierr = PetscOptionsString("-dm_plex_name", "Name of the mesh in the file", "DMPlexCreateFromFile", plexname, plexname, sizeof(plexname), &nameflg);CHKERRQ(ierr);
3161   ierr = PetscOptionsEnum("-dm_plex_cell", "Cell shape", "", DMPolytopeTypes, (PetscEnum) cell, (PetscEnum *) &cell, NULL);CHKERRQ(ierr);
3162   ierr = PetscOptionsBool("-dm_plex_reference_cell_domain", "Use a reference cell domain", "", refDomain, &refDomain, NULL);CHKERRQ(ierr);
3163   ierr = PetscOptionsEnum("-dm_plex_shape", "Shape for built-in mesh", "", DMPlexShapes, (PetscEnum) shape, (PetscEnum *) &shape, &flg);CHKERRQ(ierr);
3164   ierr = PetscOptionsBoundedInt("-dm_plex_dim", "Topological dimension of the mesh", "DMGetDimension", dim, &dim, &flg, 0);CHKERRQ(ierr);
3165   PetscCheckFalse((dim < 0) || (dim > 3),comm, PETSC_ERR_ARG_OUTOFRANGE, "Dimension %D should be in [1, 3]", dim);
3166   ierr = PetscOptionsBool("-dm_plex_simplex", "Mesh cell shape", "", simplex,  &simplex, &flg);CHKERRQ(ierr);
3167   ierr = PetscOptionsBool("-dm_plex_interpolate", "Flag to create edges and faces automatically", "", interpolate, &interpolate, &flg);CHKERRQ(ierr);
3168   ierr = PetscOptionsBool("-dm_plex_adj_cone", "Set adjacency direction", "DMSetBasicAdjacency", adjCone,  &adjCone, &flg);CHKERRQ(ierr);
3169   ierr = PetscOptionsBool("-dm_plex_adj_closure", "Set adjacency size", "DMSetBasicAdjacency", adjClosure,  &adjClosure, &flg2);CHKERRQ(ierr);
3170   if (flg || flg2) {ierr = DMSetBasicAdjacency(dm, adjCone, adjClosure);CHKERRQ(ierr);}
3171 
3172   switch (cell) {
3173     case DM_POLYTOPE_POINT:
3174     case DM_POLYTOPE_SEGMENT:
3175     case DM_POLYTOPE_POINT_PRISM_TENSOR:
3176     case DM_POLYTOPE_TRIANGLE:
3177     case DM_POLYTOPE_QUADRILATERAL:
3178     case DM_POLYTOPE_TETRAHEDRON:
3179     case DM_POLYTOPE_HEXAHEDRON:
3180       *useCoordSpace = PETSC_TRUE;break;
3181     default: *useCoordSpace = PETSC_FALSE;break;
3182   }
3183 
3184   if (fflg) {
3185     DM dmnew;
3186 
3187     ierr = DMPlexCreateFromFile(PetscObjectComm((PetscObject) dm), filename, plexname, interpolate, &dmnew);CHKERRQ(ierr);
3188     ierr = DMPlexCopy_Internal(dm, PETSC_FALSE, dmnew);CHKERRQ(ierr);
3189     ierr = DMPlexReplace_Static(dm, &dmnew);CHKERRQ(ierr);
3190   } else if (refDomain) {
3191     ierr = DMPlexCreateReferenceCell_Internal(dm, cell);CHKERRQ(ierr);
3192   } else if (bdfflg) {
3193     DM bdm, dmnew;
3194 
3195     ierr = DMPlexCreateFromFile(PetscObjectComm((PetscObject) dm), bdFilename, plexname, interpolate, &bdm);CHKERRQ(ierr);
3196     ierr = PetscObjectSetOptionsPrefix((PetscObject) bdm, "bd_");CHKERRQ(ierr);
3197     ierr = DMSetFromOptions(bdm);CHKERRQ(ierr);
3198     ierr = DMPlexGenerate(bdm, NULL, interpolate, &dmnew);CHKERRQ(ierr);
3199     ierr = DMDestroy(&bdm);CHKERRQ(ierr);
3200     ierr = DMPlexCopy_Internal(dm, PETSC_FALSE, dmnew);CHKERRQ(ierr);
3201     ierr = DMPlexReplace_Static(dm, &dmnew);CHKERRQ(ierr);
3202   } else {
3203     ierr = PetscObjectSetName((PetscObject) dm, DMPlexShapes[shape]);CHKERRQ(ierr);
3204     switch (shape) {
3205       case DM_SHAPE_BOX:
3206       {
3207         PetscInt       faces[3] = {0, 0, 0};
3208         PetscReal      lower[3] = {0, 0, 0};
3209         PetscReal      upper[3] = {1, 1, 1};
3210         DMBoundaryType bdt[3]   = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
3211         PetscInt       i, n;
3212 
3213         n    = dim;
3214         for (i = 0; i < dim; ++i) faces[i] = (dim == 1 ? 1 : 4-dim);
3215         ierr = PetscOptionsIntArray("-dm_plex_box_faces", "Number of faces along each dimension", "", faces, &n, &flg);CHKERRQ(ierr);
3216         n    = 3;
3217         ierr = PetscOptionsRealArray("-dm_plex_box_lower", "Lower left corner of box", "", lower, &n, &flg);CHKERRQ(ierr);
3218         PetscCheckFalse(flg && (n != dim),comm, PETSC_ERR_ARG_SIZ, "Lower box point had %D values, should have been %D", n, dim);
3219         n    = 3;
3220         ierr = PetscOptionsRealArray("-dm_plex_box_upper", "Upper right corner of box", "", upper, &n, &flg);CHKERRQ(ierr);
3221         PetscCheckFalse(flg && (n != dim),comm, PETSC_ERR_ARG_SIZ, "Upper box point had %D values, should have been %D", n, dim);
3222         n    = 3;
3223         ierr = PetscOptionsEnumArray("-dm_plex_box_bd", "Boundary type for each dimension", "", DMBoundaryTypes, (PetscEnum *) bdt, &n, &flg);CHKERRQ(ierr);
3224         PetscCheckFalse(flg && (n != dim),comm, PETSC_ERR_ARG_SIZ, "Box boundary types had %D values, should have been %D", n, dim);
3225         switch (cell) {
3226           case DM_POLYTOPE_TRI_PRISM_TENSOR:
3227             ierr = DMPlexCreateWedgeBoxMesh_Internal(dm, faces, lower, upper, bdt);CHKERRQ(ierr);
3228             if (!interpolate) {
3229               DM udm;
3230 
3231               ierr = DMPlexUninterpolate(dm, &udm);CHKERRQ(ierr);
3232               ierr = DMPlexReplace_Static(dm, &udm);CHKERRQ(ierr);
3233             }
3234             break;
3235           default:
3236             ierr = DMPlexCreateBoxMesh_Internal(dm, dim, simplex, faces, lower, upper, bdt, interpolate);CHKERRQ(ierr);
3237             break;
3238         }
3239       }
3240       break;
3241       case DM_SHAPE_BOX_SURFACE:
3242       {
3243         PetscInt  faces[3] = {0, 0, 0};
3244         PetscReal lower[3] = {0, 0, 0};
3245         PetscReal upper[3] = {1, 1, 1};
3246         PetscInt  i, n;
3247 
3248         n    = dim+1;
3249         for (i = 0; i < dim+1; ++i) faces[i] = (dim+1 == 1 ? 1 : 4-(dim+1));
3250         ierr = PetscOptionsIntArray("-dm_plex_box_faces", "Number of faces along each dimension", "", faces, &n, &flg);CHKERRQ(ierr);
3251         n    = 3;
3252         ierr = PetscOptionsRealArray("-dm_plex_box_lower", "Lower left corner of box", "", lower, &n, &flg);CHKERRQ(ierr);
3253         PetscCheckFalse(flg && (n != dim+1),comm, PETSC_ERR_ARG_SIZ, "Lower box point had %D values, should have been %D", n, dim+1);
3254         n    = 3;
3255         ierr = PetscOptionsRealArray("-dm_plex_box_upper", "Upper right corner of box", "", upper, &n, &flg);CHKERRQ(ierr);
3256         PetscCheckFalse(flg && (n != dim+1),comm, PETSC_ERR_ARG_SIZ, "Upper box point had %D values, should have been %D", n, dim+1);
3257         ierr = DMPlexCreateBoxSurfaceMesh_Internal(dm, dim+1, faces, lower, upper, interpolate);CHKERRQ(ierr);
3258       }
3259       break;
3260       case DM_SHAPE_SPHERE:
3261       {
3262         PetscReal R = 1.0;
3263 
3264         ierr = PetscOptionsReal("-dm_plex_sphere_radius", "Radius of the sphere", "", R,  &R, &flg);CHKERRQ(ierr);
3265         ierr = DMPlexCreateSphereMesh_Internal(dm, dim, simplex, R);CHKERRQ(ierr);
3266       }
3267       break;
3268       case DM_SHAPE_BALL:
3269       {
3270         PetscReal R = 1.0;
3271 
3272         ierr = PetscOptionsReal("-dm_plex_ball_radius", "Radius of the ball", "", R,  &R, &flg);CHKERRQ(ierr);
3273         ierr = DMPlexCreateBallMesh_Internal(dm, dim, R);CHKERRQ(ierr);
3274       }
3275       break;
3276       case DM_SHAPE_CYLINDER:
3277       {
3278         DMBoundaryType bdt = DM_BOUNDARY_NONE;
3279         PetscInt       Nw  = 6;
3280 
3281         ierr = PetscOptionsEnum("-dm_plex_cylinder_bd", "Boundary type in the z direction", "", DMBoundaryTypes, (PetscEnum) bdt, (PetscEnum *) &bdt, NULL);CHKERRQ(ierr);
3282         ierr = PetscOptionsInt("-dm_plex_cylinder_num_wedges", "Number of wedges around the cylinder", "", Nw, &Nw, NULL);CHKERRQ(ierr);
3283         switch (cell) {
3284           case DM_POLYTOPE_TRI_PRISM_TENSOR:
3285             ierr = DMPlexCreateWedgeCylinderMesh_Internal(dm, Nw, interpolate);CHKERRQ(ierr);
3286             break;
3287           default:
3288             ierr = DMPlexCreateHexCylinderMesh_Internal(dm, bdt);CHKERRQ(ierr);
3289             break;
3290         }
3291       }
3292       break;
3293       case DM_SHAPE_SCHWARZ_P: // fallthrough
3294       case DM_SHAPE_GYROID:
3295       {
3296         PetscInt       extent[3] = {1,1,1}, refine = 0, layers = 0, three;
3297         PetscReal      thickness = 0.;
3298         DMBoundaryType periodic[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
3299         DMPlexTPSType  tps_type = shape == DM_SHAPE_SCHWARZ_P ? DMPLEX_TPS_SCHWARZ_P : DMPLEX_TPS_GYROID;
3300         ierr = PetscOptionsIntArray("-dm_plex_tps_extent", "Number of replicas for each of three dimensions", NULL, extent, (three=3, &three), NULL);CHKERRQ(ierr);
3301         ierr = PetscOptionsInt("-dm_plex_tps_refine", "Number of refinements", NULL, refine, &refine, NULL);CHKERRQ(ierr);
3302         ierr = PetscOptionsEnumArray("-dm_plex_tps_periodic", "Periodicity in each of three dimensions", NULL, DMBoundaryTypes, (PetscEnum*)periodic, (three=3, &three), NULL);CHKERRQ(ierr);
3303         ierr = PetscOptionsInt("-dm_plex_tps_layers", "Number of layers in volumetric extrusion (or zero to not extrude)", NULL, layers, &layers, NULL);CHKERRQ(ierr);
3304         ierr = PetscOptionsReal("-dm_plex_tps_thickness", "Thickness of volumetric extrusion", NULL, thickness, &thickness, NULL);CHKERRQ(ierr);
3305 
3306         ierr = DMPlexCreateTPSMesh_Internal(dm, tps_type, extent, periodic, refine, layers, thickness);CHKERRQ(ierr);
3307       }
3308       break;
3309       default: SETERRQ(comm, PETSC_ERR_SUP, "Domain shape %s is unsupported", DMPlexShapes[shape]);
3310     }
3311   }
3312   ierr = DMPlexSetRefinementUniform(dm, PETSC_TRUE);CHKERRQ(ierr);
3313   if (!((PetscObject)dm)->name && nameflg) {
3314     ierr = PetscObjectSetName((PetscObject)dm, plexname);CHKERRQ(ierr);
3315   }
3316   PetscFunctionReturn(0);
3317 }
3318 
3319 PetscErrorCode DMSetFromOptions_NonRefinement_Plex(PetscOptionItems *PetscOptionsObject, DM dm)
3320 {
3321   DM_Plex       *mesh = (DM_Plex*) dm->data;
3322   PetscBool      flg;
3323   char           bdLabel[PETSC_MAX_PATH_LEN];
3324   PetscErrorCode ierr;
3325 
3326   PetscFunctionBegin;
3327   /* Handle viewing */
3328   ierr = PetscOptionsBool("-dm_plex_print_set_values", "Output all set values info", "DMPlexMatSetClosure", PETSC_FALSE, &mesh->printSetValues, NULL);CHKERRQ(ierr);
3329   ierr = PetscOptionsBoundedInt("-dm_plex_print_fem", "Debug output level all fem computations", "DMPlexSNESComputeResidualFEM", 0, &mesh->printFEM, NULL,0);CHKERRQ(ierr);
3330   ierr = PetscOptionsReal("-dm_plex_print_tol", "Tolerance for FEM output", "DMPlexSNESComputeResidualFEM", mesh->printTol, &mesh->printTol, NULL);CHKERRQ(ierr);
3331   ierr = PetscOptionsBoundedInt("-dm_plex_print_l2", "Debug output level all L2 diff computations", "DMComputeL2Diff", 0, &mesh->printL2, NULL,0);CHKERRQ(ierr);
3332   ierr = DMMonitorSetFromOptions(dm, "-dm_plex_monitor_throughput", "Monitor the simulation throughput", "DMPlexMonitorThroughput", DMPlexMonitorThroughput, NULL, &flg);CHKERRQ(ierr);
3333   if (flg) {ierr = PetscLogDefaultBegin();CHKERRQ(ierr);}
3334   /* Labeling */
3335   ierr = PetscOptionsString("-dm_plex_boundary_label", "Label to mark the mesh boundary", "", bdLabel, bdLabel, sizeof(bdLabel), &flg);CHKERRQ(ierr);
3336   if (flg) {ierr = DMPlexCreateBoundaryLabel_Private(dm, bdLabel);CHKERRQ(ierr);}
3337   /* Point Location */
3338   ierr = PetscOptionsBool("-dm_plex_hash_location", "Use grid hashing for point location", "DMInterpolate", PETSC_FALSE, &mesh->useHashLocation, NULL);CHKERRQ(ierr);
3339   /* Partitioning and distribution */
3340   ierr = PetscOptionsBool("-dm_plex_partition_balance", "Attempt to evenly divide points on partition boundary between processes", "DMPlexSetPartitionBalance", PETSC_FALSE, &mesh->partitionBalance, NULL);CHKERRQ(ierr);
3341   /* Generation and remeshing */
3342   ierr = PetscOptionsBool("-dm_plex_remesh_bd", "Allow changes to the boundary on remeshing", "DMAdapt", PETSC_FALSE, &mesh->remeshBd, NULL);CHKERRQ(ierr);
3343   /* Projection behavior */
3344   ierr = PetscOptionsBoundedInt("-dm_plex_max_projection_height", "Maxmimum mesh point height used to project locally", "DMPlexSetMaxProjectionHeight", 0, &mesh->maxProjectionHeight, NULL,0);CHKERRQ(ierr);
3345   ierr = PetscOptionsBool("-dm_plex_regular_refinement", "Use special nested projection algorithm for regular refinement", "DMPlexSetRegularRefinement", mesh->regularRefinement, &mesh->regularRefinement, NULL);CHKERRQ(ierr);
3346   /* Checking structure */
3347   {
3348     PetscBool   flg = PETSC_FALSE, flg2 = PETSC_FALSE, all = PETSC_FALSE;
3349 
3350     ierr = PetscOptionsBool("-dm_plex_check_all", "Perform all checks", NULL, PETSC_FALSE, &all, &flg2);CHKERRQ(ierr);
3351     ierr = PetscOptionsBool("-dm_plex_check_symmetry", "Check that the adjacency information in the mesh is symmetric", "DMPlexCheckSymmetry", PETSC_FALSE, &flg, &flg2);CHKERRQ(ierr);
3352     if (all || (flg && flg2)) {ierr = DMPlexCheckSymmetry(dm);CHKERRQ(ierr);}
3353     ierr = PetscOptionsBool("-dm_plex_check_skeleton", "Check that each cell has the correct number of vertices (only for homogeneous simplex or tensor meshes)", "DMPlexCheckSkeleton", PETSC_FALSE, &flg, &flg2);CHKERRQ(ierr);
3354     if (all || (flg && flg2)) {ierr = DMPlexCheckSkeleton(dm, 0);CHKERRQ(ierr);}
3355     ierr = PetscOptionsBool("-dm_plex_check_faces", "Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type", "DMPlexCheckFaces", PETSC_FALSE, &flg, &flg2);CHKERRQ(ierr);
3356     if (all || (flg && flg2)) {ierr = DMPlexCheckFaces(dm, 0);CHKERRQ(ierr);}
3357     ierr = PetscOptionsBool("-dm_plex_check_geometry", "Check that cells have positive volume", "DMPlexCheckGeometry", PETSC_FALSE, &flg, &flg2);CHKERRQ(ierr);
3358     if (all || (flg && flg2)) {ierr = DMPlexCheckGeometry(dm);CHKERRQ(ierr);}
3359     ierr = PetscOptionsBool("-dm_plex_check_pointsf", "Check some necessary conditions for PointSF", "DMPlexCheckPointSF", PETSC_FALSE, &flg, &flg2);CHKERRQ(ierr);
3360     if (all || (flg && flg2)) {ierr = DMPlexCheckPointSF(dm);CHKERRQ(ierr);}
3361     ierr = PetscOptionsBool("-dm_plex_check_interface_cones", "Check points on inter-partition interfaces have conforming order of cone points", "DMPlexCheckInterfaceCones", PETSC_FALSE, &flg, &flg2);CHKERRQ(ierr);
3362     if (all || (flg && flg2)) {ierr = DMPlexCheckInterfaceCones(dm);CHKERRQ(ierr);}
3363     ierr = PetscOptionsBool("-dm_plex_check_cell_shape", "Check cell shape", "DMPlexCheckCellShape", PETSC_FALSE, &flg, &flg2);CHKERRQ(ierr);
3364     if (flg && flg2) {ierr = DMPlexCheckCellShape(dm, PETSC_TRUE, PETSC_DETERMINE);CHKERRQ(ierr);}
3365   }
3366   {
3367     PetscReal scale = 1.0;
3368 
3369     ierr = PetscOptionsReal("-dm_plex_scale", "Scale factor for mesh coordinates", "DMPlexScale", scale, &scale, &flg);CHKERRQ(ierr);
3370     if (flg) {
3371       Vec coordinates, coordinatesLocal;
3372 
3373       ierr = DMGetCoordinates(dm, &coordinates);CHKERRQ(ierr);
3374       ierr = DMGetCoordinatesLocal(dm, &coordinatesLocal);CHKERRQ(ierr);
3375       ierr = VecScale(coordinates, scale);CHKERRQ(ierr);
3376       ierr = VecScale(coordinatesLocal, scale);CHKERRQ(ierr);
3377     }
3378   }
3379   ierr = PetscPartitionerSetFromOptions(mesh->partitioner);CHKERRQ(ierr);
3380   PetscFunctionReturn(0);
3381 }
3382 
3383 static PetscErrorCode DMSetFromOptions_Plex(PetscOptionItems *PetscOptionsObject,DM dm)
3384 {
3385   PetscFunctionList ordlist;
3386   char              oname[256];
3387   PetscReal         volume = -1.0;
3388   PetscInt          prerefine = 0, refine = 0, r, coarsen = 0, overlap = 0, extLayers = 0, dim;
3389   PetscBool         uniformOrig, created = PETSC_FALSE, uniform = PETSC_TRUE, distribute, interpolate = PETSC_TRUE, coordSpace = PETSC_TRUE, remap = PETSC_TRUE, ghostCells = PETSC_FALSE, isHierarchy, ignoreModel = PETSC_FALSE, flg;
3390   PetscErrorCode ierr;
3391 
3392   PetscFunctionBegin;
3393   PetscValidHeaderSpecific(dm, DM_CLASSID, 2);
3394   ierr = PetscOptionsHead(PetscOptionsObject,"DMPlex Options");CHKERRQ(ierr);
3395   /* Handle automatic creation */
3396   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
3397   if (dim < 0) {ierr = DMPlexCreateFromOptions_Internal(PetscOptionsObject, &coordSpace, dm);CHKERRQ(ierr);created = PETSC_TRUE;}
3398   /* Handle interpolation before distribution */
3399   ierr = PetscOptionsBool("-dm_plex_interpolate_pre", "Flag to interpolate mesh before distribution", "", interpolate, &interpolate, &flg);CHKERRQ(ierr);
3400   if (flg) {
3401     DMPlexInterpolatedFlag interpolated;
3402 
3403     ierr = DMPlexIsInterpolated(dm, &interpolated);CHKERRQ(ierr);
3404     if (interpolated == DMPLEX_INTERPOLATED_FULL && !interpolate) {
3405       DM udm;
3406 
3407       ierr = DMPlexUninterpolate(dm, &udm);CHKERRQ(ierr);
3408       ierr = DMPlexReplace_Static(dm, &udm);CHKERRQ(ierr);
3409     } else if (interpolated != DMPLEX_INTERPOLATED_FULL && interpolate) {
3410       DM idm;
3411 
3412       ierr = DMPlexInterpolate(dm, &idm);CHKERRQ(ierr);
3413       ierr = DMPlexReplace_Static(dm, &idm);CHKERRQ(ierr);
3414     }
3415   }
3416   /* Handle DMPlex refinement before distribution */
3417   ierr = PetscOptionsBool("-dm_refine_ignore_model", "Flag to ignore the geometry model when refining", "DMCreate", ignoreModel, &ignoreModel, &flg);CHKERRQ(ierr);
3418   if (flg) {((DM_Plex *) dm->data)->ignoreModel = ignoreModel;}
3419   ierr = DMPlexGetRefinementUniform(dm, &uniformOrig);CHKERRQ(ierr);
3420   ierr = PetscOptionsBoundedInt("-dm_refine_pre", "The number of refinements before distribution", "DMCreate", prerefine, &prerefine, NULL,0);CHKERRQ(ierr);
3421   ierr = PetscOptionsBool("-dm_refine_remap_pre", "Flag to control coordinate remapping", "DMCreate", remap, &remap, NULL);CHKERRQ(ierr);
3422   ierr = PetscOptionsBool("-dm_refine_uniform_pre", "Flag for uniform refinement before distribution", "DMCreate", uniform, &uniform, &flg);CHKERRQ(ierr);
3423   if (flg) {ierr = DMPlexSetRefinementUniform(dm, uniform);CHKERRQ(ierr);}
3424   ierr = PetscOptionsReal("-dm_refine_volume_limit_pre", "The maximum cell volume after refinement before distribution", "DMCreate", volume, &volume, &flg);CHKERRQ(ierr);
3425   if (flg) {
3426     ierr = DMPlexSetRefinementUniform(dm, PETSC_FALSE);CHKERRQ(ierr);
3427     ierr = DMPlexSetRefinementLimit(dm, volume);CHKERRQ(ierr);
3428     prerefine = PetscMax(prerefine, 1);
3429   }
3430   for (r = 0; r < prerefine; ++r) {
3431     DM             rdm;
3432     PetscPointFunc coordFunc = ((DM_Plex*) dm->data)->coordFunc;
3433 
3434     ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dm);CHKERRQ(ierr);
3435     ierr = DMRefine(dm, PetscObjectComm((PetscObject) dm), &rdm);CHKERRQ(ierr);
3436     ierr = DMPlexReplace_Static(dm, &rdm);CHKERRQ(ierr);
3437     ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dm);CHKERRQ(ierr);
3438     if (coordFunc && remap) {
3439       ierr = DMPlexRemapGeometry(dm, 0.0, coordFunc);CHKERRQ(ierr);
3440       ((DM_Plex*) dm->data)->coordFunc = coordFunc;
3441     }
3442   }
3443   ierr = DMPlexSetRefinementUniform(dm, uniformOrig);CHKERRQ(ierr);
3444   /* Handle DMPlex extrusion before distribution */
3445   ierr = PetscOptionsBoundedInt("-dm_extrude", "The number of layers to extrude", "", extLayers, &extLayers, NULL, 0);CHKERRQ(ierr);
3446   if (extLayers) {
3447     DM edm;
3448 
3449     ierr = DMExtrude(dm, extLayers, &edm);CHKERRQ(ierr);
3450     ierr = DMPlexReplace_Static(dm, &edm);CHKERRQ(ierr);
3451     ((DM_Plex *) dm->data)->coordFunc = NULL;
3452     ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dm);CHKERRQ(ierr);
3453     extLayers = 0;
3454   }
3455   /* Handle DMPlex reordering before distribution */
3456   ierr = MatGetOrderingList(&ordlist);CHKERRQ(ierr);
3457   ierr = PetscOptionsFList("-dm_plex_reorder", "Set mesh reordering type", "DMPlexGetOrdering", ordlist, MATORDERINGNATURAL, oname, sizeof(oname), &flg);CHKERRQ(ierr);
3458   if (flg) {
3459     DM pdm;
3460     IS perm;
3461 
3462     ierr = DMPlexGetOrdering(dm, oname, NULL, &perm);CHKERRQ(ierr);
3463     ierr = DMPlexPermute(dm, perm, &pdm);CHKERRQ(ierr);
3464     ierr = ISDestroy(&perm);CHKERRQ(ierr);
3465     ierr = DMPlexReplace_Static(dm, &pdm);CHKERRQ(ierr);
3466     ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dm);CHKERRQ(ierr);
3467   }
3468   /* Handle DMPlex distribution */
3469   ierr = DMPlexDistributeGetDefault(dm, &distribute);CHKERRQ(ierr);
3470   ierr = PetscOptionsBool("-dm_distribute", "Flag to redistribute a mesh among processes", "DMCreate", distribute, &distribute, NULL);CHKERRQ(ierr);
3471   ierr = PetscOptionsBoundedInt("-dm_distribute_overlap", "The size of the overlap halo", "DMCreate", overlap, &overlap, NULL, 0);CHKERRQ(ierr);
3472   if (distribute) {
3473     DM               pdm = NULL;
3474     PetscPartitioner part;
3475 
3476     ierr = DMPlexGetPartitioner(dm, &part);CHKERRQ(ierr);
3477     ierr = PetscPartitionerSetFromOptions(part);CHKERRQ(ierr);
3478     ierr = DMPlexDistribute(dm, overlap, NULL, &pdm);CHKERRQ(ierr);
3479     if (pdm) {
3480       ierr = DMPlexReplace_Static(dm, &pdm);CHKERRQ(ierr);
3481     }
3482   }
3483   /* Create coordinate space */
3484   if (created) {
3485     DM_Plex  *mesh = (DM_Plex *) dm->data;
3486     PetscInt  degree = 1;
3487     PetscBool periodic, flg;
3488 
3489     ierr = PetscOptionsBool("-dm_coord_space", "Use an FEM space for coordinates", "", coordSpace, &coordSpace, &flg);CHKERRQ(ierr);
3490     ierr = PetscOptionsInt("-dm_coord_petscspace_degree", "FEM degree for coordinate space", "", degree, &degree, NULL);CHKERRQ(ierr);
3491     if (coordSpace) {ierr = DMPlexCreateCoordinateSpace(dm, degree, mesh->coordFunc);CHKERRQ(ierr);}
3492     if (flg && !coordSpace) {
3493       DM           cdm;
3494       PetscDS      cds;
3495       PetscObject  obj;
3496       PetscClassId id;
3497 
3498       ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
3499       ierr = DMGetDS(cdm, &cds);CHKERRQ(ierr);
3500       ierr = PetscDSGetDiscretization(cds, 0, &obj);CHKERRQ(ierr);
3501       ierr = PetscObjectGetClassId(obj, &id);CHKERRQ(ierr);
3502       if (id == PETSCFE_CLASSID) {
3503         PetscContainer dummy;
3504 
3505         ierr = PetscContainerCreate(PETSC_COMM_SELF, &dummy);CHKERRQ(ierr);
3506         ierr = PetscObjectSetName((PetscObject) dummy, "coordinates");CHKERRQ(ierr);
3507         ierr = DMSetField(cdm, 0, NULL, (PetscObject) dummy);CHKERRQ(ierr);
3508         ierr = PetscContainerDestroy(&dummy);CHKERRQ(ierr);
3509         ierr = DMClearDS(cdm);CHKERRQ(ierr);
3510       }
3511       mesh->coordFunc = NULL;
3512     }
3513     ierr = DMLocalizeCoordinates(dm);CHKERRQ(ierr);
3514     ierr = DMGetPeriodicity(dm, &periodic, NULL, NULL, NULL);CHKERRQ(ierr);
3515     if (periodic) {ierr = DMSetPeriodicity(dm, PETSC_TRUE, NULL, NULL, NULL);CHKERRQ(ierr);}
3516   }
3517   /* Handle DMPlex refinement */
3518   remap = PETSC_TRUE;
3519   ierr = PetscOptionsBoundedInt("-dm_refine", "The number of uniform refinements", "DMCreate", refine, &refine, NULL,0);CHKERRQ(ierr);
3520   ierr = PetscOptionsBool("-dm_refine_remap", "Flag to control coordinate remapping", "DMCreate", remap, &remap, NULL);CHKERRQ(ierr);
3521   ierr = PetscOptionsBoundedInt("-dm_refine_hierarchy", "The number of uniform refinements", "DMCreate", refine, &refine, &isHierarchy,0);CHKERRQ(ierr);
3522   if (refine) {ierr = DMPlexSetRefinementUniform(dm, PETSC_TRUE);CHKERRQ(ierr);}
3523   if (refine && isHierarchy) {
3524     DM *dms, coarseDM;
3525 
3526     ierr = DMGetCoarseDM(dm, &coarseDM);CHKERRQ(ierr);
3527     ierr = PetscObjectReference((PetscObject)coarseDM);CHKERRQ(ierr);
3528     ierr = PetscMalloc1(refine,&dms);CHKERRQ(ierr);
3529     ierr = DMRefineHierarchy(dm, refine, dms);CHKERRQ(ierr);
3530     /* Total hack since we do not pass in a pointer */
3531     ierr = DMPlexSwap_Static(dm, dms[refine-1]);CHKERRQ(ierr);
3532     if (refine == 1) {
3533       ierr = DMSetCoarseDM(dm, dms[0]);CHKERRQ(ierr);
3534       ierr = DMPlexSetRegularRefinement(dm, PETSC_TRUE);CHKERRQ(ierr);
3535     } else {
3536       ierr = DMSetCoarseDM(dm, dms[refine-2]);CHKERRQ(ierr);
3537       ierr = DMPlexSetRegularRefinement(dm, PETSC_TRUE);CHKERRQ(ierr);
3538       ierr = DMSetCoarseDM(dms[0], dms[refine-1]);CHKERRQ(ierr);
3539       ierr = DMPlexSetRegularRefinement(dms[0], PETSC_TRUE);CHKERRQ(ierr);
3540     }
3541     ierr = DMSetCoarseDM(dms[refine-1], coarseDM);CHKERRQ(ierr);
3542     ierr = PetscObjectDereference((PetscObject)coarseDM);CHKERRQ(ierr);
3543     /* Free DMs */
3544     for (r = 0; r < refine; ++r) {
3545       ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dms[r]);CHKERRQ(ierr);
3546       ierr = DMDestroy(&dms[r]);CHKERRQ(ierr);
3547     }
3548     ierr = PetscFree(dms);CHKERRQ(ierr);
3549   } else {
3550     for (r = 0; r < refine; ++r) {
3551       DM             rdm;
3552       PetscPointFunc coordFunc = ((DM_Plex*) dm->data)->coordFunc;
3553 
3554       ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dm);CHKERRQ(ierr);
3555       ierr = DMRefine(dm, PetscObjectComm((PetscObject) dm), &rdm);CHKERRQ(ierr);
3556       /* Total hack since we do not pass in a pointer */
3557       ierr = DMPlexReplace_Static(dm, &rdm);CHKERRQ(ierr);
3558       ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dm);CHKERRQ(ierr);
3559       if (coordFunc && remap) {
3560         ierr = DMPlexRemapGeometry(dm, 0.0, coordFunc);CHKERRQ(ierr);
3561         ((DM_Plex*) dm->data)->coordFunc = coordFunc;
3562       }
3563     }
3564   }
3565   /* Handle DMPlex coarsening */
3566   ierr = PetscOptionsBoundedInt("-dm_coarsen", "Coarsen the mesh", "DMCreate", coarsen, &coarsen, NULL,0);CHKERRQ(ierr);
3567   ierr = PetscOptionsBoundedInt("-dm_coarsen_hierarchy", "The number of coarsenings", "DMCreate", coarsen, &coarsen, &isHierarchy,0);CHKERRQ(ierr);
3568   if (coarsen && isHierarchy) {
3569     DM *dms;
3570 
3571     ierr = PetscMalloc1(coarsen, &dms);CHKERRQ(ierr);
3572     ierr = DMCoarsenHierarchy(dm, coarsen, dms);CHKERRQ(ierr);
3573     /* Free DMs */
3574     for (r = 0; r < coarsen; ++r) {
3575       ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dms[r]);CHKERRQ(ierr);
3576       ierr = DMDestroy(&dms[r]);CHKERRQ(ierr);
3577     }
3578     ierr = PetscFree(dms);CHKERRQ(ierr);
3579   } else {
3580     for (r = 0; r < coarsen; ++r) {
3581       DM             cdm;
3582       PetscPointFunc coordFunc = ((DM_Plex*) dm->data)->coordFunc;
3583 
3584       ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dm);CHKERRQ(ierr);
3585       ierr = DMCoarsen(dm, PetscObjectComm((PetscObject) dm), &cdm);CHKERRQ(ierr);
3586       /* Total hack since we do not pass in a pointer */
3587       ierr = DMPlexReplace_Static(dm, &cdm);CHKERRQ(ierr);
3588       ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dm);CHKERRQ(ierr);
3589       if (coordFunc) {
3590         ierr = DMPlexRemapGeometry(dm, 0.0, coordFunc);CHKERRQ(ierr);
3591         ((DM_Plex*) dm->data)->coordFunc = coordFunc;
3592       }
3593     }
3594   }
3595   /* Handle ghost cells */
3596   ierr = PetscOptionsBool("-dm_plex_create_fv_ghost_cells", "Flag to create finite volume ghost cells on the boundary", "DMCreate", ghostCells, &ghostCells, NULL);CHKERRQ(ierr);
3597   if (ghostCells) {
3598     DM   gdm;
3599     char lname[PETSC_MAX_PATH_LEN];
3600 
3601     lname[0] = '\0';
3602     ierr = PetscOptionsString("-dm_plex_fv_ghost_cells_label", "Label name for ghost cells boundary", "DMCreate", lname, lname, sizeof(lname), &flg);CHKERRQ(ierr);
3603     ierr = DMPlexConstructGhostCells(dm, flg ? lname : NULL, NULL, &gdm);CHKERRQ(ierr);
3604     ierr = DMPlexReplace_Static(dm, &gdm);CHKERRQ(ierr);
3605   }
3606   /* Handle */
3607   ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dm);CHKERRQ(ierr);
3608   ierr = PetscOptionsTail();CHKERRQ(ierr);
3609   PetscFunctionReturn(0);
3610 }
3611 
3612 static PetscErrorCode DMCreateGlobalVector_Plex(DM dm,Vec *vec)
3613 {
3614   PetscErrorCode ierr;
3615 
3616   PetscFunctionBegin;
3617   ierr = DMCreateGlobalVector_Section_Private(dm,vec);CHKERRQ(ierr);
3618   /* ierr = VecSetOperation(*vec, VECOP_DUPLICATE, (void(*)(void)) VecDuplicate_MPI_DM);CHKERRQ(ierr); */
3619   ierr = VecSetOperation(*vec, VECOP_VIEW, (void (*)(void)) VecView_Plex);CHKERRQ(ierr);
3620   ierr = VecSetOperation(*vec, VECOP_VIEWNATIVE, (void (*)(void)) VecView_Plex_Native);CHKERRQ(ierr);
3621   ierr = VecSetOperation(*vec, VECOP_LOAD, (void (*)(void)) VecLoad_Plex);CHKERRQ(ierr);
3622   ierr = VecSetOperation(*vec, VECOP_LOADNATIVE, (void (*)(void)) VecLoad_Plex_Native);CHKERRQ(ierr);
3623   PetscFunctionReturn(0);
3624 }
3625 
3626 static PetscErrorCode DMCreateLocalVector_Plex(DM dm,Vec *vec)
3627 {
3628   PetscErrorCode ierr;
3629 
3630   PetscFunctionBegin;
3631   ierr = DMCreateLocalVector_Section_Private(dm,vec);CHKERRQ(ierr);
3632   ierr = VecSetOperation(*vec, VECOP_VIEW, (void (*)(void)) VecView_Plex_Local);CHKERRQ(ierr);
3633   ierr = VecSetOperation(*vec, VECOP_LOAD, (void (*)(void)) VecLoad_Plex_Local);CHKERRQ(ierr);
3634   PetscFunctionReturn(0);
3635 }
3636 
3637 static PetscErrorCode DMGetDimPoints_Plex(DM dm, PetscInt dim, PetscInt *pStart, PetscInt *pEnd)
3638 {
3639   PetscInt       depth, d;
3640   PetscErrorCode ierr;
3641 
3642   PetscFunctionBegin;
3643   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3644   if (depth == 1) {
3645     ierr = DMGetDimension(dm, &d);CHKERRQ(ierr);
3646     if (dim == 0)      {ierr = DMPlexGetDepthStratum(dm, dim, pStart, pEnd);CHKERRQ(ierr);}
3647     else if (dim == d) {ierr = DMPlexGetDepthStratum(dm, 1, pStart, pEnd);CHKERRQ(ierr);}
3648     else               {*pStart = 0; *pEnd = 0;}
3649   } else {
3650     ierr = DMPlexGetDepthStratum(dm, dim, pStart, pEnd);CHKERRQ(ierr);
3651   }
3652   PetscFunctionReturn(0);
3653 }
3654 
3655 static PetscErrorCode DMGetNeighbors_Plex(DM dm, PetscInt *nranks, const PetscMPIInt *ranks[])
3656 {
3657   PetscSF           sf;
3658   PetscInt          niranks, njranks, n;
3659   const PetscMPIInt *iranks, *jranks;
3660   DM_Plex           *data = (DM_Plex*) dm->data;
3661   PetscErrorCode    ierr;
3662 
3663   PetscFunctionBegin;
3664   ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr);
3665   if (!data->neighbors) {
3666     ierr = PetscSFSetUp(sf);CHKERRQ(ierr);
3667     ierr = PetscSFGetRootRanks(sf, &njranks, &jranks, NULL, NULL, NULL);CHKERRQ(ierr);
3668     ierr = PetscSFGetLeafRanks(sf, &niranks, &iranks, NULL, NULL);CHKERRQ(ierr);
3669     ierr = PetscMalloc1(njranks + niranks + 1, &data->neighbors);CHKERRQ(ierr);
3670     ierr = PetscArraycpy(data->neighbors + 1, jranks, njranks);CHKERRQ(ierr);
3671     ierr = PetscArraycpy(data->neighbors + njranks + 1, iranks, niranks);CHKERRQ(ierr);
3672     n = njranks + niranks;
3673     ierr = PetscSortRemoveDupsMPIInt(&n, data->neighbors + 1);CHKERRQ(ierr);
3674     /* The following cast should never fail: can't have more neighbors than PETSC_MPI_INT_MAX */
3675     ierr = PetscMPIIntCast(n, data->neighbors);CHKERRQ(ierr);
3676   }
3677   if (nranks) *nranks = data->neighbors[0];
3678   if (ranks) {
3679     if (data->neighbors[0]) *ranks = data->neighbors + 1;
3680     else                    *ranks = NULL;
3681   }
3682   PetscFunctionReturn(0);
3683 }
3684 
3685 PETSC_INTERN PetscErrorCode DMInterpolateSolution_Plex(DM, DM, Mat, Vec, Vec);
3686 
3687 static PetscErrorCode DMInitialize_Plex(DM dm)
3688 {
3689   PetscErrorCode ierr;
3690 
3691   PetscFunctionBegin;
3692   dm->ops->view                            = DMView_Plex;
3693   dm->ops->load                            = DMLoad_Plex;
3694   dm->ops->setfromoptions                  = DMSetFromOptions_Plex;
3695   dm->ops->clone                           = DMClone_Plex;
3696   dm->ops->setup                           = DMSetUp_Plex;
3697   dm->ops->createlocalsection              = DMCreateLocalSection_Plex;
3698   dm->ops->createdefaultconstraints        = DMCreateDefaultConstraints_Plex;
3699   dm->ops->createglobalvector              = DMCreateGlobalVector_Plex;
3700   dm->ops->createlocalvector               = DMCreateLocalVector_Plex;
3701   dm->ops->getlocaltoglobalmapping         = NULL;
3702   dm->ops->createfieldis                   = NULL;
3703   dm->ops->createcoordinatedm              = DMCreateCoordinateDM_Plex;
3704   dm->ops->createcoordinatefield           = DMCreateCoordinateField_Plex;
3705   dm->ops->getcoloring                     = NULL;
3706   dm->ops->creatematrix                    = DMCreateMatrix_Plex;
3707   dm->ops->createinterpolation             = DMCreateInterpolation_Plex;
3708   dm->ops->createmassmatrix                = DMCreateMassMatrix_Plex;
3709   dm->ops->createmassmatrixlumped          = DMCreateMassMatrixLumped_Plex;
3710   dm->ops->createinjection                 = DMCreateInjection_Plex;
3711   dm->ops->refine                          = DMRefine_Plex;
3712   dm->ops->coarsen                         = DMCoarsen_Plex;
3713   dm->ops->refinehierarchy                 = DMRefineHierarchy_Plex;
3714   dm->ops->coarsenhierarchy                = DMCoarsenHierarchy_Plex;
3715   dm->ops->extrude                         = DMExtrude_Plex;
3716   dm->ops->globaltolocalbegin              = NULL;
3717   dm->ops->globaltolocalend                = NULL;
3718   dm->ops->localtoglobalbegin              = NULL;
3719   dm->ops->localtoglobalend                = NULL;
3720   dm->ops->destroy                         = DMDestroy_Plex;
3721   dm->ops->createsubdm                     = DMCreateSubDM_Plex;
3722   dm->ops->createsuperdm                   = DMCreateSuperDM_Plex;
3723   dm->ops->getdimpoints                    = DMGetDimPoints_Plex;
3724   dm->ops->locatepoints                    = DMLocatePoints_Plex;
3725   dm->ops->projectfunctionlocal            = DMProjectFunctionLocal_Plex;
3726   dm->ops->projectfunctionlabellocal       = DMProjectFunctionLabelLocal_Plex;
3727   dm->ops->projectfieldlocal               = DMProjectFieldLocal_Plex;
3728   dm->ops->projectfieldlabellocal          = DMProjectFieldLabelLocal_Plex;
3729   dm->ops->projectbdfieldlabellocal        = DMProjectBdFieldLabelLocal_Plex;
3730   dm->ops->computel2diff                   = DMComputeL2Diff_Plex;
3731   dm->ops->computel2gradientdiff           = DMComputeL2GradientDiff_Plex;
3732   dm->ops->computel2fielddiff              = DMComputeL2FieldDiff_Plex;
3733   dm->ops->getneighbors                    = DMGetNeighbors_Plex;
3734   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMPlexInsertBoundaryValues_C",DMPlexInsertBoundaryValues_Plex);CHKERRQ(ierr);
3735   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMPlexInsertTimeDerviativeBoundaryValues_C",DMPlexInsertTimeDerivativeBoundaryValues_Plex);CHKERRQ(ierr);
3736   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMSetUpGLVisViewer_C",DMSetUpGLVisViewer_Plex);CHKERRQ(ierr);
3737   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMCreateNeumannOverlap_C",DMCreateNeumannOverlap_Plex);CHKERRQ(ierr);
3738   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMPlexGetOverlap_C",DMPlexGetOverlap_Plex);CHKERRQ(ierr);
3739   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMPlexDistributeGetDefault_C",DMPlexDistributeGetDefault_Plex);CHKERRQ(ierr);
3740   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMPlexDistributeSetDefault_C",DMPlexDistributeSetDefault_Plex);CHKERRQ(ierr);
3741   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMInterpolateSolution_C",DMInterpolateSolution_Plex);CHKERRQ(ierr);
3742   PetscFunctionReturn(0);
3743 }
3744 
3745 PETSC_INTERN PetscErrorCode DMClone_Plex(DM dm, DM *newdm)
3746 {
3747   DM_Plex        *mesh = (DM_Plex *) dm->data;
3748   PetscErrorCode ierr;
3749 
3750   PetscFunctionBegin;
3751   mesh->refct++;
3752   (*newdm)->data = mesh;
3753   ierr = PetscObjectChangeTypeName((PetscObject) *newdm, DMPLEX);CHKERRQ(ierr);
3754   ierr = DMInitialize_Plex(*newdm);CHKERRQ(ierr);
3755   PetscFunctionReturn(0);
3756 }
3757 
3758 /*MC
3759   DMPLEX = "plex" - A DM object that encapsulates an unstructured mesh, or CW Complex, which can be expressed using a Hasse Diagram.
3760                     In the local representation, Vecs contain all unknowns in the interior and shared boundary. This is
3761                     specified by a PetscSection object. Ownership in the global representation is determined by
3762                     ownership of the underlying DMPlex points. This is specified by another PetscSection object.
3763 
3764   Options Database Keys:
3765 + -dm_refine_pre                     - Refine mesh before distribution
3766 + -dm_refine_uniform_pre             - Choose uniform or generator-based refinement
3767 + -dm_refine_volume_limit_pre        - Cell volume limit after pre-refinement using generator
3768 . -dm_distribute                     - Distribute mesh across processes
3769 . -dm_distribute_overlap             - Number of cells to overlap for distribution
3770 . -dm_refine                         - Refine mesh after distribution
3771 . -dm_plex_hash_location             - Use grid hashing for point location
3772 . -dm_plex_hash_box_faces <n,m,p>    - The number of divisions in each direction of the grid hash
3773 . -dm_plex_partition_balance         - Attempt to evenly divide points on partition boundary between processes
3774 . -dm_plex_remesh_bd                 - Allow changes to the boundary on remeshing
3775 . -dm_plex_max_projection_height     - Maxmimum mesh point height used to project locally
3776 . -dm_plex_regular_refinement        - Use special nested projection algorithm for regular refinement
3777 . -dm_plex_check_all                 - Perform all shecks below
3778 . -dm_plex_check_symmetry            - Check that the adjacency information in the mesh is symmetric
3779 . -dm_plex_check_skeleton <celltype> - Check that each cell has the correct number of vertices
3780 . -dm_plex_check_faces <celltype>    - Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type
3781 . -dm_plex_check_geometry            - Check that cells have positive volume
3782 . -dm_view :mesh.tex:ascii_latex     - View the mesh in LaTeX/TikZ
3783 . -dm_plex_view_scale <num>          - Scale the TikZ
3784 - -dm_plex_print_fem <num>           - View FEM assembly information, such as element vectors and matrices
3785 
3786   Level: intermediate
3787 
3788 .seealso: DMType, DMPlexCreate(), DMCreate(), DMSetType()
3789 M*/
3790 
3791 PETSC_EXTERN PetscErrorCode DMCreate_Plex(DM dm)
3792 {
3793   DM_Plex       *mesh;
3794   PetscInt       unit;
3795   PetscErrorCode ierr;
3796 
3797   PetscFunctionBegin;
3798   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3799   ierr     = PetscNewLog(dm,&mesh);CHKERRQ(ierr);
3800   dm->data = mesh;
3801 
3802   mesh->refct             = 1;
3803   ierr                    = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &mesh->coneSection);CHKERRQ(ierr);
3804   mesh->maxConeSize       = 0;
3805   mesh->cones             = NULL;
3806   mesh->coneOrientations  = NULL;
3807   ierr                    = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &mesh->supportSection);CHKERRQ(ierr);
3808   mesh->maxSupportSize    = 0;
3809   mesh->supports          = NULL;
3810   mesh->refinementUniform = PETSC_TRUE;
3811   mesh->refinementLimit   = -1.0;
3812   mesh->distDefault       = PETSC_TRUE;
3813   mesh->interpolated      = DMPLEX_INTERPOLATED_INVALID;
3814   mesh->interpolatedCollective = DMPLEX_INTERPOLATED_INVALID;
3815 
3816   mesh->facesTmp = NULL;
3817 
3818   mesh->tetgenOpts   = NULL;
3819   mesh->triangleOpts = NULL;
3820   ierr = PetscPartitionerCreate(PetscObjectComm((PetscObject)dm), &mesh->partitioner);CHKERRQ(ierr);
3821   mesh->remeshBd     = PETSC_FALSE;
3822 
3823   mesh->subpointMap = NULL;
3824 
3825   for (unit = 0; unit < NUM_PETSC_UNITS; ++unit) mesh->scale[unit] = 1.0;
3826 
3827   mesh->regularRefinement   = PETSC_FALSE;
3828   mesh->depthState          = -1;
3829   mesh->celltypeState       = -1;
3830   mesh->globalVertexNumbers = NULL;
3831   mesh->globalCellNumbers   = NULL;
3832   mesh->anchorSection       = NULL;
3833   mesh->anchorIS            = NULL;
3834   mesh->createanchors       = NULL;
3835   mesh->computeanchormatrix = NULL;
3836   mesh->parentSection       = NULL;
3837   mesh->parents             = NULL;
3838   mesh->childIDs            = NULL;
3839   mesh->childSection        = NULL;
3840   mesh->children            = NULL;
3841   mesh->referenceTree       = NULL;
3842   mesh->getchildsymmetry    = NULL;
3843   mesh->vtkCellHeight       = 0;
3844   mesh->useAnchors          = PETSC_FALSE;
3845 
3846   mesh->maxProjectionHeight = 0;
3847 
3848   mesh->neighbors           = NULL;
3849 
3850   mesh->printSetValues = PETSC_FALSE;
3851   mesh->printFEM       = 0;
3852   mesh->printTol       = 1.0e-10;
3853 
3854   ierr = DMInitialize_Plex(dm);CHKERRQ(ierr);
3855   PetscFunctionReturn(0);
3856 }
3857 
3858 /*@
3859   DMPlexCreate - Creates a DMPlex object, which encapsulates an unstructured mesh, or CW complex, which can be expressed using a Hasse Diagram.
3860 
3861   Collective
3862 
3863   Input Parameter:
3864 . comm - The communicator for the DMPlex object
3865 
3866   Output Parameter:
3867 . mesh  - The DMPlex object
3868 
3869   Level: beginner
3870 
3871 @*/
3872 PetscErrorCode DMPlexCreate(MPI_Comm comm, DM *mesh)
3873 {
3874   PetscErrorCode ierr;
3875 
3876   PetscFunctionBegin;
3877   PetscValidPointer(mesh,2);
3878   ierr = DMCreate(comm, mesh);CHKERRQ(ierr);
3879   ierr = DMSetType(*mesh, DMPLEX);CHKERRQ(ierr);
3880   PetscFunctionReturn(0);
3881 }
3882 
3883 /*@C
3884   DMPlexBuildFromCellListParallel - Build distributed DMPLEX topology from a list of vertices for each cell (common mesh generator output)
3885 
3886   Input Parameters:
3887 + dm - The DM
3888 . numCells - The number of cells owned by this process
3889 . numVertices - The number of vertices to be owned by this process, or PETSC_DECIDE
3890 . NVertices - The global number of vertices, or PETSC_DETERMINE
3891 . numCorners - The number of vertices for each cell
3892 - cells - An array of numCells*numCorners numbers, the global vertex numbers for each cell
3893 
3894   Output Parameters:
3895 + vertexSF - (Optional) SF describing complete vertex ownership
3896 - verticesAdjSaved - (Optional) vertex adjacency array
3897 
3898   Notes:
3899   Two triangles sharing a face
3900 $
3901 $        2
3902 $      / | \
3903 $     /  |  \
3904 $    /   |   \
3905 $   0  0 | 1  3
3906 $    \   |   /
3907 $     \  |  /
3908 $      \ | /
3909 $        1
3910 would have input
3911 $  numCells = 2, numVertices = 4
3912 $  cells = [0 1 2  1 3 2]
3913 $
3914 which would result in the DMPlex
3915 $
3916 $        4
3917 $      / | \
3918 $     /  |  \
3919 $    /   |   \
3920 $   2  0 | 1  5
3921 $    \   |   /
3922 $     \  |  /
3923 $      \ | /
3924 $        3
3925 
3926   Vertices are implicitly numbered consecutively 0,...,NVertices.
3927   Each rank owns a chunk of numVertices consecutive vertices.
3928   If numVertices is PETSC_DECIDE, PETSc will distribute them as evenly as possible using PetscLayout.
3929   If NVertices is PETSC_DETERMINE and numVertices is PETSC_DECIDE, NVertices is computed by PETSc as the maximum vertex index in cells + 1.
3930   If only NVertices is PETSC_DETERMINE, it is computed as the sum of numVertices over all ranks.
3931 
3932   The cell distribution is arbitrary non-overlapping, independent of the vertex distribution.
3933 
3934   Not currently supported in Fortran.
3935 
3936   Level: advanced
3937 
3938 .seealso: DMPlexBuildFromCellList(), DMPlexCreateFromCellListParallelPetsc(), DMPlexBuildCoordinatesFromCellListParallel()
3939 @*/
3940 PetscErrorCode DMPlexBuildFromCellListParallel(DM dm, PetscInt numCells, PetscInt numVertices, PetscInt NVertices, PetscInt numCorners, const PetscInt cells[], PetscSF *vertexSF, PetscInt **verticesAdjSaved)
3941 {
3942   PetscSF         sfPoint;
3943   PetscLayout     layout;
3944   PetscInt        numVerticesAdj, *verticesAdj, *cones, c, p, dim;
3945   PetscMPIInt     rank, size;
3946   PetscErrorCode  ierr;
3947 
3948   PetscFunctionBegin;
3949   PetscValidLogicalCollectiveInt(dm,NVertices,4);
3950   ierr = PetscLogEventBegin(DMPLEX_BuildFromCellList,dm,0,0,0);CHKERRQ(ierr);
3951   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRMPI(ierr);
3952   ierr = MPI_Comm_size(PetscObjectComm((PetscObject) dm), &size);CHKERRMPI(ierr);
3953   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
3954   /* Get/check global number of vertices */
3955   {
3956     PetscInt NVerticesInCells, i;
3957     const PetscInt len = numCells * numCorners;
3958 
3959     /* NVerticesInCells = max(cells) + 1 */
3960     NVerticesInCells = PETSC_MIN_INT;
3961     for (i=0; i<len; i++) if (cells[i] > NVerticesInCells) NVerticesInCells = cells[i];
3962     ++NVerticesInCells;
3963     ierr = MPI_Allreduce(MPI_IN_PLACE, &NVerticesInCells, 1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject) dm));CHKERRMPI(ierr);
3964 
3965     if (numVertices == PETSC_DECIDE && NVertices == PETSC_DECIDE) NVertices = NVerticesInCells;
3966     else PetscCheckFalse(NVertices != PETSC_DECIDE && NVertices < NVerticesInCells,PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Specified global number of vertices %D must be greater than or equal to the number of vertices in cells %D",NVertices,NVerticesInCells);
3967   }
3968   /* Count locally unique vertices */
3969   {
3970     PetscHSetI vhash;
3971     PetscInt off = 0;
3972 
3973     ierr = PetscHSetICreate(&vhash);CHKERRQ(ierr);
3974     for (c = 0; c < numCells; ++c) {
3975       for (p = 0; p < numCorners; ++p) {
3976         ierr = PetscHSetIAdd(vhash, cells[c*numCorners+p]);CHKERRQ(ierr);
3977       }
3978     }
3979     ierr = PetscHSetIGetSize(vhash, &numVerticesAdj);CHKERRQ(ierr);
3980     if (!verticesAdjSaved) { ierr = PetscMalloc1(numVerticesAdj, &verticesAdj);CHKERRQ(ierr); }
3981     else { verticesAdj = *verticesAdjSaved; }
3982     ierr = PetscHSetIGetElems(vhash, &off, verticesAdj);CHKERRQ(ierr);
3983     ierr = PetscHSetIDestroy(&vhash);CHKERRQ(ierr);
3984     PetscCheckFalse(off != numVerticesAdj,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid number of local vertices %D should be %D", off, numVerticesAdj);
3985   }
3986   ierr = PetscSortInt(numVerticesAdj, verticesAdj);CHKERRQ(ierr);
3987   /* Create cones */
3988   ierr = DMPlexSetChart(dm, 0, numCells+numVerticesAdj);CHKERRQ(ierr);
3989   for (c = 0; c < numCells; ++c) {ierr = DMPlexSetConeSize(dm, c, numCorners);CHKERRQ(ierr);}
3990   ierr = DMSetUp(dm);CHKERRQ(ierr);
3991   ierr = DMPlexGetCones(dm,&cones);CHKERRQ(ierr);
3992   for (c = 0; c < numCells; ++c) {
3993     for (p = 0; p < numCorners; ++p) {
3994       const PetscInt gv = cells[c*numCorners+p];
3995       PetscInt       lv;
3996 
3997       /* Positions within verticesAdj form 0-based local vertex numbering;
3998          we need to shift it by numCells to get correct DAG points (cells go first) */
3999       ierr = PetscFindInt(gv, numVerticesAdj, verticesAdj, &lv);CHKERRQ(ierr);
4000       PetscCheckFalse(lv < 0,PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Could not find global vertex %D in local connectivity", gv);
4001       cones[c*numCorners+p] = lv+numCells;
4002     }
4003   }
4004   /* Build point sf */
4005   ierr = PetscLayoutCreate(PetscObjectComm((PetscObject)dm), &layout);CHKERRQ(ierr);
4006   ierr = PetscLayoutSetSize(layout, NVertices);CHKERRQ(ierr);
4007   ierr = PetscLayoutSetLocalSize(layout, numVertices);CHKERRQ(ierr);
4008   ierr = PetscLayoutSetBlockSize(layout, 1);CHKERRQ(ierr);
4009   ierr = PetscSFCreateByMatchingIndices(layout, numVerticesAdj, verticesAdj, NULL, numCells, numVerticesAdj, verticesAdj, NULL, numCells, vertexSF, &sfPoint);CHKERRQ(ierr);
4010   ierr = PetscLayoutDestroy(&layout);CHKERRQ(ierr);
4011   if (!verticesAdjSaved) { ierr = PetscFree(verticesAdj);CHKERRQ(ierr); }
4012   ierr = PetscObjectSetName((PetscObject) sfPoint, "point SF");CHKERRQ(ierr);
4013   if (dm->sf) {
4014     const char *prefix;
4015 
4016     ierr = PetscObjectGetOptionsPrefix((PetscObject)dm->sf, &prefix);CHKERRQ(ierr);
4017     ierr = PetscObjectSetOptionsPrefix((PetscObject)sfPoint, prefix);CHKERRQ(ierr);
4018   }
4019   ierr = DMSetPointSF(dm, sfPoint);CHKERRQ(ierr);
4020   ierr = PetscSFDestroy(&sfPoint);CHKERRQ(ierr);
4021   if (vertexSF) {ierr = PetscObjectSetName((PetscObject)(*vertexSF), "Vertex Ownership SF");CHKERRQ(ierr);}
4022   /* Fill in the rest of the topology structure */
4023   ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
4024   ierr = DMPlexStratify(dm);CHKERRQ(ierr);
4025   ierr = PetscLogEventEnd(DMPLEX_BuildFromCellList,dm,0,0,0);CHKERRQ(ierr);
4026   PetscFunctionReturn(0);
4027 }
4028 
4029 /*@C
4030   DMPlexBuildCoordinatesFromCellListParallel - Build DM coordinates from a list of coordinates for each owned vertex (common mesh generator output)
4031 
4032   Input Parameters:
4033 + dm - The DM
4034 . spaceDim - The spatial dimension used for coordinates
4035 . sfVert - SF describing complete vertex ownership
4036 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
4037 
4038   Level: advanced
4039 
4040   Notes:
4041   Not currently supported in Fortran.
4042 
4043 .seealso: DMPlexBuildCoordinatesFromCellList(), DMPlexCreateFromCellListParallelPetsc(), DMPlexBuildFromCellListParallel()
4044 @*/
4045 PetscErrorCode DMPlexBuildCoordinatesFromCellListParallel(DM dm, PetscInt spaceDim, PetscSF sfVert, const PetscReal vertexCoords[])
4046 {
4047   PetscSection   coordSection;
4048   Vec            coordinates;
4049   PetscScalar   *coords;
4050   PetscInt       numVertices, numVerticesAdj, coordSize, v, vStart, vEnd;
4051   PetscErrorCode ierr;
4052 
4053   PetscFunctionBegin;
4054   ierr = PetscLogEventBegin(DMPLEX_BuildCoordinatesFromCellList,dm,0,0,0);CHKERRQ(ierr);
4055   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
4056   PetscCheckFalse(vStart < 0 || vEnd < 0,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM is not set up properly. DMPlexBuildFromCellList() should be called first.");
4057   ierr = DMSetCoordinateDim(dm, spaceDim);CHKERRQ(ierr);
4058   ierr = PetscSFGetGraph(sfVert, &numVertices, &numVerticesAdj, NULL, NULL);CHKERRQ(ierr);
4059   PetscCheckFalse(vEnd - vStart != numVerticesAdj,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Supplied sfVert has wrong number of leaves = %D != %D = vEnd - vStart",numVerticesAdj,vEnd - vStart);
4060   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
4061   ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
4062   ierr = PetscSectionSetFieldComponents(coordSection, 0, spaceDim);CHKERRQ(ierr);
4063   ierr = PetscSectionSetChart(coordSection, vStart, vEnd);CHKERRQ(ierr);
4064   for (v = vStart; v < vEnd; ++v) {
4065     ierr = PetscSectionSetDof(coordSection, v, spaceDim);CHKERRQ(ierr);
4066     ierr = PetscSectionSetFieldDof(coordSection, v, 0, spaceDim);CHKERRQ(ierr);
4067   }
4068   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
4069   ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
4070   ierr = VecCreate(PetscObjectComm((PetscObject)dm), &coordinates);CHKERRQ(ierr);
4071   ierr = VecSetBlockSize(coordinates, spaceDim);CHKERRQ(ierr);
4072   ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
4073   ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
4074   ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
4075   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
4076   {
4077     MPI_Datatype coordtype;
4078 
4079     /* Need a temp buffer for coords if we have complex/single */
4080     ierr = MPI_Type_contiguous(spaceDim, MPIU_SCALAR, &coordtype);CHKERRMPI(ierr);
4081     ierr = MPI_Type_commit(&coordtype);CHKERRMPI(ierr);
4082 #if defined(PETSC_USE_COMPLEX)
4083     {
4084     PetscScalar *svertexCoords;
4085     PetscInt    i;
4086     ierr = PetscMalloc1(numVertices*spaceDim,&svertexCoords);CHKERRQ(ierr);
4087     for (i=0; i<numVertices*spaceDim; i++) svertexCoords[i] = vertexCoords[i];
4088     ierr = PetscSFBcastBegin(sfVert, coordtype, svertexCoords, coords,MPI_REPLACE);CHKERRQ(ierr);
4089     ierr = PetscSFBcastEnd(sfVert, coordtype, svertexCoords, coords,MPI_REPLACE);CHKERRQ(ierr);
4090     ierr = PetscFree(svertexCoords);CHKERRQ(ierr);
4091     }
4092 #else
4093     ierr = PetscSFBcastBegin(sfVert, coordtype, vertexCoords, coords,MPI_REPLACE);CHKERRQ(ierr);
4094     ierr = PetscSFBcastEnd(sfVert, coordtype, vertexCoords, coords,MPI_REPLACE);CHKERRQ(ierr);
4095 #endif
4096     ierr = MPI_Type_free(&coordtype);CHKERRMPI(ierr);
4097   }
4098   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
4099   ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
4100   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
4101   ierr = PetscLogEventEnd(DMPLEX_BuildCoordinatesFromCellList,dm,0,0,0);CHKERRQ(ierr);
4102   PetscFunctionReturn(0);
4103 }
4104 
4105 /*@
4106   DMPlexCreateFromCellListParallelPetsc - Create distributed DMPLEX from a list of vertices for each cell (common mesh generator output)
4107 
4108   Input Parameters:
4109 + comm - The communicator
4110 . dim - The topological dimension of the mesh
4111 . numCells - The number of cells owned by this process
4112 . numVertices - The number of vertices owned by this process, or PETSC_DECIDE
4113 . NVertices - The global number of vertices, or PETSC_DECIDE
4114 . numCorners - The number of vertices for each cell
4115 . interpolate - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
4116 . cells - An array of numCells*numCorners numbers, the global vertex numbers for each cell
4117 . spaceDim - The spatial dimension used for coordinates
4118 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
4119 
4120   Output Parameters:
4121 + dm - The DM
4122 . vertexSF - (Optional) SF describing complete vertex ownership
4123 - verticesAdjSaved - (Optional) vertex adjacency array
4124 
4125   Notes:
4126   This function is just a convenient sequence of DMCreate(), DMSetType(), DMSetDimension(),
4127   DMPlexBuildFromCellListParallel(), DMPlexInterpolate(), DMPlexBuildCoordinatesFromCellListParallel()
4128 
4129   See DMPlexBuildFromCellListParallel() for an example and details about the topology-related parameters.
4130   See DMPlexBuildCoordinatesFromCellListParallel() for details about the geometry-related parameters.
4131 
4132   Level: intermediate
4133 
4134 .seealso: DMPlexCreateFromCellListPetsc(), DMPlexBuildFromCellListParallel(), DMPlexBuildCoordinatesFromCellListParallel(), DMPlexCreateFromDAG(), DMPlexCreate()
4135 @*/
4136 PetscErrorCode DMPlexCreateFromCellListParallelPetsc(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt NVertices, PetscInt numCorners, PetscBool interpolate, const PetscInt cells[], PetscInt spaceDim, const PetscReal vertexCoords[], PetscSF *vertexSF, PetscInt **verticesAdj, DM *dm)
4137 {
4138   PetscSF        sfVert;
4139   PetscErrorCode ierr;
4140 
4141   PetscFunctionBegin;
4142   ierr = DMCreate(comm, dm);CHKERRQ(ierr);
4143   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
4144   PetscValidLogicalCollectiveInt(*dm, dim, 2);
4145   PetscValidLogicalCollectiveInt(*dm, spaceDim, 9);
4146   ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
4147   ierr = DMPlexBuildFromCellListParallel(*dm, numCells, numVertices, NVertices, numCorners, cells, &sfVert, verticesAdj);CHKERRQ(ierr);
4148   if (interpolate) {
4149     DM idm;
4150 
4151     ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
4152     ierr = DMDestroy(dm);CHKERRQ(ierr);
4153     *dm  = idm;
4154   }
4155   ierr = DMPlexBuildCoordinatesFromCellListParallel(*dm, spaceDim, sfVert, vertexCoords);CHKERRQ(ierr);
4156   if (vertexSF) *vertexSF = sfVert;
4157   else {ierr = PetscSFDestroy(&sfVert);CHKERRQ(ierr);}
4158   PetscFunctionReturn(0);
4159 }
4160 
4161 /*@
4162   DMPlexCreateFromCellListParallel - Deprecated, use DMPlexCreateFromCellListParallelPetsc()
4163 
4164   Level: deprecated
4165 
4166 .seealso: DMPlexCreateFromCellListParallelPetsc()
4167 @*/
4168 PetscErrorCode DMPlexCreateFromCellListParallel(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, PetscBool interpolate, const int cells[], PetscInt spaceDim, const PetscReal vertexCoords[], PetscSF *vertexSF, DM *dm)
4169 {
4170   PetscErrorCode ierr;
4171   PetscInt       i;
4172   PetscInt       *pintCells;
4173 
4174   PetscFunctionBegin;
4175   PetscCheckFalse(sizeof(int) > sizeof(PetscInt),comm, PETSC_ERR_ARG_SIZ, "Size of int %zd greater than size of PetscInt %zd. Reconfigure PETSc --with-64-bit-indices=1", sizeof(int), sizeof(PetscInt));
4176   if (sizeof(int) == sizeof(PetscInt)) {
4177     pintCells = (PetscInt *) cells;
4178   } else {
4179     ierr = PetscMalloc1(numCells*numCorners, &pintCells);CHKERRQ(ierr);
4180     for (i = 0; i < numCells*numCorners; i++) {
4181       pintCells[i] = (PetscInt) cells[i];
4182     }
4183   }
4184   ierr = DMPlexCreateFromCellListParallelPetsc(comm, dim, numCells, numVertices, PETSC_DECIDE, numCorners, interpolate, pintCells, spaceDim, vertexCoords, vertexSF, NULL, dm);CHKERRQ(ierr);
4185   if (sizeof(int) != sizeof(PetscInt)) {
4186     ierr = PetscFree(pintCells);CHKERRQ(ierr);
4187   }
4188   PetscFunctionReturn(0);
4189 }
4190 
4191 /*@C
4192   DMPlexBuildFromCellList - Build DMPLEX topology from a list of vertices for each cell (common mesh generator output)
4193 
4194   Input Parameters:
4195 + dm - The DM
4196 . numCells - The number of cells owned by this process
4197 . numVertices - The number of vertices owned by this process, or PETSC_DETERMINE
4198 . numCorners - The number of vertices for each cell
4199 - cells - An array of numCells*numCorners numbers, the global vertex numbers for each cell
4200 
4201   Level: advanced
4202 
4203   Notes:
4204   Two triangles sharing a face
4205 $
4206 $        2
4207 $      / | \
4208 $     /  |  \
4209 $    /   |   \
4210 $   0  0 | 1  3
4211 $    \   |   /
4212 $     \  |  /
4213 $      \ | /
4214 $        1
4215 would have input
4216 $  numCells = 2, numVertices = 4
4217 $  cells = [0 1 2  1 3 2]
4218 $
4219 which would result in the DMPlex
4220 $
4221 $        4
4222 $      / | \
4223 $     /  |  \
4224 $    /   |   \
4225 $   2  0 | 1  5
4226 $    \   |   /
4227 $     \  |  /
4228 $      \ | /
4229 $        3
4230 
4231   If numVertices is PETSC_DETERMINE, it is computed by PETSc as the maximum vertex index in cells + 1.
4232 
4233   Not currently supported in Fortran.
4234 
4235 .seealso: DMPlexBuildFromCellListParallel(), DMPlexBuildCoordinatesFromCellList(), DMPlexCreateFromCellListPetsc()
4236 @*/
4237 PetscErrorCode DMPlexBuildFromCellList(DM dm, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, const PetscInt cells[])
4238 {
4239   PetscInt      *cones, c, p, dim;
4240   PetscErrorCode ierr;
4241 
4242   PetscFunctionBegin;
4243   ierr = PetscLogEventBegin(DMPLEX_BuildFromCellList,dm,0,0,0);CHKERRQ(ierr);
4244   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
4245   /* Get/check global number of vertices */
4246   {
4247     PetscInt NVerticesInCells, i;
4248     const PetscInt len = numCells * numCorners;
4249 
4250     /* NVerticesInCells = max(cells) + 1 */
4251     NVerticesInCells = PETSC_MIN_INT;
4252     for (i=0; i<len; i++) if (cells[i] > NVerticesInCells) NVerticesInCells = cells[i];
4253     ++NVerticesInCells;
4254 
4255     if (numVertices == PETSC_DECIDE) numVertices = NVerticesInCells;
4256     else PetscCheckFalse(numVertices < NVerticesInCells,PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Specified number of vertices %D must be greater than or equal to the number of vertices in cells %D",numVertices,NVerticesInCells);
4257   }
4258   ierr = DMPlexSetChart(dm, 0, numCells+numVertices);CHKERRQ(ierr);
4259   for (c = 0; c < numCells; ++c) {
4260     ierr = DMPlexSetConeSize(dm, c, numCorners);CHKERRQ(ierr);
4261   }
4262   ierr = DMSetUp(dm);CHKERRQ(ierr);
4263   ierr = DMPlexGetCones(dm,&cones);CHKERRQ(ierr);
4264   for (c = 0; c < numCells; ++c) {
4265     for (p = 0; p < numCorners; ++p) {
4266       cones[c*numCorners+p] = cells[c*numCorners+p]+numCells;
4267     }
4268   }
4269   ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
4270   ierr = DMPlexStratify(dm);CHKERRQ(ierr);
4271   ierr = PetscLogEventEnd(DMPLEX_BuildFromCellList,dm,0,0,0);CHKERRQ(ierr);
4272   PetscFunctionReturn(0);
4273 }
4274 
4275 /*@C
4276   DMPlexBuildCoordinatesFromCellList - Build DM coordinates from a list of coordinates for each owned vertex (common mesh generator output)
4277 
4278   Input Parameters:
4279 + dm - The DM
4280 . spaceDim - The spatial dimension used for coordinates
4281 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
4282 
4283   Level: advanced
4284 
4285   Notes:
4286   Not currently supported in Fortran.
4287 
4288 .seealso: DMPlexBuildCoordinatesFromCellListParallel(), DMPlexCreateFromCellListPetsc(), DMPlexBuildFromCellList()
4289 @*/
4290 PetscErrorCode DMPlexBuildCoordinatesFromCellList(DM dm, PetscInt spaceDim, const PetscReal vertexCoords[])
4291 {
4292   PetscSection   coordSection;
4293   Vec            coordinates;
4294   DM             cdm;
4295   PetscScalar   *coords;
4296   PetscInt       v, vStart, vEnd, d;
4297   PetscErrorCode ierr;
4298 
4299   PetscFunctionBegin;
4300   ierr = PetscLogEventBegin(DMPLEX_BuildCoordinatesFromCellList,dm,0,0,0);CHKERRQ(ierr);
4301   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
4302   PetscCheckFalse(vStart < 0 || vEnd < 0,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM is not set up properly. DMPlexBuildFromCellList() should be called first.");
4303   ierr = DMSetCoordinateDim(dm, spaceDim);CHKERRQ(ierr);
4304   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
4305   ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
4306   ierr = PetscSectionSetFieldComponents(coordSection, 0, spaceDim);CHKERRQ(ierr);
4307   ierr = PetscSectionSetChart(coordSection, vStart, vEnd);CHKERRQ(ierr);
4308   for (v = vStart; v < vEnd; ++v) {
4309     ierr = PetscSectionSetDof(coordSection, v, spaceDim);CHKERRQ(ierr);
4310     ierr = PetscSectionSetFieldDof(coordSection, v, 0, spaceDim);CHKERRQ(ierr);
4311   }
4312   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
4313 
4314   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
4315   ierr = DMCreateLocalVector(cdm, &coordinates);CHKERRQ(ierr);
4316   ierr = VecSetBlockSize(coordinates, spaceDim);CHKERRQ(ierr);
4317   ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
4318   ierr = VecGetArrayWrite(coordinates, &coords);CHKERRQ(ierr);
4319   for (v = 0; v < vEnd-vStart; ++v) {
4320     for (d = 0; d < spaceDim; ++d) {
4321       coords[v*spaceDim+d] = vertexCoords[v*spaceDim+d];
4322     }
4323   }
4324   ierr = VecRestoreArrayWrite(coordinates, &coords);CHKERRQ(ierr);
4325   ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
4326   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
4327   ierr = PetscLogEventEnd(DMPLEX_BuildCoordinatesFromCellList,dm,0,0,0);CHKERRQ(ierr);
4328   PetscFunctionReturn(0);
4329 }
4330 
4331 /*@
4332   DMPlexCreateFromCellListPetsc - Create DMPLEX from a list of vertices for each cell (common mesh generator output), but only process 0 takes in the input
4333 
4334   Collective on comm
4335 
4336   Input Parameters:
4337 + comm - The communicator
4338 . dim - The topological dimension of the mesh
4339 . numCells - The number of cells, only on process 0
4340 . numVertices - The number of vertices owned by this process, or PETSC_DECIDE, only on process 0
4341 . numCorners - The number of vertices for each cell, only on process 0
4342 . interpolate - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
4343 . cells - An array of numCells*numCorners numbers, the vertices for each cell, only on process 0
4344 . spaceDim - The spatial dimension used for coordinates
4345 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex, only on process 0
4346 
4347   Output Parameter:
4348 . dm - The DM, which only has points on process 0
4349 
4350   Notes:
4351   This function is just a convenient sequence of DMCreate(), DMSetType(), DMSetDimension(), DMPlexBuildFromCellList(),
4352   DMPlexInterpolate(), DMPlexBuildCoordinatesFromCellList()
4353 
4354   See DMPlexBuildFromCellList() for an example and details about the topology-related parameters.
4355   See DMPlexBuildCoordinatesFromCellList() for details about the geometry-related parameters.
4356   See DMPlexCreateFromCellListParallelPetsc() for parallel input
4357 
4358   Level: intermediate
4359 
4360 .seealso: DMPlexCreateFromCellListParallelPetsc(), DMPlexBuildFromCellList(), DMPlexBuildCoordinatesFromCellList(), DMPlexCreateFromDAG(), DMPlexCreate()
4361 @*/
4362 PetscErrorCode DMPlexCreateFromCellListPetsc(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, PetscBool interpolate, const PetscInt cells[], PetscInt spaceDim, const PetscReal vertexCoords[], DM *dm)
4363 {
4364   PetscMPIInt    rank;
4365   PetscErrorCode ierr;
4366 
4367   PetscFunctionBegin;
4368   PetscCheckFalse(!dim,comm, PETSC_ERR_ARG_OUTOFRANGE, "This is not appropriate for 0-dimensional meshes. Consider either creating the DM using DMPlexCreateFromDAG(), by hand, or using DMSwarm.");
4369   ierr = MPI_Comm_rank(comm, &rank);CHKERRMPI(ierr);
4370   ierr = DMCreate(comm, dm);CHKERRQ(ierr);
4371   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
4372   ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
4373   if (!rank) {ierr = DMPlexBuildFromCellList(*dm, numCells, numVertices, numCorners, cells);CHKERRQ(ierr);}
4374   else       {ierr = DMPlexBuildFromCellList(*dm, 0, 0, 0, NULL);CHKERRQ(ierr);}
4375   if (interpolate) {
4376     DM idm;
4377 
4378     ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
4379     ierr = DMDestroy(dm);CHKERRQ(ierr);
4380     *dm  = idm;
4381   }
4382   if (!rank) {ierr = DMPlexBuildCoordinatesFromCellList(*dm, spaceDim, vertexCoords);CHKERRQ(ierr);}
4383   else       {ierr = DMPlexBuildCoordinatesFromCellList(*dm, spaceDim, NULL);CHKERRQ(ierr);}
4384   PetscFunctionReturn(0);
4385 }
4386 
4387 /*@
4388   DMPlexCreateFromCellList - Deprecated, use DMPlexCreateFromCellListPetsc()
4389 
4390   Level: deprecated
4391 
4392 .seealso: DMPlexCreateFromCellListPetsc()
4393 @*/
4394 PetscErrorCode DMPlexCreateFromCellList(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, PetscBool interpolate, const int cells[], PetscInt spaceDim, const double vertexCoords[], DM *dm)
4395 {
4396   PetscErrorCode ierr;
4397   PetscInt       i;
4398   PetscInt       *pintCells;
4399   PetscReal      *prealVC;
4400 
4401   PetscFunctionBegin;
4402   PetscCheckFalse(sizeof(int) > sizeof(PetscInt),comm, PETSC_ERR_ARG_SIZ, "Size of int %zd greater than size of PetscInt %zd. Reconfigure PETSc --with-64-bit-indices=1", sizeof(int), sizeof(PetscInt));
4403   if (sizeof(int) == sizeof(PetscInt)) {
4404     pintCells = (PetscInt *) cells;
4405   } else {
4406     ierr = PetscMalloc1(numCells*numCorners, &pintCells);CHKERRQ(ierr);
4407     for (i = 0; i < numCells*numCorners; i++) {
4408       pintCells[i] = (PetscInt) cells[i];
4409     }
4410   }
4411   PetscCheckFalse(sizeof(double) > sizeof(PetscReal),comm, PETSC_ERR_ARG_SIZ, "Size of double %zd greater than size of PetscReal %zd. Reconfigure PETSc --with-precision=<higher precision>.", sizeof(double), sizeof(PetscReal));
4412   if (sizeof(double) == sizeof(PetscReal)) {
4413     prealVC = (PetscReal *) vertexCoords;
4414   } else {
4415     ierr = PetscMalloc1(numVertices*spaceDim, &prealVC);CHKERRQ(ierr);
4416     for (i = 0; i < numVertices*spaceDim; i++) {
4417       prealVC[i] = (PetscReal) vertexCoords[i];
4418     }
4419   }
4420   ierr = DMPlexCreateFromCellListPetsc(comm, dim, numCells, numVertices, numCorners, interpolate, pintCells, spaceDim, prealVC, dm);CHKERRQ(ierr);
4421   if (sizeof(int) != sizeof(PetscInt)) {
4422     ierr = PetscFree(pintCells);CHKERRQ(ierr);
4423   }
4424   if (sizeof(double) != sizeof(PetscReal)) {
4425     ierr = PetscFree(prealVC);CHKERRQ(ierr);
4426   }
4427   PetscFunctionReturn(0);
4428 }
4429 
4430 /*@
4431   DMPlexCreateFromDAG - This takes as input the adjacency-list representation of the Directed Acyclic Graph (Hasse Diagram) encoding a mesh, and produces a DM
4432 
4433   Input Parameters:
4434 + dm - The empty DM object, usually from DMCreate() and DMSetDimension()
4435 . depth - The depth of the DAG
4436 . numPoints - Array of size depth + 1 containing the number of points at each depth
4437 . coneSize - The cone size of each point
4438 . cones - The concatenation of the cone points for each point, the cone list must be oriented correctly for each point
4439 . coneOrientations - The orientation of each cone point
4440 - vertexCoords - An array of numPoints[0]*spacedim numbers representing the coordinates of each vertex, with spacedim the value set via DMSetCoordinateDim()
4441 
4442   Output Parameter:
4443 . dm - The DM
4444 
4445   Note: Two triangles sharing a face would have input
4446 $  depth = 1, numPoints = [4 2], coneSize = [3 3 0 0 0 0]
4447 $  cones = [2 3 4  3 5 4], coneOrientations = [0 0 0  0 0 0]
4448 $ vertexCoords = [-1.0 0.0  0.0 -1.0  0.0 1.0  1.0 0.0]
4449 $
4450 which would result in the DMPlex
4451 $
4452 $        4
4453 $      / | \
4454 $     /  |  \
4455 $    /   |   \
4456 $   2  0 | 1  5
4457 $    \   |   /
4458 $     \  |  /
4459 $      \ | /
4460 $        3
4461 $
4462 $ Notice that all points are numbered consecutively, unlike DMPlexCreateFromCellListPetsc()
4463 
4464   Level: advanced
4465 
4466 .seealso: DMPlexCreateFromCellListPetsc(), DMPlexCreate()
4467 @*/
4468 PetscErrorCode DMPlexCreateFromDAG(DM dm, PetscInt depth, const PetscInt numPoints[], const PetscInt coneSize[], const PetscInt cones[], const PetscInt coneOrientations[], const PetscScalar vertexCoords[])
4469 {
4470   Vec            coordinates;
4471   PetscSection   coordSection;
4472   PetscScalar    *coords;
4473   PetscInt       coordSize, firstVertex = -1, pStart = 0, pEnd = 0, p, v, dim, dimEmbed, d, off;
4474   PetscErrorCode ierr;
4475 
4476   PetscFunctionBegin;
4477   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
4478   ierr = DMGetCoordinateDim(dm, &dimEmbed);CHKERRQ(ierr);
4479   PetscCheckFalse(dimEmbed < dim,PETSC_COMM_SELF,PETSC_ERR_PLIB,"Embedding dimension %D cannot be less than intrinsic dimension %d",dimEmbed,dim);
4480   for (d = 0; d <= depth; ++d) pEnd += numPoints[d];
4481   ierr = DMPlexSetChart(dm, pStart, pEnd);CHKERRQ(ierr);
4482   for (p = pStart; p < pEnd; ++p) {
4483     ierr = DMPlexSetConeSize(dm, p, coneSize[p-pStart]);CHKERRQ(ierr);
4484     if (firstVertex < 0 && !coneSize[p - pStart]) {
4485       firstVertex = p - pStart;
4486     }
4487   }
4488   PetscCheckFalse(firstVertex < 0 && numPoints[0],PETSC_COMM_SELF,PETSC_ERR_ARG_WRONG,"Expected %D vertices but could not find any", numPoints[0]);
4489   ierr = DMSetUp(dm);CHKERRQ(ierr); /* Allocate space for cones */
4490   for (p = pStart, off = 0; p < pEnd; off += coneSize[p-pStart], ++p) {
4491     ierr = DMPlexSetCone(dm, p, &cones[off]);CHKERRQ(ierr);
4492     ierr = DMPlexSetConeOrientation(dm, p, &coneOrientations[off]);CHKERRQ(ierr);
4493   }
4494   ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
4495   ierr = DMPlexStratify(dm);CHKERRQ(ierr);
4496   /* Build coordinates */
4497   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
4498   ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
4499   ierr = PetscSectionSetFieldComponents(coordSection, 0, dimEmbed);CHKERRQ(ierr);
4500   ierr = PetscSectionSetChart(coordSection, firstVertex, firstVertex+numPoints[0]);CHKERRQ(ierr);
4501   for (v = firstVertex; v < firstVertex+numPoints[0]; ++v) {
4502     ierr = PetscSectionSetDof(coordSection, v, dimEmbed);CHKERRQ(ierr);
4503     ierr = PetscSectionSetFieldDof(coordSection, v, 0, dimEmbed);CHKERRQ(ierr);
4504   }
4505   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
4506   ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
4507   ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
4508   ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
4509   ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
4510   ierr = VecSetBlockSize(coordinates, dimEmbed);CHKERRQ(ierr);
4511   ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
4512   if (vertexCoords) {
4513     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
4514     for (v = 0; v < numPoints[0]; ++v) {
4515       PetscInt off;
4516 
4517       ierr = PetscSectionGetOffset(coordSection, v+firstVertex, &off);CHKERRQ(ierr);
4518       for (d = 0; d < dimEmbed; ++d) {
4519         coords[off+d] = vertexCoords[v*dimEmbed+d];
4520       }
4521     }
4522   }
4523   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
4524   ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
4525   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
4526   PetscFunctionReturn(0);
4527 }
4528 
4529 /*@C
4530   DMPlexCreateCellVertexFromFile - Create a DMPlex mesh from a simple cell-vertex file.
4531 
4532 + comm        - The MPI communicator
4533 . filename    - Name of the .dat file
4534 - interpolate - Create faces and edges in the mesh
4535 
4536   Output Parameter:
4537 . dm  - The DM object representing the mesh
4538 
4539   Note: The format is the simplest possible:
4540 $ Ne
4541 $ v0 v1 ... vk
4542 $ Nv
4543 $ x y z marker
4544 
4545   Level: beginner
4546 
4547 .seealso: DMPlexCreateFromFile(), DMPlexCreateMedFromFile(), DMPlexCreateGmsh(), DMPlexCreate()
4548 @*/
4549 PetscErrorCode DMPlexCreateCellVertexFromFile(MPI_Comm comm, const char filename[], PetscBool interpolate, DM *dm)
4550 {
4551   DMLabel         marker;
4552   PetscViewer     viewer;
4553   Vec             coordinates;
4554   PetscSection    coordSection;
4555   PetscScalar    *coords;
4556   char            line[PETSC_MAX_PATH_LEN];
4557   PetscInt        dim = 3, cdim = 3, coordSize, v, c, d;
4558   PetscMPIInt     rank;
4559   int             snum, Nv, Nc, Ncn, Nl;
4560   PetscErrorCode  ierr;
4561 
4562   PetscFunctionBegin;
4563   ierr = MPI_Comm_rank(comm, &rank);CHKERRMPI(ierr);
4564   ierr = PetscViewerCreate(comm, &viewer);CHKERRQ(ierr);
4565   ierr = PetscViewerSetType(viewer, PETSCVIEWERASCII);CHKERRQ(ierr);
4566   ierr = PetscViewerFileSetMode(viewer, FILE_MODE_READ);CHKERRQ(ierr);
4567   ierr = PetscViewerFileSetName(viewer, filename);CHKERRQ(ierr);
4568   if (rank == 0) {
4569     ierr = PetscViewerRead(viewer, line, 4, NULL, PETSC_STRING);CHKERRQ(ierr);
4570     snum = sscanf(line, "%d %d %d %d", &Nc, &Nv, &Ncn, &Nl);
4571     PetscCheckFalse(snum != 4,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
4572   } else {
4573     Nc = Nv = Ncn = Nl = 0;
4574   }
4575   ierr = DMCreate(comm, dm);CHKERRQ(ierr);
4576   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
4577   ierr = DMPlexSetChart(*dm, 0, Nc+Nv);CHKERRQ(ierr);
4578   ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
4579   ierr = DMSetCoordinateDim(*dm, cdim);CHKERRQ(ierr);
4580   /* Read topology */
4581   if (rank == 0) {
4582     char     format[PETSC_MAX_PATH_LEN];
4583     PetscInt cone[8];
4584     int      vbuf[8], v;
4585 
4586     for (c = 0; c < Ncn; ++c) {format[c*3+0] = '%'; format[c*3+1] = 'd'; format[c*3+2] = ' ';}
4587     format[Ncn*3-1] = '\0';
4588     for (c = 0; c < Nc; ++c) {ierr = DMPlexSetConeSize(*dm, c, Ncn);CHKERRQ(ierr);}
4589     ierr = DMSetUp(*dm);CHKERRQ(ierr);
4590     for (c = 0; c < Nc; ++c) {
4591       ierr = PetscViewerRead(viewer, line, Ncn, NULL, PETSC_STRING);CHKERRQ(ierr);
4592       switch (Ncn) {
4593         case 2: snum = sscanf(line, format, &vbuf[0], &vbuf[1]);break;
4594         case 3: snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2]);break;
4595         case 4: snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3]);break;
4596         case 6: snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3], &vbuf[4], &vbuf[5]);break;
4597         case 8: snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3], &vbuf[4], &vbuf[5], &vbuf[6], &vbuf[7]);break;
4598         default: SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "No cell shape with %D vertices", Ncn);
4599       }
4600       PetscCheckFalse(snum != Ncn,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
4601       for (v = 0; v < Ncn; ++v) cone[v] = vbuf[v] + Nc;
4602       /* Hexahedra are inverted */
4603       if (Ncn == 8) {
4604         PetscInt tmp = cone[1];
4605         cone[1] = cone[3];
4606         cone[3] = tmp;
4607       }
4608       ierr = DMPlexSetCone(*dm, c, cone);CHKERRQ(ierr);
4609     }
4610   }
4611   ierr = DMPlexSymmetrize(*dm);CHKERRQ(ierr);
4612   ierr = DMPlexStratify(*dm);CHKERRQ(ierr);
4613   /* Read coordinates */
4614   ierr = DMGetCoordinateSection(*dm, &coordSection);CHKERRQ(ierr);
4615   ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
4616   ierr = PetscSectionSetFieldComponents(coordSection, 0, cdim);CHKERRQ(ierr);
4617   ierr = PetscSectionSetChart(coordSection, Nc, Nc + Nv);CHKERRQ(ierr);
4618   for (v = Nc; v < Nc+Nv; ++v) {
4619     ierr = PetscSectionSetDof(coordSection, v, cdim);CHKERRQ(ierr);
4620     ierr = PetscSectionSetFieldDof(coordSection, v, 0, cdim);CHKERRQ(ierr);
4621   }
4622   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
4623   ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
4624   ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
4625   ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
4626   ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
4627   ierr = VecSetBlockSize(coordinates, cdim);CHKERRQ(ierr);
4628   ierr = VecSetType(coordinates, VECSTANDARD);CHKERRQ(ierr);
4629   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
4630   if (rank == 0) {
4631     char   format[PETSC_MAX_PATH_LEN];
4632     double x[3];
4633     int    l, val[3];
4634 
4635     if (Nl) {
4636       for (l = 0; l < Nl; ++l) {format[l*3+0] = '%'; format[l*3+1] = 'd'; format[l*3+2] = ' ';}
4637       format[Nl*3-1] = '\0';
4638       ierr = DMCreateLabel(*dm, "marker");CHKERRQ(ierr);
4639       ierr = DMGetLabel(*dm, "marker", &marker);CHKERRQ(ierr);
4640     }
4641     for (v = 0; v < Nv; ++v) {
4642       ierr = PetscViewerRead(viewer, line, 3+Nl, NULL, PETSC_STRING);CHKERRQ(ierr);
4643       snum = sscanf(line, "%lg %lg %lg", &x[0], &x[1], &x[2]);
4644       PetscCheckFalse(snum != 3,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
4645       switch (Nl) {
4646         case 0: snum = 0;break;
4647         case 1: snum = sscanf(line, format, &val[0]);break;
4648         case 2: snum = sscanf(line, format, &val[0], &val[1]);break;
4649         case 3: snum = sscanf(line, format, &val[0], &val[1], &val[2]);break;
4650         default: SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Request support for %D labels", Nl);
4651       }
4652       PetscCheckFalse(snum != Nl,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
4653       for (d = 0; d < cdim; ++d) coords[v*cdim+d] = x[d];
4654       for (l = 0; l < Nl; ++l) {ierr = DMLabelSetValue(marker, v+Nc, val[l]);CHKERRQ(ierr);}
4655     }
4656   }
4657   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
4658   ierr = DMSetCoordinatesLocal(*dm, coordinates);CHKERRQ(ierr);
4659   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
4660   ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
4661   if (interpolate) {
4662     DM      idm;
4663     DMLabel bdlabel;
4664 
4665     ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
4666     ierr = DMDestroy(dm);CHKERRQ(ierr);
4667     *dm  = idm;
4668 
4669     if (!Nl) {
4670       ierr = DMCreateLabel(*dm, "marker");CHKERRQ(ierr);
4671       ierr = DMGetLabel(*dm, "marker", &bdlabel);CHKERRQ(ierr);
4672       ierr = DMPlexMarkBoundaryFaces(*dm, PETSC_DETERMINE, bdlabel);CHKERRQ(ierr);
4673       ierr = DMPlexLabelComplete(*dm, bdlabel);CHKERRQ(ierr);
4674     }
4675   }
4676   PetscFunctionReturn(0);
4677 }
4678 
4679 /*@C
4680   DMPlexCreateFromFile - This takes a filename and produces a DM
4681 
4682   Input Parameters:
4683 + comm - The communicator
4684 . filename - A file name
4685 . plexname - The object name of the resulting DM, also used for intra-datafile lookup by some formats
4686 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
4687 
4688   Output Parameter:
4689 . dm - The DM
4690 
4691   Options Database Keys:
4692 . -dm_plex_create_from_hdf5_xdmf - use the PETSC_VIEWER_HDF5_XDMF format for reading HDF5
4693 
4694   Use -dm_plex_create_ prefix to pass options to the internal PetscViewer, e.g.
4695 $ -dm_plex_create_viewer_hdf5_collective
4696 
4697   Notes:
4698   Using PETSCVIEWERHDF5 type with PETSC_VIEWER_HDF5_PETSC format, one can save multiple DMPlex
4699   meshes in a single HDF5 file. This in turn requires one to name the DMPlex object with PetscObjectSetName()
4700   before saving it with DMView() and before loading it with DMLoad() for identification of the mesh object.
4701   The input parameter name is thus used to name the DMPlex object when DMPlexCreateFromFile() internally
4702   calls DMLoad(). Currently, name is ignored for other viewer types and/or formats.
4703 
4704   Level: beginner
4705 
4706 .seealso: DMPlexCreateFromDAG(), DMPlexCreateFromCellListPetsc(), DMPlexCreate(), PetscObjectSetName(), DMView(), DMLoad()
4707 @*/
4708 PetscErrorCode DMPlexCreateFromFile(MPI_Comm comm, const char filename[], const char plexname[], PetscBool interpolate, DM *dm)
4709 {
4710   const char    *extGmsh      = ".msh";
4711   const char    *extGmsh2     = ".msh2";
4712   const char    *extGmsh4     = ".msh4";
4713   const char    *extCGNS      = ".cgns";
4714   const char    *extExodus    = ".exo";
4715   const char    *extExodus_e  = ".e";
4716   const char    *extGenesis   = ".gen";
4717   const char    *extFluent    = ".cas";
4718   const char    *extHDF5      = ".h5";
4719   const char    *extMed       = ".med";
4720   const char    *extPLY       = ".ply";
4721   const char    *extEGADSLite = ".egadslite";
4722   const char    *extEGADS     = ".egads";
4723   const char    *extIGES      = ".igs";
4724   const char    *extSTEP      = ".stp";
4725   const char    *extCV        = ".dat";
4726   size_t         len;
4727   PetscBool      isGmsh, isGmsh2, isGmsh4, isCGNS, isExodus, isGenesis, isFluent, isHDF5, isMed, isPLY, isEGADSLite, isEGADS, isIGES, isSTEP, isCV;
4728   PetscMPIInt    rank;
4729   PetscErrorCode ierr;
4730 
4731   PetscFunctionBegin;
4732   PetscValidCharPointer(filename, 2);
4733   if (plexname) PetscValidCharPointer(plexname, 3);
4734   PetscValidPointer(dm, 5);
4735   ierr = DMInitializePackage();CHKERRQ(ierr);
4736   ierr = PetscLogEventBegin(DMPLEX_CreateFromFile,0,0,0,0);CHKERRQ(ierr);
4737   ierr = MPI_Comm_rank(comm, &rank);CHKERRMPI(ierr);
4738   ierr = PetscStrlen(filename, &len);CHKERRQ(ierr);
4739   PetscCheckFalse(!len,comm, PETSC_ERR_ARG_WRONG, "Filename must be a valid path");
4740   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)],  extGmsh,      4, &isGmsh);CHKERRQ(ierr);
4741   ierr = PetscStrncmp(&filename[PetscMax(0,len-5)],  extGmsh2,     5, &isGmsh2);CHKERRQ(ierr);
4742   ierr = PetscStrncmp(&filename[PetscMax(0,len-5)],  extGmsh4,     5, &isGmsh4);CHKERRQ(ierr);
4743   ierr = PetscStrncmp(&filename[PetscMax(0,len-5)],  extCGNS,      5, &isCGNS);CHKERRQ(ierr);
4744   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)],  extExodus,    4, &isExodus);CHKERRQ(ierr);
4745   if (!isExodus) {
4746     ierr = PetscStrncmp(&filename[PetscMax(0,len-2)],  extExodus_e,    2, &isExodus);CHKERRQ(ierr);
4747   }
4748   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)],  extGenesis,   4, &isGenesis);CHKERRQ(ierr);
4749   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)],  extFluent,    4, &isFluent);CHKERRQ(ierr);
4750   ierr = PetscStrncmp(&filename[PetscMax(0,len-3)],  extHDF5,      3, &isHDF5);CHKERRQ(ierr);
4751   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)],  extMed,       4, &isMed);CHKERRQ(ierr);
4752   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)],  extPLY,       4, &isPLY);CHKERRQ(ierr);
4753   ierr = PetscStrncmp(&filename[PetscMax(0,len-10)], extEGADSLite, 10, &isEGADSLite);CHKERRQ(ierr);
4754   ierr = PetscStrncmp(&filename[PetscMax(0,len-6)],  extEGADS,     6, &isEGADS);CHKERRQ(ierr);
4755   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)],  extIGES,      4, &isIGES);CHKERRQ(ierr);
4756   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)],  extSTEP,      4, &isSTEP);CHKERRQ(ierr);
4757   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)],  extCV,        4, &isCV);CHKERRQ(ierr);
4758   if (isGmsh || isGmsh2 || isGmsh4) {
4759     ierr = DMPlexCreateGmshFromFile(comm, filename, interpolate, dm);CHKERRQ(ierr);
4760   } else if (isCGNS) {
4761     ierr = DMPlexCreateCGNSFromFile(comm, filename, interpolate, dm);CHKERRQ(ierr);
4762   } else if (isExodus || isGenesis) {
4763     ierr = DMPlexCreateExodusFromFile(comm, filename, interpolate, dm);CHKERRQ(ierr);
4764   } else if (isFluent) {
4765     ierr = DMPlexCreateFluentFromFile(comm, filename, interpolate, dm);CHKERRQ(ierr);
4766   } else if (isHDF5) {
4767     PetscBool      load_hdf5_xdmf = PETSC_FALSE;
4768     PetscViewer viewer;
4769 
4770     /* PETSC_VIEWER_HDF5_XDMF is used if the filename ends with .xdmf.h5, or if -dm_plex_create_from_hdf5_xdmf option is present */
4771     ierr = PetscStrncmp(&filename[PetscMax(0,len-8)], ".xdmf",  5, &load_hdf5_xdmf);CHKERRQ(ierr);
4772     ierr = PetscOptionsGetBool(NULL, NULL, "-dm_plex_create_from_hdf5_xdmf", &load_hdf5_xdmf, NULL);CHKERRQ(ierr);
4773     ierr = PetscViewerCreate(comm, &viewer);CHKERRQ(ierr);
4774     ierr = PetscViewerSetType(viewer, PETSCVIEWERHDF5);CHKERRQ(ierr);
4775     ierr = PetscViewerSetOptionsPrefix(viewer, "dm_plex_create_");CHKERRQ(ierr);
4776     ierr = PetscViewerSetFromOptions(viewer);CHKERRQ(ierr);
4777     ierr = PetscViewerFileSetMode(viewer, FILE_MODE_READ);CHKERRQ(ierr);
4778     ierr = PetscViewerFileSetName(viewer, filename);CHKERRQ(ierr);
4779 
4780     ierr = DMCreate(comm, dm);CHKERRQ(ierr);
4781     ierr = PetscObjectSetName((PetscObject)(*dm), plexname);CHKERRQ(ierr);
4782     ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
4783     if (load_hdf5_xdmf) {ierr = PetscViewerPushFormat(viewer, PETSC_VIEWER_HDF5_XDMF);CHKERRQ(ierr);}
4784     ierr = DMLoad(*dm, viewer);CHKERRQ(ierr);
4785     if (load_hdf5_xdmf) {ierr = PetscViewerPopFormat(viewer);CHKERRQ(ierr);}
4786     ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
4787 
4788     if (interpolate) {
4789       DM idm;
4790 
4791       ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
4792       ierr = DMDestroy(dm);CHKERRQ(ierr);
4793       *dm  = idm;
4794     }
4795   } else if (isMed) {
4796     ierr = DMPlexCreateMedFromFile(comm, filename, interpolate, dm);CHKERRQ(ierr);
4797   } else if (isPLY) {
4798     ierr = DMPlexCreatePLYFromFile(comm, filename, interpolate, dm);CHKERRQ(ierr);
4799   } else if (isEGADSLite || isEGADS || isIGES || isSTEP) {
4800     if (isEGADSLite) {ierr = DMPlexCreateEGADSLiteFromFile(comm, filename, dm);CHKERRQ(ierr);}
4801     else             {ierr = DMPlexCreateEGADSFromFile(comm, filename, dm);CHKERRQ(ierr);}
4802     if (!interpolate) {
4803       DM udm;
4804 
4805       ierr = DMPlexUninterpolate(*dm, &udm);CHKERRQ(ierr);
4806       ierr = DMDestroy(dm);CHKERRQ(ierr);
4807       *dm  = udm;
4808     }
4809   } else if (isCV) {
4810     ierr = DMPlexCreateCellVertexFromFile(comm, filename, interpolate, dm);CHKERRQ(ierr);
4811   } else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cannot load file %s: unrecognized extension", filename);
4812   ierr = PetscStrlen(plexname, &len);CHKERRQ(ierr);
4813   if (len) {ierr = PetscObjectSetName((PetscObject)(*dm), plexname);CHKERRQ(ierr);}
4814   ierr = PetscLogEventEnd(DMPLEX_CreateFromFile,0,0,0,0);CHKERRQ(ierr);
4815   PetscFunctionReturn(0);
4816 }
4817