xref: /petsc/src/dm/impls/plex/plexcreate.c (revision cfc5cb97f1a97a4a16fd5ccc9b928cf515f106b5) !
1 #define PETSCDM_DLL
2 #include <petsc/private/dmpleximpl.h>    /*I   "petscdmplex.h"   I*/
3 #include <petsc/private/hashseti.h>          /*I   "petscdmplex.h"   I*/
4 #include <petscsf.h>
5 #include <petsc/private/kernels/blockmatmult.h>
6 #include <petsc/private/kernels/blockinvert.h>
7 
8 PetscLogEvent DMPLEX_CreateFromFile, DMPLEX_BuildFromCellList, DMPLEX_BuildCoordinatesFromCellList;
9 
10 /* External function declarations here */
11 static PetscErrorCode DMInitialize_Plex(DM dm);
12 
13 /* This copies internal things in the Plex structure that we generally want when making a new, related Plex */
14 PetscErrorCode DMPlexCopy_Internal(DM dmin, PetscBool copyPeriodicity, DM dmout)
15 {
16   const DMBoundaryType *bd;
17   const PetscReal      *maxCell, *L;
18   PetscBool             isper, dist;
19   PetscErrorCode        ierr;
20 
21   PetscFunctionBegin;
22   if (copyPeriodicity) {
23     ierr = DMGetPeriodicity(dmin, &isper, &maxCell, &L, &bd);CHKERRQ(ierr);
24     ierr = DMSetPeriodicity(dmout, isper,  maxCell,  L,  bd);CHKERRQ(ierr);
25   }
26   ierr = DMPlexDistributeGetDefault(dmin, &dist);CHKERRQ(ierr);
27   ierr = DMPlexDistributeSetDefault(dmout, dist);CHKERRQ(ierr);
28   ((DM_Plex *) dmout->data)->useHashLocation = ((DM_Plex *) dmin->data)->useHashLocation;
29   PetscFunctionReturn(0);
30 }
31 
32 /* Replace dm with the contents of ndm, and then destroy ndm
33    - Share the DM_Plex structure
34    - Share the coordinates
35    - Share the SF
36 */
37 static PetscErrorCode DMPlexReplace_Static(DM dm, DM *ndm)
38 {
39   PetscSF               sf;
40   DM                    dmNew = *ndm, coordDM, coarseDM;
41   Vec                   coords;
42   PetscBool             isper;
43   const PetscReal      *maxCell, *L;
44   const DMBoundaryType *bd;
45   PetscInt              dim, cdim;
46   PetscErrorCode        ierr;
47 
48   PetscFunctionBegin;
49   if (dm == dmNew) {
50     ierr = DMDestroy(ndm);CHKERRQ(ierr);
51     PetscFunctionReturn(0);
52   }
53   dm->setupcalled = dmNew->setupcalled;
54   ierr = DMGetDimension(dmNew, &dim);CHKERRQ(ierr);
55   ierr = DMSetDimension(dm, dim);CHKERRQ(ierr);
56   ierr = DMGetCoordinateDim(dmNew, &cdim);CHKERRQ(ierr);
57   ierr = DMSetCoordinateDim(dm, cdim);CHKERRQ(ierr);
58   ierr = DMGetPointSF(dmNew, &sf);CHKERRQ(ierr);
59   ierr = DMSetPointSF(dm, sf);CHKERRQ(ierr);
60   ierr = DMGetCoordinateDM(dmNew, &coordDM);CHKERRQ(ierr);
61   ierr = DMGetCoordinatesLocal(dmNew, &coords);CHKERRQ(ierr);
62   ierr = DMSetCoordinateDM(dm, coordDM);CHKERRQ(ierr);
63   ierr = DMSetCoordinatesLocal(dm, coords);CHKERRQ(ierr);
64   /* Do not want to create the coordinate field if it does not already exist, so do not call DMGetCoordinateField() */
65   ierr = DMFieldDestroy(&dm->coordinateField);CHKERRQ(ierr);
66   dm->coordinateField = dmNew->coordinateField;
67   ((DM_Plex *) dmNew->data)->coordFunc = ((DM_Plex *) dm->data)->coordFunc;
68   ierr = DMGetPeriodicity(dmNew, &isper, &maxCell, &L, &bd);CHKERRQ(ierr);
69   ierr = DMSetPeriodicity(dm, isper, maxCell, L, bd);CHKERRQ(ierr);
70   ierr = DMDestroy_Plex(dm);CHKERRQ(ierr);
71   ierr = DMInitialize_Plex(dm);CHKERRQ(ierr);
72   dm->data = dmNew->data;
73   ((DM_Plex *) dmNew->data)->refct++;
74   ierr = DMDestroyLabelLinkList_Internal(dm);CHKERRQ(ierr);
75   ierr = DMCopyLabels(dmNew, dm, PETSC_OWN_POINTER, PETSC_TRUE, DM_COPY_LABELS_FAIL);CHKERRQ(ierr);
76   ierr = DMGetCoarseDM(dmNew,&coarseDM);CHKERRQ(ierr);
77   ierr = DMSetCoarseDM(dm,coarseDM);CHKERRQ(ierr);
78   ierr = DMDestroy(ndm);CHKERRQ(ierr);
79   PetscFunctionReturn(0);
80 }
81 
82 /* Swap dm with the contents of dmNew
83    - Swap the DM_Plex structure
84    - Swap the coordinates
85    - Swap the point PetscSF
86 */
87 static PetscErrorCode DMPlexSwap_Static(DM dmA, DM dmB)
88 {
89   DM              coordDMA, coordDMB;
90   Vec             coordsA,  coordsB;
91   PetscSF         sfA,      sfB;
92   DMField         fieldTmp;
93   void            *tmp;
94   DMLabelLink     listTmp;
95   DMLabel         depthTmp;
96   PetscInt        tmpI;
97   PetscErrorCode  ierr;
98 
99   PetscFunctionBegin;
100   if (dmA == dmB) PetscFunctionReturn(0);
101   ierr = DMGetPointSF(dmA, &sfA);CHKERRQ(ierr);
102   ierr = DMGetPointSF(dmB, &sfB);CHKERRQ(ierr);
103   ierr = PetscObjectReference((PetscObject) sfA);CHKERRQ(ierr);
104   ierr = DMSetPointSF(dmA, sfB);CHKERRQ(ierr);
105   ierr = DMSetPointSF(dmB, sfA);CHKERRQ(ierr);
106   ierr = PetscObjectDereference((PetscObject) sfA);CHKERRQ(ierr);
107 
108   ierr = DMGetCoordinateDM(dmA, &coordDMA);CHKERRQ(ierr);
109   ierr = DMGetCoordinateDM(dmB, &coordDMB);CHKERRQ(ierr);
110   ierr = PetscObjectReference((PetscObject) coordDMA);CHKERRQ(ierr);
111   ierr = DMSetCoordinateDM(dmA, coordDMB);CHKERRQ(ierr);
112   ierr = DMSetCoordinateDM(dmB, coordDMA);CHKERRQ(ierr);
113   ierr = PetscObjectDereference((PetscObject) coordDMA);CHKERRQ(ierr);
114 
115   ierr = DMGetCoordinatesLocal(dmA, &coordsA);CHKERRQ(ierr);
116   ierr = DMGetCoordinatesLocal(dmB, &coordsB);CHKERRQ(ierr);
117   ierr = PetscObjectReference((PetscObject) coordsA);CHKERRQ(ierr);
118   ierr = DMSetCoordinatesLocal(dmA, coordsB);CHKERRQ(ierr);
119   ierr = DMSetCoordinatesLocal(dmB, coordsA);CHKERRQ(ierr);
120   ierr = PetscObjectDereference((PetscObject) coordsA);CHKERRQ(ierr);
121 
122   fieldTmp             = dmA->coordinateField;
123   dmA->coordinateField = dmB->coordinateField;
124   dmB->coordinateField = fieldTmp;
125   tmp       = dmA->data;
126   dmA->data = dmB->data;
127   dmB->data = tmp;
128   listTmp   = dmA->labels;
129   dmA->labels = dmB->labels;
130   dmB->labels = listTmp;
131   depthTmp  = dmA->depthLabel;
132   dmA->depthLabel = dmB->depthLabel;
133   dmB->depthLabel = depthTmp;
134   depthTmp  = dmA->celltypeLabel;
135   dmA->celltypeLabel = dmB->celltypeLabel;
136   dmB->celltypeLabel = depthTmp;
137   tmpI         = dmA->levelup;
138   dmA->levelup = dmB->levelup;
139   dmB->levelup = tmpI;
140   PetscFunctionReturn(0);
141 }
142 
143 static PetscErrorCode DMPlexInterpolateInPlace_Internal(DM dm)
144 {
145   DM             idm;
146   PetscErrorCode ierr;
147 
148   PetscFunctionBegin;
149   ierr = DMPlexInterpolate(dm, &idm);CHKERRQ(ierr);
150   ierr = DMPlexCopyCoordinates(dm, idm);CHKERRQ(ierr);
151   ierr = DMPlexReplace_Static(dm, &idm);CHKERRQ(ierr);
152   PetscFunctionReturn(0);
153 }
154 
155 /*@C
156   DMPlexCreateCoordinateSpace - Creates a finite element space for the coordinates
157 
158   Collective
159 
160   Input Parameters:
161 + DM        - The DM
162 . degree    - The degree of the finite element or PETSC_DECIDE
163 - coordFunc - An optional function to map new points from refinement to the surface
164 
165   Level: advanced
166 
167 .seealso: PetscFECreateLagrange(), DMGetCoordinateDM()
168 @*/
169 PetscErrorCode DMPlexCreateCoordinateSpace(DM dm, PetscInt degree, PetscPointFunc coordFunc)
170 {
171   DM_Plex       *mesh = (DM_Plex *) dm->data;
172   DM             cdm;
173   PetscDS        cds;
174   PetscFE        fe;
175   PetscClassId   id;
176   PetscErrorCode ierr;
177 
178   PetscFunctionBegin;
179   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
180   ierr = DMGetDS(cdm, &cds);CHKERRQ(ierr);
181   ierr = PetscDSGetDiscretization(cds, 0, (PetscObject *) &fe);CHKERRQ(ierr);
182   ierr = PetscObjectGetClassId((PetscObject) fe, &id);CHKERRQ(ierr);
183   if (id != PETSCFE_CLASSID) {
184     PetscBool simplex;
185     PetscInt  dim, dE, qorder;
186 
187     ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
188     ierr = DMGetCoordinateDim(dm, &dE);CHKERRQ(ierr);
189     ierr = DMPlexIsSimplex(dm, &simplex);CHKERRQ(ierr);
190     qorder = degree;
191     ierr = PetscObjectOptionsBegin((PetscObject) cdm);CHKERRQ(ierr);
192     ierr = PetscOptionsBoundedInt("-coord_dm_default_quadrature_order", "Quadrature order is one less than quadrature points per edge", "DMPlexCreateCoordinateSpace", qorder, &qorder, NULL, 0);CHKERRQ(ierr);
193     ierr = PetscOptionsEnd();CHKERRQ(ierr);
194     if (degree == PETSC_DECIDE) fe = NULL;
195     else {
196       ierr = PetscFECreateLagrange(PETSC_COMM_SELF, dim, dE, simplex, degree, qorder, &fe);CHKERRQ(ierr);
197       ierr = DMSetField(cdm, 0, NULL, (PetscObject) fe);CHKERRQ(ierr);
198       ierr = DMCreateDS(cdm);CHKERRQ(ierr);
199     }
200     ierr = DMProjectCoordinates(dm, fe);CHKERRQ(ierr);
201     ierr = PetscFEDestroy(&fe);CHKERRQ(ierr);
202   }
203   mesh->coordFunc = coordFunc;
204   PetscFunctionReturn(0);
205 }
206 
207 /*@
208   DMPlexCreateDoublet - Creates a mesh of two cells of the specified type, optionally with later refinement.
209 
210   Collective
211 
212   Input Parameters:
213 + comm - The communicator for the DM object
214 . dim - The spatial dimension
215 . simplex - Flag for simplicial cells, otherwise they are tensor product cells
216 . interpolate - Flag to create intermediate mesh pieces (edges, faces)
217 - refinementLimit - A nonzero number indicates the largest admissible volume for a refined cell
218 
219   Output Parameter:
220 . dm - The DM object
221 
222   Level: beginner
223 
224 .seealso: DMSetType(), DMCreate()
225 @*/
226 PetscErrorCode DMPlexCreateDoublet(MPI_Comm comm, PetscInt dim, PetscBool simplex, PetscBool interpolate, PetscReal refinementLimit, DM *newdm)
227 {
228   DM             dm;
229   PetscMPIInt    rank;
230   PetscErrorCode ierr;
231 
232   PetscFunctionBegin;
233   ierr = DMCreate(comm, &dm);CHKERRQ(ierr);
234   ierr = DMSetType(dm, DMPLEX);CHKERRQ(ierr);
235   ierr = DMSetDimension(dm, dim);CHKERRQ(ierr);
236   ierr = MPI_Comm_rank(comm, &rank);CHKERRMPI(ierr);
237   switch (dim) {
238   case 2:
239     if (simplex) {ierr = PetscObjectSetName((PetscObject) dm, "triangular");CHKERRQ(ierr);}
240     else         {ierr = PetscObjectSetName((PetscObject) dm, "quadrilateral");CHKERRQ(ierr);}
241     break;
242   case 3:
243     if (simplex) {ierr = PetscObjectSetName((PetscObject) dm, "tetrahedral");CHKERRQ(ierr);}
244     else         {ierr = PetscObjectSetName((PetscObject) dm, "hexahedral");CHKERRQ(ierr);}
245     break;
246   default:
247     SETERRQ(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make meshes for dimension %D", dim);
248   }
249   if (rank) {
250     PetscInt numPoints[2] = {0, 0};
251     ierr = DMPlexCreateFromDAG(dm, 1, numPoints, NULL, NULL, NULL, NULL);CHKERRQ(ierr);
252   } else {
253     switch (dim) {
254     case 2:
255       if (simplex) {
256         PetscInt    numPoints[2]        = {4, 2};
257         PetscInt    coneSize[6]         = {3, 3, 0, 0, 0, 0};
258         PetscInt    cones[6]            = {2, 3, 4,  5, 4, 3};
259         PetscInt    coneOrientations[6] = {0, 0, 0,  0, 0, 0};
260         PetscScalar vertexCoords[8]     = {-0.5, 0.5, 0.0, 0.0, 0.0, 1.0, 0.5, 0.5};
261 
262         ierr = DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
263       } else {
264         PetscInt    numPoints[2]        = {6, 2};
265         PetscInt    coneSize[8]         = {4, 4, 0, 0, 0, 0, 0, 0};
266         PetscInt    cones[8]            = {2, 3, 4, 5,  3, 6, 7, 4};
267         PetscInt    coneOrientations[8] = {0, 0, 0, 0,  0, 0, 0, 0};
268         PetscScalar vertexCoords[12]    = {-1.0, -0.5,  0.0, -0.5,  0.0, 0.5,  -1.0, 0.5,  1.0, -0.5,  1.0, 0.5};
269 
270         ierr = DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
271       }
272       break;
273     case 3:
274       if (simplex) {
275         PetscInt    numPoints[2]        = {5, 2};
276         PetscInt    coneSize[7]         = {4, 4, 0, 0, 0, 0, 0};
277         PetscInt    cones[8]            = {4, 3, 5, 2,  5, 3, 4, 6};
278         PetscInt    coneOrientations[8] = {0, 0, 0, 0,  0, 0, 0, 0};
279         PetscScalar vertexCoords[15]    = {-1.0, 0.0, 0.0,  0.0, -1.0, 0.0,  0.0, 0.0, 1.0,  0.0, 1.0, 0.0,  1.0, 0.0, 0.0};
280 
281         ierr = DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
282       } else {
283         PetscInt    numPoints[2]         = {12, 2};
284         PetscInt    coneSize[14]         = {8, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
285         PetscInt    cones[16]            = {2, 3, 4, 5, 6, 7, 8, 9,  5, 4, 10, 11, 7, 12, 13, 8};
286         PetscInt    coneOrientations[16] = {0, 0, 0, 0, 0, 0, 0, 0,  0, 0,  0,  0, 0,  0,  0, 0};
287         PetscScalar vertexCoords[36]     = {-1.0, -0.5, -0.5,  -1.0,  0.5, -0.5,  0.0,  0.5, -0.5,   0.0, -0.5, -0.5,
288                                             -1.0, -0.5,  0.5,   0.0, -0.5,  0.5,  0.0,  0.5,  0.5,  -1.0,  0.5,  0.5,
289                                              1.0,  0.5, -0.5,   1.0, -0.5, -0.5,  1.0, -0.5,  0.5,   1.0,  0.5,  0.5};
290 
291         ierr = DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
292       }
293       break;
294     default:
295       SETERRQ(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make meshes for dimension %D", dim);
296     }
297   }
298   *newdm = dm;
299   if (refinementLimit > 0.0) {
300     DM rdm;
301     const char *name;
302 
303     ierr = DMPlexSetRefinementUniform(*newdm, PETSC_FALSE);CHKERRQ(ierr);
304     ierr = DMPlexSetRefinementLimit(*newdm, refinementLimit);CHKERRQ(ierr);
305     ierr = DMRefine(*newdm, comm, &rdm);CHKERRQ(ierr);
306     ierr = PetscObjectGetName((PetscObject) *newdm, &name);CHKERRQ(ierr);
307     ierr = PetscObjectSetName((PetscObject)    rdm,  name);CHKERRQ(ierr);
308     ierr = DMDestroy(newdm);CHKERRQ(ierr);
309     *newdm = rdm;
310   }
311   if (interpolate) {
312     DM idm;
313 
314     ierr = DMPlexInterpolate(*newdm, &idm);CHKERRQ(ierr);
315     ierr = DMDestroy(newdm);CHKERRQ(ierr);
316     *newdm = idm;
317   }
318   PetscFunctionReturn(0);
319 }
320 
321 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_1D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[])
322 {
323   const PetscInt numVertices    = 2;
324   PetscInt       markerRight    = 1;
325   PetscInt       markerLeft     = 1;
326   PetscBool      markerSeparate = PETSC_FALSE;
327   Vec            coordinates;
328   PetscSection   coordSection;
329   PetscScalar   *coords;
330   PetscInt       coordSize;
331   PetscMPIInt    rank;
332   PetscInt       cdim = 1, v;
333   PetscErrorCode ierr;
334 
335   PetscFunctionBegin;
336   ierr = PetscOptionsGetBool(((PetscObject) dm)->options,((PetscObject) dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL);CHKERRQ(ierr);
337   if (markerSeparate) {
338     markerRight  = 2;
339     markerLeft   = 1;
340   }
341   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRMPI(ierr);
342   if (!rank) {
343     ierr = DMPlexSetChart(dm, 0, numVertices);CHKERRQ(ierr);
344     ierr = DMSetUp(dm);CHKERRQ(ierr); /* Allocate space for cones */
345     ierr = DMSetLabelValue(dm, "marker", 0, markerLeft);CHKERRQ(ierr);
346     ierr = DMSetLabelValue(dm, "marker", 1, markerRight);CHKERRQ(ierr);
347   }
348   ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
349   ierr = DMPlexStratify(dm);CHKERRQ(ierr);
350   /* Build coordinates */
351   ierr = DMSetCoordinateDim(dm, cdim);CHKERRQ(ierr);
352   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
353   ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
354   ierr = PetscSectionSetChart(coordSection, 0, numVertices);CHKERRQ(ierr);
355   ierr = PetscSectionSetFieldComponents(coordSection, 0, cdim);CHKERRQ(ierr);
356   for (v = 0; v < numVertices; ++v) {
357     ierr = PetscSectionSetDof(coordSection, v, cdim);CHKERRQ(ierr);
358     ierr = PetscSectionSetFieldDof(coordSection, v, 0, cdim);CHKERRQ(ierr);
359   }
360   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
361   ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
362   ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
363   ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
364   ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
365   ierr = VecSetBlockSize(coordinates, cdim);CHKERRQ(ierr);
366   ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
367   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
368   coords[0] = lower[0];
369   coords[1] = upper[0];
370   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
371   ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
372   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
373   PetscFunctionReturn(0);
374 }
375 
376 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_2D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[])
377 {
378   const PetscInt numVertices    = (edges[0]+1)*(edges[1]+1);
379   const PetscInt numEdges       = edges[0]*(edges[1]+1) + (edges[0]+1)*edges[1];
380   PetscInt       markerTop      = 1;
381   PetscInt       markerBottom   = 1;
382   PetscInt       markerRight    = 1;
383   PetscInt       markerLeft     = 1;
384   PetscBool      markerSeparate = PETSC_FALSE;
385   Vec            coordinates;
386   PetscSection   coordSection;
387   PetscScalar    *coords;
388   PetscInt       coordSize;
389   PetscMPIInt    rank;
390   PetscInt       v, vx, vy;
391   PetscErrorCode ierr;
392 
393   PetscFunctionBegin;
394   ierr = PetscOptionsGetBool(((PetscObject) dm)->options,((PetscObject) dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL);CHKERRQ(ierr);
395   if (markerSeparate) {
396     markerTop    = 3;
397     markerBottom = 1;
398     markerRight  = 2;
399     markerLeft   = 4;
400   }
401   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRMPI(ierr);
402   if (rank == 0) {
403     PetscInt e, ex, ey;
404 
405     ierr = DMPlexSetChart(dm, 0, numEdges+numVertices);CHKERRQ(ierr);
406     for (e = 0; e < numEdges; ++e) {
407       ierr = DMPlexSetConeSize(dm, e, 2);CHKERRQ(ierr);
408     }
409     ierr = DMSetUp(dm);CHKERRQ(ierr); /* Allocate space for cones */
410     for (vx = 0; vx <= edges[0]; vx++) {
411       for (ey = 0; ey < edges[1]; ey++) {
412         PetscInt edge   = vx*edges[1] + ey + edges[0]*(edges[1]+1);
413         PetscInt vertex = ey*(edges[0]+1) + vx + numEdges;
414         PetscInt cone[2];
415 
416         cone[0] = vertex; cone[1] = vertex+edges[0]+1;
417         ierr    = DMPlexSetCone(dm, edge, cone);CHKERRQ(ierr);
418         if (vx == edges[0]) {
419           ierr = DMSetLabelValue(dm, "marker", edge,    markerRight);CHKERRQ(ierr);
420           ierr = DMSetLabelValue(dm, "marker", cone[0], markerRight);CHKERRQ(ierr);
421           if (ey == edges[1]-1) {
422             ierr = DMSetLabelValue(dm, "marker", cone[1], markerRight);CHKERRQ(ierr);
423             ierr = DMSetLabelValue(dm, "Face Sets", cone[1], markerRight);CHKERRQ(ierr);
424           }
425         } else if (vx == 0) {
426           ierr = DMSetLabelValue(dm, "marker", edge,    markerLeft);CHKERRQ(ierr);
427           ierr = DMSetLabelValue(dm, "marker", cone[0], markerLeft);CHKERRQ(ierr);
428           if (ey == edges[1]-1) {
429             ierr = DMSetLabelValue(dm, "marker", cone[1], markerLeft);CHKERRQ(ierr);
430             ierr = DMSetLabelValue(dm, "Face Sets", cone[1], markerLeft);CHKERRQ(ierr);
431           }
432         }
433       }
434     }
435     for (vy = 0; vy <= edges[1]; vy++) {
436       for (ex = 0; ex < edges[0]; ex++) {
437         PetscInt edge   = vy*edges[0]     + ex;
438         PetscInt vertex = vy*(edges[0]+1) + ex + numEdges;
439         PetscInt cone[2];
440 
441         cone[0] = vertex; cone[1] = vertex+1;
442         ierr    = DMPlexSetCone(dm, edge, cone);CHKERRQ(ierr);
443         if (vy == edges[1]) {
444           ierr = DMSetLabelValue(dm, "marker", edge,    markerTop);CHKERRQ(ierr);
445           ierr = DMSetLabelValue(dm, "marker", cone[0], markerTop);CHKERRQ(ierr);
446           if (ex == edges[0]-1) {
447             ierr = DMSetLabelValue(dm, "marker", cone[1], markerTop);CHKERRQ(ierr);
448             ierr = DMSetLabelValue(dm, "Face Sets", cone[1], markerTop);CHKERRQ(ierr);
449           }
450         } else if (vy == 0) {
451           ierr = DMSetLabelValue(dm, "marker", edge,    markerBottom);CHKERRQ(ierr);
452           ierr = DMSetLabelValue(dm, "marker", cone[0], markerBottom);CHKERRQ(ierr);
453           if (ex == edges[0]-1) {
454             ierr = DMSetLabelValue(dm, "marker", cone[1], markerBottom);CHKERRQ(ierr);
455             ierr = DMSetLabelValue(dm, "Face Sets", cone[1], markerBottom);CHKERRQ(ierr);
456           }
457         }
458       }
459     }
460   }
461   ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
462   ierr = DMPlexStratify(dm);CHKERRQ(ierr);
463   /* Build coordinates */
464   ierr = DMSetCoordinateDim(dm, 2);CHKERRQ(ierr);
465   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
466   ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
467   ierr = PetscSectionSetChart(coordSection, numEdges, numEdges + numVertices);CHKERRQ(ierr);
468   ierr = PetscSectionSetFieldComponents(coordSection, 0, 2);CHKERRQ(ierr);
469   for (v = numEdges; v < numEdges+numVertices; ++v) {
470     ierr = PetscSectionSetDof(coordSection, v, 2);CHKERRQ(ierr);
471     ierr = PetscSectionSetFieldDof(coordSection, v, 0, 2);CHKERRQ(ierr);
472   }
473   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
474   ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
475   ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
476   ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
477   ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
478   ierr = VecSetBlockSize(coordinates, 2);CHKERRQ(ierr);
479   ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
480   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
481   for (vy = 0; vy <= edges[1]; ++vy) {
482     for (vx = 0; vx <= edges[0]; ++vx) {
483       coords[(vy*(edges[0]+1)+vx)*2+0] = lower[0] + ((upper[0] - lower[0])/edges[0])*vx;
484       coords[(vy*(edges[0]+1)+vx)*2+1] = lower[1] + ((upper[1] - lower[1])/edges[1])*vy;
485     }
486   }
487   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
488   ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
489   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
490   PetscFunctionReturn(0);
491 }
492 
493 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_3D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt faces[])
494 {
495   PetscInt       vertices[3], numVertices;
496   PetscInt       numFaces    = 2*faces[0]*faces[1] + 2*faces[1]*faces[2] + 2*faces[0]*faces[2];
497   Vec            coordinates;
498   PetscSection   coordSection;
499   PetscScalar    *coords;
500   PetscInt       coordSize;
501   PetscMPIInt    rank;
502   PetscInt       v, vx, vy, vz;
503   PetscInt       voffset, iface=0, cone[4];
504   PetscErrorCode ierr;
505 
506   PetscFunctionBegin;
507   PetscCheckFalse((faces[0] < 1) || (faces[1] < 1) || (faces[2] < 1),PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Must have at least 1 face per side");
508   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRMPI(ierr);
509   vertices[0] = faces[0]+1; vertices[1] = faces[1]+1; vertices[2] = faces[2]+1;
510   numVertices = vertices[0]*vertices[1]*vertices[2];
511   if (rank == 0) {
512     PetscInt f;
513 
514     ierr = DMPlexSetChart(dm, 0, numFaces+numVertices);CHKERRQ(ierr);
515     for (f = 0; f < numFaces; ++f) {
516       ierr = DMPlexSetConeSize(dm, f, 4);CHKERRQ(ierr);
517     }
518     ierr = DMSetUp(dm);CHKERRQ(ierr); /* Allocate space for cones */
519 
520     /* Side 0 (Top) */
521     for (vy = 0; vy < faces[1]; vy++) {
522       for (vx = 0; vx < faces[0]; vx++) {
523         voffset = numFaces + vertices[0]*vertices[1]*(vertices[2]-1) + vy*vertices[0] + vx;
524         cone[0] = voffset; cone[1] = voffset+1; cone[2] = voffset+vertices[0]+1; cone[3] = voffset+vertices[0];
525         ierr    = DMPlexSetCone(dm, iface, cone);CHKERRQ(ierr);
526         ierr    = DMSetLabelValue(dm, "marker", iface, 1);CHKERRQ(ierr);
527         ierr    = DMSetLabelValue(dm, "marker", voffset+0, 1);CHKERRQ(ierr);
528         ierr    = DMSetLabelValue(dm, "marker", voffset+1, 1);CHKERRQ(ierr);
529         ierr    = DMSetLabelValue(dm, "marker", voffset+vertices[0]+0, 1);CHKERRQ(ierr);
530         ierr    = DMSetLabelValue(dm, "marker", voffset+vertices[0]+1, 1);CHKERRQ(ierr);
531         iface++;
532       }
533     }
534 
535     /* Side 1 (Bottom) */
536     for (vy = 0; vy < faces[1]; vy++) {
537       for (vx = 0; vx < faces[0]; vx++) {
538         voffset = numFaces + vy*(faces[0]+1) + vx;
539         cone[0] = voffset+1; cone[1] = voffset; cone[2] = voffset+vertices[0]; cone[3] = voffset+vertices[0]+1;
540         ierr    = DMPlexSetCone(dm, iface, cone);CHKERRQ(ierr);
541         ierr    = DMSetLabelValue(dm, "marker", iface, 1);CHKERRQ(ierr);
542         ierr    = DMSetLabelValue(dm, "marker", voffset+0, 1);CHKERRQ(ierr);
543         ierr    = DMSetLabelValue(dm, "marker", voffset+1, 1);CHKERRQ(ierr);
544         ierr    = DMSetLabelValue(dm, "marker", voffset+vertices[0]+0, 1);CHKERRQ(ierr);
545         ierr    = DMSetLabelValue(dm, "marker", voffset+vertices[0]+1, 1);CHKERRQ(ierr);
546         iface++;
547       }
548     }
549 
550     /* Side 2 (Front) */
551     for (vz = 0; vz < faces[2]; vz++) {
552       for (vx = 0; vx < faces[0]; vx++) {
553         voffset = numFaces + vz*vertices[0]*vertices[1] + vx;
554         cone[0] = voffset; cone[1] = voffset+1; cone[2] = voffset+vertices[0]*vertices[1]+1; cone[3] = voffset+vertices[0]*vertices[1];
555         ierr    = DMPlexSetCone(dm, iface, cone);CHKERRQ(ierr);
556         ierr    = DMSetLabelValue(dm, "marker", iface, 1);CHKERRQ(ierr);
557         ierr    = DMSetLabelValue(dm, "marker", voffset+0, 1);CHKERRQ(ierr);
558         ierr    = DMSetLabelValue(dm, "marker", voffset+1, 1);CHKERRQ(ierr);
559         ierr    = DMSetLabelValue(dm, "marker", voffset+vertices[0]*vertices[1]+0, 1);CHKERRQ(ierr);
560         ierr    = DMSetLabelValue(dm, "marker", voffset+vertices[0]*vertices[1]+1, 1);CHKERRQ(ierr);
561         iface++;
562       }
563     }
564 
565     /* Side 3 (Back) */
566     for (vz = 0; vz < faces[2]; vz++) {
567       for (vx = 0; vx < faces[0]; vx++) {
568         voffset = numFaces + vz*vertices[0]*vertices[1] + vertices[0]*(vertices[1]-1) + vx;
569         cone[0] = voffset+vertices[0]*vertices[1]; cone[1] = voffset+vertices[0]*vertices[1]+1;
570         cone[2] = voffset+1; cone[3] = voffset;
571         ierr    = DMPlexSetCone(dm, iface, cone);CHKERRQ(ierr);
572         ierr    = DMSetLabelValue(dm, "marker", iface, 1);CHKERRQ(ierr);
573         ierr    = DMSetLabelValue(dm, "marker", voffset+0, 1);CHKERRQ(ierr);
574         ierr    = DMSetLabelValue(dm, "marker", voffset+1, 1);CHKERRQ(ierr);
575         ierr    = DMSetLabelValue(dm, "marker", voffset+vertices[0]*vertices[1]+0, 1);CHKERRQ(ierr);
576         ierr    = DMSetLabelValue(dm, "marker", voffset+vertices[0]*vertices[1]+1, 1);CHKERRQ(ierr);
577         iface++;
578       }
579     }
580 
581     /* Side 4 (Left) */
582     for (vz = 0; vz < faces[2]; vz++) {
583       for (vy = 0; vy < faces[1]; vy++) {
584         voffset = numFaces + vz*vertices[0]*vertices[1] + vy*vertices[0];
585         cone[0] = voffset; cone[1] = voffset+vertices[0]*vertices[1];
586         cone[2] = voffset+vertices[0]*vertices[1]+vertices[0]; cone[3] = voffset+vertices[0];
587         ierr    = DMPlexSetCone(dm, iface, cone);CHKERRQ(ierr);
588         ierr    = DMSetLabelValue(dm, "marker", iface, 1);CHKERRQ(ierr);
589         ierr    = DMSetLabelValue(dm, "marker", voffset+0, 1);CHKERRQ(ierr);
590         ierr    = DMSetLabelValue(dm, "marker", voffset+vertices[0]+0, 1);CHKERRQ(ierr);
591         ierr    = DMSetLabelValue(dm, "marker", voffset+vertices[1]+0, 1);CHKERRQ(ierr);
592         ierr    = DMSetLabelValue(dm, "marker", voffset+vertices[0]*vertices[1]+vertices[0], 1);CHKERRQ(ierr);
593         iface++;
594       }
595     }
596 
597     /* Side 5 (Right) */
598     for (vz = 0; vz < faces[2]; vz++) {
599       for (vy = 0; vy < faces[1]; vy++) {
600         voffset = numFaces + vz*vertices[0]*vertices[1] + vy*vertices[0] + faces[0];
601         cone[0] = voffset+vertices[0]*vertices[1]; cone[1] = voffset;
602         cone[2] = voffset+vertices[0]; cone[3] = voffset+vertices[0]*vertices[1]+vertices[0];
603         ierr    = DMPlexSetCone(dm, iface, cone);CHKERRQ(ierr);
604         ierr    = DMSetLabelValue(dm, "marker", iface, 1);CHKERRQ(ierr);
605         ierr    = DMSetLabelValue(dm, "marker", voffset+0, 1);CHKERRQ(ierr);
606         ierr    = DMSetLabelValue(dm, "marker", voffset+vertices[0]+0, 1);CHKERRQ(ierr);
607         ierr    = DMSetLabelValue(dm, "marker", voffset+vertices[0]*vertices[1]+0, 1);CHKERRQ(ierr);
608         ierr    = DMSetLabelValue(dm, "marker", voffset+vertices[0]*vertices[1]+vertices[0], 1);CHKERRQ(ierr);
609         iface++;
610       }
611     }
612   }
613   ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
614   ierr = DMPlexStratify(dm);CHKERRQ(ierr);
615   /* Build coordinates */
616   ierr = DMSetCoordinateDim(dm, 3);CHKERRQ(ierr);
617   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
618   ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
619   ierr = PetscSectionSetChart(coordSection, numFaces, numFaces + numVertices);CHKERRQ(ierr);
620   ierr = PetscSectionSetFieldComponents(coordSection, 0, 3);CHKERRQ(ierr);
621   for (v = numFaces; v < numFaces+numVertices; ++v) {
622     ierr = PetscSectionSetDof(coordSection, v, 3);CHKERRQ(ierr);
623     ierr = PetscSectionSetFieldDof(coordSection, v, 0, 3);CHKERRQ(ierr);
624   }
625   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
626   ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
627   ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
628   ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
629   ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
630   ierr = VecSetBlockSize(coordinates, 3);CHKERRQ(ierr);
631   ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
632   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
633   for (vz = 0; vz <= faces[2]; ++vz) {
634     for (vy = 0; vy <= faces[1]; ++vy) {
635       for (vx = 0; vx <= faces[0]; ++vx) {
636         coords[((vz*(faces[1]+1)+vy)*(faces[0]+1)+vx)*3+0] = lower[0] + ((upper[0] - lower[0])/faces[0])*vx;
637         coords[((vz*(faces[1]+1)+vy)*(faces[0]+1)+vx)*3+1] = lower[1] + ((upper[1] - lower[1])/faces[1])*vy;
638         coords[((vz*(faces[1]+1)+vy)*(faces[0]+1)+vx)*3+2] = lower[2] + ((upper[2] - lower[2])/faces[2])*vz;
639       }
640     }
641   }
642   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
643   ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
644   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
645   PetscFunctionReturn(0);
646 }
647 
648 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], PetscBool interpolate)
649 {
650   PetscErrorCode ierr;
651 
652   PetscFunctionBegin;
653   PetscValidLogicalCollectiveInt(dm, dim, 2);
654   ierr = DMSetDimension(dm, dim-1);CHKERRQ(ierr);
655   ierr = DMSetCoordinateDim(dm, dim);CHKERRQ(ierr);
656   switch (dim) {
657     case 1: ierr = DMPlexCreateBoxSurfaceMesh_Tensor_1D_Internal(dm, lower, upper, faces);CHKERRQ(ierr);break;
658     case 2: ierr = DMPlexCreateBoxSurfaceMesh_Tensor_2D_Internal(dm, lower, upper, faces);CHKERRQ(ierr);break;
659     case 3: ierr = DMPlexCreateBoxSurfaceMesh_Tensor_3D_Internal(dm, lower, upper, faces);CHKERRQ(ierr);break;
660     default: SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Dimension not supported: %D", dim);
661   }
662   if (interpolate) {ierr = DMPlexInterpolateInPlace_Internal(dm);CHKERRQ(ierr);}
663   PetscFunctionReturn(0);
664 }
665 
666 /*@C
667   DMPlexCreateBoxSurfaceMesh - Creates a mesh on the surface of the tensor product of unit intervals (box) using tensor cells (hexahedra).
668 
669   Collective
670 
671   Input Parameters:
672 + comm        - The communicator for the DM object
673 . dim         - The spatial dimension of the box, so the resulting mesh is has dimension dim-1
674 . faces       - Number of faces per dimension, or NULL for (1,) in 1D and (2, 2) in 2D and (1, 1, 1) in 3D
675 . lower       - The lower left corner, or NULL for (0, 0, 0)
676 . upper       - The upper right corner, or NULL for (1, 1, 1)
677 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
678 
679   Output Parameter:
680 . dm  - The DM object
681 
682   Level: beginner
683 
684 .seealso: DMSetFromOptions(), DMPlexCreateBoxMesh(), DMPlexCreateFromFile(), DMSetType(), DMCreate()
685 @*/
686 PetscErrorCode DMPlexCreateBoxSurfaceMesh(MPI_Comm comm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], PetscBool interpolate, DM *dm)
687 {
688   PetscInt       fac[3] = {1, 1, 1};
689   PetscReal      low[3] = {0, 0, 0};
690   PetscReal      upp[3] = {1, 1, 1};
691   PetscErrorCode ierr;
692 
693   PetscFunctionBegin;
694   ierr = DMCreate(comm,dm);CHKERRQ(ierr);
695   ierr = DMSetType(*dm,DMPLEX);CHKERRQ(ierr);
696   ierr = DMPlexCreateBoxSurfaceMesh_Internal(*dm, dim, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, interpolate);CHKERRQ(ierr);
697   PetscFunctionReturn(0);
698 }
699 
700 static PetscErrorCode DMPlexCreateLineMesh_Internal(DM dm,PetscInt segments,PetscReal lower,PetscReal upper,DMBoundaryType bd)
701 {
702   PetscInt       i,fStart,fEnd,numCells = 0,numVerts = 0;
703   PetscInt       numPoints[2],*coneSize,*cones,*coneOrientations;
704   PetscScalar    *vertexCoords;
705   PetscReal      L,maxCell;
706   PetscBool      markerSeparate = PETSC_FALSE;
707   PetscInt       markerLeft  = 1, faceMarkerLeft  = 1;
708   PetscInt       markerRight = 1, faceMarkerRight = 2;
709   PetscBool      wrap = (bd == DM_BOUNDARY_PERIODIC || bd == DM_BOUNDARY_TWIST) ? PETSC_TRUE : PETSC_FALSE;
710   PetscMPIInt    rank;
711   PetscErrorCode ierr;
712 
713   PetscFunctionBegin;
714   PetscValidPointer(dm,1);
715 
716   ierr = DMSetDimension(dm,1);CHKERRQ(ierr);
717   ierr = DMCreateLabel(dm,"marker");CHKERRQ(ierr);
718   ierr = DMCreateLabel(dm,"Face Sets");CHKERRQ(ierr);
719 
720   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm),&rank);CHKERRMPI(ierr);
721   if (rank == 0) numCells = segments;
722   if (rank == 0) numVerts = segments + (wrap ? 0 : 1);
723 
724   numPoints[0] = numVerts ; numPoints[1] = numCells;
725   ierr = PetscMalloc4(numCells+numVerts,&coneSize,numCells*2,&cones,numCells+numVerts,&coneOrientations,numVerts,&vertexCoords);CHKERRQ(ierr);
726   ierr = PetscArrayzero(coneOrientations,numCells+numVerts);CHKERRQ(ierr);
727   for (i = 0; i < numCells; ++i) { coneSize[i] = 2; }
728   for (i = 0; i < numVerts; ++i) { coneSize[numCells+i] = 0; }
729   for (i = 0; i < numCells; ++i) { cones[2*i] = numCells + i%numVerts; cones[2*i+1] = numCells + (i+1)%numVerts; }
730   for (i = 0; i < numVerts; ++i) { vertexCoords[i] = lower + (upper-lower)*((PetscReal)i/(PetscReal)numCells); }
731   ierr = DMPlexCreateFromDAG(dm,1,numPoints,coneSize,cones,coneOrientations,vertexCoords);CHKERRQ(ierr);
732   ierr = PetscFree4(coneSize,cones,coneOrientations,vertexCoords);CHKERRQ(ierr);
733 
734   ierr = PetscOptionsGetBool(((PetscObject)dm)->options,((PetscObject)dm)->prefix,"-dm_plex_separate_marker",&markerSeparate,NULL);CHKERRQ(ierr);
735   if (markerSeparate) { markerLeft = faceMarkerLeft; markerRight = faceMarkerRight;}
736   if (!wrap && rank == 0) {
737     ierr = DMPlexGetHeightStratum(dm,1,&fStart,&fEnd);CHKERRQ(ierr);
738     ierr = DMSetLabelValue(dm,"marker",fStart,markerLeft);CHKERRQ(ierr);
739     ierr = DMSetLabelValue(dm,"marker",fEnd-1,markerRight);CHKERRQ(ierr);
740     ierr = DMSetLabelValue(dm,"Face Sets",fStart,faceMarkerLeft);CHKERRQ(ierr);
741     ierr = DMSetLabelValue(dm,"Face Sets",fEnd-1,faceMarkerRight);CHKERRQ(ierr);
742   }
743   if (wrap) {
744     L       = upper - lower;
745     maxCell = (PetscReal)1.1*(L/(PetscReal)PetscMax(1,segments));
746     ierr = DMSetPeriodicity(dm,PETSC_TRUE,&maxCell,&L,&bd);CHKERRQ(ierr);
747   }
748   ierr = DMPlexSetRefinementUniform(dm, PETSC_TRUE);CHKERRQ(ierr);
749   PetscFunctionReturn(0);
750 }
751 
752 static PetscErrorCode DMPlexCreateBoxMesh_Simplex_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate)
753 {
754   DM             boundary, vol;
755   PetscInt       i;
756   PetscErrorCode ierr;
757 
758   PetscFunctionBegin;
759   PetscValidPointer(dm, 1);
760   for (i = 0; i < dim; ++i) PetscCheckFalse(periodicity[i] != DM_BOUNDARY_NONE,PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Periodicity is not supported for simplex meshes");
761   ierr = DMCreate(PetscObjectComm((PetscObject) dm), &boundary);CHKERRQ(ierr);
762   ierr = DMSetType(boundary, DMPLEX);CHKERRQ(ierr);
763   ierr = DMPlexCreateBoxSurfaceMesh_Internal(boundary, dim, faces, lower, upper, PETSC_FALSE);CHKERRQ(ierr);
764   ierr = DMPlexGenerate(boundary, NULL, interpolate, &vol);CHKERRQ(ierr);
765   ierr = DMPlexCopy_Internal(dm, PETSC_TRUE, vol);CHKERRQ(ierr);
766   ierr = DMPlexReplace_Static(dm, &vol);CHKERRQ(ierr);
767   ierr = DMDestroy(&boundary);CHKERRQ(ierr);
768   PetscFunctionReturn(0);
769 }
770 
771 static PetscErrorCode DMPlexCreateCubeMesh_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[], DMBoundaryType bdX, DMBoundaryType bdY, DMBoundaryType bdZ)
772 {
773   DMLabel        cutLabel = NULL;
774   PetscInt       markerTop      = 1, faceMarkerTop      = 1;
775   PetscInt       markerBottom   = 1, faceMarkerBottom   = 1;
776   PetscInt       markerFront    = 1, faceMarkerFront    = 1;
777   PetscInt       markerBack     = 1, faceMarkerBack     = 1;
778   PetscInt       markerRight    = 1, faceMarkerRight    = 1;
779   PetscInt       markerLeft     = 1, faceMarkerLeft     = 1;
780   PetscInt       dim;
781   PetscBool      markerSeparate = PETSC_FALSE, cutMarker = PETSC_FALSE;
782   PetscMPIInt    rank;
783   PetscErrorCode ierr;
784 
785   PetscFunctionBegin;
786   ierr = DMGetDimension(dm,&dim);CHKERRQ(ierr);
787   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRMPI(ierr);
788   ierr = DMCreateLabel(dm,"marker");CHKERRQ(ierr);
789   ierr = DMCreateLabel(dm,"Face Sets");CHKERRQ(ierr);
790   ierr = PetscOptionsGetBool(((PetscObject) dm)->options,((PetscObject) dm)->prefix, "-dm_plex_periodic_cut", &cutMarker, NULL);CHKERRQ(ierr);
791   if (bdX == DM_BOUNDARY_PERIODIC || bdX == DM_BOUNDARY_TWIST ||
792       bdY == DM_BOUNDARY_PERIODIC || bdY == DM_BOUNDARY_TWIST ||
793       bdZ == DM_BOUNDARY_PERIODIC || bdZ == DM_BOUNDARY_TWIST) {
794 
795     if (cutMarker) {ierr = DMCreateLabel(dm, "periodic_cut");CHKERRQ(ierr); ierr = DMGetLabel(dm, "periodic_cut", &cutLabel);CHKERRQ(ierr);}
796   }
797   switch (dim) {
798   case 2:
799     faceMarkerTop    = 3;
800     faceMarkerBottom = 1;
801     faceMarkerRight  = 2;
802     faceMarkerLeft   = 4;
803     break;
804   case 3:
805     faceMarkerBottom = 1;
806     faceMarkerTop    = 2;
807     faceMarkerFront  = 3;
808     faceMarkerBack   = 4;
809     faceMarkerRight  = 5;
810     faceMarkerLeft   = 6;
811     break;
812   default:
813     SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"Dimension %D not supported",dim);
814   }
815   ierr = PetscOptionsGetBool(((PetscObject) dm)->options,((PetscObject) dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL);CHKERRQ(ierr);
816   if (markerSeparate) {
817     markerBottom = faceMarkerBottom;
818     markerTop    = faceMarkerTop;
819     markerFront  = faceMarkerFront;
820     markerBack   = faceMarkerBack;
821     markerRight  = faceMarkerRight;
822     markerLeft   = faceMarkerLeft;
823   }
824   {
825     const PetscInt numXEdges    = rank == 0 ? edges[0] : 0;
826     const PetscInt numYEdges    = rank == 0 ? edges[1] : 0;
827     const PetscInt numZEdges    = rank == 0 ? edges[2] : 0;
828     const PetscInt numXVertices = rank == 0 ? (bdX == DM_BOUNDARY_PERIODIC || bdX == DM_BOUNDARY_TWIST ? edges[0] : edges[0]+1) : 0;
829     const PetscInt numYVertices = rank == 0 ? (bdY == DM_BOUNDARY_PERIODIC || bdY == DM_BOUNDARY_TWIST ? edges[1] : edges[1]+1) : 0;
830     const PetscInt numZVertices = rank == 0 ? (bdZ == DM_BOUNDARY_PERIODIC || bdZ == DM_BOUNDARY_TWIST ? edges[2] : edges[2]+1) : 0;
831     const PetscInt numCells     = numXEdges*numYEdges*numZEdges;
832     const PetscInt numXFaces    = numYEdges*numZEdges;
833     const PetscInt numYFaces    = numXEdges*numZEdges;
834     const PetscInt numZFaces    = numXEdges*numYEdges;
835     const PetscInt numTotXFaces = numXVertices*numXFaces;
836     const PetscInt numTotYFaces = numYVertices*numYFaces;
837     const PetscInt numTotZFaces = numZVertices*numZFaces;
838     const PetscInt numFaces     = numTotXFaces + numTotYFaces + numTotZFaces;
839     const PetscInt numTotXEdges = numXEdges*numYVertices*numZVertices;
840     const PetscInt numTotYEdges = numYEdges*numXVertices*numZVertices;
841     const PetscInt numTotZEdges = numZEdges*numXVertices*numYVertices;
842     const PetscInt numVertices  = numXVertices*numYVertices*numZVertices;
843     const PetscInt numEdges     = numTotXEdges + numTotYEdges + numTotZEdges;
844     const PetscInt firstVertex  = (dim == 2) ? numFaces : numCells;
845     const PetscInt firstXFace   = (dim == 2) ? 0 : numCells + numVertices;
846     const PetscInt firstYFace   = firstXFace + numTotXFaces;
847     const PetscInt firstZFace   = firstYFace + numTotYFaces;
848     const PetscInt firstXEdge   = numCells + numFaces + numVertices;
849     const PetscInt firstYEdge   = firstXEdge + numTotXEdges;
850     const PetscInt firstZEdge   = firstYEdge + numTotYEdges;
851     Vec            coordinates;
852     PetscSection   coordSection;
853     PetscScalar   *coords;
854     PetscInt       coordSize;
855     PetscInt       v, vx, vy, vz;
856     PetscInt       c, f, fx, fy, fz, e, ex, ey, ez;
857 
858     ierr = DMPlexSetChart(dm, 0, numCells+numFaces+numEdges+numVertices);CHKERRQ(ierr);
859     for (c = 0; c < numCells; c++) {
860       ierr = DMPlexSetConeSize(dm, c, 6);CHKERRQ(ierr);
861     }
862     for (f = firstXFace; f < firstXFace+numFaces; ++f) {
863       ierr = DMPlexSetConeSize(dm, f, 4);CHKERRQ(ierr);
864     }
865     for (e = firstXEdge; e < firstXEdge+numEdges; ++e) {
866       ierr = DMPlexSetConeSize(dm, e, 2);CHKERRQ(ierr);
867     }
868     ierr = DMSetUp(dm);CHKERRQ(ierr); /* Allocate space for cones */
869     /* Build cells */
870     for (fz = 0; fz < numZEdges; ++fz) {
871       for (fy = 0; fy < numYEdges; ++fy) {
872         for (fx = 0; fx < numXEdges; ++fx) {
873           PetscInt cell    = (fz*numYEdges + fy)*numXEdges + fx;
874           PetscInt faceB   = firstZFace + (fy*numXEdges+fx)*numZVertices +   fz;
875           PetscInt faceT   = firstZFace + (fy*numXEdges+fx)*numZVertices + ((fz+1)%numZVertices);
876           PetscInt faceF   = firstYFace + (fz*numXEdges+fx)*numYVertices +   fy;
877           PetscInt faceK   = firstYFace + (fz*numXEdges+fx)*numYVertices + ((fy+1)%numYVertices);
878           PetscInt faceL   = firstXFace + (fz*numYEdges+fy)*numXVertices +   fx;
879           PetscInt faceR   = firstXFace + (fz*numYEdges+fy)*numXVertices + ((fx+1)%numXVertices);
880                             /* B,  T,  F,  K,  R,  L */
881           PetscInt ornt[6] = {-2,  0,  0, -3,  0, -2}; /* ??? */
882           PetscInt cone[6];
883 
884           /* no boundary twisting in 3D */
885           cone[0] = faceB; cone[1] = faceT; cone[2] = faceF; cone[3] = faceK; cone[4] = faceR; cone[5] = faceL;
886           ierr    = DMPlexSetCone(dm, cell, cone);CHKERRQ(ierr);
887           ierr    = DMPlexSetConeOrientation(dm, cell, ornt);CHKERRQ(ierr);
888           if (bdX != DM_BOUNDARY_NONE && fx == numXEdges-1 && cutLabel) {ierr = DMLabelSetValue(cutLabel, cell, 2);CHKERRQ(ierr);}
889           if (bdY != DM_BOUNDARY_NONE && fy == numYEdges-1 && cutLabel) {ierr = DMLabelSetValue(cutLabel, cell, 2);CHKERRQ(ierr);}
890           if (bdZ != DM_BOUNDARY_NONE && fz == numZEdges-1 && cutLabel) {ierr = DMLabelSetValue(cutLabel, cell, 2);CHKERRQ(ierr);}
891         }
892       }
893     }
894     /* Build x faces */
895     for (fz = 0; fz < numZEdges; ++fz) {
896       for (fy = 0; fy < numYEdges; ++fy) {
897         for (fx = 0; fx < numXVertices; ++fx) {
898           PetscInt face    = firstXFace + (fz*numYEdges+fy)     *numXVertices+fx;
899           PetscInt edgeL   = firstZEdge + (fy                   *numXVertices+fx)*numZEdges + fz;
900           PetscInt edgeR   = firstZEdge + (((fy+1)%numYVertices)*numXVertices+fx)*numZEdges + fz;
901           PetscInt edgeB   = firstYEdge + (fz                   *numXVertices+fx)*numYEdges + fy;
902           PetscInt edgeT   = firstYEdge + (((fz+1)%numZVertices)*numXVertices+fx)*numYEdges + fy;
903           PetscInt ornt[4] = {0, 0, -1, -1};
904           PetscInt cone[4];
905 
906           if (dim == 3) {
907             /* markers */
908             if (bdX != DM_BOUNDARY_PERIODIC) {
909               if (fx == numXVertices-1) {
910                 ierr = DMSetLabelValue(dm, "Face Sets", face, faceMarkerRight);CHKERRQ(ierr);
911                 ierr = DMSetLabelValue(dm, "marker", face, markerRight);CHKERRQ(ierr);
912               }
913               else if (fx == 0) {
914                 ierr = DMSetLabelValue(dm, "Face Sets", face, faceMarkerLeft);CHKERRQ(ierr);
915                 ierr = DMSetLabelValue(dm, "marker", face, markerLeft);CHKERRQ(ierr);
916               }
917             }
918           }
919           cone[0] = edgeB; cone[1] = edgeR; cone[2] = edgeT; cone[3] = edgeL;
920           ierr    = DMPlexSetCone(dm, face, cone);CHKERRQ(ierr);
921           ierr    = DMPlexSetConeOrientation(dm, face, ornt);CHKERRQ(ierr);
922         }
923       }
924     }
925     /* Build y faces */
926     for (fz = 0; fz < numZEdges; ++fz) {
927       for (fx = 0; fx < numXEdges; ++fx) {
928         for (fy = 0; fy < numYVertices; ++fy) {
929           PetscInt face    = firstYFace + (fz*numXEdges+fx)*numYVertices + fy;
930           PetscInt edgeL   = firstZEdge + (fy*numXVertices+  fx)*numZEdges + fz;
931           PetscInt edgeR   = firstZEdge + (fy*numXVertices+((fx+1)%numXVertices))*numZEdges + fz;
932           PetscInt edgeB   = firstXEdge + (fz                   *numYVertices+fy)*numXEdges + fx;
933           PetscInt edgeT   = firstXEdge + (((fz+1)%numZVertices)*numYVertices+fy)*numXEdges + fx;
934           PetscInt ornt[4] = {0, 0, -1, -1};
935           PetscInt cone[4];
936 
937           if (dim == 3) {
938             /* markers */
939             if (bdY != DM_BOUNDARY_PERIODIC) {
940               if (fy == numYVertices-1) {
941                 ierr = DMSetLabelValue(dm, "Face Sets", face, faceMarkerBack);CHKERRQ(ierr);
942                 ierr = DMSetLabelValue(dm, "marker", face, markerBack);CHKERRQ(ierr);
943               }
944               else if (fy == 0) {
945                 ierr = DMSetLabelValue(dm, "Face Sets", face, faceMarkerFront);CHKERRQ(ierr);
946                 ierr = DMSetLabelValue(dm, "marker", face, markerFront);CHKERRQ(ierr);
947               }
948             }
949           }
950           cone[0] = edgeB; cone[1] = edgeR; cone[2] = edgeT; cone[3] = edgeL;
951           ierr    = DMPlexSetCone(dm, face, cone);CHKERRQ(ierr);
952           ierr    = DMPlexSetConeOrientation(dm, face, ornt);CHKERRQ(ierr);
953         }
954       }
955     }
956     /* Build z faces */
957     for (fy = 0; fy < numYEdges; ++fy) {
958       for (fx = 0; fx < numXEdges; ++fx) {
959         for (fz = 0; fz < numZVertices; fz++) {
960           PetscInt face    = firstZFace + (fy*numXEdges+fx)*numZVertices + fz;
961           PetscInt edgeL   = firstYEdge + (fz*numXVertices+  fx)*numYEdges + fy;
962           PetscInt edgeR   = firstYEdge + (fz*numXVertices+((fx+1)%numXVertices))*numYEdges + fy;
963           PetscInt edgeB   = firstXEdge + (fz*numYVertices+  fy)*numXEdges + fx;
964           PetscInt edgeT   = firstXEdge + (fz*numYVertices+((fy+1)%numYVertices))*numXEdges + fx;
965           PetscInt ornt[4] = {0, 0, -1, -1};
966           PetscInt cone[4];
967 
968           if (dim == 2) {
969             if (bdX == DM_BOUNDARY_TWIST && fx == numXEdges-1) {edgeR += numYEdges-1-2*fy; ornt[1] = -1;}
970             if (bdY == DM_BOUNDARY_TWIST && fy == numYEdges-1) {edgeT += numXEdges-1-2*fx; ornt[2] =  0;}
971             if (bdX != DM_BOUNDARY_NONE && fx == numXEdges-1 && cutLabel) {ierr = DMLabelSetValue(cutLabel, face, 2);CHKERRQ(ierr);}
972             if (bdY != DM_BOUNDARY_NONE && fy == numYEdges-1 && cutLabel) {ierr = DMLabelSetValue(cutLabel, face, 2);CHKERRQ(ierr);}
973           } else {
974             /* markers */
975             if (bdZ != DM_BOUNDARY_PERIODIC) {
976               if (fz == numZVertices-1) {
977                 ierr = DMSetLabelValue(dm, "Face Sets", face, faceMarkerTop);CHKERRQ(ierr);
978                 ierr = DMSetLabelValue(dm, "marker", face, markerTop);CHKERRQ(ierr);
979               }
980               else if (fz == 0) {
981                 ierr = DMSetLabelValue(dm, "Face Sets", face, faceMarkerBottom);CHKERRQ(ierr);
982                 ierr = DMSetLabelValue(dm, "marker", face, markerBottom);CHKERRQ(ierr);
983               }
984             }
985           }
986           cone[0] = edgeB; cone[1] = edgeR; cone[2] = edgeT; cone[3] = edgeL;
987           ierr    = DMPlexSetCone(dm, face, cone);CHKERRQ(ierr);
988           ierr    = DMPlexSetConeOrientation(dm, face, ornt);CHKERRQ(ierr);
989         }
990       }
991     }
992     /* Build Z edges*/
993     for (vy = 0; vy < numYVertices; vy++) {
994       for (vx = 0; vx < numXVertices; vx++) {
995         for (ez = 0; ez < numZEdges; ez++) {
996           const PetscInt edge    = firstZEdge  + (vy*numXVertices+vx)*numZEdges + ez;
997           const PetscInt vertexB = firstVertex + (ez                   *numYVertices+vy)*numXVertices + vx;
998           const PetscInt vertexT = firstVertex + (((ez+1)%numZVertices)*numYVertices+vy)*numXVertices + vx;
999           PetscInt       cone[2];
1000 
1001           if (dim == 3) {
1002             if (bdX != DM_BOUNDARY_PERIODIC) {
1003               if (vx == numXVertices-1) {
1004                 ierr = DMSetLabelValue(dm, "marker", edge, markerRight);CHKERRQ(ierr);
1005               }
1006               else if (vx == 0) {
1007                 ierr = DMSetLabelValue(dm, "marker", edge, markerLeft);CHKERRQ(ierr);
1008               }
1009             }
1010             if (bdY != DM_BOUNDARY_PERIODIC) {
1011               if (vy == numYVertices-1) {
1012                 ierr = DMSetLabelValue(dm, "marker", edge, markerBack);CHKERRQ(ierr);
1013               }
1014               else if (vy == 0) {
1015                 ierr = DMSetLabelValue(dm, "marker", edge, markerFront);CHKERRQ(ierr);
1016               }
1017             }
1018           }
1019           cone[0] = vertexB; cone[1] = vertexT;
1020           ierr = DMPlexSetCone(dm, edge, cone);CHKERRQ(ierr);
1021         }
1022       }
1023     }
1024     /* Build Y edges*/
1025     for (vz = 0; vz < numZVertices; vz++) {
1026       for (vx = 0; vx < numXVertices; vx++) {
1027         for (ey = 0; ey < numYEdges; ey++) {
1028           const PetscInt nextv   = (dim == 2 && bdY == DM_BOUNDARY_TWIST && ey == numYEdges-1) ? (numXVertices-vx-1) : (vz*numYVertices+((ey+1)%numYVertices))*numXVertices + vx;
1029           const PetscInt edge    = firstYEdge  + (vz*numXVertices+vx)*numYEdges + ey;
1030           const PetscInt vertexF = firstVertex + (vz*numYVertices+ey)*numXVertices + vx;
1031           const PetscInt vertexK = firstVertex + nextv;
1032           PetscInt       cone[2];
1033 
1034           cone[0] = vertexF; cone[1] = vertexK;
1035           ierr = DMPlexSetCone(dm, edge, cone);CHKERRQ(ierr);
1036           if (dim == 2) {
1037             if ((bdX != DM_BOUNDARY_PERIODIC) && (bdX != DM_BOUNDARY_TWIST)) {
1038               if (vx == numXVertices-1) {
1039                 ierr = DMSetLabelValue(dm, "Face Sets", edge, faceMarkerRight);CHKERRQ(ierr);
1040                 ierr = DMSetLabelValue(dm, "marker", edge,    markerRight);CHKERRQ(ierr);
1041                 ierr = DMSetLabelValue(dm, "marker", cone[0], markerRight);CHKERRQ(ierr);
1042                 if (ey == numYEdges-1) {
1043                   ierr = DMSetLabelValue(dm, "marker", cone[1], markerRight);CHKERRQ(ierr);
1044                 }
1045               } else if (vx == 0) {
1046                 ierr = DMSetLabelValue(dm, "Face Sets", edge, faceMarkerLeft);CHKERRQ(ierr);
1047                 ierr = DMSetLabelValue(dm, "marker", edge,    markerLeft);CHKERRQ(ierr);
1048                 ierr = DMSetLabelValue(dm, "marker", cone[0], markerLeft);CHKERRQ(ierr);
1049                 if (ey == numYEdges-1) {
1050                   ierr = DMSetLabelValue(dm, "marker", cone[1], markerLeft);CHKERRQ(ierr);
1051                 }
1052               }
1053             } else {
1054               if (vx == 0 && cutLabel) {
1055                 ierr = DMLabelSetValue(cutLabel, edge,    1);CHKERRQ(ierr);
1056                 ierr = DMLabelSetValue(cutLabel, cone[0], 1);CHKERRQ(ierr);
1057                 if (ey == numYEdges-1) {
1058                   ierr = DMLabelSetValue(cutLabel, cone[1], 1);CHKERRQ(ierr);
1059                 }
1060               }
1061             }
1062           } else {
1063             if (bdX != DM_BOUNDARY_PERIODIC) {
1064               if (vx == numXVertices-1) {
1065                 ierr = DMSetLabelValue(dm, "marker", edge, markerRight);CHKERRQ(ierr);
1066               } else if (vx == 0) {
1067                 ierr = DMSetLabelValue(dm, "marker", edge, markerLeft);CHKERRQ(ierr);
1068               }
1069             }
1070             if (bdZ != DM_BOUNDARY_PERIODIC) {
1071               if (vz == numZVertices-1) {
1072                 ierr = DMSetLabelValue(dm, "marker", edge, markerTop);CHKERRQ(ierr);
1073               } else if (vz == 0) {
1074                 ierr = DMSetLabelValue(dm, "marker", edge, markerBottom);CHKERRQ(ierr);
1075               }
1076             }
1077           }
1078         }
1079       }
1080     }
1081     /* Build X edges*/
1082     for (vz = 0; vz < numZVertices; vz++) {
1083       for (vy = 0; vy < numYVertices; vy++) {
1084         for (ex = 0; ex < numXEdges; ex++) {
1085           const PetscInt nextv   = (dim == 2 && bdX == DM_BOUNDARY_TWIST && ex == numXEdges-1) ? (numYVertices-vy-1)*numXVertices : (vz*numYVertices+vy)*numXVertices + (ex+1)%numXVertices;
1086           const PetscInt edge    = firstXEdge  + (vz*numYVertices+vy)*numXEdges + ex;
1087           const PetscInt vertexL = firstVertex + (vz*numYVertices+vy)*numXVertices + ex;
1088           const PetscInt vertexR = firstVertex + nextv;
1089           PetscInt       cone[2];
1090 
1091           cone[0] = vertexL; cone[1] = vertexR;
1092           ierr = DMPlexSetCone(dm, edge, cone);CHKERRQ(ierr);
1093           if (dim == 2) {
1094             if ((bdY != DM_BOUNDARY_PERIODIC) && (bdY != DM_BOUNDARY_TWIST)) {
1095               if (vy == numYVertices-1) {
1096                 ierr = DMSetLabelValue(dm, "Face Sets", edge, faceMarkerTop);CHKERRQ(ierr);
1097                 ierr = DMSetLabelValue(dm, "marker", edge,    markerTop);CHKERRQ(ierr);
1098                 ierr = DMSetLabelValue(dm, "marker", cone[0], markerTop);CHKERRQ(ierr);
1099                 if (ex == numXEdges-1) {
1100                   ierr = DMSetLabelValue(dm, "marker", cone[1], markerTop);CHKERRQ(ierr);
1101                 }
1102               } else if (vy == 0) {
1103                 ierr = DMSetLabelValue(dm, "Face Sets", edge, faceMarkerBottom);CHKERRQ(ierr);
1104                 ierr = DMSetLabelValue(dm, "marker", edge,    markerBottom);CHKERRQ(ierr);
1105                 ierr = DMSetLabelValue(dm, "marker", cone[0], markerBottom);CHKERRQ(ierr);
1106                 if (ex == numXEdges-1) {
1107                   ierr = DMSetLabelValue(dm, "marker", cone[1], markerBottom);CHKERRQ(ierr);
1108                 }
1109               }
1110             } else {
1111               if (vy == 0 && cutLabel) {
1112                 ierr = DMLabelSetValue(cutLabel, edge,    1);CHKERRQ(ierr);
1113                 ierr = DMLabelSetValue(cutLabel, cone[0], 1);CHKERRQ(ierr);
1114                 if (ex == numXEdges-1) {
1115                   ierr = DMLabelSetValue(cutLabel, cone[1], 1);CHKERRQ(ierr);
1116                 }
1117               }
1118             }
1119           } else {
1120             if (bdY != DM_BOUNDARY_PERIODIC) {
1121               if (vy == numYVertices-1) {
1122                 ierr = DMSetLabelValue(dm, "marker", edge, markerBack);CHKERRQ(ierr);
1123               }
1124               else if (vy == 0) {
1125                 ierr = DMSetLabelValue(dm, "marker", edge, markerFront);CHKERRQ(ierr);
1126               }
1127             }
1128             if (bdZ != DM_BOUNDARY_PERIODIC) {
1129               if (vz == numZVertices-1) {
1130                 ierr = DMSetLabelValue(dm, "marker", edge, markerTop);CHKERRQ(ierr);
1131               }
1132               else if (vz == 0) {
1133                 ierr = DMSetLabelValue(dm, "marker", edge, markerBottom);CHKERRQ(ierr);
1134               }
1135             }
1136           }
1137         }
1138       }
1139     }
1140     ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
1141     ierr = DMPlexStratify(dm);CHKERRQ(ierr);
1142     /* Build coordinates */
1143     ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
1144     ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
1145     ierr = PetscSectionSetFieldComponents(coordSection, 0, dim);CHKERRQ(ierr);
1146     ierr = PetscSectionSetChart(coordSection, firstVertex, firstVertex+numVertices);CHKERRQ(ierr);
1147     for (v = firstVertex; v < firstVertex+numVertices; ++v) {
1148       ierr = PetscSectionSetDof(coordSection, v, dim);CHKERRQ(ierr);
1149       ierr = PetscSectionSetFieldDof(coordSection, v, 0, dim);CHKERRQ(ierr);
1150     }
1151     ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
1152     ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
1153     ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
1154     ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
1155     ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
1156     ierr = VecSetBlockSize(coordinates, dim);CHKERRQ(ierr);
1157     ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
1158     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
1159     for (vz = 0; vz < numZVertices; ++vz) {
1160       for (vy = 0; vy < numYVertices; ++vy) {
1161         for (vx = 0; vx < numXVertices; ++vx) {
1162           coords[((vz*numYVertices+vy)*numXVertices+vx)*dim+0] = lower[0] + ((upper[0] - lower[0])/numXEdges)*vx;
1163           coords[((vz*numYVertices+vy)*numXVertices+vx)*dim+1] = lower[1] + ((upper[1] - lower[1])/numYEdges)*vy;
1164           if (dim == 3) {
1165             coords[((vz*numYVertices+vy)*numXVertices+vx)*dim+2] = lower[2] + ((upper[2] - lower[2])/numZEdges)*vz;
1166           }
1167         }
1168       }
1169     }
1170     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
1171     ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
1172     ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
1173   }
1174   PetscFunctionReturn(0);
1175 }
1176 
1177 static PetscErrorCode DMPlexCreateBoxMesh_Tensor_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[])
1178 {
1179   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1180   PetscInt       fac[3] = {0, 0, 0}, d;
1181   PetscErrorCode ierr;
1182 
1183   PetscFunctionBegin;
1184   PetscValidPointer(dm, 1);
1185   PetscValidLogicalCollectiveInt(dm, dim, 2);
1186   ierr = DMSetDimension(dm, dim);CHKERRQ(ierr);
1187   for (d = 0; d < dim; ++d) {fac[d] = faces[d]; bdt[d] = periodicity[d];}
1188   ierr = DMPlexCreateCubeMesh_Internal(dm, lower, upper, fac, bdt[0], bdt[1], bdt[2]);CHKERRQ(ierr);
1189   if (periodicity[0] == DM_BOUNDARY_PERIODIC || periodicity[0] == DM_BOUNDARY_TWIST ||
1190       periodicity[1] == DM_BOUNDARY_PERIODIC || periodicity[1] == DM_BOUNDARY_TWIST ||
1191       (dim > 2 && (periodicity[2] == DM_BOUNDARY_PERIODIC || periodicity[2] == DM_BOUNDARY_TWIST))) {
1192     PetscReal L[3];
1193     PetscReal maxCell[3];
1194 
1195     for (d = 0; d < dim; ++d) {
1196       L[d]       = upper[d] - lower[d];
1197       maxCell[d] = 1.1 * (L[d] / PetscMax(1, faces[d]));
1198     }
1199     ierr = DMSetPeriodicity(dm, PETSC_TRUE, maxCell, L, periodicity);CHKERRQ(ierr);
1200   }
1201   ierr = DMPlexSetRefinementUniform(dm, PETSC_TRUE);CHKERRQ(ierr);
1202   PetscFunctionReturn(0);
1203 }
1204 
1205 static PetscErrorCode DMPlexCreateBoxMesh_Internal(DM dm, PetscInt dim, PetscBool simplex, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate)
1206 {
1207   PetscErrorCode ierr;
1208 
1209   PetscFunctionBegin;
1210   if (dim == 1)      {ierr = DMPlexCreateLineMesh_Internal(dm, faces[0], lower[0], upper[0], periodicity[0]);CHKERRQ(ierr);}
1211   else if (simplex)  {ierr = DMPlexCreateBoxMesh_Simplex_Internal(dm, dim, faces, lower, upper, periodicity, interpolate);CHKERRQ(ierr);}
1212   else               {ierr = DMPlexCreateBoxMesh_Tensor_Internal(dm, dim, faces, lower, upper, periodicity);CHKERRQ(ierr);}
1213   if (!interpolate && dim > 1 && !simplex) {
1214     DM udm;
1215 
1216     ierr = DMPlexUninterpolate(dm, &udm);CHKERRQ(ierr);
1217     ierr = DMPlexCopyCoordinates(dm, udm);CHKERRQ(ierr);
1218     ierr = DMPlexReplace_Static(dm, &udm);CHKERRQ(ierr);
1219   }
1220   PetscFunctionReturn(0);
1221 }
1222 
1223 /*@C
1224   DMPlexCreateBoxMesh - Creates a mesh on the tensor product of unit intervals (box) using simplices or tensor cells (hexahedra).
1225 
1226   Collective
1227 
1228   Input Parameters:
1229 + comm        - The communicator for the DM object
1230 . dim         - The spatial dimension
1231 . simplex     - PETSC_TRUE for simplices, PETSC_FALSE for tensor cells
1232 . faces       - Number of faces per dimension, or NULL for (1,) in 1D and (2, 2) in 2D and (1, 1, 1) in 3D
1233 . lower       - The lower left corner, or NULL for (0, 0, 0)
1234 . upper       - The upper right corner, or NULL for (1, 1, 1)
1235 . periodicity - The boundary type for the X,Y,Z direction, or NULL for DM_BOUNDARY_NONE
1236 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
1237 
1238   Output Parameter:
1239 . dm  - The DM object
1240 
1241   Note: If you want to customize this mesh using options, you just need to
1242 $  DMCreate(comm, &dm);
1243 $  DMSetType(dm, DMPLEX);
1244 $  DMSetFromOptions(dm);
1245 and use the options on the DMSetFromOptions() page.
1246 
1247   Here is the numbering returned for 2 faces in each direction for tensor cells:
1248 $ 10---17---11---18----12
1249 $  |         |         |
1250 $  |         |         |
1251 $ 20    2   22    3    24
1252 $  |         |         |
1253 $  |         |         |
1254 $  7---15----8---16----9
1255 $  |         |         |
1256 $  |         |         |
1257 $ 19    0   21    1   23
1258 $  |         |         |
1259 $  |         |         |
1260 $  4---13----5---14----6
1261 
1262 and for simplicial cells
1263 
1264 $ 14----8---15----9----16
1265 $  |\     5  |\      7 |
1266 $  | \       | \       |
1267 $ 13   2    14    3    15
1268 $  | 4   \   | 6   \   |
1269 $  |       \ |       \ |
1270 $ 11----6---12----7----13
1271 $  |\        |\        |
1272 $  | \    1  | \     3 |
1273 $ 10   0    11    1    12
1274 $  | 0   \   | 2   \   |
1275 $  |       \ |       \ |
1276 $  8----4----9----5----10
1277 
1278   Level: beginner
1279 
1280 .seealso: DMSetFromOptions(), DMPlexCreateFromFile(), DMPlexCreateHexCylinderMesh(), DMSetType(), DMCreate()
1281 @*/
1282 PetscErrorCode DMPlexCreateBoxMesh(MPI_Comm comm, PetscInt dim, PetscBool simplex, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate, DM *dm)
1283 {
1284   PetscInt       fac[3] = {1, 1, 1};
1285   PetscReal      low[3] = {0, 0, 0};
1286   PetscReal      upp[3] = {1, 1, 1};
1287   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1288   PetscErrorCode ierr;
1289 
1290   PetscFunctionBegin;
1291   ierr = DMCreate(comm,dm);CHKERRQ(ierr);
1292   ierr = DMSetType(*dm,DMPLEX);CHKERRQ(ierr);
1293   ierr = DMPlexCreateBoxMesh_Internal(*dm, dim, simplex, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, periodicity ? periodicity : bdt, interpolate);CHKERRQ(ierr);
1294   PetscFunctionReturn(0);
1295 }
1296 
1297 static PetscErrorCode DMPlexCreateWedgeBoxMesh_Internal(DM dm, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[])
1298 {
1299   DM             bdm, vol;
1300   PetscInt       i;
1301   PetscErrorCode ierr;
1302 
1303   PetscFunctionBegin;
1304   for (i = 0; i < 3; ++i) PetscCheckFalse(periodicity[i] != DM_BOUNDARY_NONE,PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Periodicity not yet supported");
1305   ierr = DMCreate(PetscObjectComm((PetscObject) dm), &bdm);CHKERRQ(ierr);
1306   ierr = DMSetType(bdm, DMPLEX);CHKERRQ(ierr);
1307   ierr = DMSetDimension(bdm, 2);CHKERRQ(ierr);
1308   ierr = DMPlexCreateBoxMesh_Simplex_Internal(bdm, 2, faces, lower, upper, periodicity, PETSC_TRUE);CHKERRQ(ierr);
1309   ierr = DMPlexExtrude(bdm, faces[2], upper[2] - lower[2], PETSC_TRUE, PETSC_FALSE, NULL, NULL, &vol);CHKERRQ(ierr);
1310   ierr = DMDestroy(&bdm);CHKERRQ(ierr);
1311   ierr = DMPlexReplace_Static(dm, &vol);CHKERRQ(ierr);
1312   if (lower[2] != 0.0) {
1313     Vec          v;
1314     PetscScalar *x;
1315     PetscInt     cDim, n;
1316 
1317     ierr = DMGetCoordinatesLocal(dm, &v);CHKERRQ(ierr);
1318     ierr = VecGetBlockSize(v, &cDim);CHKERRQ(ierr);
1319     ierr = VecGetLocalSize(v, &n);CHKERRQ(ierr);
1320     ierr = VecGetArray(v, &x);CHKERRQ(ierr);
1321     x   += cDim;
1322     for (i = 0; i < n; i += cDim) x[i] += lower[2];
1323     ierr = VecRestoreArray(v,&x);CHKERRQ(ierr);
1324     ierr = DMSetCoordinatesLocal(dm, v);CHKERRQ(ierr);
1325   }
1326   PetscFunctionReturn(0);
1327 }
1328 
1329 /*@
1330   DMPlexCreateWedgeBoxMesh - Creates a 3-D mesh tesselating the (x,y) plane and extruding in the third direction using wedge cells.
1331 
1332   Collective
1333 
1334   Input Parameters:
1335 + comm        - The communicator for the DM object
1336 . faces       - Number of faces per dimension, or NULL for (1, 1, 1)
1337 . lower       - The lower left corner, or NULL for (0, 0, 0)
1338 . upper       - The upper right corner, or NULL for (1, 1, 1)
1339 . periodicity - The boundary type for the X,Y,Z direction, or NULL for DM_BOUNDARY_NONE
1340 . orderHeight - If PETSC_TRUE, orders the extruded cells in the height first. Otherwise, orders the cell on the layers first
1341 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
1342 
1343   Output Parameter:
1344 . dm  - The DM object
1345 
1346   Level: beginner
1347 
1348 .seealso: DMPlexCreateHexCylinderMesh(), DMPlexCreateWedgeCylinderMesh(), DMExtrude(), DMPlexCreateBoxMesh(), DMSetType(), DMCreate()
1349 @*/
1350 PetscErrorCode DMPlexCreateWedgeBoxMesh(MPI_Comm comm, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool orderHeight, PetscBool interpolate, DM *dm)
1351 {
1352   PetscInt       fac[3] = {1, 1, 1};
1353   PetscReal      low[3] = {0, 0, 0};
1354   PetscReal      upp[3] = {1, 1, 1};
1355   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1356   PetscErrorCode ierr;
1357 
1358   PetscFunctionBegin;
1359   ierr = DMCreate(comm,dm);CHKERRQ(ierr);
1360   ierr = DMSetType(*dm,DMPLEX);CHKERRQ(ierr);
1361   ierr = DMPlexCreateWedgeBoxMesh_Internal(*dm, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, periodicity ? periodicity : bdt);CHKERRQ(ierr);
1362   if (!interpolate) {
1363     DM udm;
1364 
1365     ierr = DMPlexUninterpolate(*dm, &udm);CHKERRQ(ierr);
1366     ierr = DMPlexReplace_Static(*dm, &udm);CHKERRQ(ierr);
1367   }
1368   PetscFunctionReturn(0);
1369 }
1370 
1371 /*@C
1372   DMPlexSetOptionsPrefix - Sets the prefix used for searching for all DM options in the database.
1373 
1374   Logically Collective on dm
1375 
1376   Input Parameters:
1377 + dm - the DM context
1378 - prefix - the prefix to prepend to all option names
1379 
1380   Notes:
1381   A hyphen (-) must NOT be given at the beginning of the prefix name.
1382   The first character of all runtime options is AUTOMATICALLY the hyphen.
1383 
1384   Level: advanced
1385 
1386 .seealso: SNESSetFromOptions()
1387 @*/
1388 PetscErrorCode DMPlexSetOptionsPrefix(DM dm, const char prefix[])
1389 {
1390   DM_Plex       *mesh = (DM_Plex *) dm->data;
1391   PetscErrorCode ierr;
1392 
1393   PetscFunctionBegin;
1394   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1395   ierr = PetscObjectSetOptionsPrefix((PetscObject) dm, prefix);CHKERRQ(ierr);
1396   ierr = PetscObjectSetOptionsPrefix((PetscObject) mesh->partitioner, prefix);CHKERRQ(ierr);
1397   PetscFunctionReturn(0);
1398 }
1399 
1400 /* Remap geometry to cylinder
1401    TODO: This only works for a single refinement, then it is broken
1402 
1403      Interior square: Linear interpolation is correct
1404      The other cells all have vertices on rays from the origin. We want to uniformly expand the spacing
1405      such that the last vertex is on the unit circle. So the closest and farthest vertices are at distance
1406 
1407        phi     = arctan(y/x)
1408        d_close = sqrt(1/8 + 1/4 sin^2(phi))
1409        d_far   = sqrt(1/2 + sin^2(phi))
1410 
1411      so we remap them using
1412 
1413        x_new = x_close + (x - x_close) (1 - d_close) / (d_far - d_close)
1414        y_new = y_close + (y - y_close) (1 - d_close) / (d_far - d_close)
1415 
1416      If pi/4 < phi < 3pi/4 or -3pi/4 < phi < -pi/4, then we switch x and y.
1417 */
1418 static void snapToCylinder(PetscInt dim, PetscInt Nf, PetscInt NfAux,
1419                            const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[],
1420                            const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[],
1421                            PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
1422 {
1423   const PetscReal dis = 1.0/PetscSqrtReal(2.0);
1424   const PetscReal ds2 = 0.5*dis;
1425 
1426   if ((PetscAbsScalar(u[0]) <= ds2) && (PetscAbsScalar(u[1]) <= ds2)) {
1427     f0[0] = u[0];
1428     f0[1] = u[1];
1429   } else {
1430     PetscReal phi, sinp, cosp, dc, df, x, y, xc, yc;
1431 
1432     x    = PetscRealPart(u[0]);
1433     y    = PetscRealPart(u[1]);
1434     phi  = PetscAtan2Real(y, x);
1435     sinp = PetscSinReal(phi);
1436     cosp = PetscCosReal(phi);
1437     if ((PetscAbsReal(phi) > PETSC_PI/4.0) && (PetscAbsReal(phi) < 3.0*PETSC_PI/4.0)) {
1438       dc = PetscAbsReal(ds2/sinp);
1439       df = PetscAbsReal(dis/sinp);
1440       xc = ds2*x/PetscAbsReal(y);
1441       yc = ds2*PetscSignReal(y);
1442     } else {
1443       dc = PetscAbsReal(ds2/cosp);
1444       df = PetscAbsReal(dis/cosp);
1445       xc = ds2*PetscSignReal(x);
1446       yc = ds2*y/PetscAbsReal(x);
1447     }
1448     f0[0] = xc + (u[0] - xc)*(1.0 - dc)/(df - dc);
1449     f0[1] = yc + (u[1] - yc)*(1.0 - dc)/(df - dc);
1450   }
1451   f0[2] = u[2];
1452 }
1453 
1454 static PetscErrorCode DMPlexCreateHexCylinderMesh_Internal(DM dm, DMBoundaryType periodicZ)
1455 {
1456   const PetscInt dim = 3;
1457   PetscInt       numCells, numVertices;
1458   PetscMPIInt    rank;
1459   PetscErrorCode ierr;
1460 
1461   PetscFunctionBegin;
1462   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRMPI(ierr);
1463   ierr = DMSetDimension(dm, dim);CHKERRQ(ierr);
1464   /* Create topology */
1465   {
1466     PetscInt cone[8], c;
1467 
1468     numCells    = rank == 0 ?  5 : 0;
1469     numVertices = rank == 0 ? 16 : 0;
1470     if (periodicZ == DM_BOUNDARY_PERIODIC) {
1471       numCells   *= 3;
1472       numVertices = rank == 0 ? 24 : 0;
1473     }
1474     ierr = DMPlexSetChart(dm, 0, numCells+numVertices);CHKERRQ(ierr);
1475     for (c = 0; c < numCells; c++) {ierr = DMPlexSetConeSize(dm, c, 8);CHKERRQ(ierr);}
1476     ierr = DMSetUp(dm);CHKERRQ(ierr);
1477     if (rank == 0) {
1478       if (periodicZ == DM_BOUNDARY_PERIODIC) {
1479         cone[0] = 15; cone[1] = 18; cone[2] = 17; cone[3] = 16;
1480         cone[4] = 31; cone[5] = 32; cone[6] = 33; cone[7] = 34;
1481         ierr = DMPlexSetCone(dm, 0, cone);CHKERRQ(ierr);
1482         cone[0] = 16; cone[1] = 17; cone[2] = 24; cone[3] = 23;
1483         cone[4] = 32; cone[5] = 36; cone[6] = 37; cone[7] = 33; /* 22 25 26 21 */
1484         ierr = DMPlexSetCone(dm, 1, cone);CHKERRQ(ierr);
1485         cone[0] = 18; cone[1] = 27; cone[2] = 24; cone[3] = 17;
1486         cone[4] = 34; cone[5] = 33; cone[6] = 37; cone[7] = 38;
1487         ierr = DMPlexSetCone(dm, 2, cone);CHKERRQ(ierr);
1488         cone[0] = 29; cone[1] = 27; cone[2] = 18; cone[3] = 15;
1489         cone[4] = 35; cone[5] = 31; cone[6] = 34; cone[7] = 38;
1490         ierr = DMPlexSetCone(dm, 3, cone);CHKERRQ(ierr);
1491         cone[0] = 29; cone[1] = 15; cone[2] = 16; cone[3] = 23;
1492         cone[4] = 35; cone[5] = 36; cone[6] = 32; cone[7] = 31;
1493         ierr = DMPlexSetCone(dm, 4, cone);CHKERRQ(ierr);
1494 
1495         cone[0] = 31; cone[1] = 34; cone[2] = 33; cone[3] = 32;
1496         cone[4] = 19; cone[5] = 22; cone[6] = 21; cone[7] = 20;
1497         ierr = DMPlexSetCone(dm, 5, cone);CHKERRQ(ierr);
1498         cone[0] = 32; cone[1] = 33; cone[2] = 37; cone[3] = 36;
1499         cone[4] = 22; cone[5] = 25; cone[6] = 26; cone[7] = 21;
1500         ierr = DMPlexSetCone(dm, 6, cone);CHKERRQ(ierr);
1501         cone[0] = 34; cone[1] = 38; cone[2] = 37; cone[3] = 33;
1502         cone[4] = 20; cone[5] = 21; cone[6] = 26; cone[7] = 28;
1503         ierr = DMPlexSetCone(dm, 7, cone);CHKERRQ(ierr);
1504         cone[0] = 35; cone[1] = 38; cone[2] = 34; cone[3] = 31;
1505         cone[4] = 30; cone[5] = 19; cone[6] = 20; cone[7] = 28;
1506         ierr = DMPlexSetCone(dm, 8, cone);CHKERRQ(ierr);
1507         cone[0] = 35; cone[1] = 31; cone[2] = 32; cone[3] = 36;
1508         cone[4] = 30; cone[5] = 25; cone[6] = 22; cone[7] = 19;
1509         ierr = DMPlexSetCone(dm, 9, cone);CHKERRQ(ierr);
1510 
1511         cone[0] = 19; cone[1] = 20; cone[2] = 21; cone[3] = 22;
1512         cone[4] = 15; cone[5] = 16; cone[6] = 17; cone[7] = 18;
1513         ierr = DMPlexSetCone(dm, 10, cone);CHKERRQ(ierr);
1514         cone[0] = 22; cone[1] = 21; cone[2] = 26; cone[3] = 25;
1515         cone[4] = 16; cone[5] = 23; cone[6] = 24; cone[7] = 17;
1516         ierr = DMPlexSetCone(dm, 11, cone);CHKERRQ(ierr);
1517         cone[0] = 20; cone[1] = 28; cone[2] = 26; cone[3] = 21;
1518         cone[4] = 18; cone[5] = 17; cone[6] = 24; cone[7] = 27;
1519         ierr = DMPlexSetCone(dm, 12, cone);CHKERRQ(ierr);
1520         cone[0] = 30; cone[1] = 28; cone[2] = 20; cone[3] = 19;
1521         cone[4] = 29; cone[5] = 15; cone[6] = 18; cone[7] = 27;
1522         ierr = DMPlexSetCone(dm, 13, cone);CHKERRQ(ierr);
1523         cone[0] = 30; cone[1] = 19; cone[2] = 22; cone[3] = 25;
1524         cone[4] = 29; cone[5] = 23; cone[6] = 16; cone[7] = 15;
1525         ierr = DMPlexSetCone(dm, 14, cone);CHKERRQ(ierr);
1526       } else {
1527         cone[0] =  5; cone[1] =  8; cone[2] =  7; cone[3] =  6;
1528         cone[4] =  9; cone[5] = 12; cone[6] = 11; cone[7] = 10;
1529         ierr = DMPlexSetCone(dm, 0, cone);CHKERRQ(ierr);
1530         cone[0] =  6; cone[1] =  7; cone[2] = 14; cone[3] = 13;
1531         cone[4] = 12; cone[5] = 15; cone[6] = 16; cone[7] = 11;
1532         ierr = DMPlexSetCone(dm, 1, cone);CHKERRQ(ierr);
1533         cone[0] =  8; cone[1] = 17; cone[2] = 14; cone[3] =  7;
1534         cone[4] = 10; cone[5] = 11; cone[6] = 16; cone[7] = 18;
1535         ierr = DMPlexSetCone(dm, 2, cone);CHKERRQ(ierr);
1536         cone[0] = 19; cone[1] = 17; cone[2] =  8; cone[3] =  5;
1537         cone[4] = 20; cone[5] =  9; cone[6] = 10; cone[7] = 18;
1538         ierr = DMPlexSetCone(dm, 3, cone);CHKERRQ(ierr);
1539         cone[0] = 19; cone[1] =  5; cone[2] =  6; cone[3] = 13;
1540         cone[4] = 20; cone[5] = 15; cone[6] = 12; cone[7] =  9;
1541         ierr = DMPlexSetCone(dm, 4, cone);CHKERRQ(ierr);
1542       }
1543     }
1544     ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
1545     ierr = DMPlexStratify(dm);CHKERRQ(ierr);
1546   }
1547   /* Create cube geometry */
1548   {
1549     Vec             coordinates;
1550     PetscSection    coordSection;
1551     PetscScalar    *coords;
1552     PetscInt        coordSize, v;
1553     const PetscReal dis = 1.0/PetscSqrtReal(2.0);
1554     const PetscReal ds2 = dis/2.0;
1555 
1556     /* Build coordinates */
1557     ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
1558     ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
1559     ierr = PetscSectionSetFieldComponents(coordSection, 0, dim);CHKERRQ(ierr);
1560     ierr = PetscSectionSetChart(coordSection, numCells, numCells+numVertices);CHKERRQ(ierr);
1561     for (v = numCells; v < numCells+numVertices; ++v) {
1562       ierr = PetscSectionSetDof(coordSection, v, dim);CHKERRQ(ierr);
1563       ierr = PetscSectionSetFieldDof(coordSection, v, 0, dim);CHKERRQ(ierr);
1564     }
1565     ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
1566     ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
1567     ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
1568     ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
1569     ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
1570     ierr = VecSetBlockSize(coordinates, dim);CHKERRQ(ierr);
1571     ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
1572     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
1573     if (rank == 0) {
1574       coords[0*dim+0] = -ds2; coords[0*dim+1] = -ds2; coords[0*dim+2] = 0.0;
1575       coords[1*dim+0] =  ds2; coords[1*dim+1] = -ds2; coords[1*dim+2] = 0.0;
1576       coords[2*dim+0] =  ds2; coords[2*dim+1] =  ds2; coords[2*dim+2] = 0.0;
1577       coords[3*dim+0] = -ds2; coords[3*dim+1] =  ds2; coords[3*dim+2] = 0.0;
1578       coords[4*dim+0] = -ds2; coords[4*dim+1] = -ds2; coords[4*dim+2] = 1.0;
1579       coords[5*dim+0] = -ds2; coords[5*dim+1] =  ds2; coords[5*dim+2] = 1.0;
1580       coords[6*dim+0] =  ds2; coords[6*dim+1] =  ds2; coords[6*dim+2] = 1.0;
1581       coords[7*dim+0] =  ds2; coords[7*dim+1] = -ds2; coords[7*dim+2] = 1.0;
1582       coords[ 8*dim+0] =  dis; coords[ 8*dim+1] = -dis; coords[ 8*dim+2] = 0.0;
1583       coords[ 9*dim+0] =  dis; coords[ 9*dim+1] =  dis; coords[ 9*dim+2] = 0.0;
1584       coords[10*dim+0] =  dis; coords[10*dim+1] = -dis; coords[10*dim+2] = 1.0;
1585       coords[11*dim+0] =  dis; coords[11*dim+1] =  dis; coords[11*dim+2] = 1.0;
1586       coords[12*dim+0] = -dis; coords[12*dim+1] =  dis; coords[12*dim+2] = 0.0;
1587       coords[13*dim+0] = -dis; coords[13*dim+1] =  dis; coords[13*dim+2] = 1.0;
1588       coords[14*dim+0] = -dis; coords[14*dim+1] = -dis; coords[14*dim+2] = 0.0;
1589       coords[15*dim+0] = -dis; coords[15*dim+1] = -dis; coords[15*dim+2] = 1.0;
1590       if (periodicZ == DM_BOUNDARY_PERIODIC) {
1591         /* 15 31 19 */ coords[16*dim+0] = -ds2; coords[16*dim+1] = -ds2; coords[16*dim+2] = 0.5;
1592         /* 16 32 22 */ coords[17*dim+0] =  ds2; coords[17*dim+1] = -ds2; coords[17*dim+2] = 0.5;
1593         /* 17 33 21 */ coords[18*dim+0] =  ds2; coords[18*dim+1] =  ds2; coords[18*dim+2] = 0.5;
1594         /* 18 34 20 */ coords[19*dim+0] = -ds2; coords[19*dim+1] =  ds2; coords[19*dim+2] = 0.5;
1595         /* 29 35 30 */ coords[20*dim+0] = -dis; coords[20*dim+1] = -dis; coords[20*dim+2] = 0.5;
1596         /* 23 36 25 */ coords[21*dim+0] =  dis; coords[21*dim+1] = -dis; coords[21*dim+2] = 0.5;
1597         /* 24 37 26 */ coords[22*dim+0] =  dis; coords[22*dim+1] =  dis; coords[22*dim+2] = 0.5;
1598         /* 27 38 28 */ coords[23*dim+0] = -dis; coords[23*dim+1] =  dis; coords[23*dim+2] = 0.5;
1599       }
1600     }
1601     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
1602     ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
1603     ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
1604   }
1605   /* Create periodicity */
1606   if (periodicZ == DM_BOUNDARY_PERIODIC || periodicZ == DM_BOUNDARY_TWIST) {
1607     PetscReal      L[3];
1608     PetscReal      maxCell[3];
1609     DMBoundaryType bdType[3];
1610     PetscReal      lower[3] = {0.0, 0.0, 0.0};
1611     PetscReal      upper[3] = {1.0, 1.0, 1.5};
1612     PetscInt       i, numZCells = 3;
1613 
1614     bdType[0] = DM_BOUNDARY_NONE;
1615     bdType[1] = DM_BOUNDARY_NONE;
1616     bdType[2] = periodicZ;
1617     for (i = 0; i < dim; i++) {
1618       L[i]       = upper[i] - lower[i];
1619       maxCell[i] = 1.1 * (L[i] / numZCells);
1620     }
1621     ierr = DMSetPeriodicity(dm, PETSC_TRUE, maxCell, L, bdType);CHKERRQ(ierr);
1622   }
1623   {
1624     DM          cdm;
1625     PetscDS     cds;
1626     PetscScalar c[2] = {1.0, 1.0};
1627 
1628     ierr = DMPlexCreateCoordinateSpace(dm, 1, snapToCylinder);CHKERRQ(ierr);
1629     ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
1630     ierr = DMGetDS(cdm, &cds);CHKERRQ(ierr);
1631     ierr = PetscDSSetConstants(cds, 2, c);CHKERRQ(ierr);
1632   }
1633   /* Wait for coordinate creation before doing in-place modification */
1634   ierr = DMPlexInterpolateInPlace_Internal(dm);CHKERRQ(ierr);
1635   PetscFunctionReturn(0);
1636 }
1637 
1638 /*@
1639   DMPlexCreateHexCylinderMesh - Creates a mesh on the tensor product of the unit interval with the circle (cylinder) using hexahedra.
1640 
1641   Collective
1642 
1643   Input Parameters:
1644 + comm      - The communicator for the DM object
1645 - periodicZ - The boundary type for the Z direction
1646 
1647   Output Parameter:
1648 . dm  - The DM object
1649 
1650   Note:
1651   Here is the output numbering looking from the bottom of the cylinder:
1652 $       17-----14
1653 $        |     |
1654 $        |  2  |
1655 $        |     |
1656 $ 17-----8-----7-----14
1657 $  |     |     |     |
1658 $  |  3  |  0  |  1  |
1659 $  |     |     |     |
1660 $ 19-----5-----6-----13
1661 $        |     |
1662 $        |  4  |
1663 $        |     |
1664 $       19-----13
1665 $
1666 $ and up through the top
1667 $
1668 $       18-----16
1669 $        |     |
1670 $        |  2  |
1671 $        |     |
1672 $ 18----10----11-----16
1673 $  |     |     |     |
1674 $  |  3  |  0  |  1  |
1675 $  |     |     |     |
1676 $ 20-----9----12-----15
1677 $        |     |
1678 $        |  4  |
1679 $        |     |
1680 $       20-----15
1681 
1682   Level: beginner
1683 
1684 .seealso: DMPlexCreateBoxMesh(), DMSetType(), DMCreate()
1685 @*/
1686 PetscErrorCode DMPlexCreateHexCylinderMesh(MPI_Comm comm, DMBoundaryType periodicZ, DM *dm)
1687 {
1688   PetscErrorCode ierr;
1689 
1690   PetscFunctionBegin;
1691   PetscValidPointer(dm, 3);
1692   ierr = DMCreate(comm, dm);CHKERRQ(ierr);
1693   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
1694   ierr = DMPlexCreateHexCylinderMesh_Internal(*dm, periodicZ);CHKERRQ(ierr);
1695   PetscFunctionReturn(0);
1696 }
1697 
1698 static PetscErrorCode DMPlexCreateWedgeCylinderMesh_Internal(DM dm, PetscInt n, PetscBool interpolate)
1699 {
1700   const PetscInt dim = 3;
1701   PetscInt       numCells, numVertices, v;
1702   PetscMPIInt    rank;
1703   PetscErrorCode ierr;
1704 
1705   PetscFunctionBegin;
1706   PetscCheckFalse(n < 0,PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of wedges %D cannot be negative", n);
1707   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRMPI(ierr);
1708   ierr = DMSetDimension(dm, dim);CHKERRQ(ierr);
1709   /* Must create the celltype label here so that we do not automatically try to compute the types */
1710   ierr = DMCreateLabel(dm, "celltype");CHKERRQ(ierr);
1711   /* Create topology */
1712   {
1713     PetscInt cone[6], c;
1714 
1715     numCells    = rank == 0 ?        n : 0;
1716     numVertices = rank == 0 ?  2*(n+1) : 0;
1717     ierr = DMPlexSetChart(dm, 0, numCells+numVertices);CHKERRQ(ierr);
1718     for (c = 0; c < numCells; c++) {ierr = DMPlexSetConeSize(dm, c, 6);CHKERRQ(ierr);}
1719     ierr = DMSetUp(dm);CHKERRQ(ierr);
1720     for (c = 0; c < numCells; c++) {
1721       cone[0] =  c+n*1; cone[1] = (c+1)%n+n*1; cone[2] = 0+3*n;
1722       cone[3] =  c+n*2; cone[4] = (c+1)%n+n*2; cone[5] = 1+3*n;
1723       ierr = DMPlexSetCone(dm, c, cone);CHKERRQ(ierr);
1724       ierr = DMPlexSetCellType(dm, c, DM_POLYTOPE_TRI_PRISM_TENSOR);CHKERRQ(ierr);
1725     }
1726     ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
1727     ierr = DMPlexStratify(dm);CHKERRQ(ierr);
1728   }
1729   for (v = numCells; v < numCells+numVertices; ++v) {
1730     ierr = DMPlexSetCellType(dm, v, DM_POLYTOPE_POINT);CHKERRQ(ierr);
1731   }
1732   /* Create cylinder geometry */
1733   {
1734     Vec          coordinates;
1735     PetscSection coordSection;
1736     PetscScalar *coords;
1737     PetscInt     coordSize, c;
1738 
1739     /* Build coordinates */
1740     ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
1741     ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
1742     ierr = PetscSectionSetFieldComponents(coordSection, 0, dim);CHKERRQ(ierr);
1743     ierr = PetscSectionSetChart(coordSection, numCells, numCells+numVertices);CHKERRQ(ierr);
1744     for (v = numCells; v < numCells+numVertices; ++v) {
1745       ierr = PetscSectionSetDof(coordSection, v, dim);CHKERRQ(ierr);
1746       ierr = PetscSectionSetFieldDof(coordSection, v, 0, dim);CHKERRQ(ierr);
1747     }
1748     ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
1749     ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
1750     ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
1751     ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
1752     ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
1753     ierr = VecSetBlockSize(coordinates, dim);CHKERRQ(ierr);
1754     ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
1755     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
1756     for (c = 0; c < numCells; c++) {
1757       coords[(c+0*n)*dim+0] = PetscCosReal(2.0*c*PETSC_PI/n); coords[(c+0*n)*dim+1] = PetscSinReal(2.0*c*PETSC_PI/n); coords[(c+0*n)*dim+2] = 1.0;
1758       coords[(c+1*n)*dim+0] = PetscCosReal(2.0*c*PETSC_PI/n); coords[(c+1*n)*dim+1] = PetscSinReal(2.0*c*PETSC_PI/n); coords[(c+1*n)*dim+2] = 0.0;
1759     }
1760     if (rank == 0) {
1761       coords[(2*n+0)*dim+0] = 0.0; coords[(2*n+0)*dim+1] = 0.0; coords[(2*n+0)*dim+2] = 1.0;
1762       coords[(2*n+1)*dim+0] = 0.0; coords[(2*n+1)*dim+1] = 0.0; coords[(2*n+1)*dim+2] = 0.0;
1763     }
1764     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
1765     ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
1766     ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
1767   }
1768   /* Interpolate */
1769   if (interpolate) {ierr = DMPlexInterpolateInPlace_Internal(dm);CHKERRQ(ierr);}
1770   PetscFunctionReturn(0);
1771 }
1772 
1773 /*@
1774   DMPlexCreateWedgeCylinderMesh - Creates a mesh on the tensor product of the unit interval with the circle (cylinder) using wedges.
1775 
1776   Collective
1777 
1778   Input Parameters:
1779 + comm - The communicator for the DM object
1780 . n    - The number of wedges around the origin
1781 - interpolate - Create edges and faces
1782 
1783   Output Parameter:
1784 . dm  - The DM object
1785 
1786   Level: beginner
1787 
1788 .seealso: DMPlexCreateHexCylinderMesh(), DMPlexCreateBoxMesh(), DMSetType(), DMCreate()
1789 @*/
1790 PetscErrorCode DMPlexCreateWedgeCylinderMesh(MPI_Comm comm, PetscInt n, PetscBool interpolate, DM *dm)
1791 {
1792   PetscErrorCode ierr;
1793 
1794   PetscFunctionBegin;
1795   PetscValidPointer(dm, 4);
1796   ierr = DMCreate(comm, dm);CHKERRQ(ierr);
1797   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
1798   ierr = DMPlexCreateWedgeCylinderMesh_Internal(*dm, n, interpolate);CHKERRQ(ierr);
1799   PetscFunctionReturn(0);
1800 }
1801 
1802 static inline PetscReal DiffNormReal(PetscInt dim, const PetscReal x[], const PetscReal y[])
1803 {
1804   PetscReal prod = 0.0;
1805   PetscInt  i;
1806   for (i = 0; i < dim; ++i) prod += PetscSqr(x[i] - y[i]);
1807   return PetscSqrtReal(prod);
1808 }
1809 static inline PetscReal DotReal(PetscInt dim, const PetscReal x[], const PetscReal y[])
1810 {
1811   PetscReal prod = 0.0;
1812   PetscInt  i;
1813   for (i = 0; i < dim; ++i) prod += x[i]*y[i];
1814   return prod;
1815 }
1816 
1817 /* The first constant is the sphere radius */
1818 static void snapToSphere(PetscInt dim, PetscInt Nf, PetscInt NfAux,
1819                          const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[],
1820                          const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[],
1821                          PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
1822 {
1823   PetscReal r = PetscRealPart(constants[0]);
1824   PetscReal norm2 = 0.0, fac;
1825   PetscInt  n = uOff[1] - uOff[0], d;
1826 
1827   for (d = 0; d < n; ++d) norm2 += PetscSqr(PetscRealPart(u[d]));
1828   fac = r/PetscSqrtReal(norm2);
1829   for (d = 0; d < n; ++d) f0[d] = u[d]*fac;
1830 }
1831 
1832 static PetscErrorCode DMPlexCreateSphereMesh_Internal(DM dm, PetscInt dim, PetscBool simplex, PetscReal R)
1833 {
1834   const PetscInt  embedDim = dim+1;
1835   PetscSection    coordSection;
1836   Vec             coordinates;
1837   PetscScalar    *coords;
1838   PetscReal      *coordsIn;
1839   PetscInt        numCells, numEdges, numVerts, firstVertex, v, firstEdge, coordSize, d, c, e;
1840   PetscMPIInt     rank;
1841   PetscErrorCode  ierr;
1842 
1843   PetscFunctionBegin;
1844   PetscValidLogicalCollectiveBool(dm, simplex, 3);
1845   ierr = DMSetDimension(dm, dim);CHKERRQ(ierr);
1846   ierr = DMSetCoordinateDim(dm, dim+1);CHKERRQ(ierr);
1847   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRMPI(ierr);
1848   switch (dim) {
1849   case 2:
1850     if (simplex) {
1851       const PetscReal radius    = PetscSqrtReal(1 + PETSC_PHI*PETSC_PHI)/(1.0 + PETSC_PHI);
1852       const PetscReal edgeLen   = 2.0/(1.0 + PETSC_PHI) * (R/radius);
1853       const PetscInt  degree    = 5;
1854       PetscReal       vertex[3] = {0.0, 1.0/(1.0 + PETSC_PHI), PETSC_PHI/(1.0 + PETSC_PHI)};
1855       PetscInt        s[3]      = {1, 1, 1};
1856       PetscInt        cone[3];
1857       PetscInt       *graph, p, i, j, k;
1858 
1859       vertex[0] *= R/radius; vertex[1] *= R/radius; vertex[2] *= R/radius;
1860       numCells    = rank == 0 ? 20 : 0;
1861       numVerts    = rank == 0 ? 12 : 0;
1862       firstVertex = numCells;
1863       /* Use icosahedron, which for a R-sphere has coordinates which are all cyclic permutations of
1864 
1865            (0, \pm 1/\phi+1, \pm \phi/\phi+1)
1866 
1867          where \phi^2 - \phi - 1 = 0, meaning \phi is the golden ratio \frac{1 + \sqrt{5}}{2}. The edge
1868          length is then given by 2/(1+\phi) = 2 * 0.38197 = 0.76393.
1869       */
1870       /* Construct vertices */
1871       ierr = PetscCalloc1(numVerts * embedDim, &coordsIn);CHKERRQ(ierr);
1872       if (rank == 0) {
1873         for (p = 0, i = 0; p < embedDim; ++p) {
1874           for (s[1] = -1; s[1] < 2; s[1] += 2) {
1875             for (s[2] = -1; s[2] < 2; s[2] += 2) {
1876               for (d = 0; d < embedDim; ++d) coordsIn[i*embedDim+d] = s[(d+p)%embedDim]*vertex[(d+p)%embedDim];
1877               ++i;
1878             }
1879           }
1880         }
1881       }
1882       /* Construct graph */
1883       ierr = PetscCalloc1(numVerts * numVerts, &graph);CHKERRQ(ierr);
1884       for (i = 0; i < numVerts; ++i) {
1885         for (j = 0, k = 0; j < numVerts; ++j) {
1886           if (PetscAbsReal(DiffNormReal(embedDim, &coordsIn[i*embedDim], &coordsIn[j*embedDim]) - edgeLen) < PETSC_SMALL) {graph[i*numVerts+j] = 1; ++k;}
1887         }
1888         PetscCheckFalse(k != degree,PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Invalid icosahedron, vertex %D degree %D != %D", i, k, degree);
1889       }
1890       /* Build Topology */
1891       ierr = DMPlexSetChart(dm, 0, numCells+numVerts);CHKERRQ(ierr);
1892       for (c = 0; c < numCells; c++) {
1893         ierr = DMPlexSetConeSize(dm, c, embedDim);CHKERRQ(ierr);
1894       }
1895       ierr = DMSetUp(dm);CHKERRQ(ierr); /* Allocate space for cones */
1896       /* Cells */
1897       for (i = 0, c = 0; i < numVerts; ++i) {
1898         for (j = 0; j < i; ++j) {
1899           for (k = 0; k < j; ++k) {
1900             if (graph[i*numVerts+j] && graph[j*numVerts+k] && graph[k*numVerts+i]) {
1901               cone[0] = firstVertex+i; cone[1] = firstVertex+j; cone[2] = firstVertex+k;
1902               /* Check orientation */
1903               {
1904                 const PetscInt epsilon[3][3][3] = {{{0, 0, 0}, {0, 0, 1}, {0, -1, 0}}, {{0, 0, -1}, {0, 0, 0}, {1, 0, 0}}, {{0, 1, 0}, {-1, 0, 0}, {0, 0, 0}}};
1905                 PetscReal normal[3];
1906                 PetscInt  e, f;
1907 
1908                 for (d = 0; d < embedDim; ++d) {
1909                   normal[d] = 0.0;
1910                   for (e = 0; e < embedDim; ++e) {
1911                     for (f = 0; f < embedDim; ++f) {
1912                       normal[d] += epsilon[d][e][f]*(coordsIn[j*embedDim+e] - coordsIn[i*embedDim+e])*(coordsIn[k*embedDim+f] - coordsIn[i*embedDim+f]);
1913                     }
1914                   }
1915                 }
1916                 if (DotReal(embedDim, normal, &coordsIn[i*embedDim]) < 0) {PetscInt tmp = cone[1]; cone[1] = cone[2]; cone[2] = tmp;}
1917               }
1918               ierr = DMPlexSetCone(dm, c++, cone);CHKERRQ(ierr);
1919             }
1920           }
1921         }
1922       }
1923       ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
1924       ierr = DMPlexStratify(dm);CHKERRQ(ierr);
1925       ierr = PetscFree(graph);CHKERRQ(ierr);
1926     } else {
1927       /*
1928         12-21--13
1929          |     |
1930         25  4  24
1931          |     |
1932   12-25--9-16--8-24--13
1933    |     |     |     |
1934   23  5 17  0 15  3  22
1935    |     |     |     |
1936   10-20--6-14--7-19--11
1937          |     |
1938         20  1  19
1939          |     |
1940         10-18--11
1941          |     |
1942         23  2  22
1943          |     |
1944         12-21--13
1945        */
1946       PetscInt cone[4], ornt[4];
1947 
1948       numCells    = rank == 0 ?  6 : 0;
1949       numEdges    = rank == 0 ? 12 : 0;
1950       numVerts    = rank == 0 ?  8 : 0;
1951       firstVertex = numCells;
1952       firstEdge   = numCells + numVerts;
1953       /* Build Topology */
1954       ierr = DMPlexSetChart(dm, 0, numCells+numEdges+numVerts);CHKERRQ(ierr);
1955       for (c = 0; c < numCells; c++) {
1956         ierr = DMPlexSetConeSize(dm, c, 4);CHKERRQ(ierr);
1957       }
1958       for (e = firstEdge; e < firstEdge+numEdges; ++e) {
1959         ierr = DMPlexSetConeSize(dm, e, 2);CHKERRQ(ierr);
1960       }
1961       ierr = DMSetUp(dm);CHKERRQ(ierr); /* Allocate space for cones */
1962       if (rank == 0) {
1963         /* Cell 0 */
1964         cone[0] = 14; cone[1] = 15; cone[2] = 16; cone[3] = 17;
1965         ierr = DMPlexSetCone(dm, 0, cone);CHKERRQ(ierr);
1966         ornt[0] = 0; ornt[1] = 0; ornt[2] = 0; ornt[3] = 0;
1967         ierr = DMPlexSetConeOrientation(dm, 0, ornt);CHKERRQ(ierr);
1968         /* Cell 1 */
1969         cone[0] = 18; cone[1] = 19; cone[2] = 14; cone[3] = 20;
1970         ierr = DMPlexSetCone(dm, 1, cone);CHKERRQ(ierr);
1971         ornt[0] = 0; ornt[1] = 0; ornt[2] = -1; ornt[3] = 0;
1972         ierr = DMPlexSetConeOrientation(dm, 1, ornt);CHKERRQ(ierr);
1973         /* Cell 2 */
1974         cone[0] = 21; cone[1] = 22; cone[2] = 18; cone[3] = 23;
1975         ierr = DMPlexSetCone(dm, 2, cone);CHKERRQ(ierr);
1976         ornt[0] = 0; ornt[1] = 0; ornt[2] = -1; ornt[3] = 0;
1977         ierr = DMPlexSetConeOrientation(dm, 2, ornt);CHKERRQ(ierr);
1978         /* Cell 3 */
1979         cone[0] = 19; cone[1] = 22; cone[2] = 24; cone[3] = 15;
1980         ierr = DMPlexSetCone(dm, 3, cone);CHKERRQ(ierr);
1981         ornt[0] = -1; ornt[1] = -1; ornt[2] = 0; ornt[3] = -1;
1982         ierr = DMPlexSetConeOrientation(dm, 3, ornt);CHKERRQ(ierr);
1983         /* Cell 4 */
1984         cone[0] = 16; cone[1] = 24; cone[2] = 21; cone[3] = 25;
1985         ierr = DMPlexSetCone(dm, 4, cone);CHKERRQ(ierr);
1986         ornt[0] = -1; ornt[1] = -1; ornt[2] = -1; ornt[3] = 0;
1987         ierr = DMPlexSetConeOrientation(dm, 4, ornt);CHKERRQ(ierr);
1988         /* Cell 5 */
1989         cone[0] = 20; cone[1] = 17; cone[2] = 25; cone[3] = 23;
1990         ierr = DMPlexSetCone(dm, 5, cone);CHKERRQ(ierr);
1991         ornt[0] = -1; ornt[1] = -1; ornt[2] = -1; ornt[3] = -1;
1992         ierr = DMPlexSetConeOrientation(dm, 5, ornt);CHKERRQ(ierr);
1993         /* Edges */
1994         cone[0] =  6; cone[1] =  7;
1995         ierr = DMPlexSetCone(dm, 14, cone);CHKERRQ(ierr);
1996         cone[0] =  7; cone[1] =  8;
1997         ierr = DMPlexSetCone(dm, 15, cone);CHKERRQ(ierr);
1998         cone[0] =  8; cone[1] =  9;
1999         ierr = DMPlexSetCone(dm, 16, cone);CHKERRQ(ierr);
2000         cone[0] =  9; cone[1] =  6;
2001         ierr = DMPlexSetCone(dm, 17, cone);CHKERRQ(ierr);
2002         cone[0] = 10; cone[1] = 11;
2003         ierr = DMPlexSetCone(dm, 18, cone);CHKERRQ(ierr);
2004         cone[0] = 11; cone[1] =  7;
2005         ierr = DMPlexSetCone(dm, 19, cone);CHKERRQ(ierr);
2006         cone[0] =  6; cone[1] = 10;
2007         ierr = DMPlexSetCone(dm, 20, cone);CHKERRQ(ierr);
2008         cone[0] = 12; cone[1] = 13;
2009         ierr = DMPlexSetCone(dm, 21, cone);CHKERRQ(ierr);
2010         cone[0] = 13; cone[1] = 11;
2011         ierr = DMPlexSetCone(dm, 22, cone);CHKERRQ(ierr);
2012         cone[0] = 10; cone[1] = 12;
2013         ierr = DMPlexSetCone(dm, 23, cone);CHKERRQ(ierr);
2014         cone[0] = 13; cone[1] =  8;
2015         ierr = DMPlexSetCone(dm, 24, cone);CHKERRQ(ierr);
2016         cone[0] = 12; cone[1] =  9;
2017         ierr = DMPlexSetCone(dm, 25, cone);CHKERRQ(ierr);
2018       }
2019       ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
2020       ierr = DMPlexStratify(dm);CHKERRQ(ierr);
2021       /* Build coordinates */
2022       ierr = PetscCalloc1(numVerts * embedDim, &coordsIn);CHKERRQ(ierr);
2023       if (rank == 0) {
2024         coordsIn[0*embedDim+0] = -R; coordsIn[0*embedDim+1] =  R; coordsIn[0*embedDim+2] = -R;
2025         coordsIn[1*embedDim+0] =  R; coordsIn[1*embedDim+1] =  R; coordsIn[1*embedDim+2] = -R;
2026         coordsIn[2*embedDim+0] =  R; coordsIn[2*embedDim+1] = -R; coordsIn[2*embedDim+2] = -R;
2027         coordsIn[3*embedDim+0] = -R; coordsIn[3*embedDim+1] = -R; coordsIn[3*embedDim+2] = -R;
2028         coordsIn[4*embedDim+0] = -R; coordsIn[4*embedDim+1] =  R; coordsIn[4*embedDim+2] =  R;
2029         coordsIn[5*embedDim+0] =  R; coordsIn[5*embedDim+1] =  R; coordsIn[5*embedDim+2] =  R;
2030         coordsIn[6*embedDim+0] = -R; coordsIn[6*embedDim+1] = -R; coordsIn[6*embedDim+2] =  R;
2031         coordsIn[7*embedDim+0] =  R; coordsIn[7*embedDim+1] = -R; coordsIn[7*embedDim+2] =  R;
2032       }
2033     }
2034     break;
2035   case 3:
2036     if (simplex) {
2037       const PetscReal edgeLen         = 1.0/PETSC_PHI;
2038       PetscReal       vertexA[4]      = {0.5, 0.5, 0.5, 0.5};
2039       PetscReal       vertexB[4]      = {1.0, 0.0, 0.0, 0.0};
2040       PetscReal       vertexC[4]      = {0.5, 0.5*PETSC_PHI, 0.5/PETSC_PHI, 0.0};
2041       const PetscInt  degree          = 12;
2042       PetscInt        s[4]            = {1, 1, 1};
2043       PetscInt        evenPerm[12][4] = {{0, 1, 2, 3}, {0, 2, 3, 1}, {0, 3, 1, 2}, {1, 0, 3, 2}, {1, 2, 0, 3}, {1, 3, 2, 0},
2044                                          {2, 0, 1, 3}, {2, 1, 3, 0}, {2, 3, 0, 1}, {3, 0, 2, 1}, {3, 1, 0, 2}, {3, 2, 1, 0}};
2045       PetscInt        cone[4];
2046       PetscInt       *graph, p, i, j, k, l;
2047 
2048       vertexA[0] *= R; vertexA[1] *= R; vertexA[2] *= R; vertexA[3] *= R;
2049       vertexB[0] *= R; vertexB[1] *= R; vertexB[2] *= R; vertexB[3] *= R;
2050       vertexC[0] *= R; vertexC[1] *= R; vertexC[2] *= R; vertexC[3] *= R;
2051       numCells    = rank == 0 ? 600 : 0;
2052       numVerts    = rank == 0 ? 120 : 0;
2053       firstVertex = numCells;
2054       /* Use the 600-cell, which for a unit sphere has coordinates which are
2055 
2056            1/2 (\pm 1, \pm 1,    \pm 1, \pm 1)                          16
2057                (\pm 1,    0,       0,      0)  all cyclic permutations   8
2058            1/2 (\pm 1, \pm phi, \pm 1/phi, 0)  all even permutations    96
2059 
2060          where \phi^2 - \phi - 1 = 0, meaning \phi is the golden ratio \frac{1 + \sqrt{5}}{2}. The edge
2061          length is then given by 1/\phi = 0.61803.
2062 
2063          http://buzzard.pugetsound.edu/sage-practice/ch03s03.html
2064          http://mathworld.wolfram.com/600-Cell.html
2065       */
2066       /* Construct vertices */
2067       ierr = PetscCalloc1(numVerts * embedDim, &coordsIn);CHKERRQ(ierr);
2068       i    = 0;
2069       if (rank == 0) {
2070         for (s[0] = -1; s[0] < 2; s[0] += 2) {
2071           for (s[1] = -1; s[1] < 2; s[1] += 2) {
2072             for (s[2] = -1; s[2] < 2; s[2] += 2) {
2073               for (s[3] = -1; s[3] < 2; s[3] += 2) {
2074                 for (d = 0; d < embedDim; ++d) coordsIn[i*embedDim+d] = s[d]*vertexA[d];
2075                 ++i;
2076               }
2077             }
2078           }
2079         }
2080         for (p = 0; p < embedDim; ++p) {
2081           s[1] = s[2] = s[3] = 1;
2082           for (s[0] = -1; s[0] < 2; s[0] += 2) {
2083             for (d = 0; d < embedDim; ++d) coordsIn[i*embedDim+d] = s[(d+p)%embedDim]*vertexB[(d+p)%embedDim];
2084             ++i;
2085           }
2086         }
2087         for (p = 0; p < 12; ++p) {
2088           s[3] = 1;
2089           for (s[0] = -1; s[0] < 2; s[0] += 2) {
2090             for (s[1] = -1; s[1] < 2; s[1] += 2) {
2091               for (s[2] = -1; s[2] < 2; s[2] += 2) {
2092                 for (d = 0; d < embedDim; ++d) coordsIn[i*embedDim+d] = s[evenPerm[p][d]]*vertexC[evenPerm[p][d]];
2093                 ++i;
2094               }
2095             }
2096           }
2097         }
2098       }
2099       PetscCheckFalse(i != numVerts,PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Invalid 600-cell, vertices %D != %D", i, numVerts);
2100       /* Construct graph */
2101       ierr = PetscCalloc1(numVerts * numVerts, &graph);CHKERRQ(ierr);
2102       for (i = 0; i < numVerts; ++i) {
2103         for (j = 0, k = 0; j < numVerts; ++j) {
2104           if (PetscAbsReal(DiffNormReal(embedDim, &coordsIn[i*embedDim], &coordsIn[j*embedDim]) - edgeLen) < PETSC_SMALL) {graph[i*numVerts+j] = 1; ++k;}
2105         }
2106         PetscCheckFalse(k != degree,PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Invalid 600-cell, vertex %D degree %D != %D", i, k, degree);
2107       }
2108       /* Build Topology */
2109       ierr = DMPlexSetChart(dm, 0, numCells+numVerts);CHKERRQ(ierr);
2110       for (c = 0; c < numCells; c++) {
2111         ierr = DMPlexSetConeSize(dm, c, embedDim);CHKERRQ(ierr);
2112       }
2113       ierr = DMSetUp(dm);CHKERRQ(ierr); /* Allocate space for cones */
2114       /* Cells */
2115       if (rank == 0) {
2116         for (i = 0, c = 0; i < numVerts; ++i) {
2117           for (j = 0; j < i; ++j) {
2118             for (k = 0; k < j; ++k) {
2119               for (l = 0; l < k; ++l) {
2120                 if (graph[i*numVerts+j] && graph[j*numVerts+k] && graph[k*numVerts+i] &&
2121                     graph[l*numVerts+i] && graph[l*numVerts+j] && graph[l*numVerts+k]) {
2122                   cone[0] = firstVertex+i; cone[1] = firstVertex+j; cone[2] = firstVertex+k; cone[3] = firstVertex+l;
2123                   /* Check orientation: https://ef.gy/linear-algebra:normal-vectors-in-higher-dimensional-spaces */
2124                   {
2125                     const PetscInt epsilon[4][4][4][4] = {{{{0,  0,  0,  0}, { 0, 0,  0,  0}, { 0,  0, 0,  0}, { 0,  0,  0, 0}},
2126                                                            {{0,  0,  0,  0}, { 0, 0,  0,  0}, { 0,  0, 0,  1}, { 0,  0, -1, 0}},
2127                                                            {{0,  0,  0,  0}, { 0, 0,  0, -1}, { 0,  0, 0,  0}, { 0,  1,  0, 0}},
2128                                                            {{0,  0,  0,  0}, { 0, 0,  1,  0}, { 0, -1, 0,  0}, { 0,  0,  0, 0}}},
2129 
2130                                                           {{{0,  0,  0,  0}, { 0, 0,  0,  0}, { 0,  0, 0, -1}, { 0,  0,  1, 0}},
2131                                                            {{0,  0,  0,  0}, { 0, 0,  0,  0}, { 0,  0, 0,  0}, { 0,  0,  0, 0}},
2132                                                            {{0,  0,  0,  1}, { 0, 0,  0,  0}, { 0,  0, 0,  0}, {-1,  0,  0, 0}},
2133                                                            {{0,  0, -1,  0}, { 0, 0,  0,  0}, { 1,  0, 0,  0}, { 0,  0,  0, 0}}},
2134 
2135                                                           {{{0,  0,  0,  0}, { 0, 0,  0,  1}, { 0,  0, 0,  0}, { 0, -1,  0, 0}},
2136                                                            {{0,  0,  0, -1}, { 0, 0,  0,  0}, { 0,  0, 0,  0}, { 1,  0,  0, 0}},
2137                                                            {{0,  0,  0,  0}, { 0, 0,  0,  0}, { 0,  0, 0,  0}, { 0,  0,  0, 0}},
2138                                                            {{0,  1,  0,  0}, {-1, 0,  0,  0}, { 0,  0, 0,  0}, { 0,  0,  0, 0}}},
2139 
2140                                                           {{{0,  0,  0,  0}, { 0, 0, -1,  0}, { 0,  1, 0,  0}, { 0,  0,  0, 0}},
2141                                                            {{0,  0,  1,  0}, { 0, 0,  0,  0}, {-1,  0, 0,  0}, { 0,  0,  0, 0}},
2142                                                            {{0, -1,  0,  0}, { 1, 0,  0,  0}, { 0,  0, 0,  0}, { 0,  0,  0, 0}},
2143                                                            {{0,  0,  0,  0}, { 0, 0,  0,  0}, { 0,  0, 0,  0}, { 0,  0,  0, 0}}}};
2144                     PetscReal normal[4];
2145                     PetscInt  e, f, g;
2146 
2147                     for (d = 0; d < embedDim; ++d) {
2148                       normal[d] = 0.0;
2149                       for (e = 0; e < embedDim; ++e) {
2150                         for (f = 0; f < embedDim; ++f) {
2151                           for (g = 0; g < embedDim; ++g) {
2152                             normal[d] += epsilon[d][e][f][g]*(coordsIn[j*embedDim+e] - coordsIn[i*embedDim+e])*(coordsIn[k*embedDim+f] - coordsIn[i*embedDim+f])*(coordsIn[l*embedDim+f] - coordsIn[i*embedDim+f]);
2153                           }
2154                         }
2155                       }
2156                     }
2157                     if (DotReal(embedDim, normal, &coordsIn[i*embedDim]) < 0) {PetscInt tmp = cone[1]; cone[1] = cone[2]; cone[2] = tmp;}
2158                   }
2159                   ierr = DMPlexSetCone(dm, c++, cone);CHKERRQ(ierr);
2160                 }
2161               }
2162             }
2163           }
2164         }
2165       }
2166       ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
2167       ierr = DMPlexStratify(dm);CHKERRQ(ierr);
2168       ierr = PetscFree(graph);CHKERRQ(ierr);
2169       break;
2170     }
2171   default: SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Unsupported dimension for sphere: %D", dim);
2172   }
2173   /* Create coordinates */
2174   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
2175   ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
2176   ierr = PetscSectionSetFieldComponents(coordSection, 0, embedDim);CHKERRQ(ierr);
2177   ierr = PetscSectionSetChart(coordSection, firstVertex, firstVertex+numVerts);CHKERRQ(ierr);
2178   for (v = firstVertex; v < firstVertex+numVerts; ++v) {
2179     ierr = PetscSectionSetDof(coordSection, v, embedDim);CHKERRQ(ierr);
2180     ierr = PetscSectionSetFieldDof(coordSection, v, 0, embedDim);CHKERRQ(ierr);
2181   }
2182   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
2183   ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
2184   ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
2185   ierr = VecSetBlockSize(coordinates, embedDim);CHKERRQ(ierr);
2186   ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
2187   ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
2188   ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
2189   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
2190   for (v = 0; v < numVerts; ++v) for (d = 0; d < embedDim; ++d) {coords[v*embedDim+d] = coordsIn[v*embedDim+d];}
2191   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
2192   ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
2193   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
2194   ierr = PetscFree(coordsIn);CHKERRQ(ierr);
2195   {
2196     DM          cdm;
2197     PetscDS     cds;
2198     PetscScalar c = R;
2199 
2200     ierr = DMPlexCreateCoordinateSpace(dm, 1, snapToSphere);CHKERRQ(ierr);
2201     ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
2202     ierr = DMGetDS(cdm, &cds);CHKERRQ(ierr);
2203     ierr = PetscDSSetConstants(cds, 1, &c);CHKERRQ(ierr);
2204   }
2205   /* Wait for coordinate creation before doing in-place modification */
2206   if (simplex) {ierr = DMPlexInterpolateInPlace_Internal(dm);CHKERRQ(ierr);}
2207   PetscFunctionReturn(0);
2208 }
2209 
2210 typedef void (*TPSEvaluateFunc)(const PetscReal[], PetscReal*, PetscReal[], PetscReal(*)[3]);
2211 
2212 /*
2213  The Schwarz P implicit surface is
2214 
2215      f(x) = cos(x0) + cos(x1) + cos(x2) = 0
2216 */
2217 static void TPSEvaluate_SchwarzP(const PetscReal y[3], PetscReal *f, PetscReal grad[], PetscReal (*hess)[3])
2218 {
2219   PetscReal c[3] = {PetscCosReal(y[0] * PETSC_PI), PetscCosReal(y[1] * PETSC_PI), PetscCosReal(y[2] * PETSC_PI)};
2220   PetscReal g[3] = {-PetscSinReal(y[0] * PETSC_PI), -PetscSinReal(y[1] * PETSC_PI), -PetscSinReal(y[2] * PETSC_PI)};
2221   f[0] = c[0] + c[1] + c[2];
2222   for (PetscInt i=0; i<3; i++) {
2223     grad[i] = PETSC_PI * g[i];
2224     for (PetscInt j=0; j<3; j++) {
2225       hess[i][j] = (i == j) ? -PetscSqr(PETSC_PI) * c[i] : 0.;
2226     }
2227   }
2228 }
2229 
2230 /*
2231  The Gyroid implicit surface is
2232 
2233  f(x,y,z) = sin(pi * x) * cos (pi * (y + 1/2))  + sin(pi * (y + 1/2)) * cos(pi * (z + 1/4)) + sin(pi * (z + 1/4)) * cos(pi * x)
2234 
2235 */
2236 static void TPSEvaluate_Gyroid(const PetscReal y[3], PetscReal *f, PetscReal grad[], PetscReal (*hess)[3])
2237 {
2238   PetscReal s[3] = {PetscSinReal(PETSC_PI * y[0]), PetscSinReal(PETSC_PI * (y[1] + .5)), PetscSinReal(PETSC_PI * (y[2] + .25))};
2239   PetscReal c[3] = {PetscCosReal(PETSC_PI * y[0]), PetscCosReal(PETSC_PI * (y[1] + .5)), PetscCosReal(PETSC_PI * (y[2] + .25))};
2240   f[0] = s[0] * c[1] + s[1] * c[2] + s[2] * c[0];
2241   grad[0] = PETSC_PI * (c[0] * c[1] - s[2] * s[0]);
2242   grad[1] = PETSC_PI * (c[1] * c[2] - s[0] * s[1]);
2243   grad[2] = PETSC_PI * (c[2] * c[0] - s[1] * s[2]);
2244   hess[0][0] = -PetscSqr(PETSC_PI) * (s[0] * c[1] + s[2] * c[0]);
2245   hess[0][1] = -PetscSqr(PETSC_PI) * (c[0] * s[1]);
2246   hess[0][2] = -PetscSqr(PETSC_PI) * (c[2] * s[0]);
2247   hess[1][0] = -PetscSqr(PETSC_PI) * (s[1] * c[2] + s[0] * c[1]);
2248   hess[1][1] = -PetscSqr(PETSC_PI) * (c[1] * s[2]);
2249   hess[2][2] = -PetscSqr(PETSC_PI) * (c[0] * s[1]);
2250   hess[2][0] = -PetscSqr(PETSC_PI) * (s[2] * c[0] + s[1] * c[2]);
2251   hess[2][1] = -PetscSqr(PETSC_PI) * (c[2] * s[0]);
2252   hess[2][2] = -PetscSqr(PETSC_PI) * (c[1] * s[2]);
2253 }
2254 
2255 /*
2256    We wish to solve
2257 
2258          min_y || y - x ||^2  subject to f(y) = 0
2259 
2260    Let g(y) = grad(f).  The minimization problem is equivalent to asking to satisfy
2261    f(y) = 0 and (y-x) is parallel to g(y).  We do this by using Householder QR to obtain a basis for the
2262    tangent space and ask for both components in the tangent space to be zero.
2263 
2264    Take g to be a column vector and compute the "full QR" factorization Q R = g,
2265    where Q = I - 2 n n^T is a symmetric orthogonal matrix.
2266    The first column of Q is parallel to g so the remaining two columns span the null space.
2267    Let Qn = Q[:,1:] be those remaining columns.  Then Qn Qn^T is an orthogonal projector into the tangent space.
2268    Since Q is symmetric, this is equivalent to multipyling by Q and taking the last two entries.
2269    In total, we have a system of 3 equations in 3 unknowns:
2270 
2271      f(y) = 0                       1 equation
2272      Qn^T (y - x) = 0               2 equations
2273 
2274    Here, we compute the residual and Jacobian of this system.
2275 */
2276 static void TPSNearestPointResJac(TPSEvaluateFunc feval, const PetscScalar x[], const PetscScalar y[], PetscScalar res[], PetscScalar J[])
2277 {
2278   PetscReal yreal[3] = {PetscRealPart(y[0]), PetscRealPart(y[1]), PetscRealPart(y[2])};
2279   PetscReal d[3] = {PetscRealPart(y[0] - x[0]), PetscRealPart(y[1] - x[1]), PetscRealPart(y[2] - x[2])};
2280   PetscReal f, grad[3], n[3], n_y[3][3], norm, norm_y[3], nd, nd_y[3], sign;
2281 
2282   feval(yreal, &f, grad, n_y);
2283 
2284   for (PetscInt i=0; i<3; i++) n[i] = grad[i];
2285   norm = PetscSqrtReal(PetscSqr(n[0]) + PetscSqr(n[1]) + PetscSqr(n[2]));
2286   for (PetscInt i=0; i<3; i++) {
2287     norm_y[i] = 1. / norm * n[i] * n_y[i][i];
2288   }
2289 
2290   // Define the Householder reflector
2291   sign = n[0] >= 0 ? 1. : -1.;
2292   n[0] += norm * sign;
2293   for (PetscInt i=0; i<3; i++) n_y[0][i] += norm_y[i] * sign;
2294 
2295   norm = PetscSqrtReal(PetscSqr(n[0]) + PetscSqr(n[1]) + PetscSqr(n[2]));
2296   norm_y[0] = 1. / norm * (n[0] * n_y[0][0]);
2297   norm_y[1] = 1. / norm * (n[0] * n_y[0][1] + n[1] * n_y[1][1]);
2298   norm_y[2] = 1. / norm * (n[0] * n_y[0][2] + n[2] * n_y[2][2]);
2299 
2300   for (PetscInt i=0; i<3; i++) {
2301     n[i] /= norm;
2302     for (PetscInt j=0; j<3; j++) {
2303       // note that n[i] is n_old[i]/norm when executing the code below
2304       n_y[i][j] = n_y[i][j] / norm - n[i] / norm * norm_y[j];
2305     }
2306   }
2307 
2308   nd = n[0] * d[0] + n[1] * d[1] + n[2] * d[2];
2309   for (PetscInt i=0; i<3; i++) nd_y[i] = n[i] + n_y[0][i] * d[0] + n_y[1][i] * d[1] + n_y[2][i] * d[2];
2310 
2311   res[0] = f;
2312   res[1] = d[1] - 2 * n[1] * nd;
2313   res[2] = d[2] - 2 * n[2] * nd;
2314   // J[j][i] is J_{ij} (column major)
2315   for (PetscInt j=0; j<3; j++) {
2316     J[0 + j*3] = grad[j];
2317     J[1 + j*3] = (j == 1)*1. - 2 * (n_y[1][j] * nd + n[1] * nd_y[j]);
2318     J[2 + j*3] = (j == 2)*1. - 2 * (n_y[2][j] * nd + n[2] * nd_y[j]);
2319   }
2320 }
2321 
2322 /*
2323    Project x to the nearest point on the implicit surface using Newton's method.
2324 */
2325 static PetscErrorCode TPSNearestPoint(TPSEvaluateFunc feval, PetscScalar x[])
2326 {
2327   PetscScalar y[3] = {x[0], x[1], x[2]}; // Initial guess
2328   PetscErrorCode ierr;
2329 
2330   PetscFunctionBegin;
2331   for (PetscInt iter=0; iter<10; iter++) {
2332     PetscScalar res[3], J[9];
2333     PetscReal resnorm;
2334     TPSNearestPointResJac(feval, x, y, res, J);
2335     resnorm = PetscSqrtReal(PetscSqr(PetscRealPart(res[0])) + PetscSqr(PetscRealPart(res[1])) + PetscSqr(PetscRealPart(res[2])));
2336     if (0) { // Turn on this monitor if you need to confirm quadratic convergence
2337       ierr = PetscPrintf(PETSC_COMM_SELF, "[%D] res [%g %g %g]\n", iter, PetscRealPart(res[0]), PetscRealPart(res[1]), PetscRealPart(res[2]));CHKERRQ(ierr);
2338     }
2339     if (resnorm < PETSC_SMALL) break;
2340 
2341     // Take the Newton step
2342     ierr = PetscKernel_A_gets_inverse_A_3(J, 0., PETSC_FALSE, NULL);CHKERRQ(ierr);
2343     PetscKernel_v_gets_v_minus_A_times_w_3(y, J, res);
2344   }
2345   for (PetscInt i=0; i<3; i++) x[i] = y[i];
2346   PetscFunctionReturn(0);
2347 }
2348 
2349 const char *const DMPlexTPSTypes[] = {"SCHWARZ_P", "GYROID", "DMPlexTPSType", "DMPLEX_TPS_", NULL};
2350 
2351 static PetscErrorCode DMPlexCreateTPSMesh_Internal(DM dm, DMPlexTPSType tpstype, const PetscInt extent[], const DMBoundaryType periodic[], PetscBool tps_distribute, PetscInt refinements, PetscInt layers, PetscReal thickness)
2352 {
2353   PetscErrorCode ierr;
2354   PetscMPIInt rank;
2355   PetscInt topoDim = 2, spaceDim = 3, numFaces = 0, numVertices = 0, numEdges = 0;
2356   PetscInt (*edges)[2] = NULL, *edgeSets = NULL;
2357   PetscInt *cells_flat = NULL;
2358   PetscReal *vtxCoords = NULL;
2359   TPSEvaluateFunc evalFunc = NULL;
2360   DMLabel label;
2361 
2362   PetscFunctionBegin;
2363   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRMPI(ierr);
2364   PetscCheck((layers != 0) ^ (thickness == 0.), PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_INCOMP, "Layers %D must be nonzero iff thickness %g is nonzero", layers, (double)thickness);
2365   switch (tpstype) {
2366   case DMPLEX_TPS_SCHWARZ_P:
2367     PetscCheck(!periodic || (periodic[0] == DM_BOUNDARY_NONE && periodic[1] == DM_BOUNDARY_NONE && periodic[2] == DM_BOUNDARY_NONE), PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Schwarz P does not support periodic meshes");
2368     if (!rank) {
2369       PetscInt (*cells)[6][4][4] = NULL; // [junction, junction-face, cell, conn]
2370       PetscInt Njunctions = 0, Ncuts = 0, Npipes[3], vcount;
2371       PetscReal L = 1;
2372 
2373       Npipes[0] = (extent[0] + 1) * extent[1] * extent[2];
2374       Npipes[1] = extent[0] * (extent[1] + 1) * extent[2];
2375       Npipes[2] = extent[0] * extent[1] * (extent[2] + 1);
2376       Njunctions = extent[0] * extent[1] * extent[2];
2377       Ncuts = 2 * (extent[0] * extent[1] + extent[1] * extent[2] + extent[2] * extent[0]);
2378       numVertices = 4 * (Npipes[0] + Npipes[1] + Npipes[2]) + 8 * Njunctions;
2379       ierr = PetscMalloc1(3*numVertices, &vtxCoords);CHKERRQ(ierr);
2380       ierr = PetscMalloc1(Njunctions, &cells);CHKERRQ(ierr);
2381       ierr = PetscMalloc1(Ncuts*4, &edges);CHKERRQ(ierr);
2382       ierr = PetscMalloc1(Ncuts*4, &edgeSets);CHKERRQ(ierr);
2383       // x-normal pipes
2384       vcount = 0;
2385       for (PetscInt i=0; i<extent[0]+1; i++) {
2386         for (PetscInt j=0; j<extent[1]; j++) {
2387           for (PetscInt k=0; k<extent[2]; k++) {
2388             for (PetscInt l=0; l<4; l++) {
2389               vtxCoords[vcount++] = (2*i - 1) * L;
2390               vtxCoords[vcount++] = 2 * j * L + PetscCosReal((2*l + 1) * PETSC_PI / 4) * L / 2;
2391               vtxCoords[vcount++] = 2 * k * L + PetscSinReal((2*l + 1) * PETSC_PI / 4) * L / 2;
2392             }
2393           }
2394         }
2395       }
2396       // y-normal pipes
2397       for (PetscInt i=0; i<extent[0]; i++) {
2398         for (PetscInt j=0; j<extent[1]+1; j++) {
2399           for (PetscInt k=0; k<extent[2]; k++) {
2400             for (PetscInt l=0; l<4; l++) {
2401               vtxCoords[vcount++] = 2 * i * L + PetscSinReal((2*l + 1) * PETSC_PI / 4) * L / 2;
2402               vtxCoords[vcount++] = (2*j - 1) * L;
2403               vtxCoords[vcount++] = 2 * k * L + PetscCosReal((2*l + 1) * PETSC_PI / 4) * L / 2;
2404             }
2405           }
2406         }
2407       }
2408       // z-normal pipes
2409       for (PetscInt i=0; i<extent[0]; i++) {
2410         for (PetscInt j=0; j<extent[1]; j++) {
2411           for (PetscInt k=0; k<extent[2]+1; k++) {
2412             for (PetscInt l=0; l<4; l++) {
2413               vtxCoords[vcount++] = 2 * i * L + PetscCosReal((2*l + 1) * PETSC_PI / 4) * L / 2;
2414               vtxCoords[vcount++] = 2 * j * L + PetscSinReal((2*l + 1) * PETSC_PI / 4) * L / 2;
2415               vtxCoords[vcount++] = (2*k - 1) * L;
2416             }
2417           }
2418         }
2419       }
2420       // junctions
2421       for (PetscInt i=0; i<extent[0]; i++) {
2422         for (PetscInt j=0; j<extent[1]; j++) {
2423           for (PetscInt k=0; k<extent[2]; k++) {
2424             const PetscInt J = (i*extent[1] + j)*extent[2] + k, Jvoff = (Npipes[0] + Npipes[1] + Npipes[2])*4 + J*8;
2425             PetscCheck(vcount / 3 == Jvoff, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Unexpected vertex count");
2426             for (PetscInt ii=0; ii<2; ii++) {
2427               for (PetscInt jj=0; jj<2; jj++) {
2428                 for (PetscInt kk=0; kk<2; kk++) {
2429                   double Ls = (1 - sqrt(2) / 4) * L;
2430                   vtxCoords[vcount++] = 2*i*L + (2*ii-1) * Ls;
2431                   vtxCoords[vcount++] = 2*j*L + (2*jj-1) * Ls;
2432                   vtxCoords[vcount++] = 2*k*L + (2*kk-1) * Ls;
2433                 }
2434               }
2435             }
2436             const PetscInt jfaces[3][2][4] = {
2437               {{3,1,0,2}, {7,5,4,6}}, // x-aligned
2438               {{5,4,0,1}, {7,6,2,3}}, // y-aligned
2439               {{6,2,0,4}, {7,3,1,5}}  // z-aligned
2440             };
2441             const PetscInt pipe_lo[3] = { // vertex numbers of pipes
2442               ((i * extent[1] + j) * extent[2] + k)*4,
2443               ((i * (extent[1] + 1) + j) * extent[2] + k + Npipes[0])*4,
2444               ((i * extent[1] + j) * (extent[2]+1) + k + Npipes[0] + Npipes[1])*4
2445             };
2446             const PetscInt pipe_hi[3] = { // vertex numbers of pipes
2447               (((i + 1) * extent[1] + j) * extent[2] + k)*4,
2448               ((i * (extent[1] + 1) + j + 1) * extent[2] + k + Npipes[0])*4,
2449               ((i * extent[1] + j) * (extent[2]+1) + k + 1 + Npipes[0] + Npipes[1])*4
2450             };
2451             for (PetscInt dir=0; dir<3; dir++) { // x,y,z
2452               const PetscInt ijk[3] = {i, j, k};
2453               for (PetscInt l=0; l<4; l++) { // rotations
2454                 cells[J][dir*2+0][l][0] = pipe_lo[dir] + l;
2455                 cells[J][dir*2+0][l][1] = Jvoff + jfaces[dir][0][l];
2456                 cells[J][dir*2+0][l][2] = Jvoff + jfaces[dir][0][(l-1+4)%4];
2457                 cells[J][dir*2+0][l][3] = pipe_lo[dir] + (l-1+4)%4;
2458                 cells[J][dir*2+1][l][0] = Jvoff + jfaces[dir][1][l];
2459                 cells[J][dir*2+1][l][1] = pipe_hi[dir] + l;
2460                 cells[J][dir*2+1][l][2] = pipe_hi[dir] + (l-1+4)%4;
2461                 cells[J][dir*2+1][l][3] = Jvoff + jfaces[dir][1][(l-1+4)%4];
2462                 if (ijk[dir] == 0) {
2463                   edges[numEdges][0] = pipe_lo[dir] + l;
2464                   edges[numEdges][1] = pipe_lo[dir] + (l+1) % 4;
2465                   edgeSets[numEdges] = dir*2 + 1;
2466                   numEdges++;
2467                 }
2468                 if (ijk[dir] + 1 == extent[dir]) {
2469                   edges[numEdges][0] = pipe_hi[dir] + l;
2470                   edges[numEdges][1] = pipe_hi[dir] + (l+1) % 4;
2471                   edgeSets[numEdges] = dir*2 + 2;
2472                   numEdges++;
2473                 }
2474               }
2475             }
2476           }
2477         }
2478       }
2479       PetscCheck(numEdges == Ncuts * 4, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Edge count %D incompatible with number of cuts %D", numEdges, Ncuts);
2480       numFaces = 24 * Njunctions;
2481       cells_flat = cells[0][0][0];
2482     }
2483     evalFunc = TPSEvaluate_SchwarzP;
2484     break;
2485   case DMPLEX_TPS_GYROID:
2486     if (!rank) {
2487       // This is a coarse mesh approximation of the gyroid shifted to being the zero of the level set
2488       //
2489       //     sin(pi*x)*cos(pi*(y+1/2)) + sin(pi*(y+1/2))*cos(pi*(z+1/4)) + sin(pi*(z+1/4))*cos(x)
2490       //
2491       // on the cell [0,2]^3.
2492       //
2493       // Think about dividing that cell into four columns, and focus on the column [0,1]x[0,1]x[0,2].
2494       // If you looked at the gyroid in that column at different slices of z you would see that it kind of spins
2495       // like a boomerang:
2496       //
2497       //     z = 0          z = 1/4        z = 1/2        z = 3/4     //
2498       //     -----          -------        -------        -------     //
2499       //                                                              //
2500       //     +       +      +       +      +       +      +   \   +   //
2501       //      \                                   /            \      //
2502       //       \            `-_   _-'            /              }     //
2503       //        *-_            `-'            _-'              /      //
2504       //     +     `-+      +       +      +-'     +      +   /   +   //
2505       //                                                              //
2506       //                                                              //
2507       //     z = 1          z = 5/4        z = 3/2        z = 7/4     //
2508       //     -----          -------        -------        -------     //
2509       //                                                              //
2510       //     +-_     +      +       +      +     _-+      +   /   +   //
2511       //        `-_            _-_            _-`            /        //
2512       //           \        _-'   `-_        /              {         //
2513       //            \                       /                \        //
2514       //     +       +      +       +      +       +      +   \   +   //
2515       //
2516       //
2517       // This course mesh approximates each of these slices by two line segments,
2518       // and then connects the segments in consecutive layers with quadrilateral faces.
2519       // All of the end points of the segments are multiples of 1/4 except for the
2520       // point * in the picture for z = 0 above and the similar points in other layers.
2521       // That point is at (gamma, gamma, 0), where gamma is calculated below.
2522       //
2523       // The column  [1,2]x[1,2]x[0,2] looks the same as this column;
2524       // The columns [1,2]x[0,1]x[0,2] and [0,1]x[1,2]x[0,2] are mirror images.
2525       //
2526       // As for how this method turned into the names given to the vertices:
2527       // that was not systematic, it was just the way it worked out in my handwritten notes.
2528 
2529       PetscInt facesPerBlock = 64;
2530       PetscInt vertsPerBlock = 56;
2531       PetscInt extentPlus[3];
2532       PetscInt numBlocks, numBlocksPlus;
2533       const PetscInt A =  0,   B =  1,   C =  2,   D =  3,   E =  4,   F =  5,   G =  6,   H =  7,
2534         II =  8,   J =  9,   K = 10,   L = 11,   M = 12,   N = 13,   O = 14,   P = 15,
2535         Q = 16,   R = 17,   S = 18,   T = 19,   U = 20,   V = 21,   W = 22,   X = 23,
2536         Y = 24,   Z = 25,  Ap = 26,  Bp = 27,  Cp = 28,  Dp = 29,  Ep = 30,  Fp = 31,
2537         Gp = 32,  Hp = 33,  Ip = 34,  Jp = 35,  Kp = 36,  Lp = 37,  Mp = 38,  Np = 39,
2538         Op = 40,  Pp = 41,  Qp = 42,  Rp = 43,  Sp = 44,  Tp = 45,  Up = 46,  Vp = 47,
2539         Wp = 48,  Xp = 49,  Yp = 50,  Zp = 51,  Aq = 52,  Bq = 53,  Cq = 54,  Dq = 55;
2540       const PetscInt pattern[64][4] =
2541         { /* face to vertex within the coarse discretization of a single gyroid block */
2542           /* layer 0 */
2543           {A,C,K,G},{C,B,II,K},{D,A,H,L},{B+56*1,D,L,J},{E,B+56*1,J,N},{A+56*2,E,N,H+56*2},{F,A+56*2,G+56*2,M},{B,F,M,II},
2544           /* layer 1 */
2545           {G,K,Q,O},{K,II,P,Q},{L,H,O+56*1,R},{J,L,R,P},{N,J,P,S},{H+56*2,N,S,O+56*3},{M,G+56*2,O+56*2,T},{II,M,T,P},
2546           /* layer 2 */
2547           {O,Q,Y,U},{Q,P,W,Y},{R,O+56*1,U+56*1,Ap},{P,R,Ap,W},{S,P,X,Bp},{O+56*3,S,Bp,V+56*1},{T,O+56*2,V,Z},{P,T,Z,X},
2548           /* layer 3 */
2549           {U,Y,Ep,Dp},{Y,W,Cp,Ep},{Ap,U+56*1,Dp+56*1,Gp},{W,Ap,Gp,Cp},{Bp,X,Cp+56*2,Fp},{V+56*1,Bp,Fp,Dp+56*1},{Z,V,Dp,Hp},{X,Z,Hp,Cp+56*2},
2550           /* layer 4 */
2551           {Dp,Ep,Mp,Kp},{Ep,Cp,Ip,Mp},{Gp,Dp+56*1,Lp,Np},{Cp,Gp,Np,Jp},{Fp,Cp+56*2,Jp+56*2,Pp},{Dp+56*1,Fp,Pp,Lp},{Hp,Dp,Kp,Op},{Cp+56*2,Hp,Op,Ip+56*2},
2552           /* layer 5 */
2553           {Kp,Mp,Sp,Rp},{Mp,Ip,Qp,Sp},{Np,Lp,Rp,Tp},{Jp,Np,Tp,Qp+56*1},{Pp,Jp+56*2,Qp+56*3,Up},{Lp,Pp,Up,Rp},{Op,Kp,Rp,Vp},{Ip+56*2,Op,Vp,Qp+56*2},
2554           /* layer 6 */
2555           {Rp,Sp,Aq,Yp},{Sp,Qp,Wp,Aq},{Tp,Rp,Yp,Cq},{Qp+56*1,Tp,Cq,Wp+56*1},{Up,Qp+56*3,Xp+56*1,Dq},{Rp,Up,Dq,Zp},{Vp,Rp,Zp,Bq},{Qp+56*2,Vp,Bq,Xp},
2556           /* layer 7 (the top is the periodic image of the bottom of layer 0) */
2557           {Yp,Aq,C+56*4,A+56*4},{Aq,Wp,B+56*4,C+56*4},{Cq,Yp,A+56*4,D+56*4},{Wp+56*1,Cq,D+56*4,B+56*5},{Dq,Xp+56*1,B+56*5,E+56*4},{Zp,Dq,E+56*4,A+56*6},{Bq,Zp,A+56*6,F+56*4},{Xp,Bq,F+56*4,B+56*4}
2558         };
2559       const PetscReal gamma = PetscAcosReal((PetscSqrtReal(3.)-1.) / PetscSqrtReal(2.)) / PETSC_PI;
2560       const PetscReal patternCoords[56][3] =
2561         {
2562           /* A  */ {1.,0.,0.},
2563           /* B  */ {0.,1.,0.},
2564           /* C  */ {gamma,gamma,0.},
2565           /* D  */ {1+gamma,1-gamma,0.},
2566           /* E  */ {2-gamma,2-gamma,0.},
2567           /* F  */ {1-gamma,1+gamma,0.},
2568 
2569           /* G  */ {.5,0,.25},
2570           /* H  */ {1.5,0.,.25},
2571           /* II */ {.5,1.,.25},
2572           /* J  */ {1.5,1.,.25},
2573           /* K  */ {.25,.5,.25},
2574           /* L  */ {1.25,.5,.25},
2575           /* M  */ {.75,1.5,.25},
2576           /* N  */ {1.75,1.5,.25},
2577 
2578           /* O  */ {0.,0.,.5},
2579           /* P  */ {1.,1.,.5},
2580           /* Q  */ {gamma,1-gamma,.5},
2581           /* R  */ {1+gamma,gamma,.5},
2582           /* S  */ {2-gamma,1+gamma,.5},
2583           /* T  */ {1-gamma,2-gamma,.5},
2584 
2585           /* U  */ {0.,.5,.75},
2586           /* V  */ {0.,1.5,.75},
2587           /* W  */ {1.,.5,.75},
2588           /* X  */ {1.,1.5,.75},
2589           /* Y  */ {.5,.75,.75},
2590           /* Z  */ {.5,1.75,.75},
2591           /* Ap */ {1.5,.25,.75},
2592           /* Bp */ {1.5,1.25,.75},
2593 
2594           /* Cp */ {1.,0.,1.},
2595           /* Dp */ {0.,1.,1.},
2596           /* Ep */ {1-gamma,1-gamma,1.},
2597           /* Fp */ {1+gamma,1+gamma,1.},
2598           /* Gp */ {2-gamma,gamma,1.},
2599           /* Hp */ {gamma,2-gamma,1.},
2600 
2601           /* Ip */ {.5,0.,1.25},
2602           /* Jp */ {1.5,0.,1.25},
2603           /* Kp */ {.5,1.,1.25},
2604           /* Lp */ {1.5,1.,1.25},
2605           /* Mp */ {.75,.5,1.25},
2606           /* Np */ {1.75,.5,1.25},
2607           /* Op */ {.25,1.5,1.25},
2608           /* Pp */ {1.25,1.5,1.25},
2609 
2610           /* Qp */ {0.,0.,1.5},
2611           /* Rp */ {1.,1.,1.5},
2612           /* Sp */ {1-gamma,gamma,1.5},
2613           /* Tp */ {2-gamma,1-gamma,1.5},
2614           /* Up */ {1+gamma,2-gamma,1.5},
2615           /* Vp */ {gamma,1+gamma,1.5},
2616 
2617           /* Wp */ {0.,.5,1.75},
2618           /* Xp */ {0.,1.5,1.75},
2619           /* Yp */ {1.,.5,1.75},
2620           /* Zp */ {1.,1.5,1.75},
2621           /* Aq */ {.5,.25,1.75},
2622           /* Bq */ {.5,1.25,1.75},
2623           /* Cq */ {1.5,.75,1.75},
2624           /* Dq */ {1.5,1.75,1.75},
2625         };
2626       PetscInt  (*cells)[64][4] = NULL;
2627       PetscBool *seen;
2628       PetscInt  *vertToTrueVert;
2629       PetscInt  count;
2630 
2631       for (PetscInt i = 0; i < 3; i++) extentPlus[i]  = extent[i] + 1;
2632       numBlocks = 1;
2633       for (PetscInt i = 0; i < 3; i++)     numBlocks *= extent[i];
2634       numBlocksPlus = 1;
2635       for (PetscInt i = 0; i < 3; i++) numBlocksPlus *= extentPlus[i];
2636       numFaces = numBlocks * facesPerBlock;
2637       ierr = PetscMalloc1(numBlocks, &cells);CHKERRQ(ierr);
2638       ierr = PetscCalloc1(numBlocksPlus * vertsPerBlock,&seen);CHKERRQ(ierr);
2639       for (PetscInt k = 0; k < extent[2]; k++) {
2640         for (PetscInt j = 0; j < extent[1]; j++) {
2641           for (PetscInt i = 0; i < extent[0]; i++) {
2642             for (PetscInt f = 0; f < facesPerBlock; f++) {
2643               for (PetscInt v = 0; v < 4; v++) {
2644                 PetscInt vertRaw = pattern[f][v];
2645                 PetscInt blockidx = vertRaw / 56;
2646                 PetscInt patternvert = vertRaw % 56;
2647                 PetscInt xplus = (blockidx & 1);
2648                 PetscInt yplus = (blockidx & 2) >> 1;
2649                 PetscInt zplus = (blockidx & 4) >> 2;
2650                 PetscInt zcoord = (periodic && periodic[2] == DM_BOUNDARY_PERIODIC) ? ((k + zplus) % extent[2]) : (k + zplus);
2651                 PetscInt ycoord = (periodic && periodic[1] == DM_BOUNDARY_PERIODIC) ? ((j + yplus) % extent[1]) : (j + yplus);
2652                 PetscInt xcoord = (periodic && periodic[0] == DM_BOUNDARY_PERIODIC) ? ((i + xplus) % extent[0]) : (i + xplus);
2653                 PetscInt vert = ((zcoord * extentPlus[1] + ycoord) * extentPlus[0] + xcoord) * 56 + patternvert;
2654 
2655                 cells[(k * extent[1] + j) * extent[0] + i][f][v] = vert;
2656                 seen[vert] = PETSC_TRUE;
2657               }
2658             }
2659           }
2660         }
2661       }
2662       for (PetscInt i = 0; i < numBlocksPlus * vertsPerBlock; i++) if (seen[i]) numVertices++;
2663       count = 0;
2664       ierr = PetscMalloc1(numBlocksPlus * vertsPerBlock, &vertToTrueVert);CHKERRQ(ierr);
2665       ierr = PetscMalloc1(numVertices * 3, &vtxCoords);CHKERRQ(ierr);
2666       for (PetscInt i = 0; i < numBlocksPlus * vertsPerBlock; i++) vertToTrueVert[i] = -1;
2667       for (PetscInt k = 0; k < extentPlus[2]; k++) {
2668         for (PetscInt j = 0; j < extentPlus[1]; j++) {
2669           for (PetscInt i = 0; i < extentPlus[0]; i++) {
2670             for (PetscInt v = 0; v < vertsPerBlock; v++) {
2671               PetscInt vIdx = ((k * extentPlus[1] + j) * extentPlus[0] + i) * vertsPerBlock + v;
2672 
2673               if (seen[vIdx]) {
2674                 PetscInt thisVert;
2675 
2676                 vertToTrueVert[vIdx] = thisVert = count++;
2677 
2678                 for (PetscInt d = 0; d < 3; d++) vtxCoords[3 * thisVert + d] = patternCoords[v][d];
2679                 vtxCoords[3 * thisVert + 0] += i * 2;
2680                 vtxCoords[3 * thisVert + 1] += j * 2;
2681                 vtxCoords[3 * thisVert + 2] += k * 2;
2682               }
2683             }
2684           }
2685         }
2686       }
2687       for (PetscInt i = 0; i < numBlocks; i++) {
2688         for (PetscInt f = 0; f < facesPerBlock; f++) {
2689           for (PetscInt v = 0; v < 4; v++) {
2690             cells[i][f][v] = vertToTrueVert[cells[i][f][v]];
2691           }
2692         }
2693       }
2694       ierr = PetscFree(vertToTrueVert);CHKERRQ(ierr);
2695       ierr = PetscFree(seen);CHKERRQ(ierr);
2696       cells_flat = cells[0][0];
2697       numEdges = 0;
2698       for (PetscInt i = 0; i < numFaces; i++) {
2699         for (PetscInt e = 0; e < 4; e++) {
2700           PetscInt ev[] = {cells_flat[i*4 + e], cells_flat[i*4 + ((e+1)%4)]};
2701           const PetscReal *evCoords[] = {&vtxCoords[3*ev[0]], &vtxCoords[3*ev[1]]};
2702 
2703           for (PetscInt d = 0; d < 3; d++) {
2704             if (!periodic || periodic[0] != DM_BOUNDARY_PERIODIC) {
2705               if (evCoords[0][d] == 0. && evCoords[1][d] == 0.) numEdges++;
2706               if (evCoords[0][d] == 2.*extent[d] && evCoords[1][d] == 2.*extent[d]) numEdges++;
2707             }
2708           }
2709         }
2710       }
2711       ierr = PetscMalloc1(numEdges, &edges);CHKERRQ(ierr);
2712       ierr = PetscMalloc1(numEdges, &edgeSets);CHKERRQ(ierr);
2713       for (PetscInt edge = 0, i = 0; i < numFaces; i++) {
2714         for (PetscInt e = 0; e < 4; e++) {
2715           PetscInt ev[] = {cells_flat[i*4 + e], cells_flat[i*4 + ((e+1)%4)]};
2716           const PetscReal *evCoords[] = {&vtxCoords[3*ev[0]], &vtxCoords[3*ev[1]]};
2717 
2718           for (PetscInt d = 0; d < 3; d++) {
2719             if (!periodic || periodic[d] != DM_BOUNDARY_PERIODIC) {
2720               if (evCoords[0][d] == 0. && evCoords[1][d] == 0.) {
2721                 edges[edge][0] = ev[0];
2722                 edges[edge][1] = ev[1];
2723                 edgeSets[edge++] = 2 * d;
2724               }
2725               if (evCoords[0][d] == 2.*extent[d] && evCoords[1][d] == 2.*extent[d]) {
2726                 edges[edge][0] = ev[0];
2727                 edges[edge][1] = ev[1];
2728                 edgeSets[edge++] = 2 * d + 1;
2729               }
2730             }
2731           }
2732         }
2733       }
2734     }
2735     evalFunc = TPSEvaluate_Gyroid;
2736     break;
2737   }
2738 
2739   ierr = DMSetDimension(dm, topoDim);CHKERRQ(ierr);
2740   if (!rank) {ierr = DMPlexBuildFromCellList(dm, numFaces, numVertices, 4, cells_flat);CHKERRQ(ierr);}
2741   else       {ierr = DMPlexBuildFromCellList(dm, 0, 0, 0, NULL);CHKERRQ(ierr);}
2742   ierr = PetscFree(cells_flat);CHKERRQ(ierr);
2743   {
2744     DM idm;
2745     ierr = DMPlexInterpolate(dm, &idm);CHKERRQ(ierr);
2746     ierr = DMPlexReplace_Static(dm, &idm);CHKERRQ(ierr);
2747   }
2748   if (!rank) {ierr = DMPlexBuildCoordinatesFromCellList(dm, spaceDim, vtxCoords);CHKERRQ(ierr);}
2749   else       {ierr = DMPlexBuildCoordinatesFromCellList(dm, spaceDim, NULL);CHKERRQ(ierr);}
2750   ierr = PetscFree(vtxCoords);CHKERRQ(ierr);
2751 
2752   ierr = DMCreateLabel(dm, "Face Sets");CHKERRQ(ierr);
2753   ierr = DMGetLabel(dm, "Face Sets", &label);CHKERRQ(ierr);
2754   for (PetscInt e=0; e<numEdges; e++) {
2755     PetscInt njoin;
2756     const PetscInt *join, verts[] = {numFaces + edges[e][0], numFaces + edges[e][1]};
2757     ierr = DMPlexGetJoin(dm, 2, verts, &njoin, &join);CHKERRQ(ierr);
2758     PetscCheck(njoin == 1, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Expected unique join of vertices %D and %D", edges[e][0], edges[e][1]);
2759     ierr = DMLabelSetValue(label, join[0], edgeSets[e]);CHKERRQ(ierr);
2760     ierr = DMPlexRestoreJoin(dm, 2, verts, &njoin, &join);CHKERRQ(ierr);
2761   }
2762   ierr = PetscFree(edges);CHKERRQ(ierr);
2763   ierr = PetscFree(edgeSets);CHKERRQ(ierr);
2764   if (tps_distribute) {
2765     DM               pdm = NULL;
2766     PetscPartitioner part;
2767 
2768     ierr = DMPlexGetPartitioner(dm, &part);CHKERRQ(ierr);
2769     ierr = PetscPartitionerSetFromOptions(part);CHKERRQ(ierr);
2770     ierr = DMPlexDistribute(dm, 0, NULL, &pdm);CHKERRQ(ierr);
2771     if (pdm) {
2772       ierr = DMPlexReplace_Static(dm, &pdm);CHKERRQ(ierr);
2773     }
2774     // Do not auto-distribute again
2775     ierr = DMPlexDistributeSetDefault(dm, PETSC_FALSE);CHKERRQ(ierr);
2776   }
2777 
2778   ierr = DMPlexSetRefinementUniform(dm, PETSC_TRUE);CHKERRQ(ierr);
2779   for (PetscInt refine=0; refine<refinements; refine++) {
2780     PetscInt m;
2781     DM dmf;
2782     Vec X;
2783     PetscScalar *x;
2784     ierr = DMRefine(dm, MPI_COMM_NULL, &dmf);CHKERRQ(ierr);
2785     ierr = DMPlexReplace_Static(dm, &dmf);CHKERRQ(ierr);
2786 
2787     ierr = DMGetCoordinatesLocal(dm, &X);CHKERRQ(ierr);
2788     ierr = VecGetLocalSize(X, &m);CHKERRQ(ierr);
2789     ierr = VecGetArray(X, &x);CHKERRQ(ierr);
2790     for (PetscInt i=0; i<m; i+=3) {
2791       ierr = TPSNearestPoint(evalFunc, &x[i]);CHKERRQ(ierr);
2792     }
2793     ierr = VecRestoreArray(X, &x);CHKERRQ(ierr);
2794   }
2795 
2796   // Face Sets has already been propagated to new vertices during refinement; this propagates to the initial vertices.
2797   ierr = DMGetLabel(dm, "Face Sets", &label);CHKERRQ(ierr);
2798   ierr = DMPlexLabelComplete(dm, label);CHKERRQ(ierr);
2799 
2800   if (thickness > 0) {
2801     DM dm3;
2802     ierr = DMPlexExtrude(dm, layers, thickness, PETSC_FALSE, PETSC_TRUE, NULL, NULL, &dm3);CHKERRQ(ierr);
2803     ierr = DMPlexReplace_Static(dm, &dm3);CHKERRQ(ierr);
2804   }
2805   PetscFunctionReturn(0);
2806 }
2807 
2808 /*@
2809   DMPlexCreateTPSMesh - Create a distributed, interpolated mesh of a triply-periodic surface
2810 
2811   Collective
2812 
2813   Input Parameters:
2814 + comm   - The communicator for the DM object
2815 . tpstype - Type of triply-periodic surface
2816 . extent - Array of length 3 containing number of periods in each direction
2817 . periodic - array of length 3 with periodicity, or NULL for non-periodic
2818 . tps_distribute - Distribute 2D manifold mesh prior to refinement and extrusion (more scalable)
2819 . refinements - Number of factor-of-2 refinements of 2D manifold mesh
2820 . layers - Number of cell layers extruded in normal direction
2821 - thickness - Thickness in normal direction
2822 
2823   Output Parameter:
2824 . dm  - The DM object
2825 
2826   Notes:
2827   This meshes the surface of the Schwarz P or Gyroid surfaces.  Schwarz P is is the simplest member of the triply-periodic minimal surfaces.
2828   https://en.wikipedia.org/wiki/Schwarz_minimal_surface#Schwarz_P_(%22Primitive%22) and can be cut with "clean" boundaries.
2829   The Gyroid (https://en.wikipedia.org/wiki/Gyroid) is another triply-periodic minimal surface with applications in additive manufacturing; it is much more difficult to "cut" since there are no planes of symmetry.
2830   Our implementation creates a very coarse mesh of the surface and refines (by 4-way splitting) as many times as requested.
2831   On each refinement, all vertices are projected to their nearest point on the surface.
2832   This projection could readily be extended to related surfaces.
2833 
2834   The face (edge) sets for the Schwarz P surface are numbered 1(-x), 2(+x), 3(-y), 4(+y), 5(-z), 6(+z).
2835   When the mesh is refined, "Face Sets" contain the new vertices (created during refinement).  Use DMPlexLabelComplete() to propagate to coarse-level vertices.
2836 
2837   References:
2838 . * - Maskery et al, Insights into the mechanical properties of several triply periodic minimal surface lattice structures made by polymer additive manufacturing, 2017. https://doi.org/10.1016/j.polymer.2017.11.049
2839 
2840   Developer Notes:
2841   The Gyroid mesh does not currently mark boundary sets.
2842 
2843   Level: beginner
2844 
2845 .seealso: DMPlexCreateSphereMesh(), DMSetType(), DMCreate()
2846 @*/
2847 PetscErrorCode DMPlexCreateTPSMesh(MPI_Comm comm, DMPlexTPSType tpstype, const PetscInt extent[], const DMBoundaryType periodic[], PetscBool tps_distribute, PetscInt refinements, PetscInt layers, PetscReal thickness, DM *dm)
2848 {
2849   PetscErrorCode ierr;
2850 
2851   PetscFunctionBegin;
2852   ierr = DMCreate(comm, dm);CHKERRQ(ierr);
2853   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
2854   ierr = DMPlexCreateTPSMesh_Internal(*dm, tpstype, extent, periodic, tps_distribute, refinements, layers, thickness);CHKERRQ(ierr);
2855   PetscFunctionReturn(0);
2856 }
2857 
2858 /*@
2859   DMPlexCreateSphereMesh - Creates a mesh on the d-dimensional sphere, S^d.
2860 
2861   Collective
2862 
2863   Input Parameters:
2864 + comm    - The communicator for the DM object
2865 . dim     - The dimension
2866 . simplex - Use simplices, or tensor product cells
2867 - R       - The radius
2868 
2869   Output Parameter:
2870 . dm  - The DM object
2871 
2872   Level: beginner
2873 
2874 .seealso: DMPlexCreateBallMesh(), DMPlexCreateBoxMesh(), DMSetType(), DMCreate()
2875 @*/
2876 PetscErrorCode DMPlexCreateSphereMesh(MPI_Comm comm, PetscInt dim, PetscBool simplex, PetscReal R, DM *dm)
2877 {
2878   PetscErrorCode ierr;
2879 
2880   PetscFunctionBegin;
2881   PetscValidPointer(dm, 5);
2882   ierr = DMCreate(comm, dm);CHKERRQ(ierr);
2883   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
2884   ierr = DMPlexCreateSphereMesh_Internal(*dm, dim, simplex, R);CHKERRQ(ierr);
2885   PetscFunctionReturn(0);
2886 }
2887 
2888 static PetscErrorCode DMPlexCreateBallMesh_Internal(DM dm, PetscInt dim, PetscReal R)
2889 {
2890   DM             sdm, vol;
2891   DMLabel        bdlabel;
2892   PetscErrorCode ierr;
2893 
2894   PetscFunctionBegin;
2895   ierr = DMCreate(PetscObjectComm((PetscObject) dm), &sdm);CHKERRQ(ierr);
2896   ierr = DMSetType(sdm, DMPLEX);CHKERRQ(ierr);
2897   ierr = PetscObjectSetOptionsPrefix((PetscObject) sdm, "bd_");CHKERRQ(ierr);
2898   ierr = DMPlexCreateSphereMesh_Internal(sdm, dim-1, PETSC_TRUE, R);CHKERRQ(ierr);
2899   ierr = DMSetFromOptions(sdm);CHKERRQ(ierr);
2900   ierr = DMViewFromOptions(sdm, NULL, "-dm_view");CHKERRQ(ierr);
2901   ierr = DMPlexGenerate(sdm, NULL, PETSC_TRUE, &vol);CHKERRQ(ierr);
2902   ierr = DMDestroy(&sdm);CHKERRQ(ierr);
2903   ierr = DMPlexReplace_Static(dm, &vol);CHKERRQ(ierr);
2904   ierr = DMCreateLabel(dm, "marker");CHKERRQ(ierr);
2905   ierr = DMGetLabel(dm, "marker", &bdlabel);CHKERRQ(ierr);
2906   ierr = DMPlexMarkBoundaryFaces(dm, PETSC_DETERMINE, bdlabel);CHKERRQ(ierr);
2907   ierr = DMPlexLabelComplete(dm, bdlabel);CHKERRQ(ierr);
2908   PetscFunctionReturn(0);
2909 }
2910 
2911 /*@
2912   DMPlexCreateBallMesh - Creates a simplex mesh on the d-dimensional ball, B^d.
2913 
2914   Collective
2915 
2916   Input Parameters:
2917 + comm  - The communicator for the DM object
2918 . dim   - The dimension
2919 - R     - The radius
2920 
2921   Output Parameter:
2922 . dm  - The DM object
2923 
2924   Options Database Keys:
2925 - bd_dm_refine - This will refine the surface mesh preserving the sphere geometry
2926 
2927   Level: beginner
2928 
2929 .seealso: DMPlexCreateSphereMesh(), DMPlexCreateBoxMesh(), DMSetType(), DMCreate()
2930 @*/
2931 PetscErrorCode DMPlexCreateBallMesh(MPI_Comm comm, PetscInt dim, PetscReal R, DM *dm)
2932 {
2933   PetscErrorCode ierr;
2934 
2935   PetscFunctionBegin;
2936   ierr = DMCreate(comm, dm);CHKERRQ(ierr);
2937   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
2938   ierr = DMPlexCreateBallMesh_Internal(*dm, dim, R);CHKERRQ(ierr);
2939   PetscFunctionReturn(0);
2940 }
2941 
2942 static PetscErrorCode DMPlexCreateReferenceCell_Internal(DM rdm, DMPolytopeType ct)
2943 {
2944   PetscErrorCode ierr;
2945 
2946   PetscFunctionBegin;
2947   switch (ct) {
2948     case DM_POLYTOPE_POINT:
2949     {
2950       PetscInt    numPoints[1]        = {1};
2951       PetscInt    coneSize[1]         = {0};
2952       PetscInt    cones[1]            = {0};
2953       PetscInt    coneOrientations[1] = {0};
2954       PetscScalar vertexCoords[1]     = {0.0};
2955 
2956       ierr = DMSetDimension(rdm, 0);CHKERRQ(ierr);
2957       ierr = DMPlexCreateFromDAG(rdm, 0, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
2958     }
2959     break;
2960     case DM_POLYTOPE_SEGMENT:
2961     {
2962       PetscInt    numPoints[2]        = {2, 1};
2963       PetscInt    coneSize[3]         = {2, 0, 0};
2964       PetscInt    cones[2]            = {1, 2};
2965       PetscInt    coneOrientations[2] = {0, 0};
2966       PetscScalar vertexCoords[2]     = {-1.0,  1.0};
2967 
2968       ierr = DMSetDimension(rdm, 1);CHKERRQ(ierr);
2969       ierr = DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
2970     }
2971     break;
2972     case DM_POLYTOPE_POINT_PRISM_TENSOR:
2973     {
2974       PetscInt    numPoints[2]        = {2, 1};
2975       PetscInt    coneSize[3]         = {2, 0, 0};
2976       PetscInt    cones[2]            = {1, 2};
2977       PetscInt    coneOrientations[2] = {0, 0};
2978       PetscScalar vertexCoords[2]     = {-1.0,  1.0};
2979 
2980       ierr = DMSetDimension(rdm, 1);CHKERRQ(ierr);
2981       ierr = DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
2982     }
2983     break;
2984     case DM_POLYTOPE_TRIANGLE:
2985     {
2986       PetscInt    numPoints[2]        = {3, 1};
2987       PetscInt    coneSize[4]         = {3, 0, 0, 0};
2988       PetscInt    cones[3]            = {1, 2, 3};
2989       PetscInt    coneOrientations[3] = {0, 0, 0};
2990       PetscScalar vertexCoords[6]     = {-1.0, -1.0,  1.0, -1.0,  -1.0, 1.0};
2991 
2992       ierr = DMSetDimension(rdm, 2);CHKERRQ(ierr);
2993       ierr = DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
2994     }
2995     break;
2996     case DM_POLYTOPE_QUADRILATERAL:
2997     {
2998       PetscInt    numPoints[2]        = {4, 1};
2999       PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
3000       PetscInt    cones[4]            = {1, 2, 3, 4};
3001       PetscInt    coneOrientations[4] = {0, 0, 0, 0};
3002       PetscScalar vertexCoords[8]     = {-1.0, -1.0,  1.0, -1.0,  1.0, 1.0,  -1.0, 1.0};
3003 
3004       ierr = DMSetDimension(rdm, 2);CHKERRQ(ierr);
3005       ierr = DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
3006     }
3007     break;
3008     case DM_POLYTOPE_SEG_PRISM_TENSOR:
3009     {
3010       PetscInt    numPoints[2]        = {4, 1};
3011       PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
3012       PetscInt    cones[4]            = {1, 2, 3, 4};
3013       PetscInt    coneOrientations[4] = {0, 0, 0, 0};
3014       PetscScalar vertexCoords[8]     = {-1.0, -1.0,  1.0, -1.0,  -1.0, 1.0,  1.0, 1.0};
3015 
3016       ierr = DMSetDimension(rdm, 2);CHKERRQ(ierr);
3017       ierr = DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
3018     }
3019     break;
3020     case DM_POLYTOPE_TETRAHEDRON:
3021     {
3022       PetscInt    numPoints[2]        = {4, 1};
3023       PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
3024       PetscInt    cones[4]            = {1, 2, 3, 4};
3025       PetscInt    coneOrientations[4] = {0, 0, 0, 0};
3026       PetscScalar vertexCoords[12]    = {-1.0, -1.0, -1.0,  -1.0, 1.0, -1.0,  1.0, -1.0, -1.0,  -1.0, -1.0, 1.0};
3027 
3028       ierr = DMSetDimension(rdm, 3);CHKERRQ(ierr);
3029       ierr = DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
3030     }
3031     break;
3032     case DM_POLYTOPE_HEXAHEDRON:
3033     {
3034       PetscInt    numPoints[2]        = {8, 1};
3035       PetscInt    coneSize[9]         = {8, 0, 0, 0, 0, 0, 0, 0, 0};
3036       PetscInt    cones[8]            = {1, 2, 3, 4, 5, 6, 7, 8};
3037       PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
3038       PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0,  -1.0,  1.0, -1.0,  1.0, 1.0, -1.0,   1.0, -1.0, -1.0,
3039                                          -1.0, -1.0,  1.0,   1.0, -1.0,  1.0,  1.0, 1.0,  1.0,  -1.0,  1.0,  1.0};
3040 
3041       ierr = DMSetDimension(rdm, 3);CHKERRQ(ierr);
3042       ierr = DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
3043     }
3044     break;
3045     case DM_POLYTOPE_TRI_PRISM:
3046     {
3047       PetscInt    numPoints[2]        = {6, 1};
3048       PetscInt    coneSize[7]         = {6, 0, 0, 0, 0, 0, 0};
3049       PetscInt    cones[6]            = {1, 2, 3, 4, 5, 6};
3050       PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
3051       PetscScalar vertexCoords[18]    = {-1.0, -1.0, -1.0, -1.0,  1.0, -1.0,   1.0, -1.0, -1.0,
3052                                          -1.0, -1.0,  1.0,  1.0, -1.0,  1.0,  -1.0,  1.0,  1.0};
3053 
3054       ierr = DMSetDimension(rdm, 3);CHKERRQ(ierr);
3055       ierr = DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
3056     }
3057     break;
3058     case DM_POLYTOPE_TRI_PRISM_TENSOR:
3059     {
3060       PetscInt    numPoints[2]        = {6, 1};
3061       PetscInt    coneSize[7]         = {6, 0, 0, 0, 0, 0, 0};
3062       PetscInt    cones[6]            = {1, 2, 3, 4, 5, 6};
3063       PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
3064       PetscScalar vertexCoords[18]    = {-1.0, -1.0, -1.0,  1.0, -1.0, -1.0,  -1.0, 1.0, -1.0,
3065                                          -1.0, -1.0,  1.0,  1.0, -1.0,  1.0,  -1.0, 1.0,  1.0};
3066 
3067       ierr = DMSetDimension(rdm, 3);CHKERRQ(ierr);
3068       ierr = DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
3069     }
3070     break;
3071     case DM_POLYTOPE_QUAD_PRISM_TENSOR:
3072     {
3073       PetscInt    numPoints[2]        = {8, 1};
3074       PetscInt    coneSize[9]         = {8, 0, 0, 0, 0, 0, 0, 0, 0};
3075       PetscInt    cones[8]            = {1, 2, 3, 4, 5, 6, 7, 8};
3076       PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
3077       PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0,  1.0, -1.0, -1.0,  1.0, 1.0, -1.0,  -1.0, 1.0, -1.0,
3078                                          -1.0, -1.0,  1.0,  1.0, -1.0,  1.0,  1.0, 1.0,  1.0,  -1.0, 1.0,  1.0};
3079 
3080       ierr = DMSetDimension(rdm, 3);CHKERRQ(ierr);
3081       ierr = DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
3082     }
3083     break;
3084     case DM_POLYTOPE_PYRAMID:
3085     {
3086       PetscInt    numPoints[2]        = {5, 1};
3087       PetscInt    coneSize[6]         = {5, 0, 0, 0, 0, 0};
3088       PetscInt    cones[5]            = {1, 2, 3, 4, 5};
3089       PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
3090       PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0,  -1.0, 1.0, -1.0,  1.0, 1.0, -1.0,  1.0, -1.0, -1.0,
3091                                           0.0,  0.0,  1.0};
3092 
3093       ierr = DMSetDimension(rdm, 3);CHKERRQ(ierr);
3094       ierr = DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
3095     }
3096     break;
3097     default: SETERRQ(PetscObjectComm((PetscObject) rdm), PETSC_ERR_ARG_WRONG, "Cannot create reference cell for cell type %s", DMPolytopeTypes[ct]);
3098   }
3099   {
3100     PetscInt Nv, v;
3101 
3102     /* Must create the celltype label here so that we do not automatically try to compute the types */
3103     ierr = DMCreateLabel(rdm, "celltype");CHKERRQ(ierr);
3104     ierr = DMPlexSetCellType(rdm, 0, ct);CHKERRQ(ierr);
3105     ierr = DMPlexGetChart(rdm, NULL, &Nv);CHKERRQ(ierr);
3106     for (v = 1; v < Nv; ++v) {ierr = DMPlexSetCellType(rdm, v, DM_POLYTOPE_POINT);CHKERRQ(ierr);}
3107   }
3108   ierr = DMPlexInterpolateInPlace_Internal(rdm);CHKERRQ(ierr);
3109   ierr = PetscObjectSetName((PetscObject) rdm, DMPolytopeTypes[ct]);CHKERRQ(ierr);
3110   PetscFunctionReturn(0);
3111 }
3112 
3113 /*@
3114   DMPlexCreateReferenceCell - Create a DMPLEX with the appropriate FEM reference cell
3115 
3116   Collective
3117 
3118   Input Parameters:
3119 + comm - The communicator
3120 - ct   - The cell type of the reference cell
3121 
3122   Output Parameter:
3123 . refdm - The reference cell
3124 
3125   Level: intermediate
3126 
3127 .seealso: DMPlexCreateReferenceCell(), DMPlexCreateBoxMesh()
3128 @*/
3129 PetscErrorCode DMPlexCreateReferenceCell(MPI_Comm comm, DMPolytopeType ct, DM *refdm)
3130 {
3131   PetscErrorCode ierr;
3132 
3133   PetscFunctionBegin;
3134   ierr = DMCreate(comm, refdm);CHKERRQ(ierr);
3135   ierr = DMSetType(*refdm, DMPLEX);CHKERRQ(ierr);
3136   ierr = DMPlexCreateReferenceCell_Internal(*refdm, ct);CHKERRQ(ierr);
3137   PetscFunctionReturn(0);
3138 }
3139 
3140 static PetscErrorCode DMPlexCreateBoundaryLabel_Private(DM dm, const char name[])
3141 {
3142   DM             plex;
3143   DMLabel        label;
3144   PetscBool      hasLabel;
3145   PetscErrorCode ierr;
3146 
3147   PetscFunctionBeginUser;
3148   ierr = DMHasLabel(dm, name, &hasLabel);CHKERRQ(ierr);
3149   if (hasLabel) PetscFunctionReturn(0);
3150   ierr = DMCreateLabel(dm, name);CHKERRQ(ierr);
3151   ierr = DMGetLabel(dm, name, &label);CHKERRQ(ierr);
3152   ierr = DMConvert(dm, DMPLEX, &plex);CHKERRQ(ierr);
3153   ierr = DMPlexMarkBoundaryFaces(plex, 1, label);CHKERRQ(ierr);
3154   ierr = DMDestroy(&plex);CHKERRQ(ierr);
3155   PetscFunctionReturn(0);
3156 }
3157 
3158 const char * const DMPlexShapes[] = {"box", "box_surface", "ball", "sphere", "cylinder", "schwarz_p", "gyroid", "unknown", "DMPlexShape", "DM_SHAPE_", NULL};
3159 
3160 static PetscErrorCode DMPlexCreateFromOptions_Internal(PetscOptionItems *PetscOptionsObject, PetscBool *useCoordSpace, DM dm)
3161 {
3162   DMPlexShape    shape = DM_SHAPE_BOX;
3163   DMPolytopeType cell  = DM_POLYTOPE_TRIANGLE;
3164   PetscInt       dim   = 2;
3165   PetscBool      simplex = PETSC_TRUE, interpolate = PETSC_TRUE, adjCone = PETSC_FALSE, adjClosure = PETSC_TRUE, refDomain = PETSC_FALSE;
3166   PetscBool      flg, flg2, fflg, bdfflg, nameflg;
3167   MPI_Comm       comm;
3168   char           filename[PETSC_MAX_PATH_LEN]   = "<unspecified>";
3169   char           bdFilename[PETSC_MAX_PATH_LEN] = "<unspecified>";
3170   char           plexname[PETSC_MAX_PATH_LEN]   = "";
3171   PetscErrorCode ierr;
3172 
3173   PetscFunctionBegin;
3174   ierr = PetscObjectGetComm((PetscObject) dm, &comm);CHKERRQ(ierr);
3175   /* TODO Turn this into a registration interface */
3176   ierr = PetscOptionsString("-dm_plex_filename", "File containing a mesh", "DMPlexCreateFromFile", filename, filename, sizeof(filename), &fflg);CHKERRQ(ierr);
3177   ierr = PetscOptionsString("-dm_plex_boundary_filename", "File containing a mesh boundary", "DMPlexCreateFromFile", bdFilename, bdFilename, sizeof(bdFilename), &bdfflg);CHKERRQ(ierr);
3178   ierr = PetscOptionsString("-dm_plex_name", "Name of the mesh in the file", "DMPlexCreateFromFile", plexname, plexname, sizeof(plexname), &nameflg);CHKERRQ(ierr);
3179   ierr = PetscOptionsEnum("-dm_plex_cell", "Cell shape", "", DMPolytopeTypes, (PetscEnum) cell, (PetscEnum *) &cell, NULL);CHKERRQ(ierr);
3180   ierr = PetscOptionsBool("-dm_plex_reference_cell_domain", "Use a reference cell domain", "", refDomain, &refDomain, NULL);CHKERRQ(ierr);
3181   ierr = PetscOptionsEnum("-dm_plex_shape", "Shape for built-in mesh", "", DMPlexShapes, (PetscEnum) shape, (PetscEnum *) &shape, &flg);CHKERRQ(ierr);
3182   ierr = PetscOptionsBoundedInt("-dm_plex_dim", "Topological dimension of the mesh", "DMGetDimension", dim, &dim, &flg, 0);CHKERRQ(ierr);
3183   PetscCheckFalse((dim < 0) || (dim > 3),comm, PETSC_ERR_ARG_OUTOFRANGE, "Dimension %D should be in [1, 3]", dim);
3184   ierr = PetscOptionsBool("-dm_plex_simplex", "Mesh cell shape", "", simplex,  &simplex, &flg);CHKERRQ(ierr);
3185   ierr = PetscOptionsBool("-dm_plex_interpolate", "Flag to create edges and faces automatically", "", interpolate, &interpolate, &flg);CHKERRQ(ierr);
3186   ierr = PetscOptionsBool("-dm_plex_adj_cone", "Set adjacency direction", "DMSetBasicAdjacency", adjCone,  &adjCone, &flg);CHKERRQ(ierr);
3187   ierr = PetscOptionsBool("-dm_plex_adj_closure", "Set adjacency size", "DMSetBasicAdjacency", adjClosure,  &adjClosure, &flg2);CHKERRQ(ierr);
3188   if (flg || flg2) {ierr = DMSetBasicAdjacency(dm, adjCone, adjClosure);CHKERRQ(ierr);}
3189 
3190   switch (cell) {
3191     case DM_POLYTOPE_POINT:
3192     case DM_POLYTOPE_SEGMENT:
3193     case DM_POLYTOPE_POINT_PRISM_TENSOR:
3194     case DM_POLYTOPE_TRIANGLE:
3195     case DM_POLYTOPE_QUADRILATERAL:
3196     case DM_POLYTOPE_TETRAHEDRON:
3197     case DM_POLYTOPE_HEXAHEDRON:
3198       *useCoordSpace = PETSC_TRUE;break;
3199     default: *useCoordSpace = PETSC_FALSE;break;
3200   }
3201 
3202   if (fflg) {
3203     DM dmnew;
3204 
3205     ierr = DMPlexCreateFromFile(PetscObjectComm((PetscObject) dm), filename, plexname, interpolate, &dmnew);CHKERRQ(ierr);
3206     ierr = DMPlexCopy_Internal(dm, PETSC_FALSE, dmnew);CHKERRQ(ierr);
3207     ierr = DMPlexReplace_Static(dm, &dmnew);CHKERRQ(ierr);
3208   } else if (refDomain) {
3209     ierr = DMPlexCreateReferenceCell_Internal(dm, cell);CHKERRQ(ierr);
3210   } else if (bdfflg) {
3211     DM bdm, dmnew;
3212 
3213     ierr = DMPlexCreateFromFile(PetscObjectComm((PetscObject) dm), bdFilename, plexname, interpolate, &bdm);CHKERRQ(ierr);
3214     ierr = PetscObjectSetOptionsPrefix((PetscObject) bdm, "bd_");CHKERRQ(ierr);
3215     ierr = DMSetFromOptions(bdm);CHKERRQ(ierr);
3216     ierr = DMPlexGenerate(bdm, NULL, interpolate, &dmnew);CHKERRQ(ierr);
3217     ierr = DMDestroy(&bdm);CHKERRQ(ierr);
3218     ierr = DMPlexCopy_Internal(dm, PETSC_FALSE, dmnew);CHKERRQ(ierr);
3219     ierr = DMPlexReplace_Static(dm, &dmnew);CHKERRQ(ierr);
3220   } else {
3221     ierr = PetscObjectSetName((PetscObject) dm, DMPlexShapes[shape]);CHKERRQ(ierr);
3222     switch (shape) {
3223       case DM_SHAPE_BOX:
3224       {
3225         PetscInt       faces[3] = {0, 0, 0};
3226         PetscReal      lower[3] = {0, 0, 0};
3227         PetscReal      upper[3] = {1, 1, 1};
3228         DMBoundaryType bdt[3]   = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
3229         PetscInt       i, n;
3230 
3231         n    = dim;
3232         for (i = 0; i < dim; ++i) faces[i] = (dim == 1 ? 1 : 4-dim);
3233         ierr = PetscOptionsIntArray("-dm_plex_box_faces", "Number of faces along each dimension", "", faces, &n, &flg);CHKERRQ(ierr);
3234         n    = 3;
3235         ierr = PetscOptionsRealArray("-dm_plex_box_lower", "Lower left corner of box", "", lower, &n, &flg);CHKERRQ(ierr);
3236         PetscCheckFalse(flg && (n != dim),comm, PETSC_ERR_ARG_SIZ, "Lower box point had %D values, should have been %D", n, dim);
3237         n    = 3;
3238         ierr = PetscOptionsRealArray("-dm_plex_box_upper", "Upper right corner of box", "", upper, &n, &flg);CHKERRQ(ierr);
3239         PetscCheckFalse(flg && (n != dim),comm, PETSC_ERR_ARG_SIZ, "Upper box point had %D values, should have been %D", n, dim);
3240         n    = 3;
3241         ierr = PetscOptionsEnumArray("-dm_plex_box_bd", "Boundary type for each dimension", "", DMBoundaryTypes, (PetscEnum *) bdt, &n, &flg);CHKERRQ(ierr);
3242         PetscCheckFalse(flg && (n != dim),comm, PETSC_ERR_ARG_SIZ, "Box boundary types had %D values, should have been %D", n, dim);
3243         switch (cell) {
3244           case DM_POLYTOPE_TRI_PRISM_TENSOR:
3245             ierr = DMPlexCreateWedgeBoxMesh_Internal(dm, faces, lower, upper, bdt);CHKERRQ(ierr);
3246             if (!interpolate) {
3247               DM udm;
3248 
3249               ierr = DMPlexUninterpolate(dm, &udm);CHKERRQ(ierr);
3250               ierr = DMPlexReplace_Static(dm, &udm);CHKERRQ(ierr);
3251             }
3252             break;
3253           default:
3254             ierr = DMPlexCreateBoxMesh_Internal(dm, dim, simplex, faces, lower, upper, bdt, interpolate);CHKERRQ(ierr);
3255             break;
3256         }
3257       }
3258       break;
3259       case DM_SHAPE_BOX_SURFACE:
3260       {
3261         PetscInt  faces[3] = {0, 0, 0};
3262         PetscReal lower[3] = {0, 0, 0};
3263         PetscReal upper[3] = {1, 1, 1};
3264         PetscInt  i, n;
3265 
3266         n    = dim+1;
3267         for (i = 0; i < dim+1; ++i) faces[i] = (dim+1 == 1 ? 1 : 4-(dim+1));
3268         ierr = PetscOptionsIntArray("-dm_plex_box_faces", "Number of faces along each dimension", "", faces, &n, &flg);CHKERRQ(ierr);
3269         n    = 3;
3270         ierr = PetscOptionsRealArray("-dm_plex_box_lower", "Lower left corner of box", "", lower, &n, &flg);CHKERRQ(ierr);
3271         PetscCheckFalse(flg && (n != dim+1),comm, PETSC_ERR_ARG_SIZ, "Lower box point had %D values, should have been %D", n, dim+1);
3272         n    = 3;
3273         ierr = PetscOptionsRealArray("-dm_plex_box_upper", "Upper right corner of box", "", upper, &n, &flg);CHKERRQ(ierr);
3274         PetscCheckFalse(flg && (n != dim+1),comm, PETSC_ERR_ARG_SIZ, "Upper box point had %D values, should have been %D", n, dim+1);
3275         ierr = DMPlexCreateBoxSurfaceMesh_Internal(dm, dim+1, faces, lower, upper, interpolate);CHKERRQ(ierr);
3276       }
3277       break;
3278       case DM_SHAPE_SPHERE:
3279       {
3280         PetscReal R = 1.0;
3281 
3282         ierr = PetscOptionsReal("-dm_plex_sphere_radius", "Radius of the sphere", "", R,  &R, &flg);CHKERRQ(ierr);
3283         ierr = DMPlexCreateSphereMesh_Internal(dm, dim, simplex, R);CHKERRQ(ierr);
3284       }
3285       break;
3286       case DM_SHAPE_BALL:
3287       {
3288         PetscReal R = 1.0;
3289 
3290         ierr = PetscOptionsReal("-dm_plex_ball_radius", "Radius of the ball", "", R,  &R, &flg);CHKERRQ(ierr);
3291         ierr = DMPlexCreateBallMesh_Internal(dm, dim, R);CHKERRQ(ierr);
3292       }
3293       break;
3294       case DM_SHAPE_CYLINDER:
3295       {
3296         DMBoundaryType bdt = DM_BOUNDARY_NONE;
3297         PetscInt       Nw  = 6;
3298 
3299         ierr = PetscOptionsEnum("-dm_plex_cylinder_bd", "Boundary type in the z direction", "", DMBoundaryTypes, (PetscEnum) bdt, (PetscEnum *) &bdt, NULL);CHKERRQ(ierr);
3300         ierr = PetscOptionsInt("-dm_plex_cylinder_num_wedges", "Number of wedges around the cylinder", "", Nw, &Nw, NULL);CHKERRQ(ierr);
3301         switch (cell) {
3302           case DM_POLYTOPE_TRI_PRISM_TENSOR:
3303             ierr = DMPlexCreateWedgeCylinderMesh_Internal(dm, Nw, interpolate);CHKERRQ(ierr);
3304             break;
3305           default:
3306             ierr = DMPlexCreateHexCylinderMesh_Internal(dm, bdt);CHKERRQ(ierr);
3307             break;
3308         }
3309       }
3310       break;
3311       case DM_SHAPE_SCHWARZ_P: // fallthrough
3312       case DM_SHAPE_GYROID:
3313       {
3314         PetscInt       extent[3] = {1,1,1}, refine = 0, layers = 0, three;
3315         PetscReal      thickness = 0.;
3316         DMBoundaryType periodic[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
3317         DMPlexTPSType  tps_type = shape == DM_SHAPE_SCHWARZ_P ? DMPLEX_TPS_SCHWARZ_P : DMPLEX_TPS_GYROID;
3318         PetscBool      tps_distribute;
3319         ierr = PetscOptionsIntArray("-dm_plex_tps_extent", "Number of replicas for each of three dimensions", NULL, extent, (three=3, &three), NULL);CHKERRQ(ierr);
3320         ierr = PetscOptionsInt("-dm_plex_tps_refine", "Number of refinements", NULL, refine, &refine, NULL);CHKERRQ(ierr);
3321         ierr = PetscOptionsEnumArray("-dm_plex_tps_periodic", "Periodicity in each of three dimensions", NULL, DMBoundaryTypes, (PetscEnum*)periodic, (three=3, &three), NULL);CHKERRQ(ierr);
3322         ierr = PetscOptionsInt("-dm_plex_tps_layers", "Number of layers in volumetric extrusion (or zero to not extrude)", NULL, layers, &layers, NULL);CHKERRQ(ierr);
3323         ierr = PetscOptionsReal("-dm_plex_tps_thickness", "Thickness of volumetric extrusion", NULL, thickness, &thickness, NULL);CHKERRQ(ierr);
3324         ierr = DMPlexDistributeGetDefault(dm, &tps_distribute);CHKERRQ(ierr);
3325         ierr = PetscOptionsBool("-dm_plex_tps_distribute", "Distribute the 2D mesh prior to refinement and extrusion", NULL, tps_distribute, &tps_distribute, NULL);CHKERRQ(ierr);
3326         ierr = DMPlexCreateTPSMesh_Internal(dm, tps_type, extent, periodic, tps_distribute, refine, layers, thickness);CHKERRQ(ierr);
3327       }
3328       break;
3329       default: SETERRQ(comm, PETSC_ERR_SUP, "Domain shape %s is unsupported", DMPlexShapes[shape]);
3330     }
3331   }
3332   ierr = DMPlexSetRefinementUniform(dm, PETSC_TRUE);CHKERRQ(ierr);
3333   if (!((PetscObject)dm)->name && nameflg) {
3334     ierr = PetscObjectSetName((PetscObject)dm, plexname);CHKERRQ(ierr);
3335   }
3336   PetscFunctionReturn(0);
3337 }
3338 
3339 PetscErrorCode DMSetFromOptions_NonRefinement_Plex(PetscOptionItems *PetscOptionsObject, DM dm)
3340 {
3341   DM_Plex       *mesh = (DM_Plex*) dm->data;
3342   PetscBool      flg;
3343   char           bdLabel[PETSC_MAX_PATH_LEN];
3344   PetscErrorCode ierr;
3345 
3346   PetscFunctionBegin;
3347   /* Handle viewing */
3348   ierr = PetscOptionsBool("-dm_plex_print_set_values", "Output all set values info", "DMPlexMatSetClosure", PETSC_FALSE, &mesh->printSetValues, NULL);CHKERRQ(ierr);
3349   ierr = PetscOptionsBoundedInt("-dm_plex_print_fem", "Debug output level all fem computations", "DMPlexSNESComputeResidualFEM", 0, &mesh->printFEM, NULL,0);CHKERRQ(ierr);
3350   ierr = PetscOptionsReal("-dm_plex_print_tol", "Tolerance for FEM output", "DMPlexSNESComputeResidualFEM", mesh->printTol, &mesh->printTol, NULL);CHKERRQ(ierr);
3351   ierr = PetscOptionsBoundedInt("-dm_plex_print_l2", "Debug output level all L2 diff computations", "DMComputeL2Diff", 0, &mesh->printL2, NULL,0);CHKERRQ(ierr);
3352   ierr = DMMonitorSetFromOptions(dm, "-dm_plex_monitor_throughput", "Monitor the simulation throughput", "DMPlexMonitorThroughput", DMPlexMonitorThroughput, NULL, &flg);CHKERRQ(ierr);
3353   if (flg) {ierr = PetscLogDefaultBegin();CHKERRQ(ierr);}
3354   /* Labeling */
3355   ierr = PetscOptionsString("-dm_plex_boundary_label", "Label to mark the mesh boundary", "", bdLabel, bdLabel, sizeof(bdLabel), &flg);CHKERRQ(ierr);
3356   if (flg) {ierr = DMPlexCreateBoundaryLabel_Private(dm, bdLabel);CHKERRQ(ierr);}
3357   /* Point Location */
3358   ierr = PetscOptionsBool("-dm_plex_hash_location", "Use grid hashing for point location", "DMInterpolate", PETSC_FALSE, &mesh->useHashLocation, NULL);CHKERRQ(ierr);
3359   /* Partitioning and distribution */
3360   ierr = PetscOptionsBool("-dm_plex_partition_balance", "Attempt to evenly divide points on partition boundary between processes", "DMPlexSetPartitionBalance", PETSC_FALSE, &mesh->partitionBalance, NULL);CHKERRQ(ierr);
3361   /* Generation and remeshing */
3362   ierr = PetscOptionsBool("-dm_plex_remesh_bd", "Allow changes to the boundary on remeshing", "DMAdapt", PETSC_FALSE, &mesh->remeshBd, NULL);CHKERRQ(ierr);
3363   /* Projection behavior */
3364   ierr = PetscOptionsBoundedInt("-dm_plex_max_projection_height", "Maxmimum mesh point height used to project locally", "DMPlexSetMaxProjectionHeight", 0, &mesh->maxProjectionHeight, NULL,0);CHKERRQ(ierr);
3365   ierr = PetscOptionsBool("-dm_plex_regular_refinement", "Use special nested projection algorithm for regular refinement", "DMPlexSetRegularRefinement", mesh->regularRefinement, &mesh->regularRefinement, NULL);CHKERRQ(ierr);
3366   /* Checking structure */
3367   {
3368     PetscBool   flg = PETSC_FALSE, flg2 = PETSC_FALSE, all = PETSC_FALSE;
3369 
3370     ierr = PetscOptionsBool("-dm_plex_check_all", "Perform all checks", NULL, PETSC_FALSE, &all, &flg2);CHKERRQ(ierr);
3371     ierr = PetscOptionsBool("-dm_plex_check_symmetry", "Check that the adjacency information in the mesh is symmetric", "DMPlexCheckSymmetry", PETSC_FALSE, &flg, &flg2);CHKERRQ(ierr);
3372     if (all || (flg && flg2)) {ierr = DMPlexCheckSymmetry(dm);CHKERRQ(ierr);}
3373     ierr = PetscOptionsBool("-dm_plex_check_skeleton", "Check that each cell has the correct number of vertices (only for homogeneous simplex or tensor meshes)", "DMPlexCheckSkeleton", PETSC_FALSE, &flg, &flg2);CHKERRQ(ierr);
3374     if (all || (flg && flg2)) {ierr = DMPlexCheckSkeleton(dm, 0);CHKERRQ(ierr);}
3375     ierr = PetscOptionsBool("-dm_plex_check_faces", "Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type", "DMPlexCheckFaces", PETSC_FALSE, &flg, &flg2);CHKERRQ(ierr);
3376     if (all || (flg && flg2)) {ierr = DMPlexCheckFaces(dm, 0);CHKERRQ(ierr);}
3377     ierr = PetscOptionsBool("-dm_plex_check_geometry", "Check that cells have positive volume", "DMPlexCheckGeometry", PETSC_FALSE, &flg, &flg2);CHKERRQ(ierr);
3378     if (all || (flg && flg2)) {ierr = DMPlexCheckGeometry(dm);CHKERRQ(ierr);}
3379     ierr = PetscOptionsBool("-dm_plex_check_pointsf", "Check some necessary conditions for PointSF", "DMPlexCheckPointSF", PETSC_FALSE, &flg, &flg2);CHKERRQ(ierr);
3380     if (all || (flg && flg2)) {ierr = DMPlexCheckPointSF(dm);CHKERRQ(ierr);}
3381     ierr = PetscOptionsBool("-dm_plex_check_interface_cones", "Check points on inter-partition interfaces have conforming order of cone points", "DMPlexCheckInterfaceCones", PETSC_FALSE, &flg, &flg2);CHKERRQ(ierr);
3382     if (all || (flg && flg2)) {ierr = DMPlexCheckInterfaceCones(dm);CHKERRQ(ierr);}
3383     ierr = PetscOptionsBool("-dm_plex_check_cell_shape", "Check cell shape", "DMPlexCheckCellShape", PETSC_FALSE, &flg, &flg2);CHKERRQ(ierr);
3384     if (flg && flg2) {ierr = DMPlexCheckCellShape(dm, PETSC_TRUE, PETSC_DETERMINE);CHKERRQ(ierr);}
3385   }
3386   {
3387     PetscReal scale = 1.0;
3388 
3389     ierr = PetscOptionsReal("-dm_plex_scale", "Scale factor for mesh coordinates", "DMPlexScale", scale, &scale, &flg);CHKERRQ(ierr);
3390     if (flg) {
3391       Vec coordinates, coordinatesLocal;
3392 
3393       ierr = DMGetCoordinates(dm, &coordinates);CHKERRQ(ierr);
3394       ierr = DMGetCoordinatesLocal(dm, &coordinatesLocal);CHKERRQ(ierr);
3395       ierr = VecScale(coordinates, scale);CHKERRQ(ierr);
3396       ierr = VecScale(coordinatesLocal, scale);CHKERRQ(ierr);
3397     }
3398   }
3399   ierr = PetscPartitionerSetFromOptions(mesh->partitioner);CHKERRQ(ierr);
3400   PetscFunctionReturn(0);
3401 }
3402 
3403 static PetscErrorCode DMSetFromOptions_Plex(PetscOptionItems *PetscOptionsObject,DM dm)
3404 {
3405   PetscFunctionList ordlist;
3406   char              oname[256];
3407   PetscReal         volume = -1.0;
3408   PetscInt          prerefine = 0, refine = 0, r, coarsen = 0, overlap = 0, extLayers = 0, dim;
3409   PetscBool         uniformOrig, created = PETSC_FALSE, uniform = PETSC_TRUE, distribute, interpolate = PETSC_TRUE, coordSpace = PETSC_TRUE, remap = PETSC_TRUE, ghostCells = PETSC_FALSE, isHierarchy, ignoreModel = PETSC_FALSE, flg;
3410   PetscErrorCode ierr;
3411 
3412   PetscFunctionBegin;
3413   PetscValidHeaderSpecific(dm, DM_CLASSID, 2);
3414   ierr = PetscOptionsHead(PetscOptionsObject,"DMPlex Options");CHKERRQ(ierr);
3415   /* Handle automatic creation */
3416   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
3417   if (dim < 0) {ierr = DMPlexCreateFromOptions_Internal(PetscOptionsObject, &coordSpace, dm);CHKERRQ(ierr);created = PETSC_TRUE;}
3418   /* Handle interpolation before distribution */
3419   ierr = PetscOptionsBool("-dm_plex_interpolate_pre", "Flag to interpolate mesh before distribution", "", interpolate, &interpolate, &flg);CHKERRQ(ierr);
3420   if (flg) {
3421     DMPlexInterpolatedFlag interpolated;
3422 
3423     ierr = DMPlexIsInterpolated(dm, &interpolated);CHKERRQ(ierr);
3424     if (interpolated == DMPLEX_INTERPOLATED_FULL && !interpolate) {
3425       DM udm;
3426 
3427       ierr = DMPlexUninterpolate(dm, &udm);CHKERRQ(ierr);
3428       ierr = DMPlexReplace_Static(dm, &udm);CHKERRQ(ierr);
3429     } else if (interpolated != DMPLEX_INTERPOLATED_FULL && interpolate) {
3430       DM idm;
3431 
3432       ierr = DMPlexInterpolate(dm, &idm);CHKERRQ(ierr);
3433       ierr = DMPlexReplace_Static(dm, &idm);CHKERRQ(ierr);
3434     }
3435   }
3436   /* Handle DMPlex refinement before distribution */
3437   ierr = PetscOptionsBool("-dm_refine_ignore_model", "Flag to ignore the geometry model when refining", "DMCreate", ignoreModel, &ignoreModel, &flg);CHKERRQ(ierr);
3438   if (flg) {((DM_Plex *) dm->data)->ignoreModel = ignoreModel;}
3439   ierr = DMPlexGetRefinementUniform(dm, &uniformOrig);CHKERRQ(ierr);
3440   ierr = PetscOptionsBoundedInt("-dm_refine_pre", "The number of refinements before distribution", "DMCreate", prerefine, &prerefine, NULL,0);CHKERRQ(ierr);
3441   ierr = PetscOptionsBool("-dm_refine_remap_pre", "Flag to control coordinate remapping", "DMCreate", remap, &remap, NULL);CHKERRQ(ierr);
3442   ierr = PetscOptionsBool("-dm_refine_uniform_pre", "Flag for uniform refinement before distribution", "DMCreate", uniform, &uniform, &flg);CHKERRQ(ierr);
3443   if (flg) {ierr = DMPlexSetRefinementUniform(dm, uniform);CHKERRQ(ierr);}
3444   ierr = PetscOptionsReal("-dm_refine_volume_limit_pre", "The maximum cell volume after refinement before distribution", "DMCreate", volume, &volume, &flg);CHKERRQ(ierr);
3445   if (flg) {
3446     ierr = DMPlexSetRefinementUniform(dm, PETSC_FALSE);CHKERRQ(ierr);
3447     ierr = DMPlexSetRefinementLimit(dm, volume);CHKERRQ(ierr);
3448     prerefine = PetscMax(prerefine, 1);
3449   }
3450   for (r = 0; r < prerefine; ++r) {
3451     DM             rdm;
3452     PetscPointFunc coordFunc = ((DM_Plex*) dm->data)->coordFunc;
3453 
3454     ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dm);CHKERRQ(ierr);
3455     ierr = DMRefine(dm, PetscObjectComm((PetscObject) dm), &rdm);CHKERRQ(ierr);
3456     ierr = DMPlexReplace_Static(dm, &rdm);CHKERRQ(ierr);
3457     ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dm);CHKERRQ(ierr);
3458     if (coordFunc && remap) {
3459       ierr = DMPlexRemapGeometry(dm, 0.0, coordFunc);CHKERRQ(ierr);
3460       ((DM_Plex*) dm->data)->coordFunc = coordFunc;
3461     }
3462   }
3463   ierr = DMPlexSetRefinementUniform(dm, uniformOrig);CHKERRQ(ierr);
3464   /* Handle DMPlex extrusion before distribution */
3465   ierr = PetscOptionsBoundedInt("-dm_extrude", "The number of layers to extrude", "", extLayers, &extLayers, NULL, 0);CHKERRQ(ierr);
3466   if (extLayers) {
3467     DM edm;
3468 
3469     ierr = DMExtrude(dm, extLayers, &edm);CHKERRQ(ierr);
3470     ierr = DMPlexReplace_Static(dm, &edm);CHKERRQ(ierr);
3471     ((DM_Plex *) dm->data)->coordFunc = NULL;
3472     ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dm);CHKERRQ(ierr);
3473     extLayers = 0;
3474   }
3475   /* Handle DMPlex reordering before distribution */
3476   ierr = MatGetOrderingList(&ordlist);CHKERRQ(ierr);
3477   ierr = PetscOptionsFList("-dm_plex_reorder", "Set mesh reordering type", "DMPlexGetOrdering", ordlist, MATORDERINGNATURAL, oname, sizeof(oname), &flg);CHKERRQ(ierr);
3478   if (flg) {
3479     DM pdm;
3480     IS perm;
3481 
3482     ierr = DMPlexGetOrdering(dm, oname, NULL, &perm);CHKERRQ(ierr);
3483     ierr = DMPlexPermute(dm, perm, &pdm);CHKERRQ(ierr);
3484     ierr = ISDestroy(&perm);CHKERRQ(ierr);
3485     ierr = DMPlexReplace_Static(dm, &pdm);CHKERRQ(ierr);
3486     ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dm);CHKERRQ(ierr);
3487   }
3488   /* Handle DMPlex distribution */
3489   ierr = DMPlexDistributeGetDefault(dm, &distribute);CHKERRQ(ierr);
3490   ierr = PetscOptionsBool("-dm_distribute", "Flag to redistribute a mesh among processes", "DMCreate", distribute, &distribute, NULL);CHKERRQ(ierr);
3491   ierr = PetscOptionsBoundedInt("-dm_distribute_overlap", "The size of the overlap halo", "DMCreate", overlap, &overlap, NULL, 0);CHKERRQ(ierr);
3492   if (distribute) {
3493     DM               pdm = NULL;
3494     PetscPartitioner part;
3495 
3496     ierr = DMPlexGetPartitioner(dm, &part);CHKERRQ(ierr);
3497     ierr = PetscPartitionerSetFromOptions(part);CHKERRQ(ierr);
3498     ierr = DMPlexDistribute(dm, overlap, NULL, &pdm);CHKERRQ(ierr);
3499     if (pdm) {
3500       ierr = DMPlexReplace_Static(dm, &pdm);CHKERRQ(ierr);
3501     }
3502   }
3503   /* Create coordinate space */
3504   if (created) {
3505     DM_Plex  *mesh = (DM_Plex *) dm->data;
3506     PetscInt  degree = 1;
3507     PetscBool periodic, flg;
3508 
3509     ierr = PetscOptionsBool("-dm_coord_space", "Use an FEM space for coordinates", "", coordSpace, &coordSpace, &flg);CHKERRQ(ierr);
3510     ierr = PetscOptionsInt("-dm_coord_petscspace_degree", "FEM degree for coordinate space", "", degree, &degree, NULL);CHKERRQ(ierr);
3511     if (coordSpace) {ierr = DMPlexCreateCoordinateSpace(dm, degree, mesh->coordFunc);CHKERRQ(ierr);}
3512     if (flg && !coordSpace) {
3513       DM           cdm;
3514       PetscDS      cds;
3515       PetscObject  obj;
3516       PetscClassId id;
3517 
3518       ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
3519       ierr = DMGetDS(cdm, &cds);CHKERRQ(ierr);
3520       ierr = PetscDSGetDiscretization(cds, 0, &obj);CHKERRQ(ierr);
3521       ierr = PetscObjectGetClassId(obj, &id);CHKERRQ(ierr);
3522       if (id == PETSCFE_CLASSID) {
3523         PetscContainer dummy;
3524 
3525         ierr = PetscContainerCreate(PETSC_COMM_SELF, &dummy);CHKERRQ(ierr);
3526         ierr = PetscObjectSetName((PetscObject) dummy, "coordinates");CHKERRQ(ierr);
3527         ierr = DMSetField(cdm, 0, NULL, (PetscObject) dummy);CHKERRQ(ierr);
3528         ierr = PetscContainerDestroy(&dummy);CHKERRQ(ierr);
3529         ierr = DMClearDS(cdm);CHKERRQ(ierr);
3530       }
3531       mesh->coordFunc = NULL;
3532     }
3533     ierr = DMLocalizeCoordinates(dm);CHKERRQ(ierr);
3534     ierr = DMGetPeriodicity(dm, &periodic, NULL, NULL, NULL);CHKERRQ(ierr);
3535     if (periodic) {ierr = DMSetPeriodicity(dm, PETSC_TRUE, NULL, NULL, NULL);CHKERRQ(ierr);}
3536   }
3537   /* Handle DMPlex refinement */
3538   remap = PETSC_TRUE;
3539   ierr = PetscOptionsBoundedInt("-dm_refine", "The number of uniform refinements", "DMCreate", refine, &refine, NULL,0);CHKERRQ(ierr);
3540   ierr = PetscOptionsBool("-dm_refine_remap", "Flag to control coordinate remapping", "DMCreate", remap, &remap, NULL);CHKERRQ(ierr);
3541   ierr = PetscOptionsBoundedInt("-dm_refine_hierarchy", "The number of uniform refinements", "DMCreate", refine, &refine, &isHierarchy,0);CHKERRQ(ierr);
3542   if (refine) {ierr = DMPlexSetRefinementUniform(dm, PETSC_TRUE);CHKERRQ(ierr);}
3543   if (refine && isHierarchy) {
3544     DM *dms, coarseDM;
3545 
3546     ierr = DMGetCoarseDM(dm, &coarseDM);CHKERRQ(ierr);
3547     ierr = PetscObjectReference((PetscObject)coarseDM);CHKERRQ(ierr);
3548     ierr = PetscMalloc1(refine,&dms);CHKERRQ(ierr);
3549     ierr = DMRefineHierarchy(dm, refine, dms);CHKERRQ(ierr);
3550     /* Total hack since we do not pass in a pointer */
3551     ierr = DMPlexSwap_Static(dm, dms[refine-1]);CHKERRQ(ierr);
3552     if (refine == 1) {
3553       ierr = DMSetCoarseDM(dm, dms[0]);CHKERRQ(ierr);
3554       ierr = DMPlexSetRegularRefinement(dm, PETSC_TRUE);CHKERRQ(ierr);
3555     } else {
3556       ierr = DMSetCoarseDM(dm, dms[refine-2]);CHKERRQ(ierr);
3557       ierr = DMPlexSetRegularRefinement(dm, PETSC_TRUE);CHKERRQ(ierr);
3558       ierr = DMSetCoarseDM(dms[0], dms[refine-1]);CHKERRQ(ierr);
3559       ierr = DMPlexSetRegularRefinement(dms[0], PETSC_TRUE);CHKERRQ(ierr);
3560     }
3561     ierr = DMSetCoarseDM(dms[refine-1], coarseDM);CHKERRQ(ierr);
3562     ierr = PetscObjectDereference((PetscObject)coarseDM);CHKERRQ(ierr);
3563     /* Free DMs */
3564     for (r = 0; r < refine; ++r) {
3565       ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dms[r]);CHKERRQ(ierr);
3566       ierr = DMDestroy(&dms[r]);CHKERRQ(ierr);
3567     }
3568     ierr = PetscFree(dms);CHKERRQ(ierr);
3569   } else {
3570     for (r = 0; r < refine; ++r) {
3571       DM             rdm;
3572       PetscPointFunc coordFunc = ((DM_Plex*) dm->data)->coordFunc;
3573 
3574       ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dm);CHKERRQ(ierr);
3575       ierr = DMRefine(dm, PetscObjectComm((PetscObject) dm), &rdm);CHKERRQ(ierr);
3576       /* Total hack since we do not pass in a pointer */
3577       ierr = DMPlexReplace_Static(dm, &rdm);CHKERRQ(ierr);
3578       ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dm);CHKERRQ(ierr);
3579       if (coordFunc && remap) {
3580         ierr = DMPlexRemapGeometry(dm, 0.0, coordFunc);CHKERRQ(ierr);
3581         ((DM_Plex*) dm->data)->coordFunc = coordFunc;
3582       }
3583     }
3584   }
3585   /* Handle DMPlex coarsening */
3586   ierr = PetscOptionsBoundedInt("-dm_coarsen", "Coarsen the mesh", "DMCreate", coarsen, &coarsen, NULL,0);CHKERRQ(ierr);
3587   ierr = PetscOptionsBoundedInt("-dm_coarsen_hierarchy", "The number of coarsenings", "DMCreate", coarsen, &coarsen, &isHierarchy,0);CHKERRQ(ierr);
3588   if (coarsen && isHierarchy) {
3589     DM *dms;
3590 
3591     ierr = PetscMalloc1(coarsen, &dms);CHKERRQ(ierr);
3592     ierr = DMCoarsenHierarchy(dm, coarsen, dms);CHKERRQ(ierr);
3593     /* Free DMs */
3594     for (r = 0; r < coarsen; ++r) {
3595       ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dms[r]);CHKERRQ(ierr);
3596       ierr = DMDestroy(&dms[r]);CHKERRQ(ierr);
3597     }
3598     ierr = PetscFree(dms);CHKERRQ(ierr);
3599   } else {
3600     for (r = 0; r < coarsen; ++r) {
3601       DM             cdm;
3602       PetscPointFunc coordFunc = ((DM_Plex*) dm->data)->coordFunc;
3603 
3604       ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dm);CHKERRQ(ierr);
3605       ierr = DMCoarsen(dm, PetscObjectComm((PetscObject) dm), &cdm);CHKERRQ(ierr);
3606       /* Total hack since we do not pass in a pointer */
3607       ierr = DMPlexReplace_Static(dm, &cdm);CHKERRQ(ierr);
3608       ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dm);CHKERRQ(ierr);
3609       if (coordFunc) {
3610         ierr = DMPlexRemapGeometry(dm, 0.0, coordFunc);CHKERRQ(ierr);
3611         ((DM_Plex*) dm->data)->coordFunc = coordFunc;
3612       }
3613     }
3614   }
3615   /* Handle ghost cells */
3616   ierr = PetscOptionsBool("-dm_plex_create_fv_ghost_cells", "Flag to create finite volume ghost cells on the boundary", "DMCreate", ghostCells, &ghostCells, NULL);CHKERRQ(ierr);
3617   if (ghostCells) {
3618     DM   gdm;
3619     char lname[PETSC_MAX_PATH_LEN];
3620 
3621     lname[0] = '\0';
3622     ierr = PetscOptionsString("-dm_plex_fv_ghost_cells_label", "Label name for ghost cells boundary", "DMCreate", lname, lname, sizeof(lname), &flg);CHKERRQ(ierr);
3623     ierr = DMPlexConstructGhostCells(dm, flg ? lname : NULL, NULL, &gdm);CHKERRQ(ierr);
3624     ierr = DMPlexReplace_Static(dm, &gdm);CHKERRQ(ierr);
3625   }
3626   /* Handle 1D order */
3627   {
3628     DM           cdm, rdm;
3629     PetscDS      cds;
3630     PetscObject  obj;
3631     PetscClassId id = PETSC_OBJECT_CLASSID;
3632     IS           perm;
3633     PetscInt     dim, Nf;
3634     PetscBool    distributed;
3635 
3636     ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
3637     ierr = DMPlexIsDistributed(dm, &distributed);CHKERRQ(ierr);
3638     ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
3639     ierr = DMGetDS(cdm, &cds);CHKERRQ(ierr);
3640     ierr = PetscDSGetNumFields(cds, &Nf);CHKERRQ(ierr);
3641     if (Nf) {
3642       ierr = PetscDSGetDiscretization(cds, 0, &obj);CHKERRQ(ierr);
3643       ierr = PetscObjectGetClassId(obj, &id);CHKERRQ(ierr);
3644     }
3645     if (dim == 1 && !distributed && id != PETSCFE_CLASSID) {
3646       ierr = DMPlexGetOrdering1D(dm, &perm);CHKERRQ(ierr);
3647       ierr = DMPlexPermute(dm, perm, &rdm);CHKERRQ(ierr);
3648       ierr = DMPlexReplace_Static(dm, &rdm);CHKERRQ(ierr);
3649       ierr = ISDestroy(&perm);CHKERRQ(ierr);
3650     }
3651   }
3652   /* Handle */
3653   ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dm);CHKERRQ(ierr);
3654   ierr = PetscOptionsTail();CHKERRQ(ierr);
3655   PetscFunctionReturn(0);
3656 }
3657 
3658 static PetscErrorCode DMCreateGlobalVector_Plex(DM dm,Vec *vec)
3659 {
3660   PetscErrorCode ierr;
3661 
3662   PetscFunctionBegin;
3663   ierr = DMCreateGlobalVector_Section_Private(dm,vec);CHKERRQ(ierr);
3664   /* ierr = VecSetOperation(*vec, VECOP_DUPLICATE, (void(*)(void)) VecDuplicate_MPI_DM);CHKERRQ(ierr); */
3665   ierr = VecSetOperation(*vec, VECOP_VIEW, (void (*)(void)) VecView_Plex);CHKERRQ(ierr);
3666   ierr = VecSetOperation(*vec, VECOP_VIEWNATIVE, (void (*)(void)) VecView_Plex_Native);CHKERRQ(ierr);
3667   ierr = VecSetOperation(*vec, VECOP_LOAD, (void (*)(void)) VecLoad_Plex);CHKERRQ(ierr);
3668   ierr = VecSetOperation(*vec, VECOP_LOADNATIVE, (void (*)(void)) VecLoad_Plex_Native);CHKERRQ(ierr);
3669   PetscFunctionReturn(0);
3670 }
3671 
3672 static PetscErrorCode DMCreateLocalVector_Plex(DM dm,Vec *vec)
3673 {
3674   PetscErrorCode ierr;
3675 
3676   PetscFunctionBegin;
3677   ierr = DMCreateLocalVector_Section_Private(dm,vec);CHKERRQ(ierr);
3678   ierr = VecSetOperation(*vec, VECOP_VIEW, (void (*)(void)) VecView_Plex_Local);CHKERRQ(ierr);
3679   ierr = VecSetOperation(*vec, VECOP_LOAD, (void (*)(void)) VecLoad_Plex_Local);CHKERRQ(ierr);
3680   PetscFunctionReturn(0);
3681 }
3682 
3683 static PetscErrorCode DMGetDimPoints_Plex(DM dm, PetscInt dim, PetscInt *pStart, PetscInt *pEnd)
3684 {
3685   PetscInt       depth, d;
3686   PetscErrorCode ierr;
3687 
3688   PetscFunctionBegin;
3689   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3690   if (depth == 1) {
3691     ierr = DMGetDimension(dm, &d);CHKERRQ(ierr);
3692     if (dim == 0)      {ierr = DMPlexGetDepthStratum(dm, dim, pStart, pEnd);CHKERRQ(ierr);}
3693     else if (dim == d) {ierr = DMPlexGetDepthStratum(dm, 1, pStart, pEnd);CHKERRQ(ierr);}
3694     else               {*pStart = 0; *pEnd = 0;}
3695   } else {
3696     ierr = DMPlexGetDepthStratum(dm, dim, pStart, pEnd);CHKERRQ(ierr);
3697   }
3698   PetscFunctionReturn(0);
3699 }
3700 
3701 static PetscErrorCode DMGetNeighbors_Plex(DM dm, PetscInt *nranks, const PetscMPIInt *ranks[])
3702 {
3703   PetscSF           sf;
3704   PetscInt          niranks, njranks, n;
3705   const PetscMPIInt *iranks, *jranks;
3706   DM_Plex           *data = (DM_Plex*) dm->data;
3707   PetscErrorCode    ierr;
3708 
3709   PetscFunctionBegin;
3710   ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr);
3711   if (!data->neighbors) {
3712     ierr = PetscSFSetUp(sf);CHKERRQ(ierr);
3713     ierr = PetscSFGetRootRanks(sf, &njranks, &jranks, NULL, NULL, NULL);CHKERRQ(ierr);
3714     ierr = PetscSFGetLeafRanks(sf, &niranks, &iranks, NULL, NULL);CHKERRQ(ierr);
3715     ierr = PetscMalloc1(njranks + niranks + 1, &data->neighbors);CHKERRQ(ierr);
3716     ierr = PetscArraycpy(data->neighbors + 1, jranks, njranks);CHKERRQ(ierr);
3717     ierr = PetscArraycpy(data->neighbors + njranks + 1, iranks, niranks);CHKERRQ(ierr);
3718     n = njranks + niranks;
3719     ierr = PetscSortRemoveDupsMPIInt(&n, data->neighbors + 1);CHKERRQ(ierr);
3720     /* The following cast should never fail: can't have more neighbors than PETSC_MPI_INT_MAX */
3721     ierr = PetscMPIIntCast(n, data->neighbors);CHKERRQ(ierr);
3722   }
3723   if (nranks) *nranks = data->neighbors[0];
3724   if (ranks) {
3725     if (data->neighbors[0]) *ranks = data->neighbors + 1;
3726     else                    *ranks = NULL;
3727   }
3728   PetscFunctionReturn(0);
3729 }
3730 
3731 PETSC_INTERN PetscErrorCode DMInterpolateSolution_Plex(DM, DM, Mat, Vec, Vec);
3732 
3733 static PetscErrorCode DMInitialize_Plex(DM dm)
3734 {
3735   PetscErrorCode ierr;
3736 
3737   PetscFunctionBegin;
3738   dm->ops->view                            = DMView_Plex;
3739   dm->ops->load                            = DMLoad_Plex;
3740   dm->ops->setfromoptions                  = DMSetFromOptions_Plex;
3741   dm->ops->clone                           = DMClone_Plex;
3742   dm->ops->setup                           = DMSetUp_Plex;
3743   dm->ops->createlocalsection              = DMCreateLocalSection_Plex;
3744   dm->ops->createdefaultconstraints        = DMCreateDefaultConstraints_Plex;
3745   dm->ops->createglobalvector              = DMCreateGlobalVector_Plex;
3746   dm->ops->createlocalvector               = DMCreateLocalVector_Plex;
3747   dm->ops->getlocaltoglobalmapping         = NULL;
3748   dm->ops->createfieldis                   = NULL;
3749   dm->ops->createcoordinatedm              = DMCreateCoordinateDM_Plex;
3750   dm->ops->createcoordinatefield           = DMCreateCoordinateField_Plex;
3751   dm->ops->getcoloring                     = NULL;
3752   dm->ops->creatematrix                    = DMCreateMatrix_Plex;
3753   dm->ops->createinterpolation             = DMCreateInterpolation_Plex;
3754   dm->ops->createmassmatrix                = DMCreateMassMatrix_Plex;
3755   dm->ops->createmassmatrixlumped          = DMCreateMassMatrixLumped_Plex;
3756   dm->ops->createinjection                 = DMCreateInjection_Plex;
3757   dm->ops->refine                          = DMRefine_Plex;
3758   dm->ops->coarsen                         = DMCoarsen_Plex;
3759   dm->ops->refinehierarchy                 = DMRefineHierarchy_Plex;
3760   dm->ops->coarsenhierarchy                = DMCoarsenHierarchy_Plex;
3761   dm->ops->extrude                         = DMExtrude_Plex;
3762   dm->ops->globaltolocalbegin              = NULL;
3763   dm->ops->globaltolocalend                = NULL;
3764   dm->ops->localtoglobalbegin              = NULL;
3765   dm->ops->localtoglobalend                = NULL;
3766   dm->ops->destroy                         = DMDestroy_Plex;
3767   dm->ops->createsubdm                     = DMCreateSubDM_Plex;
3768   dm->ops->createsuperdm                   = DMCreateSuperDM_Plex;
3769   dm->ops->getdimpoints                    = DMGetDimPoints_Plex;
3770   dm->ops->locatepoints                    = DMLocatePoints_Plex;
3771   dm->ops->projectfunctionlocal            = DMProjectFunctionLocal_Plex;
3772   dm->ops->projectfunctionlabellocal       = DMProjectFunctionLabelLocal_Plex;
3773   dm->ops->projectfieldlocal               = DMProjectFieldLocal_Plex;
3774   dm->ops->projectfieldlabellocal          = DMProjectFieldLabelLocal_Plex;
3775   dm->ops->projectbdfieldlabellocal        = DMProjectBdFieldLabelLocal_Plex;
3776   dm->ops->computel2diff                   = DMComputeL2Diff_Plex;
3777   dm->ops->computel2gradientdiff           = DMComputeL2GradientDiff_Plex;
3778   dm->ops->computel2fielddiff              = DMComputeL2FieldDiff_Plex;
3779   dm->ops->getneighbors                    = DMGetNeighbors_Plex;
3780   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMPlexInsertBoundaryValues_C",DMPlexInsertBoundaryValues_Plex);CHKERRQ(ierr);
3781   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMPlexInsertTimeDerviativeBoundaryValues_C",DMPlexInsertTimeDerivativeBoundaryValues_Plex);CHKERRQ(ierr);
3782   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMSetUpGLVisViewer_C",DMSetUpGLVisViewer_Plex);CHKERRQ(ierr);
3783   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMCreateNeumannOverlap_C",DMCreateNeumannOverlap_Plex);CHKERRQ(ierr);
3784   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMPlexGetOverlap_C",DMPlexGetOverlap_Plex);CHKERRQ(ierr);
3785   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMPlexDistributeGetDefault_C",DMPlexDistributeGetDefault_Plex);CHKERRQ(ierr);
3786   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMPlexDistributeSetDefault_C",DMPlexDistributeSetDefault_Plex);CHKERRQ(ierr);
3787   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMInterpolateSolution_C",DMInterpolateSolution_Plex);CHKERRQ(ierr);
3788   PetscFunctionReturn(0);
3789 }
3790 
3791 PETSC_INTERN PetscErrorCode DMClone_Plex(DM dm, DM *newdm)
3792 {
3793   DM_Plex        *mesh = (DM_Plex *) dm->data;
3794   PetscErrorCode ierr;
3795 
3796   PetscFunctionBegin;
3797   mesh->refct++;
3798   (*newdm)->data = mesh;
3799   ierr = PetscObjectChangeTypeName((PetscObject) *newdm, DMPLEX);CHKERRQ(ierr);
3800   ierr = DMInitialize_Plex(*newdm);CHKERRQ(ierr);
3801   PetscFunctionReturn(0);
3802 }
3803 
3804 /*MC
3805   DMPLEX = "plex" - A DM object that encapsulates an unstructured mesh, or CW Complex, which can be expressed using a Hasse Diagram.
3806                     In the local representation, Vecs contain all unknowns in the interior and shared boundary. This is
3807                     specified by a PetscSection object. Ownership in the global representation is determined by
3808                     ownership of the underlying DMPlex points. This is specified by another PetscSection object.
3809 
3810   Options Database Keys:
3811 + -dm_refine_pre                     - Refine mesh before distribution
3812 + -dm_refine_uniform_pre             - Choose uniform or generator-based refinement
3813 + -dm_refine_volume_limit_pre        - Cell volume limit after pre-refinement using generator
3814 . -dm_distribute                     - Distribute mesh across processes
3815 . -dm_distribute_overlap             - Number of cells to overlap for distribution
3816 . -dm_refine                         - Refine mesh after distribution
3817 . -dm_plex_hash_location             - Use grid hashing for point location
3818 . -dm_plex_hash_box_faces <n,m,p>    - The number of divisions in each direction of the grid hash
3819 . -dm_plex_partition_balance         - Attempt to evenly divide points on partition boundary between processes
3820 . -dm_plex_remesh_bd                 - Allow changes to the boundary on remeshing
3821 . -dm_plex_max_projection_height     - Maxmimum mesh point height used to project locally
3822 . -dm_plex_regular_refinement        - Use special nested projection algorithm for regular refinement
3823 . -dm_plex_check_all                 - Perform all shecks below
3824 . -dm_plex_check_symmetry            - Check that the adjacency information in the mesh is symmetric
3825 . -dm_plex_check_skeleton <celltype> - Check that each cell has the correct number of vertices
3826 . -dm_plex_check_faces <celltype>    - Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type
3827 . -dm_plex_check_geometry            - Check that cells have positive volume
3828 . -dm_view :mesh.tex:ascii_latex     - View the mesh in LaTeX/TikZ
3829 . -dm_plex_view_scale <num>          - Scale the TikZ
3830 - -dm_plex_print_fem <num>           - View FEM assembly information, such as element vectors and matrices
3831 
3832   Level: intermediate
3833 
3834 .seealso: DMType, DMPlexCreate(), DMCreate(), DMSetType()
3835 M*/
3836 
3837 PETSC_EXTERN PetscErrorCode DMCreate_Plex(DM dm)
3838 {
3839   DM_Plex       *mesh;
3840   PetscInt       unit;
3841   PetscErrorCode ierr;
3842 
3843   PetscFunctionBegin;
3844   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3845   ierr     = PetscNewLog(dm,&mesh);CHKERRQ(ierr);
3846   dm->data = mesh;
3847 
3848   mesh->refct             = 1;
3849   ierr                    = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &mesh->coneSection);CHKERRQ(ierr);
3850   mesh->maxConeSize       = 0;
3851   mesh->cones             = NULL;
3852   mesh->coneOrientations  = NULL;
3853   ierr                    = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &mesh->supportSection);CHKERRQ(ierr);
3854   mesh->maxSupportSize    = 0;
3855   mesh->supports          = NULL;
3856   mesh->refinementUniform = PETSC_TRUE;
3857   mesh->refinementLimit   = -1.0;
3858   mesh->distDefault       = PETSC_TRUE;
3859   mesh->interpolated      = DMPLEX_INTERPOLATED_INVALID;
3860   mesh->interpolatedCollective = DMPLEX_INTERPOLATED_INVALID;
3861 
3862   mesh->facesTmp = NULL;
3863 
3864   mesh->tetgenOpts   = NULL;
3865   mesh->triangleOpts = NULL;
3866   ierr = PetscPartitionerCreate(PetscObjectComm((PetscObject)dm), &mesh->partitioner);CHKERRQ(ierr);
3867   mesh->remeshBd     = PETSC_FALSE;
3868 
3869   mesh->subpointMap = NULL;
3870 
3871   for (unit = 0; unit < NUM_PETSC_UNITS; ++unit) mesh->scale[unit] = 1.0;
3872 
3873   mesh->regularRefinement   = PETSC_FALSE;
3874   mesh->depthState          = -1;
3875   mesh->celltypeState       = -1;
3876   mesh->globalVertexNumbers = NULL;
3877   mesh->globalCellNumbers   = NULL;
3878   mesh->anchorSection       = NULL;
3879   mesh->anchorIS            = NULL;
3880   mesh->createanchors       = NULL;
3881   mesh->computeanchormatrix = NULL;
3882   mesh->parentSection       = NULL;
3883   mesh->parents             = NULL;
3884   mesh->childIDs            = NULL;
3885   mesh->childSection        = NULL;
3886   mesh->children            = NULL;
3887   mesh->referenceTree       = NULL;
3888   mesh->getchildsymmetry    = NULL;
3889   mesh->vtkCellHeight       = 0;
3890   mesh->useAnchors          = PETSC_FALSE;
3891 
3892   mesh->maxProjectionHeight = 0;
3893 
3894   mesh->neighbors           = NULL;
3895 
3896   mesh->printSetValues = PETSC_FALSE;
3897   mesh->printFEM       = 0;
3898   mesh->printTol       = 1.0e-10;
3899 
3900   ierr = DMInitialize_Plex(dm);CHKERRQ(ierr);
3901   PetscFunctionReturn(0);
3902 }
3903 
3904 /*@
3905   DMPlexCreate - Creates a DMPlex object, which encapsulates an unstructured mesh, or CW complex, which can be expressed using a Hasse Diagram.
3906 
3907   Collective
3908 
3909   Input Parameter:
3910 . comm - The communicator for the DMPlex object
3911 
3912   Output Parameter:
3913 . mesh  - The DMPlex object
3914 
3915   Level: beginner
3916 
3917 @*/
3918 PetscErrorCode DMPlexCreate(MPI_Comm comm, DM *mesh)
3919 {
3920   PetscErrorCode ierr;
3921 
3922   PetscFunctionBegin;
3923   PetscValidPointer(mesh,2);
3924   ierr = DMCreate(comm, mesh);CHKERRQ(ierr);
3925   ierr = DMSetType(*mesh, DMPLEX);CHKERRQ(ierr);
3926   PetscFunctionReturn(0);
3927 }
3928 
3929 /*@C
3930   DMPlexBuildFromCellListParallel - Build distributed DMPLEX topology from a list of vertices for each cell (common mesh generator output)
3931 
3932   Input Parameters:
3933 + dm - The DM
3934 . numCells - The number of cells owned by this process
3935 . numVertices - The number of vertices to be owned by this process, or PETSC_DECIDE
3936 . NVertices - The global number of vertices, or PETSC_DETERMINE
3937 . numCorners - The number of vertices for each cell
3938 - cells - An array of numCells*numCorners numbers, the global vertex numbers for each cell
3939 
3940   Output Parameters:
3941 + vertexSF - (Optional) SF describing complete vertex ownership
3942 - verticesAdjSaved - (Optional) vertex adjacency array
3943 
3944   Notes:
3945   Two triangles sharing a face
3946 $
3947 $        2
3948 $      / | \
3949 $     /  |  \
3950 $    /   |   \
3951 $   0  0 | 1  3
3952 $    \   |   /
3953 $     \  |  /
3954 $      \ | /
3955 $        1
3956 would have input
3957 $  numCells = 2, numVertices = 4
3958 $  cells = [0 1 2  1 3 2]
3959 $
3960 which would result in the DMPlex
3961 $
3962 $        4
3963 $      / | \
3964 $     /  |  \
3965 $    /   |   \
3966 $   2  0 | 1  5
3967 $    \   |   /
3968 $     \  |  /
3969 $      \ | /
3970 $        3
3971 
3972   Vertices are implicitly numbered consecutively 0,...,NVertices.
3973   Each rank owns a chunk of numVertices consecutive vertices.
3974   If numVertices is PETSC_DECIDE, PETSc will distribute them as evenly as possible using PetscLayout.
3975   If NVertices is PETSC_DETERMINE and numVertices is PETSC_DECIDE, NVertices is computed by PETSc as the maximum vertex index in cells + 1.
3976   If only NVertices is PETSC_DETERMINE, it is computed as the sum of numVertices over all ranks.
3977 
3978   The cell distribution is arbitrary non-overlapping, independent of the vertex distribution.
3979 
3980   Not currently supported in Fortran.
3981 
3982   Level: advanced
3983 
3984 .seealso: DMPlexBuildFromCellList(), DMPlexCreateFromCellListParallelPetsc(), DMPlexBuildCoordinatesFromCellListParallel()
3985 @*/
3986 PetscErrorCode DMPlexBuildFromCellListParallel(DM dm, PetscInt numCells, PetscInt numVertices, PetscInt NVertices, PetscInt numCorners, const PetscInt cells[], PetscSF *vertexSF, PetscInt **verticesAdjSaved)
3987 {
3988   PetscSF         sfPoint;
3989   PetscLayout     layout;
3990   PetscInt        numVerticesAdj, *verticesAdj, *cones, c, p;
3991   PetscErrorCode  ierr;
3992 
3993   PetscFunctionBegin;
3994   PetscValidLogicalCollectiveInt(dm,NVertices,4);
3995   ierr = PetscLogEventBegin(DMPLEX_BuildFromCellList,dm,0,0,0);CHKERRQ(ierr);
3996   /* Get/check global number of vertices */
3997   {
3998     PetscInt NVerticesInCells, i;
3999     const PetscInt len = numCells * numCorners;
4000 
4001     /* NVerticesInCells = max(cells) + 1 */
4002     NVerticesInCells = PETSC_MIN_INT;
4003     for (i=0; i<len; i++) if (cells[i] > NVerticesInCells) NVerticesInCells = cells[i];
4004     ++NVerticesInCells;
4005     ierr = MPI_Allreduce(MPI_IN_PLACE, &NVerticesInCells, 1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject) dm));CHKERRMPI(ierr);
4006 
4007     if (numVertices == PETSC_DECIDE && NVertices == PETSC_DECIDE) NVertices = NVerticesInCells;
4008     else PetscCheckFalse(NVertices != PETSC_DECIDE && NVertices < NVerticesInCells,PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Specified global number of vertices %D must be greater than or equal to the number of vertices in cells %D",NVertices,NVerticesInCells);
4009   }
4010   /* Count locally unique vertices */
4011   {
4012     PetscHSetI vhash;
4013     PetscInt off = 0;
4014 
4015     ierr = PetscHSetICreate(&vhash);CHKERRQ(ierr);
4016     for (c = 0; c < numCells; ++c) {
4017       for (p = 0; p < numCorners; ++p) {
4018         ierr = PetscHSetIAdd(vhash, cells[c*numCorners+p]);CHKERRQ(ierr);
4019       }
4020     }
4021     ierr = PetscHSetIGetSize(vhash, &numVerticesAdj);CHKERRQ(ierr);
4022     if (!verticesAdjSaved) { ierr = PetscMalloc1(numVerticesAdj, &verticesAdj);CHKERRQ(ierr); }
4023     else { verticesAdj = *verticesAdjSaved; }
4024     ierr = PetscHSetIGetElems(vhash, &off, verticesAdj);CHKERRQ(ierr);
4025     ierr = PetscHSetIDestroy(&vhash);CHKERRQ(ierr);
4026     PetscCheckFalse(off != numVerticesAdj,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid number of local vertices %D should be %D", off, numVerticesAdj);
4027   }
4028   ierr = PetscSortInt(numVerticesAdj, verticesAdj);CHKERRQ(ierr);
4029   /* Create cones */
4030   ierr = DMPlexSetChart(dm, 0, numCells+numVerticesAdj);CHKERRQ(ierr);
4031   for (c = 0; c < numCells; ++c) {ierr = DMPlexSetConeSize(dm, c, numCorners);CHKERRQ(ierr);}
4032   ierr = DMSetUp(dm);CHKERRQ(ierr);
4033   ierr = DMPlexGetCones(dm,&cones);CHKERRQ(ierr);
4034   for (c = 0; c < numCells; ++c) {
4035     for (p = 0; p < numCorners; ++p) {
4036       const PetscInt gv = cells[c*numCorners+p];
4037       PetscInt       lv;
4038 
4039       /* Positions within verticesAdj form 0-based local vertex numbering;
4040          we need to shift it by numCells to get correct DAG points (cells go first) */
4041       ierr = PetscFindInt(gv, numVerticesAdj, verticesAdj, &lv);CHKERRQ(ierr);
4042       PetscCheckFalse(lv < 0,PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Could not find global vertex %D in local connectivity", gv);
4043       cones[c*numCorners+p] = lv+numCells;
4044     }
4045   }
4046   /* Build point sf */
4047   ierr = PetscLayoutCreate(PetscObjectComm((PetscObject)dm), &layout);CHKERRQ(ierr);
4048   ierr = PetscLayoutSetSize(layout, NVertices);CHKERRQ(ierr);
4049   ierr = PetscLayoutSetLocalSize(layout, numVertices);CHKERRQ(ierr);
4050   ierr = PetscLayoutSetBlockSize(layout, 1);CHKERRQ(ierr);
4051   ierr = PetscSFCreateByMatchingIndices(layout, numVerticesAdj, verticesAdj, NULL, numCells, numVerticesAdj, verticesAdj, NULL, numCells, vertexSF, &sfPoint);CHKERRQ(ierr);
4052   ierr = PetscLayoutDestroy(&layout);CHKERRQ(ierr);
4053   if (!verticesAdjSaved) { ierr = PetscFree(verticesAdj);CHKERRQ(ierr); }
4054   ierr = PetscObjectSetName((PetscObject) sfPoint, "point SF");CHKERRQ(ierr);
4055   if (dm->sf) {
4056     const char *prefix;
4057 
4058     ierr = PetscObjectGetOptionsPrefix((PetscObject)dm->sf, &prefix);CHKERRQ(ierr);
4059     ierr = PetscObjectSetOptionsPrefix((PetscObject)sfPoint, prefix);CHKERRQ(ierr);
4060   }
4061   ierr = DMSetPointSF(dm, sfPoint);CHKERRQ(ierr);
4062   ierr = PetscSFDestroy(&sfPoint);CHKERRQ(ierr);
4063   if (vertexSF) {ierr = PetscObjectSetName((PetscObject)(*vertexSF), "Vertex Ownership SF");CHKERRQ(ierr);}
4064   /* Fill in the rest of the topology structure */
4065   ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
4066   ierr = DMPlexStratify(dm);CHKERRQ(ierr);
4067   ierr = PetscLogEventEnd(DMPLEX_BuildFromCellList,dm,0,0,0);CHKERRQ(ierr);
4068   PetscFunctionReturn(0);
4069 }
4070 
4071 /*@C
4072   DMPlexBuildCoordinatesFromCellListParallel - Build DM coordinates from a list of coordinates for each owned vertex (common mesh generator output)
4073 
4074   Input Parameters:
4075 + dm - The DM
4076 . spaceDim - The spatial dimension used for coordinates
4077 . sfVert - SF describing complete vertex ownership
4078 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
4079 
4080   Level: advanced
4081 
4082   Notes:
4083   Not currently supported in Fortran.
4084 
4085 .seealso: DMPlexBuildCoordinatesFromCellList(), DMPlexCreateFromCellListParallelPetsc(), DMPlexBuildFromCellListParallel()
4086 @*/
4087 PetscErrorCode DMPlexBuildCoordinatesFromCellListParallel(DM dm, PetscInt spaceDim, PetscSF sfVert, const PetscReal vertexCoords[])
4088 {
4089   PetscSection   coordSection;
4090   Vec            coordinates;
4091   PetscScalar   *coords;
4092   PetscInt       numVertices, numVerticesAdj, coordSize, v, vStart, vEnd;
4093   PetscErrorCode ierr;
4094 
4095   PetscFunctionBegin;
4096   ierr = PetscLogEventBegin(DMPLEX_BuildCoordinatesFromCellList,dm,0,0,0);CHKERRQ(ierr);
4097   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
4098   PetscCheckFalse(vStart < 0 || vEnd < 0,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM is not set up properly. DMPlexBuildFromCellList() should be called first.");
4099   ierr = DMSetCoordinateDim(dm, spaceDim);CHKERRQ(ierr);
4100   ierr = PetscSFGetGraph(sfVert, &numVertices, &numVerticesAdj, NULL, NULL);CHKERRQ(ierr);
4101   PetscCheckFalse(vEnd - vStart != numVerticesAdj,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Supplied sfVert has wrong number of leaves = %D != %D = vEnd - vStart",numVerticesAdj,vEnd - vStart);
4102   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
4103   ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
4104   ierr = PetscSectionSetFieldComponents(coordSection, 0, spaceDim);CHKERRQ(ierr);
4105   ierr = PetscSectionSetChart(coordSection, vStart, vEnd);CHKERRQ(ierr);
4106   for (v = vStart; v < vEnd; ++v) {
4107     ierr = PetscSectionSetDof(coordSection, v, spaceDim);CHKERRQ(ierr);
4108     ierr = PetscSectionSetFieldDof(coordSection, v, 0, spaceDim);CHKERRQ(ierr);
4109   }
4110   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
4111   ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
4112   ierr = VecCreate(PetscObjectComm((PetscObject)dm), &coordinates);CHKERRQ(ierr);
4113   ierr = VecSetBlockSize(coordinates, spaceDim);CHKERRQ(ierr);
4114   ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
4115   ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
4116   ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
4117   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
4118   {
4119     MPI_Datatype coordtype;
4120 
4121     /* Need a temp buffer for coords if we have complex/single */
4122     ierr = MPI_Type_contiguous(spaceDim, MPIU_SCALAR, &coordtype);CHKERRMPI(ierr);
4123     ierr = MPI_Type_commit(&coordtype);CHKERRMPI(ierr);
4124 #if defined(PETSC_USE_COMPLEX)
4125     {
4126     PetscScalar *svertexCoords;
4127     PetscInt    i;
4128     ierr = PetscMalloc1(numVertices*spaceDim,&svertexCoords);CHKERRQ(ierr);
4129     for (i=0; i<numVertices*spaceDim; i++) svertexCoords[i] = vertexCoords[i];
4130     ierr = PetscSFBcastBegin(sfVert, coordtype, svertexCoords, coords,MPI_REPLACE);CHKERRQ(ierr);
4131     ierr = PetscSFBcastEnd(sfVert, coordtype, svertexCoords, coords,MPI_REPLACE);CHKERRQ(ierr);
4132     ierr = PetscFree(svertexCoords);CHKERRQ(ierr);
4133     }
4134 #else
4135     ierr = PetscSFBcastBegin(sfVert, coordtype, vertexCoords, coords,MPI_REPLACE);CHKERRQ(ierr);
4136     ierr = PetscSFBcastEnd(sfVert, coordtype, vertexCoords, coords,MPI_REPLACE);CHKERRQ(ierr);
4137 #endif
4138     ierr = MPI_Type_free(&coordtype);CHKERRMPI(ierr);
4139   }
4140   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
4141   ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
4142   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
4143   ierr = PetscLogEventEnd(DMPLEX_BuildCoordinatesFromCellList,dm,0,0,0);CHKERRQ(ierr);
4144   PetscFunctionReturn(0);
4145 }
4146 
4147 /*@
4148   DMPlexCreateFromCellListParallelPetsc - Create distributed DMPLEX from a list of vertices for each cell (common mesh generator output)
4149 
4150   Input Parameters:
4151 + comm - The communicator
4152 . dim - The topological dimension of the mesh
4153 . numCells - The number of cells owned by this process
4154 . numVertices - The number of vertices owned by this process, or PETSC_DECIDE
4155 . NVertices - The global number of vertices, or PETSC_DECIDE
4156 . numCorners - The number of vertices for each cell
4157 . interpolate - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
4158 . cells - An array of numCells*numCorners numbers, the global vertex numbers for each cell
4159 . spaceDim - The spatial dimension used for coordinates
4160 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
4161 
4162   Output Parameters:
4163 + dm - The DM
4164 . vertexSF - (Optional) SF describing complete vertex ownership
4165 - verticesAdjSaved - (Optional) vertex adjacency array
4166 
4167   Notes:
4168   This function is just a convenient sequence of DMCreate(), DMSetType(), DMSetDimension(),
4169   DMPlexBuildFromCellListParallel(), DMPlexInterpolate(), DMPlexBuildCoordinatesFromCellListParallel()
4170 
4171   See DMPlexBuildFromCellListParallel() for an example and details about the topology-related parameters.
4172   See DMPlexBuildCoordinatesFromCellListParallel() for details about the geometry-related parameters.
4173 
4174   Level: intermediate
4175 
4176 .seealso: DMPlexCreateFromCellListPetsc(), DMPlexBuildFromCellListParallel(), DMPlexBuildCoordinatesFromCellListParallel(), DMPlexCreateFromDAG(), DMPlexCreate()
4177 @*/
4178 PetscErrorCode DMPlexCreateFromCellListParallelPetsc(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt NVertices, PetscInt numCorners, PetscBool interpolate, const PetscInt cells[], PetscInt spaceDim, const PetscReal vertexCoords[], PetscSF *vertexSF, PetscInt **verticesAdj, DM *dm)
4179 {
4180   PetscSF        sfVert;
4181   PetscErrorCode ierr;
4182 
4183   PetscFunctionBegin;
4184   ierr = DMCreate(comm, dm);CHKERRQ(ierr);
4185   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
4186   PetscValidLogicalCollectiveInt(*dm, dim, 2);
4187   PetscValidLogicalCollectiveInt(*dm, spaceDim, 9);
4188   ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
4189   ierr = DMPlexBuildFromCellListParallel(*dm, numCells, numVertices, NVertices, numCorners, cells, &sfVert, verticesAdj);CHKERRQ(ierr);
4190   if (interpolate) {
4191     DM idm;
4192 
4193     ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
4194     ierr = DMDestroy(dm);CHKERRQ(ierr);
4195     *dm  = idm;
4196   }
4197   ierr = DMPlexBuildCoordinatesFromCellListParallel(*dm, spaceDim, sfVert, vertexCoords);CHKERRQ(ierr);
4198   if (vertexSF) *vertexSF = sfVert;
4199   else {ierr = PetscSFDestroy(&sfVert);CHKERRQ(ierr);}
4200   PetscFunctionReturn(0);
4201 }
4202 
4203 /*@C
4204   DMPlexBuildFromCellList - Build DMPLEX topology from a list of vertices for each cell (common mesh generator output)
4205 
4206   Input Parameters:
4207 + dm - The DM
4208 . numCells - The number of cells owned by this process
4209 . numVertices - The number of vertices owned by this process, or PETSC_DETERMINE
4210 . numCorners - The number of vertices for each cell
4211 - cells - An array of numCells*numCorners numbers, the global vertex numbers for each cell
4212 
4213   Level: advanced
4214 
4215   Notes:
4216   Two triangles sharing a face
4217 $
4218 $        2
4219 $      / | \
4220 $     /  |  \
4221 $    /   |   \
4222 $   0  0 | 1  3
4223 $    \   |   /
4224 $     \  |  /
4225 $      \ | /
4226 $        1
4227 would have input
4228 $  numCells = 2, numVertices = 4
4229 $  cells = [0 1 2  1 3 2]
4230 $
4231 which would result in the DMPlex
4232 $
4233 $        4
4234 $      / | \
4235 $     /  |  \
4236 $    /   |   \
4237 $   2  0 | 1  5
4238 $    \   |   /
4239 $     \  |  /
4240 $      \ | /
4241 $        3
4242 
4243   If numVertices is PETSC_DETERMINE, it is computed by PETSc as the maximum vertex index in cells + 1.
4244 
4245   Not currently supported in Fortran.
4246 
4247 .seealso: DMPlexBuildFromCellListParallel(), DMPlexBuildCoordinatesFromCellList(), DMPlexCreateFromCellListPetsc()
4248 @*/
4249 PetscErrorCode DMPlexBuildFromCellList(DM dm, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, const PetscInt cells[])
4250 {
4251   PetscInt      *cones, c, p, dim;
4252   PetscErrorCode ierr;
4253 
4254   PetscFunctionBegin;
4255   ierr = PetscLogEventBegin(DMPLEX_BuildFromCellList,dm,0,0,0);CHKERRQ(ierr);
4256   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
4257   /* Get/check global number of vertices */
4258   {
4259     PetscInt NVerticesInCells, i;
4260     const PetscInt len = numCells * numCorners;
4261 
4262     /* NVerticesInCells = max(cells) + 1 */
4263     NVerticesInCells = PETSC_MIN_INT;
4264     for (i=0; i<len; i++) if (cells[i] > NVerticesInCells) NVerticesInCells = cells[i];
4265     ++NVerticesInCells;
4266 
4267     if (numVertices == PETSC_DECIDE) numVertices = NVerticesInCells;
4268     else PetscCheckFalse(numVertices < NVerticesInCells,PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Specified number of vertices %D must be greater than or equal to the number of vertices in cells %D",numVertices,NVerticesInCells);
4269   }
4270   ierr = DMPlexSetChart(dm, 0, numCells+numVertices);CHKERRQ(ierr);
4271   for (c = 0; c < numCells; ++c) {
4272     ierr = DMPlexSetConeSize(dm, c, numCorners);CHKERRQ(ierr);
4273   }
4274   ierr = DMSetUp(dm);CHKERRQ(ierr);
4275   ierr = DMPlexGetCones(dm,&cones);CHKERRQ(ierr);
4276   for (c = 0; c < numCells; ++c) {
4277     for (p = 0; p < numCorners; ++p) {
4278       cones[c*numCorners+p] = cells[c*numCorners+p]+numCells;
4279     }
4280   }
4281   ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
4282   ierr = DMPlexStratify(dm);CHKERRQ(ierr);
4283   ierr = PetscLogEventEnd(DMPLEX_BuildFromCellList,dm,0,0,0);CHKERRQ(ierr);
4284   PetscFunctionReturn(0);
4285 }
4286 
4287 /*@C
4288   DMPlexBuildCoordinatesFromCellList - Build DM coordinates from a list of coordinates for each owned vertex (common mesh generator output)
4289 
4290   Input Parameters:
4291 + dm - The DM
4292 . spaceDim - The spatial dimension used for coordinates
4293 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
4294 
4295   Level: advanced
4296 
4297   Notes:
4298   Not currently supported in Fortran.
4299 
4300 .seealso: DMPlexBuildCoordinatesFromCellListParallel(), DMPlexCreateFromCellListPetsc(), DMPlexBuildFromCellList()
4301 @*/
4302 PetscErrorCode DMPlexBuildCoordinatesFromCellList(DM dm, PetscInt spaceDim, const PetscReal vertexCoords[])
4303 {
4304   PetscSection   coordSection;
4305   Vec            coordinates;
4306   DM             cdm;
4307   PetscScalar   *coords;
4308   PetscInt       v, vStart, vEnd, d;
4309   PetscErrorCode ierr;
4310 
4311   PetscFunctionBegin;
4312   ierr = PetscLogEventBegin(DMPLEX_BuildCoordinatesFromCellList,dm,0,0,0);CHKERRQ(ierr);
4313   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
4314   PetscCheckFalse(vStart < 0 || vEnd < 0,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM is not set up properly. DMPlexBuildFromCellList() should be called first.");
4315   ierr = DMSetCoordinateDim(dm, spaceDim);CHKERRQ(ierr);
4316   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
4317   ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
4318   ierr = PetscSectionSetFieldComponents(coordSection, 0, spaceDim);CHKERRQ(ierr);
4319   ierr = PetscSectionSetChart(coordSection, vStart, vEnd);CHKERRQ(ierr);
4320   for (v = vStart; v < vEnd; ++v) {
4321     ierr = PetscSectionSetDof(coordSection, v, spaceDim);CHKERRQ(ierr);
4322     ierr = PetscSectionSetFieldDof(coordSection, v, 0, spaceDim);CHKERRQ(ierr);
4323   }
4324   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
4325 
4326   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
4327   ierr = DMCreateLocalVector(cdm, &coordinates);CHKERRQ(ierr);
4328   ierr = VecSetBlockSize(coordinates, spaceDim);CHKERRQ(ierr);
4329   ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
4330   ierr = VecGetArrayWrite(coordinates, &coords);CHKERRQ(ierr);
4331   for (v = 0; v < vEnd-vStart; ++v) {
4332     for (d = 0; d < spaceDim; ++d) {
4333       coords[v*spaceDim+d] = vertexCoords[v*spaceDim+d];
4334     }
4335   }
4336   ierr = VecRestoreArrayWrite(coordinates, &coords);CHKERRQ(ierr);
4337   ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
4338   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
4339   ierr = PetscLogEventEnd(DMPLEX_BuildCoordinatesFromCellList,dm,0,0,0);CHKERRQ(ierr);
4340   PetscFunctionReturn(0);
4341 }
4342 
4343 /*@
4344   DMPlexCreateFromCellListPetsc - Create DMPLEX from a list of vertices for each cell (common mesh generator output), but only process 0 takes in the input
4345 
4346   Collective on comm
4347 
4348   Input Parameters:
4349 + comm - The communicator
4350 . dim - The topological dimension of the mesh
4351 . numCells - The number of cells, only on process 0
4352 . numVertices - The number of vertices owned by this process, or PETSC_DECIDE, only on process 0
4353 . numCorners - The number of vertices for each cell, only on process 0
4354 . interpolate - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
4355 . cells - An array of numCells*numCorners numbers, the vertices for each cell, only on process 0
4356 . spaceDim - The spatial dimension used for coordinates
4357 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex, only on process 0
4358 
4359   Output Parameter:
4360 . dm - The DM, which only has points on process 0
4361 
4362   Notes:
4363   This function is just a convenient sequence of DMCreate(), DMSetType(), DMSetDimension(), DMPlexBuildFromCellList(),
4364   DMPlexInterpolate(), DMPlexBuildCoordinatesFromCellList()
4365 
4366   See DMPlexBuildFromCellList() for an example and details about the topology-related parameters.
4367   See DMPlexBuildCoordinatesFromCellList() for details about the geometry-related parameters.
4368   See DMPlexCreateFromCellListParallelPetsc() for parallel input
4369 
4370   Level: intermediate
4371 
4372 .seealso: DMPlexCreateFromCellListParallelPetsc(), DMPlexBuildFromCellList(), DMPlexBuildCoordinatesFromCellList(), DMPlexCreateFromDAG(), DMPlexCreate()
4373 @*/
4374 PetscErrorCode DMPlexCreateFromCellListPetsc(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, PetscBool interpolate, const PetscInt cells[], PetscInt spaceDim, const PetscReal vertexCoords[], DM *dm)
4375 {
4376   PetscMPIInt    rank;
4377   PetscErrorCode ierr;
4378 
4379   PetscFunctionBegin;
4380   PetscCheckFalse(!dim,comm, PETSC_ERR_ARG_OUTOFRANGE, "This is not appropriate for 0-dimensional meshes. Consider either creating the DM using DMPlexCreateFromDAG(), by hand, or using DMSwarm.");
4381   ierr = MPI_Comm_rank(comm, &rank);CHKERRMPI(ierr);
4382   ierr = DMCreate(comm, dm);CHKERRQ(ierr);
4383   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
4384   ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
4385   if (!rank) {ierr = DMPlexBuildFromCellList(*dm, numCells, numVertices, numCorners, cells);CHKERRQ(ierr);}
4386   else       {ierr = DMPlexBuildFromCellList(*dm, 0, 0, 0, NULL);CHKERRQ(ierr);}
4387   if (interpolate) {
4388     DM idm;
4389 
4390     ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
4391     ierr = DMDestroy(dm);CHKERRQ(ierr);
4392     *dm  = idm;
4393   }
4394   if (!rank) {ierr = DMPlexBuildCoordinatesFromCellList(*dm, spaceDim, vertexCoords);CHKERRQ(ierr);}
4395   else       {ierr = DMPlexBuildCoordinatesFromCellList(*dm, spaceDim, NULL);CHKERRQ(ierr);}
4396   PetscFunctionReturn(0);
4397 }
4398 
4399 /*@
4400   DMPlexCreateFromDAG - This takes as input the adjacency-list representation of the Directed Acyclic Graph (Hasse Diagram) encoding a mesh, and produces a DM
4401 
4402   Input Parameters:
4403 + dm - The empty DM object, usually from DMCreate() and DMSetDimension()
4404 . depth - The depth of the DAG
4405 . numPoints - Array of size depth + 1 containing the number of points at each depth
4406 . coneSize - The cone size of each point
4407 . cones - The concatenation of the cone points for each point, the cone list must be oriented correctly for each point
4408 . coneOrientations - The orientation of each cone point
4409 - vertexCoords - An array of numPoints[0]*spacedim numbers representing the coordinates of each vertex, with spacedim the value set via DMSetCoordinateDim()
4410 
4411   Output Parameter:
4412 . dm - The DM
4413 
4414   Note: Two triangles sharing a face would have input
4415 $  depth = 1, numPoints = [4 2], coneSize = [3 3 0 0 0 0]
4416 $  cones = [2 3 4  3 5 4], coneOrientations = [0 0 0  0 0 0]
4417 $ vertexCoords = [-1.0 0.0  0.0 -1.0  0.0 1.0  1.0 0.0]
4418 $
4419 which would result in the DMPlex
4420 $
4421 $        4
4422 $      / | \
4423 $     /  |  \
4424 $    /   |   \
4425 $   2  0 | 1  5
4426 $    \   |   /
4427 $     \  |  /
4428 $      \ | /
4429 $        3
4430 $
4431 $ Notice that all points are numbered consecutively, unlike DMPlexCreateFromCellListPetsc()
4432 
4433   Level: advanced
4434 
4435 .seealso: DMPlexCreateFromCellListPetsc(), DMPlexCreate()
4436 @*/
4437 PetscErrorCode DMPlexCreateFromDAG(DM dm, PetscInt depth, const PetscInt numPoints[], const PetscInt coneSize[], const PetscInt cones[], const PetscInt coneOrientations[], const PetscScalar vertexCoords[])
4438 {
4439   Vec            coordinates;
4440   PetscSection   coordSection;
4441   PetscScalar    *coords;
4442   PetscInt       coordSize, firstVertex = -1, pStart = 0, pEnd = 0, p, v, dim, dimEmbed, d, off;
4443   PetscErrorCode ierr;
4444 
4445   PetscFunctionBegin;
4446   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
4447   ierr = DMGetCoordinateDim(dm, &dimEmbed);CHKERRQ(ierr);
4448   PetscCheckFalse(dimEmbed < dim,PETSC_COMM_SELF,PETSC_ERR_PLIB,"Embedding dimension %D cannot be less than intrinsic dimension %d",dimEmbed,dim);
4449   for (d = 0; d <= depth; ++d) pEnd += numPoints[d];
4450   ierr = DMPlexSetChart(dm, pStart, pEnd);CHKERRQ(ierr);
4451   for (p = pStart; p < pEnd; ++p) {
4452     ierr = DMPlexSetConeSize(dm, p, coneSize[p-pStart]);CHKERRQ(ierr);
4453     if (firstVertex < 0 && !coneSize[p - pStart]) {
4454       firstVertex = p - pStart;
4455     }
4456   }
4457   PetscCheckFalse(firstVertex < 0 && numPoints[0],PETSC_COMM_SELF,PETSC_ERR_ARG_WRONG,"Expected %D vertices but could not find any", numPoints[0]);
4458   ierr = DMSetUp(dm);CHKERRQ(ierr); /* Allocate space for cones */
4459   for (p = pStart, off = 0; p < pEnd; off += coneSize[p-pStart], ++p) {
4460     ierr = DMPlexSetCone(dm, p, &cones[off]);CHKERRQ(ierr);
4461     ierr = DMPlexSetConeOrientation(dm, p, &coneOrientations[off]);CHKERRQ(ierr);
4462   }
4463   ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
4464   ierr = DMPlexStratify(dm);CHKERRQ(ierr);
4465   /* Build coordinates */
4466   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
4467   ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
4468   ierr = PetscSectionSetFieldComponents(coordSection, 0, dimEmbed);CHKERRQ(ierr);
4469   ierr = PetscSectionSetChart(coordSection, firstVertex, firstVertex+numPoints[0]);CHKERRQ(ierr);
4470   for (v = firstVertex; v < firstVertex+numPoints[0]; ++v) {
4471     ierr = PetscSectionSetDof(coordSection, v, dimEmbed);CHKERRQ(ierr);
4472     ierr = PetscSectionSetFieldDof(coordSection, v, 0, dimEmbed);CHKERRQ(ierr);
4473   }
4474   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
4475   ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
4476   ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
4477   ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
4478   ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
4479   ierr = VecSetBlockSize(coordinates, dimEmbed);CHKERRQ(ierr);
4480   ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
4481   if (vertexCoords) {
4482     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
4483     for (v = 0; v < numPoints[0]; ++v) {
4484       PetscInt off;
4485 
4486       ierr = PetscSectionGetOffset(coordSection, v+firstVertex, &off);CHKERRQ(ierr);
4487       for (d = 0; d < dimEmbed; ++d) {
4488         coords[off+d] = vertexCoords[v*dimEmbed+d];
4489       }
4490     }
4491   }
4492   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
4493   ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
4494   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
4495   PetscFunctionReturn(0);
4496 }
4497 
4498 /*@C
4499   DMPlexCreateCellVertexFromFile - Create a DMPlex mesh from a simple cell-vertex file.
4500 
4501 + comm        - The MPI communicator
4502 . filename    - Name of the .dat file
4503 - interpolate - Create faces and edges in the mesh
4504 
4505   Output Parameter:
4506 . dm  - The DM object representing the mesh
4507 
4508   Note: The format is the simplest possible:
4509 $ Ne
4510 $ v0 v1 ... vk
4511 $ Nv
4512 $ x y z marker
4513 
4514   Level: beginner
4515 
4516 .seealso: DMPlexCreateFromFile(), DMPlexCreateMedFromFile(), DMPlexCreateGmsh(), DMPlexCreate()
4517 @*/
4518 PetscErrorCode DMPlexCreateCellVertexFromFile(MPI_Comm comm, const char filename[], PetscBool interpolate, DM *dm)
4519 {
4520   DMLabel         marker;
4521   PetscViewer     viewer;
4522   Vec             coordinates;
4523   PetscSection    coordSection;
4524   PetscScalar    *coords;
4525   char            line[PETSC_MAX_PATH_LEN];
4526   PetscInt        dim = 3, cdim = 3, coordSize, v, c, d;
4527   PetscMPIInt     rank;
4528   int             snum, Nv, Nc, Ncn, Nl;
4529   PetscErrorCode  ierr;
4530 
4531   PetscFunctionBegin;
4532   ierr = MPI_Comm_rank(comm, &rank);CHKERRMPI(ierr);
4533   ierr = PetscViewerCreate(comm, &viewer);CHKERRQ(ierr);
4534   ierr = PetscViewerSetType(viewer, PETSCVIEWERASCII);CHKERRQ(ierr);
4535   ierr = PetscViewerFileSetMode(viewer, FILE_MODE_READ);CHKERRQ(ierr);
4536   ierr = PetscViewerFileSetName(viewer, filename);CHKERRQ(ierr);
4537   if (rank == 0) {
4538     ierr = PetscViewerRead(viewer, line, 4, NULL, PETSC_STRING);CHKERRQ(ierr);
4539     snum = sscanf(line, "%d %d %d %d", &Nc, &Nv, &Ncn, &Nl);
4540     PetscCheckFalse(snum != 4,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
4541   } else {
4542     Nc = Nv = Ncn = Nl = 0;
4543   }
4544   ierr = DMCreate(comm, dm);CHKERRQ(ierr);
4545   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
4546   ierr = DMPlexSetChart(*dm, 0, Nc+Nv);CHKERRQ(ierr);
4547   ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
4548   ierr = DMSetCoordinateDim(*dm, cdim);CHKERRQ(ierr);
4549   /* Read topology */
4550   if (rank == 0) {
4551     char     format[PETSC_MAX_PATH_LEN];
4552     PetscInt cone[8];
4553     int      vbuf[8], v;
4554 
4555     for (c = 0; c < Ncn; ++c) {format[c*3+0] = '%'; format[c*3+1] = 'd'; format[c*3+2] = ' ';}
4556     format[Ncn*3-1] = '\0';
4557     for (c = 0; c < Nc; ++c) {ierr = DMPlexSetConeSize(*dm, c, Ncn);CHKERRQ(ierr);}
4558     ierr = DMSetUp(*dm);CHKERRQ(ierr);
4559     for (c = 0; c < Nc; ++c) {
4560       ierr = PetscViewerRead(viewer, line, Ncn, NULL, PETSC_STRING);CHKERRQ(ierr);
4561       switch (Ncn) {
4562         case 2: snum = sscanf(line, format, &vbuf[0], &vbuf[1]);break;
4563         case 3: snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2]);break;
4564         case 4: snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3]);break;
4565         case 6: snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3], &vbuf[4], &vbuf[5]);break;
4566         case 8: snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3], &vbuf[4], &vbuf[5], &vbuf[6], &vbuf[7]);break;
4567         default: SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "No cell shape with %D vertices", Ncn);
4568       }
4569       PetscCheckFalse(snum != Ncn,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
4570       for (v = 0; v < Ncn; ++v) cone[v] = vbuf[v] + Nc;
4571       /* Hexahedra are inverted */
4572       if (Ncn == 8) {
4573         PetscInt tmp = cone[1];
4574         cone[1] = cone[3];
4575         cone[3] = tmp;
4576       }
4577       ierr = DMPlexSetCone(*dm, c, cone);CHKERRQ(ierr);
4578     }
4579   }
4580   ierr = DMPlexSymmetrize(*dm);CHKERRQ(ierr);
4581   ierr = DMPlexStratify(*dm);CHKERRQ(ierr);
4582   /* Read coordinates */
4583   ierr = DMGetCoordinateSection(*dm, &coordSection);CHKERRQ(ierr);
4584   ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
4585   ierr = PetscSectionSetFieldComponents(coordSection, 0, cdim);CHKERRQ(ierr);
4586   ierr = PetscSectionSetChart(coordSection, Nc, Nc + Nv);CHKERRQ(ierr);
4587   for (v = Nc; v < Nc+Nv; ++v) {
4588     ierr = PetscSectionSetDof(coordSection, v, cdim);CHKERRQ(ierr);
4589     ierr = PetscSectionSetFieldDof(coordSection, v, 0, cdim);CHKERRQ(ierr);
4590   }
4591   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
4592   ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
4593   ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
4594   ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
4595   ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
4596   ierr = VecSetBlockSize(coordinates, cdim);CHKERRQ(ierr);
4597   ierr = VecSetType(coordinates, VECSTANDARD);CHKERRQ(ierr);
4598   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
4599   if (rank == 0) {
4600     char   format[PETSC_MAX_PATH_LEN];
4601     double x[3];
4602     int    l, val[3];
4603 
4604     if (Nl) {
4605       for (l = 0; l < Nl; ++l) {format[l*3+0] = '%'; format[l*3+1] = 'd'; format[l*3+2] = ' ';}
4606       format[Nl*3-1] = '\0';
4607       ierr = DMCreateLabel(*dm, "marker");CHKERRQ(ierr);
4608       ierr = DMGetLabel(*dm, "marker", &marker);CHKERRQ(ierr);
4609     }
4610     for (v = 0; v < Nv; ++v) {
4611       ierr = PetscViewerRead(viewer, line, 3+Nl, NULL, PETSC_STRING);CHKERRQ(ierr);
4612       snum = sscanf(line, "%lg %lg %lg", &x[0], &x[1], &x[2]);
4613       PetscCheckFalse(snum != 3,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
4614       switch (Nl) {
4615         case 0: snum = 0;break;
4616         case 1: snum = sscanf(line, format, &val[0]);break;
4617         case 2: snum = sscanf(line, format, &val[0], &val[1]);break;
4618         case 3: snum = sscanf(line, format, &val[0], &val[1], &val[2]);break;
4619         default: SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Request support for %D labels", Nl);
4620       }
4621       PetscCheckFalse(snum != Nl,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
4622       for (d = 0; d < cdim; ++d) coords[v*cdim+d] = x[d];
4623       for (l = 0; l < Nl; ++l) {ierr = DMLabelSetValue(marker, v+Nc, val[l]);CHKERRQ(ierr);}
4624     }
4625   }
4626   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
4627   ierr = DMSetCoordinatesLocal(*dm, coordinates);CHKERRQ(ierr);
4628   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
4629   ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
4630   if (interpolate) {
4631     DM      idm;
4632     DMLabel bdlabel;
4633 
4634     ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
4635     ierr = DMDestroy(dm);CHKERRQ(ierr);
4636     *dm  = idm;
4637 
4638     if (!Nl) {
4639       ierr = DMCreateLabel(*dm, "marker");CHKERRQ(ierr);
4640       ierr = DMGetLabel(*dm, "marker", &bdlabel);CHKERRQ(ierr);
4641       ierr = DMPlexMarkBoundaryFaces(*dm, PETSC_DETERMINE, bdlabel);CHKERRQ(ierr);
4642       ierr = DMPlexLabelComplete(*dm, bdlabel);CHKERRQ(ierr);
4643     }
4644   }
4645   PetscFunctionReturn(0);
4646 }
4647 
4648 /*@C
4649   DMPlexCreateFromFile - This takes a filename and produces a DM
4650 
4651   Input Parameters:
4652 + comm - The communicator
4653 . filename - A file name
4654 . plexname - The object name of the resulting DM, also used for intra-datafile lookup by some formats
4655 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
4656 
4657   Output Parameter:
4658 . dm - The DM
4659 
4660   Options Database Keys:
4661 . -dm_plex_create_from_hdf5_xdmf - use the PETSC_VIEWER_HDF5_XDMF format for reading HDF5
4662 
4663   Use -dm_plex_create_ prefix to pass options to the internal PetscViewer, e.g.
4664 $ -dm_plex_create_viewer_hdf5_collective
4665 
4666   Notes:
4667   Using PETSCVIEWERHDF5 type with PETSC_VIEWER_HDF5_PETSC format, one can save multiple DMPlex
4668   meshes in a single HDF5 file. This in turn requires one to name the DMPlex object with PetscObjectSetName()
4669   before saving it with DMView() and before loading it with DMLoad() for identification of the mesh object.
4670   The input parameter name is thus used to name the DMPlex object when DMPlexCreateFromFile() internally
4671   calls DMLoad(). Currently, name is ignored for other viewer types and/or formats.
4672 
4673   Level: beginner
4674 
4675 .seealso: DMPlexCreateFromDAG(), DMPlexCreateFromCellListPetsc(), DMPlexCreate(), PetscObjectSetName(), DMView(), DMLoad()
4676 @*/
4677 PetscErrorCode DMPlexCreateFromFile(MPI_Comm comm, const char filename[], const char plexname[], PetscBool interpolate, DM *dm)
4678 {
4679   const char    *extGmsh      = ".msh";
4680   const char    *extGmsh2     = ".msh2";
4681   const char    *extGmsh4     = ".msh4";
4682   const char    *extCGNS      = ".cgns";
4683   const char    *extExodus    = ".exo";
4684   const char    *extExodus_e  = ".e";
4685   const char    *extGenesis   = ".gen";
4686   const char    *extFluent    = ".cas";
4687   const char    *extHDF5      = ".h5";
4688   const char    *extMed       = ".med";
4689   const char    *extPLY       = ".ply";
4690   const char    *extEGADSLite = ".egadslite";
4691   const char    *extEGADS     = ".egads";
4692   const char    *extIGES      = ".igs";
4693   const char    *extSTEP      = ".stp";
4694   const char    *extCV        = ".dat";
4695   size_t         len;
4696   PetscBool      isGmsh, isGmsh2, isGmsh4, isCGNS, isExodus, isGenesis, isFluent, isHDF5, isMed, isPLY, isEGADSLite, isEGADS, isIGES, isSTEP, isCV;
4697   PetscMPIInt    rank;
4698   PetscErrorCode ierr;
4699 
4700   PetscFunctionBegin;
4701   PetscValidCharPointer(filename, 2);
4702   if (plexname) PetscValidCharPointer(plexname, 3);
4703   PetscValidPointer(dm, 5);
4704   ierr = DMInitializePackage();CHKERRQ(ierr);
4705   ierr = PetscLogEventBegin(DMPLEX_CreateFromFile,0,0,0,0);CHKERRQ(ierr);
4706   ierr = MPI_Comm_rank(comm, &rank);CHKERRMPI(ierr);
4707   ierr = PetscStrlen(filename, &len);CHKERRQ(ierr);
4708   PetscCheckFalse(!len,comm, PETSC_ERR_ARG_WRONG, "Filename must be a valid path");
4709   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)],  extGmsh,      4, &isGmsh);CHKERRQ(ierr);
4710   ierr = PetscStrncmp(&filename[PetscMax(0,len-5)],  extGmsh2,     5, &isGmsh2);CHKERRQ(ierr);
4711   ierr = PetscStrncmp(&filename[PetscMax(0,len-5)],  extGmsh4,     5, &isGmsh4);CHKERRQ(ierr);
4712   ierr = PetscStrncmp(&filename[PetscMax(0,len-5)],  extCGNS,      5, &isCGNS);CHKERRQ(ierr);
4713   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)],  extExodus,    4, &isExodus);CHKERRQ(ierr);
4714   if (!isExodus) {
4715     ierr = PetscStrncmp(&filename[PetscMax(0,len-2)],  extExodus_e,    2, &isExodus);CHKERRQ(ierr);
4716   }
4717   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)],  extGenesis,   4, &isGenesis);CHKERRQ(ierr);
4718   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)],  extFluent,    4, &isFluent);CHKERRQ(ierr);
4719   ierr = PetscStrncmp(&filename[PetscMax(0,len-3)],  extHDF5,      3, &isHDF5);CHKERRQ(ierr);
4720   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)],  extMed,       4, &isMed);CHKERRQ(ierr);
4721   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)],  extPLY,       4, &isPLY);CHKERRQ(ierr);
4722   ierr = PetscStrncmp(&filename[PetscMax(0,len-10)], extEGADSLite, 10, &isEGADSLite);CHKERRQ(ierr);
4723   ierr = PetscStrncmp(&filename[PetscMax(0,len-6)],  extEGADS,     6, &isEGADS);CHKERRQ(ierr);
4724   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)],  extIGES,      4, &isIGES);CHKERRQ(ierr);
4725   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)],  extSTEP,      4, &isSTEP);CHKERRQ(ierr);
4726   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)],  extCV,        4, &isCV);CHKERRQ(ierr);
4727   if (isGmsh || isGmsh2 || isGmsh4) {
4728     ierr = DMPlexCreateGmshFromFile(comm, filename, interpolate, dm);CHKERRQ(ierr);
4729   } else if (isCGNS) {
4730     ierr = DMPlexCreateCGNSFromFile(comm, filename, interpolate, dm);CHKERRQ(ierr);
4731   } else if (isExodus || isGenesis) {
4732     ierr = DMPlexCreateExodusFromFile(comm, filename, interpolate, dm);CHKERRQ(ierr);
4733   } else if (isFluent) {
4734     ierr = DMPlexCreateFluentFromFile(comm, filename, interpolate, dm);CHKERRQ(ierr);
4735   } else if (isHDF5) {
4736     PetscBool      load_hdf5_xdmf = PETSC_FALSE;
4737     PetscViewer viewer;
4738 
4739     /* PETSC_VIEWER_HDF5_XDMF is used if the filename ends with .xdmf.h5, or if -dm_plex_create_from_hdf5_xdmf option is present */
4740     ierr = PetscStrncmp(&filename[PetscMax(0,len-8)], ".xdmf",  5, &load_hdf5_xdmf);CHKERRQ(ierr);
4741     ierr = PetscOptionsGetBool(NULL, NULL, "-dm_plex_create_from_hdf5_xdmf", &load_hdf5_xdmf, NULL);CHKERRQ(ierr);
4742     ierr = PetscViewerCreate(comm, &viewer);CHKERRQ(ierr);
4743     ierr = PetscViewerSetType(viewer, PETSCVIEWERHDF5);CHKERRQ(ierr);
4744     ierr = PetscViewerSetOptionsPrefix(viewer, "dm_plex_create_");CHKERRQ(ierr);
4745     ierr = PetscViewerSetFromOptions(viewer);CHKERRQ(ierr);
4746     ierr = PetscViewerFileSetMode(viewer, FILE_MODE_READ);CHKERRQ(ierr);
4747     ierr = PetscViewerFileSetName(viewer, filename);CHKERRQ(ierr);
4748 
4749     ierr = DMCreate(comm, dm);CHKERRQ(ierr);
4750     ierr = PetscObjectSetName((PetscObject)(*dm), plexname);CHKERRQ(ierr);
4751     ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
4752     if (load_hdf5_xdmf) {ierr = PetscViewerPushFormat(viewer, PETSC_VIEWER_HDF5_XDMF);CHKERRQ(ierr);}
4753     ierr = DMLoad(*dm, viewer);CHKERRQ(ierr);
4754     if (load_hdf5_xdmf) {ierr = PetscViewerPopFormat(viewer);CHKERRQ(ierr);}
4755     ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
4756 
4757     if (interpolate) {
4758       DM idm;
4759 
4760       ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
4761       ierr = DMDestroy(dm);CHKERRQ(ierr);
4762       *dm  = idm;
4763     }
4764   } else if (isMed) {
4765     ierr = DMPlexCreateMedFromFile(comm, filename, interpolate, dm);CHKERRQ(ierr);
4766   } else if (isPLY) {
4767     ierr = DMPlexCreatePLYFromFile(comm, filename, interpolate, dm);CHKERRQ(ierr);
4768   } else if (isEGADSLite || isEGADS || isIGES || isSTEP) {
4769     if (isEGADSLite) {ierr = DMPlexCreateEGADSLiteFromFile(comm, filename, dm);CHKERRQ(ierr);}
4770     else             {ierr = DMPlexCreateEGADSFromFile(comm, filename, dm);CHKERRQ(ierr);}
4771     if (!interpolate) {
4772       DM udm;
4773 
4774       ierr = DMPlexUninterpolate(*dm, &udm);CHKERRQ(ierr);
4775       ierr = DMDestroy(dm);CHKERRQ(ierr);
4776       *dm  = udm;
4777     }
4778   } else if (isCV) {
4779     ierr = DMPlexCreateCellVertexFromFile(comm, filename, interpolate, dm);CHKERRQ(ierr);
4780   } else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cannot load file %s: unrecognized extension", filename);
4781   ierr = PetscStrlen(plexname, &len);CHKERRQ(ierr);
4782   if (len) {ierr = PetscObjectSetName((PetscObject)(*dm), plexname);CHKERRQ(ierr);}
4783   ierr = PetscLogEventEnd(DMPLEX_CreateFromFile,0,0,0,0);CHKERRQ(ierr);
4784   PetscFunctionReturn(0);
4785 }
4786