xref: /petsc/src/dm/impls/plex/plexcreate.c (revision cb62b0caa0287db0dd6fdd08f210e9dd5648f130)
1 #define PETSCDM_DLL
2 #include <petsc/private/dmpleximpl.h>    /*I   "petscdmplex.h"   I*/
3 #include <petsc/private/hashseti.h>          /*I   "petscdmplex.h"   I*/
4 #include <petscsf.h>
5 #include <petsc/private/kernels/blockmatmult.h>
6 #include <petsc/private/kernels/blockinvert.h>
7 
8 PetscLogEvent DMPLEX_CreateFromFile, DMPLEX_BuildFromCellList, DMPLEX_BuildCoordinatesFromCellList;
9 
10 /* External function declarations here */
11 static PetscErrorCode DMInitialize_Plex(DM dm);
12 
13 /* This copies internal things in the Plex structure that we generally want when making a new, related Plex */
14 PetscErrorCode DMPlexCopy_Internal(DM dmin, PetscBool copyPeriodicity, DM dmout)
15 {
16   const DMBoundaryType *bd;
17   const PetscReal      *maxCell, *L;
18   PetscBool             isper, dist;
19   PetscErrorCode        ierr;
20 
21   PetscFunctionBegin;
22   if (copyPeriodicity) {
23     ierr = DMGetPeriodicity(dmin, &isper, &maxCell, &L, &bd);CHKERRQ(ierr);
24     ierr = DMSetPeriodicity(dmout, isper,  maxCell,  L,  bd);CHKERRQ(ierr);
25   }
26   ierr = DMPlexDistributeGetDefault(dmin, &dist);CHKERRQ(ierr);
27   ierr = DMPlexDistributeSetDefault(dmout, dist);CHKERRQ(ierr);
28   ((DM_Plex *) dmout->data)->useHashLocation = ((DM_Plex *) dmin->data)->useHashLocation;
29   PetscFunctionReturn(0);
30 }
31 
32 /* Replace dm with the contents of ndm, and then destroy ndm
33    - Share the DM_Plex structure
34    - Share the coordinates
35    - Share the SF
36 */
37 static PetscErrorCode DMPlexReplace_Static(DM dm, DM *ndm)
38 {
39   PetscSF               sf;
40   DM                    dmNew = *ndm, coordDM, coarseDM;
41   Vec                   coords;
42   PetscBool             isper;
43   const PetscReal      *maxCell, *L;
44   const DMBoundaryType *bd;
45   PetscInt              dim, cdim;
46   PetscErrorCode        ierr;
47 
48   PetscFunctionBegin;
49   if (dm == dmNew) {
50     ierr = DMDestroy(ndm);CHKERRQ(ierr);
51     PetscFunctionReturn(0);
52   }
53   dm->setupcalled = dmNew->setupcalled;
54   ierr = DMGetDimension(dmNew, &dim);CHKERRQ(ierr);
55   ierr = DMSetDimension(dm, dim);CHKERRQ(ierr);
56   ierr = DMGetCoordinateDim(dmNew, &cdim);CHKERRQ(ierr);
57   ierr = DMSetCoordinateDim(dm, cdim);CHKERRQ(ierr);
58   ierr = DMGetPointSF(dmNew, &sf);CHKERRQ(ierr);
59   ierr = DMSetPointSF(dm, sf);CHKERRQ(ierr);
60   ierr = DMGetCoordinateDM(dmNew, &coordDM);CHKERRQ(ierr);
61   ierr = DMGetCoordinatesLocal(dmNew, &coords);CHKERRQ(ierr);
62   ierr = DMSetCoordinateDM(dm, coordDM);CHKERRQ(ierr);
63   ierr = DMSetCoordinatesLocal(dm, coords);CHKERRQ(ierr);
64   /* Do not want to create the coordinate field if it does not already exist, so do not call DMGetCoordinateField() */
65   ierr = DMFieldDestroy(&dm->coordinateField);CHKERRQ(ierr);
66   dm->coordinateField = dmNew->coordinateField;
67   ((DM_Plex *) dmNew->data)->coordFunc = ((DM_Plex *) dm->data)->coordFunc;
68   ierr = DMGetPeriodicity(dmNew, &isper, &maxCell, &L, &bd);CHKERRQ(ierr);
69   ierr = DMSetPeriodicity(dm, isper, maxCell, L, bd);CHKERRQ(ierr);
70   ierr = DMDestroy_Plex(dm);CHKERRQ(ierr);
71   ierr = DMInitialize_Plex(dm);CHKERRQ(ierr);
72   dm->data = dmNew->data;
73   ((DM_Plex *) dmNew->data)->refct++;
74   ierr = DMDestroyLabelLinkList_Internal(dm);CHKERRQ(ierr);
75   ierr = DMCopyLabels(dmNew, dm, PETSC_OWN_POINTER, PETSC_TRUE, DM_COPY_LABELS_FAIL);CHKERRQ(ierr);
76   ierr = DMGetCoarseDM(dmNew,&coarseDM);CHKERRQ(ierr);
77   ierr = DMSetCoarseDM(dm,coarseDM);CHKERRQ(ierr);
78   ierr = DMDestroy(ndm);CHKERRQ(ierr);
79   PetscFunctionReturn(0);
80 }
81 
82 /* Swap dm with the contents of dmNew
83    - Swap the DM_Plex structure
84    - Swap the coordinates
85    - Swap the point PetscSF
86 */
87 static PetscErrorCode DMPlexSwap_Static(DM dmA, DM dmB)
88 {
89   DM              coordDMA, coordDMB;
90   Vec             coordsA,  coordsB;
91   PetscSF         sfA,      sfB;
92   DMField         fieldTmp;
93   void            *tmp;
94   DMLabelLink     listTmp;
95   DMLabel         depthTmp;
96   PetscInt        tmpI;
97   PetscErrorCode  ierr;
98 
99   PetscFunctionBegin;
100   if (dmA == dmB) PetscFunctionReturn(0);
101   ierr = DMGetPointSF(dmA, &sfA);CHKERRQ(ierr);
102   ierr = DMGetPointSF(dmB, &sfB);CHKERRQ(ierr);
103   ierr = PetscObjectReference((PetscObject) sfA);CHKERRQ(ierr);
104   ierr = DMSetPointSF(dmA, sfB);CHKERRQ(ierr);
105   ierr = DMSetPointSF(dmB, sfA);CHKERRQ(ierr);
106   ierr = PetscObjectDereference((PetscObject) sfA);CHKERRQ(ierr);
107 
108   ierr = DMGetCoordinateDM(dmA, &coordDMA);CHKERRQ(ierr);
109   ierr = DMGetCoordinateDM(dmB, &coordDMB);CHKERRQ(ierr);
110   ierr = PetscObjectReference((PetscObject) coordDMA);CHKERRQ(ierr);
111   ierr = DMSetCoordinateDM(dmA, coordDMB);CHKERRQ(ierr);
112   ierr = DMSetCoordinateDM(dmB, coordDMA);CHKERRQ(ierr);
113   ierr = PetscObjectDereference((PetscObject) coordDMA);CHKERRQ(ierr);
114 
115   ierr = DMGetCoordinatesLocal(dmA, &coordsA);CHKERRQ(ierr);
116   ierr = DMGetCoordinatesLocal(dmB, &coordsB);CHKERRQ(ierr);
117   ierr = PetscObjectReference((PetscObject) coordsA);CHKERRQ(ierr);
118   ierr = DMSetCoordinatesLocal(dmA, coordsB);CHKERRQ(ierr);
119   ierr = DMSetCoordinatesLocal(dmB, coordsA);CHKERRQ(ierr);
120   ierr = PetscObjectDereference((PetscObject) coordsA);CHKERRQ(ierr);
121 
122   fieldTmp             = dmA->coordinateField;
123   dmA->coordinateField = dmB->coordinateField;
124   dmB->coordinateField = fieldTmp;
125   tmp       = dmA->data;
126   dmA->data = dmB->data;
127   dmB->data = tmp;
128   listTmp   = dmA->labels;
129   dmA->labels = dmB->labels;
130   dmB->labels = listTmp;
131   depthTmp  = dmA->depthLabel;
132   dmA->depthLabel = dmB->depthLabel;
133   dmB->depthLabel = depthTmp;
134   depthTmp  = dmA->celltypeLabel;
135   dmA->celltypeLabel = dmB->celltypeLabel;
136   dmB->celltypeLabel = depthTmp;
137   tmpI         = dmA->levelup;
138   dmA->levelup = dmB->levelup;
139   dmB->levelup = tmpI;
140   PetscFunctionReturn(0);
141 }
142 
143 static PetscErrorCode DMPlexInterpolateInPlace_Internal(DM dm)
144 {
145   DM             idm;
146   PetscErrorCode ierr;
147 
148   PetscFunctionBegin;
149   ierr = DMPlexInterpolate(dm, &idm);CHKERRQ(ierr);
150   ierr = DMPlexCopyCoordinates(dm, idm);CHKERRQ(ierr);
151   ierr = DMPlexReplace_Static(dm, &idm);CHKERRQ(ierr);
152   PetscFunctionReturn(0);
153 }
154 
155 /*@C
156   DMPlexCreateCoordinateSpace - Creates a finite element space for the coordinates
157 
158   Collective
159 
160   Input Parameters:
161 + DM        - The DM
162 . degree    - The degree of the finite element or PETSC_DECIDE
163 - coordFunc - An optional function to map new points from refinement to the surface
164 
165   Level: advanced
166 
167 .seealso: PetscFECreateLagrange(), DMGetCoordinateDM()
168 @*/
169 PetscErrorCode DMPlexCreateCoordinateSpace(DM dm, PetscInt degree, PetscPointFunc coordFunc)
170 {
171   DM_Plex       *mesh = (DM_Plex *) dm->data;
172   DM             cdm;
173   PetscDS        cds;
174   PetscFE        fe;
175   PetscClassId   id;
176   PetscErrorCode ierr;
177 
178   PetscFunctionBegin;
179   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
180   ierr = DMGetDS(cdm, &cds);CHKERRQ(ierr);
181   ierr = PetscDSGetDiscretization(cds, 0, (PetscObject *) &fe);CHKERRQ(ierr);
182   ierr = PetscObjectGetClassId((PetscObject) fe, &id);CHKERRQ(ierr);
183   if (id != PETSCFE_CLASSID) {
184     PetscBool simplex;
185     PetscInt  dim, dE, qorder;
186 
187     ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
188     ierr = DMGetCoordinateDim(dm, &dE);CHKERRQ(ierr);
189     ierr = DMPlexIsSimplex(dm, &simplex);CHKERRQ(ierr);
190     qorder = degree;
191     ierr = PetscObjectOptionsBegin((PetscObject) cdm);CHKERRQ(ierr);
192     ierr = PetscOptionsBoundedInt("-coord_dm_default_quadrature_order", "Quadrature order is one less than quadrature points per edge", "DMPlexCreateCoordinateSpace", qorder, &qorder, NULL, 0);CHKERRQ(ierr);
193     ierr = PetscOptionsEnd();CHKERRQ(ierr);
194     if (degree == PETSC_DECIDE) fe = NULL;
195     else {
196       ierr = PetscFECreateLagrange(PETSC_COMM_SELF, dim, dE, simplex, degree, qorder, &fe);CHKERRQ(ierr);
197       ierr = DMSetField(cdm, 0, NULL, (PetscObject) fe);CHKERRQ(ierr);
198       ierr = DMCreateDS(cdm);CHKERRQ(ierr);
199     }
200     ierr = DMProjectCoordinates(dm, fe);CHKERRQ(ierr);
201     ierr = PetscFEDestroy(&fe);CHKERRQ(ierr);
202   }
203   mesh->coordFunc = coordFunc;
204   PetscFunctionReturn(0);
205 }
206 
207 /*@
208   DMPlexCreateDoublet - Creates a mesh of two cells of the specified type, optionally with later refinement.
209 
210   Collective
211 
212   Input Parameters:
213 + comm - The communicator for the DM object
214 . dim - The spatial dimension
215 . simplex - Flag for simplicial cells, otherwise they are tensor product cells
216 . interpolate - Flag to create intermediate mesh pieces (edges, faces)
217 - refinementLimit - A nonzero number indicates the largest admissible volume for a refined cell
218 
219   Output Parameter:
220 . dm - The DM object
221 
222   Level: beginner
223 
224 .seealso: DMSetType(), DMCreate()
225 @*/
226 PetscErrorCode DMPlexCreateDoublet(MPI_Comm comm, PetscInt dim, PetscBool simplex, PetscBool interpolate, PetscReal refinementLimit, DM *newdm)
227 {
228   DM             dm;
229   PetscMPIInt    rank;
230   PetscErrorCode ierr;
231 
232   PetscFunctionBegin;
233   ierr = DMCreate(comm, &dm);CHKERRQ(ierr);
234   ierr = DMSetType(dm, DMPLEX);CHKERRQ(ierr);
235   ierr = DMSetDimension(dm, dim);CHKERRQ(ierr);
236   ierr = MPI_Comm_rank(comm, &rank);CHKERRMPI(ierr);
237   switch (dim) {
238   case 2:
239     if (simplex) {ierr = PetscObjectSetName((PetscObject) dm, "triangular");CHKERRQ(ierr);}
240     else         {ierr = PetscObjectSetName((PetscObject) dm, "quadrilateral");CHKERRQ(ierr);}
241     break;
242   case 3:
243     if (simplex) {ierr = PetscObjectSetName((PetscObject) dm, "tetrahedral");CHKERRQ(ierr);}
244     else         {ierr = PetscObjectSetName((PetscObject) dm, "hexahedral");CHKERRQ(ierr);}
245     break;
246   default:
247     SETERRQ(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make meshes for dimension %D", dim);
248   }
249   if (rank) {
250     PetscInt numPoints[2] = {0, 0};
251     ierr = DMPlexCreateFromDAG(dm, 1, numPoints, NULL, NULL, NULL, NULL);CHKERRQ(ierr);
252   } else {
253     switch (dim) {
254     case 2:
255       if (simplex) {
256         PetscInt    numPoints[2]        = {4, 2};
257         PetscInt    coneSize[6]         = {3, 3, 0, 0, 0, 0};
258         PetscInt    cones[6]            = {2, 3, 4,  5, 4, 3};
259         PetscInt    coneOrientations[6] = {0, 0, 0,  0, 0, 0};
260         PetscScalar vertexCoords[8]     = {-0.5, 0.5, 0.0, 0.0, 0.0, 1.0, 0.5, 0.5};
261 
262         ierr = DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
263       } else {
264         PetscInt    numPoints[2]        = {6, 2};
265         PetscInt    coneSize[8]         = {4, 4, 0, 0, 0, 0, 0, 0};
266         PetscInt    cones[8]            = {2, 3, 4, 5,  3, 6, 7, 4};
267         PetscInt    coneOrientations[8] = {0, 0, 0, 0,  0, 0, 0, 0};
268         PetscScalar vertexCoords[12]    = {-1.0, -0.5,  0.0, -0.5,  0.0, 0.5,  -1.0, 0.5,  1.0, -0.5,  1.0, 0.5};
269 
270         ierr = DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
271       }
272       break;
273     case 3:
274       if (simplex) {
275         PetscInt    numPoints[2]        = {5, 2};
276         PetscInt    coneSize[7]         = {4, 4, 0, 0, 0, 0, 0};
277         PetscInt    cones[8]            = {4, 3, 5, 2,  5, 3, 4, 6};
278         PetscInt    coneOrientations[8] = {0, 0, 0, 0,  0, 0, 0, 0};
279         PetscScalar vertexCoords[15]    = {-1.0, 0.0, 0.0,  0.0, -1.0, 0.0,  0.0, 0.0, 1.0,  0.0, 1.0, 0.0,  1.0, 0.0, 0.0};
280 
281         ierr = DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
282       } else {
283         PetscInt    numPoints[2]         = {12, 2};
284         PetscInt    coneSize[14]         = {8, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
285         PetscInt    cones[16]            = {2, 3, 4, 5, 6, 7, 8, 9,  5, 4, 10, 11, 7, 12, 13, 8};
286         PetscInt    coneOrientations[16] = {0, 0, 0, 0, 0, 0, 0, 0,  0, 0,  0,  0, 0,  0,  0, 0};
287         PetscScalar vertexCoords[36]     = {-1.0, -0.5, -0.5,  -1.0,  0.5, -0.5,  0.0,  0.5, -0.5,   0.0, -0.5, -0.5,
288                                             -1.0, -0.5,  0.5,   0.0, -0.5,  0.5,  0.0,  0.5,  0.5,  -1.0,  0.5,  0.5,
289                                              1.0,  0.5, -0.5,   1.0, -0.5, -0.5,  1.0, -0.5,  0.5,   1.0,  0.5,  0.5};
290 
291         ierr = DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
292       }
293       break;
294     default:
295       SETERRQ(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make meshes for dimension %D", dim);
296     }
297   }
298   *newdm = dm;
299   if (refinementLimit > 0.0) {
300     DM rdm;
301     const char *name;
302 
303     ierr = DMPlexSetRefinementUniform(*newdm, PETSC_FALSE);CHKERRQ(ierr);
304     ierr = DMPlexSetRefinementLimit(*newdm, refinementLimit);CHKERRQ(ierr);
305     ierr = DMRefine(*newdm, comm, &rdm);CHKERRQ(ierr);
306     ierr = PetscObjectGetName((PetscObject) *newdm, &name);CHKERRQ(ierr);
307     ierr = PetscObjectSetName((PetscObject)    rdm,  name);CHKERRQ(ierr);
308     ierr = DMDestroy(newdm);CHKERRQ(ierr);
309     *newdm = rdm;
310   }
311   if (interpolate) {
312     DM idm;
313 
314     ierr = DMPlexInterpolate(*newdm, &idm);CHKERRQ(ierr);
315     ierr = DMDestroy(newdm);CHKERRQ(ierr);
316     *newdm = idm;
317   }
318   PetscFunctionReturn(0);
319 }
320 
321 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_1D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[])
322 {
323   const PetscInt numVertices    = 2;
324   PetscInt       markerRight    = 1;
325   PetscInt       markerLeft     = 1;
326   PetscBool      markerSeparate = PETSC_FALSE;
327   Vec            coordinates;
328   PetscSection   coordSection;
329   PetscScalar   *coords;
330   PetscInt       coordSize;
331   PetscMPIInt    rank;
332   PetscInt       cdim = 1, v;
333   PetscErrorCode ierr;
334 
335   PetscFunctionBegin;
336   ierr = PetscOptionsGetBool(((PetscObject) dm)->options,((PetscObject) dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL);CHKERRQ(ierr);
337   if (markerSeparate) {
338     markerRight  = 2;
339     markerLeft   = 1;
340   }
341   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRMPI(ierr);
342   if (!rank) {
343     ierr = DMPlexSetChart(dm, 0, numVertices);CHKERRQ(ierr);
344     ierr = DMSetUp(dm);CHKERRQ(ierr); /* Allocate space for cones */
345     ierr = DMSetLabelValue(dm, "marker", 0, markerLeft);CHKERRQ(ierr);
346     ierr = DMSetLabelValue(dm, "marker", 1, markerRight);CHKERRQ(ierr);
347   }
348   ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
349   ierr = DMPlexStratify(dm);CHKERRQ(ierr);
350   /* Build coordinates */
351   ierr = DMSetCoordinateDim(dm, cdim);CHKERRQ(ierr);
352   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
353   ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
354   ierr = PetscSectionSetChart(coordSection, 0, numVertices);CHKERRQ(ierr);
355   ierr = PetscSectionSetFieldComponents(coordSection, 0, cdim);CHKERRQ(ierr);
356   for (v = 0; v < numVertices; ++v) {
357     ierr = PetscSectionSetDof(coordSection, v, cdim);CHKERRQ(ierr);
358     ierr = PetscSectionSetFieldDof(coordSection, v, 0, cdim);CHKERRQ(ierr);
359   }
360   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
361   ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
362   ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
363   ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
364   ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
365   ierr = VecSetBlockSize(coordinates, cdim);CHKERRQ(ierr);
366   ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
367   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
368   coords[0] = lower[0];
369   coords[1] = upper[0];
370   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
371   ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
372   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
373   PetscFunctionReturn(0);
374 }
375 
376 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_2D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[])
377 {
378   const PetscInt numVertices    = (edges[0]+1)*(edges[1]+1);
379   const PetscInt numEdges       = edges[0]*(edges[1]+1) + (edges[0]+1)*edges[1];
380   PetscInt       markerTop      = 1;
381   PetscInt       markerBottom   = 1;
382   PetscInt       markerRight    = 1;
383   PetscInt       markerLeft     = 1;
384   PetscBool      markerSeparate = PETSC_FALSE;
385   Vec            coordinates;
386   PetscSection   coordSection;
387   PetscScalar    *coords;
388   PetscInt       coordSize;
389   PetscMPIInt    rank;
390   PetscInt       v, vx, vy;
391   PetscErrorCode ierr;
392 
393   PetscFunctionBegin;
394   ierr = PetscOptionsGetBool(((PetscObject) dm)->options,((PetscObject) dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL);CHKERRQ(ierr);
395   if (markerSeparate) {
396     markerTop    = 3;
397     markerBottom = 1;
398     markerRight  = 2;
399     markerLeft   = 4;
400   }
401   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRMPI(ierr);
402   if (rank == 0) {
403     PetscInt e, ex, ey;
404 
405     ierr = DMPlexSetChart(dm, 0, numEdges+numVertices);CHKERRQ(ierr);
406     for (e = 0; e < numEdges; ++e) {
407       ierr = DMPlexSetConeSize(dm, e, 2);CHKERRQ(ierr);
408     }
409     ierr = DMSetUp(dm);CHKERRQ(ierr); /* Allocate space for cones */
410     for (vx = 0; vx <= edges[0]; vx++) {
411       for (ey = 0; ey < edges[1]; ey++) {
412         PetscInt edge   = vx*edges[1] + ey + edges[0]*(edges[1]+1);
413         PetscInt vertex = ey*(edges[0]+1) + vx + numEdges;
414         PetscInt cone[2];
415 
416         cone[0] = vertex; cone[1] = vertex+edges[0]+1;
417         ierr    = DMPlexSetCone(dm, edge, cone);CHKERRQ(ierr);
418         if (vx == edges[0]) {
419           ierr = DMSetLabelValue(dm, "marker", edge,    markerRight);CHKERRQ(ierr);
420           ierr = DMSetLabelValue(dm, "marker", cone[0], markerRight);CHKERRQ(ierr);
421           if (ey == edges[1]-1) {
422             ierr = DMSetLabelValue(dm, "marker", cone[1], markerRight);CHKERRQ(ierr);
423             ierr = DMSetLabelValue(dm, "Face Sets", cone[1], markerRight);CHKERRQ(ierr);
424           }
425         } else if (vx == 0) {
426           ierr = DMSetLabelValue(dm, "marker", edge,    markerLeft);CHKERRQ(ierr);
427           ierr = DMSetLabelValue(dm, "marker", cone[0], markerLeft);CHKERRQ(ierr);
428           if (ey == edges[1]-1) {
429             ierr = DMSetLabelValue(dm, "marker", cone[1], markerLeft);CHKERRQ(ierr);
430             ierr = DMSetLabelValue(dm, "Face Sets", cone[1], markerLeft);CHKERRQ(ierr);
431           }
432         }
433       }
434     }
435     for (vy = 0; vy <= edges[1]; vy++) {
436       for (ex = 0; ex < edges[0]; ex++) {
437         PetscInt edge   = vy*edges[0]     + ex;
438         PetscInt vertex = vy*(edges[0]+1) + ex + numEdges;
439         PetscInt cone[2];
440 
441         cone[0] = vertex; cone[1] = vertex+1;
442         ierr    = DMPlexSetCone(dm, edge, cone);CHKERRQ(ierr);
443         if (vy == edges[1]) {
444           ierr = DMSetLabelValue(dm, "marker", edge,    markerTop);CHKERRQ(ierr);
445           ierr = DMSetLabelValue(dm, "marker", cone[0], markerTop);CHKERRQ(ierr);
446           if (ex == edges[0]-1) {
447             ierr = DMSetLabelValue(dm, "marker", cone[1], markerTop);CHKERRQ(ierr);
448             ierr = DMSetLabelValue(dm, "Face Sets", cone[1], markerTop);CHKERRQ(ierr);
449           }
450         } else if (vy == 0) {
451           ierr = DMSetLabelValue(dm, "marker", edge,    markerBottom);CHKERRQ(ierr);
452           ierr = DMSetLabelValue(dm, "marker", cone[0], markerBottom);CHKERRQ(ierr);
453           if (ex == edges[0]-1) {
454             ierr = DMSetLabelValue(dm, "marker", cone[1], markerBottom);CHKERRQ(ierr);
455             ierr = DMSetLabelValue(dm, "Face Sets", cone[1], markerBottom);CHKERRQ(ierr);
456           }
457         }
458       }
459     }
460   }
461   ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
462   ierr = DMPlexStratify(dm);CHKERRQ(ierr);
463   /* Build coordinates */
464   ierr = DMSetCoordinateDim(dm, 2);CHKERRQ(ierr);
465   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
466   ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
467   ierr = PetscSectionSetChart(coordSection, numEdges, numEdges + numVertices);CHKERRQ(ierr);
468   ierr = PetscSectionSetFieldComponents(coordSection, 0, 2);CHKERRQ(ierr);
469   for (v = numEdges; v < numEdges+numVertices; ++v) {
470     ierr = PetscSectionSetDof(coordSection, v, 2);CHKERRQ(ierr);
471     ierr = PetscSectionSetFieldDof(coordSection, v, 0, 2);CHKERRQ(ierr);
472   }
473   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
474   ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
475   ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
476   ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
477   ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
478   ierr = VecSetBlockSize(coordinates, 2);CHKERRQ(ierr);
479   ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
480   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
481   for (vy = 0; vy <= edges[1]; ++vy) {
482     for (vx = 0; vx <= edges[0]; ++vx) {
483       coords[(vy*(edges[0]+1)+vx)*2+0] = lower[0] + ((upper[0] - lower[0])/edges[0])*vx;
484       coords[(vy*(edges[0]+1)+vx)*2+1] = lower[1] + ((upper[1] - lower[1])/edges[1])*vy;
485     }
486   }
487   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
488   ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
489   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
490   PetscFunctionReturn(0);
491 }
492 
493 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_3D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt faces[])
494 {
495   PetscInt       vertices[3], numVertices;
496   PetscInt       numFaces    = 2*faces[0]*faces[1] + 2*faces[1]*faces[2] + 2*faces[0]*faces[2];
497   Vec            coordinates;
498   PetscSection   coordSection;
499   PetscScalar    *coords;
500   PetscInt       coordSize;
501   PetscMPIInt    rank;
502   PetscInt       v, vx, vy, vz;
503   PetscInt       voffset, iface=0, cone[4];
504   PetscErrorCode ierr;
505 
506   PetscFunctionBegin;
507   PetscCheckFalse((faces[0] < 1) || (faces[1] < 1) || (faces[2] < 1),PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Must have at least 1 face per side");
508   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRMPI(ierr);
509   vertices[0] = faces[0]+1; vertices[1] = faces[1]+1; vertices[2] = faces[2]+1;
510   numVertices = vertices[0]*vertices[1]*vertices[2];
511   if (rank == 0) {
512     PetscInt f;
513 
514     ierr = DMPlexSetChart(dm, 0, numFaces+numVertices);CHKERRQ(ierr);
515     for (f = 0; f < numFaces; ++f) {
516       ierr = DMPlexSetConeSize(dm, f, 4);CHKERRQ(ierr);
517     }
518     ierr = DMSetUp(dm);CHKERRQ(ierr); /* Allocate space for cones */
519 
520     /* Side 0 (Top) */
521     for (vy = 0; vy < faces[1]; vy++) {
522       for (vx = 0; vx < faces[0]; vx++) {
523         voffset = numFaces + vertices[0]*vertices[1]*(vertices[2]-1) + vy*vertices[0] + vx;
524         cone[0] = voffset; cone[1] = voffset+1; cone[2] = voffset+vertices[0]+1; cone[3] = voffset+vertices[0];
525         ierr    = DMPlexSetCone(dm, iface, cone);CHKERRQ(ierr);
526         ierr    = DMSetLabelValue(dm, "marker", iface, 1);CHKERRQ(ierr);
527         ierr    = DMSetLabelValue(dm, "marker", voffset+0, 1);CHKERRQ(ierr);
528         ierr    = DMSetLabelValue(dm, "marker", voffset+1, 1);CHKERRQ(ierr);
529         ierr    = DMSetLabelValue(dm, "marker", voffset+vertices[0]+0, 1);CHKERRQ(ierr);
530         ierr    = DMSetLabelValue(dm, "marker", voffset+vertices[0]+1, 1);CHKERRQ(ierr);
531         iface++;
532       }
533     }
534 
535     /* Side 1 (Bottom) */
536     for (vy = 0; vy < faces[1]; vy++) {
537       for (vx = 0; vx < faces[0]; vx++) {
538         voffset = numFaces + vy*(faces[0]+1) + vx;
539         cone[0] = voffset+1; cone[1] = voffset; cone[2] = voffset+vertices[0]; cone[3] = voffset+vertices[0]+1;
540         ierr    = DMPlexSetCone(dm, iface, cone);CHKERRQ(ierr);
541         ierr    = DMSetLabelValue(dm, "marker", iface, 1);CHKERRQ(ierr);
542         ierr    = DMSetLabelValue(dm, "marker", voffset+0, 1);CHKERRQ(ierr);
543         ierr    = DMSetLabelValue(dm, "marker", voffset+1, 1);CHKERRQ(ierr);
544         ierr    = DMSetLabelValue(dm, "marker", voffset+vertices[0]+0, 1);CHKERRQ(ierr);
545         ierr    = DMSetLabelValue(dm, "marker", voffset+vertices[0]+1, 1);CHKERRQ(ierr);
546         iface++;
547       }
548     }
549 
550     /* Side 2 (Front) */
551     for (vz = 0; vz < faces[2]; vz++) {
552       for (vx = 0; vx < faces[0]; vx++) {
553         voffset = numFaces + vz*vertices[0]*vertices[1] + vx;
554         cone[0] = voffset; cone[1] = voffset+1; cone[2] = voffset+vertices[0]*vertices[1]+1; cone[3] = voffset+vertices[0]*vertices[1];
555         ierr    = DMPlexSetCone(dm, iface, cone);CHKERRQ(ierr);
556         ierr    = DMSetLabelValue(dm, "marker", iface, 1);CHKERRQ(ierr);
557         ierr    = DMSetLabelValue(dm, "marker", voffset+0, 1);CHKERRQ(ierr);
558         ierr    = DMSetLabelValue(dm, "marker", voffset+1, 1);CHKERRQ(ierr);
559         ierr    = DMSetLabelValue(dm, "marker", voffset+vertices[0]*vertices[1]+0, 1);CHKERRQ(ierr);
560         ierr    = DMSetLabelValue(dm, "marker", voffset+vertices[0]*vertices[1]+1, 1);CHKERRQ(ierr);
561         iface++;
562       }
563     }
564 
565     /* Side 3 (Back) */
566     for (vz = 0; vz < faces[2]; vz++) {
567       for (vx = 0; vx < faces[0]; vx++) {
568         voffset = numFaces + vz*vertices[0]*vertices[1] + vertices[0]*(vertices[1]-1) + vx;
569         cone[0] = voffset+vertices[0]*vertices[1]; cone[1] = voffset+vertices[0]*vertices[1]+1;
570         cone[2] = voffset+1; cone[3] = voffset;
571         ierr    = DMPlexSetCone(dm, iface, cone);CHKERRQ(ierr);
572         ierr    = DMSetLabelValue(dm, "marker", iface, 1);CHKERRQ(ierr);
573         ierr    = DMSetLabelValue(dm, "marker", voffset+0, 1);CHKERRQ(ierr);
574         ierr    = DMSetLabelValue(dm, "marker", voffset+1, 1);CHKERRQ(ierr);
575         ierr    = DMSetLabelValue(dm, "marker", voffset+vertices[0]*vertices[1]+0, 1);CHKERRQ(ierr);
576         ierr    = DMSetLabelValue(dm, "marker", voffset+vertices[0]*vertices[1]+1, 1);CHKERRQ(ierr);
577         iface++;
578       }
579     }
580 
581     /* Side 4 (Left) */
582     for (vz = 0; vz < faces[2]; vz++) {
583       for (vy = 0; vy < faces[1]; vy++) {
584         voffset = numFaces + vz*vertices[0]*vertices[1] + vy*vertices[0];
585         cone[0] = voffset; cone[1] = voffset+vertices[0]*vertices[1];
586         cone[2] = voffset+vertices[0]*vertices[1]+vertices[0]; cone[3] = voffset+vertices[0];
587         ierr    = DMPlexSetCone(dm, iface, cone);CHKERRQ(ierr);
588         ierr    = DMSetLabelValue(dm, "marker", iface, 1);CHKERRQ(ierr);
589         ierr    = DMSetLabelValue(dm, "marker", voffset+0, 1);CHKERRQ(ierr);
590         ierr    = DMSetLabelValue(dm, "marker", voffset+vertices[0]+0, 1);CHKERRQ(ierr);
591         ierr    = DMSetLabelValue(dm, "marker", voffset+vertices[1]+0, 1);CHKERRQ(ierr);
592         ierr    = DMSetLabelValue(dm, "marker", voffset+vertices[0]*vertices[1]+vertices[0], 1);CHKERRQ(ierr);
593         iface++;
594       }
595     }
596 
597     /* Side 5 (Right) */
598     for (vz = 0; vz < faces[2]; vz++) {
599       for (vy = 0; vy < faces[1]; vy++) {
600         voffset = numFaces + vz*vertices[0]*vertices[1] + vy*vertices[0] + faces[0];
601         cone[0] = voffset+vertices[0]*vertices[1]; cone[1] = voffset;
602         cone[2] = voffset+vertices[0]; cone[3] = voffset+vertices[0]*vertices[1]+vertices[0];
603         ierr    = DMPlexSetCone(dm, iface, cone);CHKERRQ(ierr);
604         ierr    = DMSetLabelValue(dm, "marker", iface, 1);CHKERRQ(ierr);
605         ierr    = DMSetLabelValue(dm, "marker", voffset+0, 1);CHKERRQ(ierr);
606         ierr    = DMSetLabelValue(dm, "marker", voffset+vertices[0]+0, 1);CHKERRQ(ierr);
607         ierr    = DMSetLabelValue(dm, "marker", voffset+vertices[0]*vertices[1]+0, 1);CHKERRQ(ierr);
608         ierr    = DMSetLabelValue(dm, "marker", voffset+vertices[0]*vertices[1]+vertices[0], 1);CHKERRQ(ierr);
609         iface++;
610       }
611     }
612   }
613   ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
614   ierr = DMPlexStratify(dm);CHKERRQ(ierr);
615   /* Build coordinates */
616   ierr = DMSetCoordinateDim(dm, 3);CHKERRQ(ierr);
617   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
618   ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
619   ierr = PetscSectionSetChart(coordSection, numFaces, numFaces + numVertices);CHKERRQ(ierr);
620   ierr = PetscSectionSetFieldComponents(coordSection, 0, 3);CHKERRQ(ierr);
621   for (v = numFaces; v < numFaces+numVertices; ++v) {
622     ierr = PetscSectionSetDof(coordSection, v, 3);CHKERRQ(ierr);
623     ierr = PetscSectionSetFieldDof(coordSection, v, 0, 3);CHKERRQ(ierr);
624   }
625   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
626   ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
627   ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
628   ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
629   ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
630   ierr = VecSetBlockSize(coordinates, 3);CHKERRQ(ierr);
631   ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
632   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
633   for (vz = 0; vz <= faces[2]; ++vz) {
634     for (vy = 0; vy <= faces[1]; ++vy) {
635       for (vx = 0; vx <= faces[0]; ++vx) {
636         coords[((vz*(faces[1]+1)+vy)*(faces[0]+1)+vx)*3+0] = lower[0] + ((upper[0] - lower[0])/faces[0])*vx;
637         coords[((vz*(faces[1]+1)+vy)*(faces[0]+1)+vx)*3+1] = lower[1] + ((upper[1] - lower[1])/faces[1])*vy;
638         coords[((vz*(faces[1]+1)+vy)*(faces[0]+1)+vx)*3+2] = lower[2] + ((upper[2] - lower[2])/faces[2])*vz;
639       }
640     }
641   }
642   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
643   ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
644   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
645   PetscFunctionReturn(0);
646 }
647 
648 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], PetscBool interpolate)
649 {
650   PetscErrorCode ierr;
651 
652   PetscFunctionBegin;
653   PetscValidLogicalCollectiveInt(dm, dim, 2);
654   ierr = DMSetDimension(dm, dim-1);CHKERRQ(ierr);
655   ierr = DMSetCoordinateDim(dm, dim);CHKERRQ(ierr);
656   switch (dim) {
657     case 1: ierr = DMPlexCreateBoxSurfaceMesh_Tensor_1D_Internal(dm, lower, upper, faces);CHKERRQ(ierr);break;
658     case 2: ierr = DMPlexCreateBoxSurfaceMesh_Tensor_2D_Internal(dm, lower, upper, faces);CHKERRQ(ierr);break;
659     case 3: ierr = DMPlexCreateBoxSurfaceMesh_Tensor_3D_Internal(dm, lower, upper, faces);CHKERRQ(ierr);break;
660     default: SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Dimension not supported: %D", dim);
661   }
662   if (interpolate) {ierr = DMPlexInterpolateInPlace_Internal(dm);CHKERRQ(ierr);}
663   PetscFunctionReturn(0);
664 }
665 
666 /*@C
667   DMPlexCreateBoxSurfaceMesh - Creates a mesh on the surface of the tensor product of unit intervals (box) using tensor cells (hexahedra).
668 
669   Collective
670 
671   Input Parameters:
672 + comm        - The communicator for the DM object
673 . dim         - The spatial dimension of the box, so the resulting mesh is has dimension dim-1
674 . faces       - Number of faces per dimension, or NULL for (1,) in 1D and (2, 2) in 2D and (1, 1, 1) in 3D
675 . lower       - The lower left corner, or NULL for (0, 0, 0)
676 . upper       - The upper right corner, or NULL for (1, 1, 1)
677 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
678 
679   Output Parameter:
680 . dm  - The DM object
681 
682   Level: beginner
683 
684 .seealso: DMSetFromOptions(), DMPlexCreateBoxMesh(), DMPlexCreateFromFile(), DMSetType(), DMCreate()
685 @*/
686 PetscErrorCode DMPlexCreateBoxSurfaceMesh(MPI_Comm comm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], PetscBool interpolate, DM *dm)
687 {
688   PetscInt       fac[3] = {1, 1, 1};
689   PetscReal      low[3] = {0, 0, 0};
690   PetscReal      upp[3] = {1, 1, 1};
691   PetscErrorCode ierr;
692 
693   PetscFunctionBegin;
694   ierr = DMCreate(comm,dm);CHKERRQ(ierr);
695   ierr = DMSetType(*dm,DMPLEX);CHKERRQ(ierr);
696   ierr = DMPlexCreateBoxSurfaceMesh_Internal(*dm, dim, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, interpolate);CHKERRQ(ierr);
697   PetscFunctionReturn(0);
698 }
699 
700 static PetscErrorCode DMPlexCreateLineMesh_Internal(DM dm,PetscInt segments,PetscReal lower,PetscReal upper,DMBoundaryType bd)
701 {
702   PetscInt       i,fStart,fEnd,numCells = 0,numVerts = 0;
703   PetscInt       numPoints[2],*coneSize,*cones,*coneOrientations;
704   PetscScalar    *vertexCoords;
705   PetscReal      L,maxCell;
706   PetscBool      markerSeparate = PETSC_FALSE;
707   PetscInt       markerLeft  = 1, faceMarkerLeft  = 1;
708   PetscInt       markerRight = 1, faceMarkerRight = 2;
709   PetscBool      wrap = (bd == DM_BOUNDARY_PERIODIC || bd == DM_BOUNDARY_TWIST) ? PETSC_TRUE : PETSC_FALSE;
710   PetscMPIInt    rank;
711   PetscErrorCode ierr;
712 
713   PetscFunctionBegin;
714   PetscValidPointer(dm,1);
715 
716   ierr = DMSetDimension(dm,1);CHKERRQ(ierr);
717   ierr = DMCreateLabel(dm,"marker");CHKERRQ(ierr);
718   ierr = DMCreateLabel(dm,"Face Sets");CHKERRQ(ierr);
719 
720   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm),&rank);CHKERRMPI(ierr);
721   if (rank == 0) numCells = segments;
722   if (rank == 0) numVerts = segments + (wrap ? 0 : 1);
723 
724   numPoints[0] = numVerts ; numPoints[1] = numCells;
725   ierr = PetscMalloc4(numCells+numVerts,&coneSize,numCells*2,&cones,numCells+numVerts,&coneOrientations,numVerts,&vertexCoords);CHKERRQ(ierr);
726   ierr = PetscArrayzero(coneOrientations,numCells+numVerts);CHKERRQ(ierr);
727   for (i = 0; i < numCells; ++i) { coneSize[i] = 2; }
728   for (i = 0; i < numVerts; ++i) { coneSize[numCells+i] = 0; }
729   for (i = 0; i < numCells; ++i) { cones[2*i] = numCells + i%numVerts; cones[2*i+1] = numCells + (i+1)%numVerts; }
730   for (i = 0; i < numVerts; ++i) { vertexCoords[i] = lower + (upper-lower)*((PetscReal)i/(PetscReal)numCells); }
731   ierr = DMPlexCreateFromDAG(dm,1,numPoints,coneSize,cones,coneOrientations,vertexCoords);CHKERRQ(ierr);
732   ierr = PetscFree4(coneSize,cones,coneOrientations,vertexCoords);CHKERRQ(ierr);
733 
734   ierr = PetscOptionsGetBool(((PetscObject)dm)->options,((PetscObject)dm)->prefix,"-dm_plex_separate_marker",&markerSeparate,NULL);CHKERRQ(ierr);
735   if (markerSeparate) { markerLeft = faceMarkerLeft; markerRight = faceMarkerRight;}
736   if (!wrap && rank == 0) {
737     ierr = DMPlexGetHeightStratum(dm,1,&fStart,&fEnd);CHKERRQ(ierr);
738     ierr = DMSetLabelValue(dm,"marker",fStart,markerLeft);CHKERRQ(ierr);
739     ierr = DMSetLabelValue(dm,"marker",fEnd-1,markerRight);CHKERRQ(ierr);
740     ierr = DMSetLabelValue(dm,"Face Sets",fStart,faceMarkerLeft);CHKERRQ(ierr);
741     ierr = DMSetLabelValue(dm,"Face Sets",fEnd-1,faceMarkerRight);CHKERRQ(ierr);
742   }
743   if (wrap) {
744     L       = upper - lower;
745     maxCell = (PetscReal)1.1*(L/(PetscReal)PetscMax(1,segments));
746     ierr = DMSetPeriodicity(dm,PETSC_TRUE,&maxCell,&L,&bd);CHKERRQ(ierr);
747   }
748   ierr = DMPlexSetRefinementUniform(dm, PETSC_TRUE);CHKERRQ(ierr);
749   PetscFunctionReturn(0);
750 }
751 
752 static PetscErrorCode DMPlexCreateBoxMesh_Simplex_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate)
753 {
754   DM             boundary, vol;
755   PetscInt       i;
756   PetscErrorCode ierr;
757 
758   PetscFunctionBegin;
759   PetscValidPointer(dm, 1);
760   for (i = 0; i < dim; ++i) PetscCheckFalse(periodicity[i] != DM_BOUNDARY_NONE,PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Periodicity is not supported for simplex meshes");
761   ierr = DMCreate(PetscObjectComm((PetscObject) dm), &boundary);CHKERRQ(ierr);
762   ierr = DMSetType(boundary, DMPLEX);CHKERRQ(ierr);
763   ierr = DMPlexCreateBoxSurfaceMesh_Internal(boundary, dim, faces, lower, upper, PETSC_FALSE);CHKERRQ(ierr);
764   ierr = DMPlexGenerate(boundary, NULL, interpolate, &vol);CHKERRQ(ierr);
765   ierr = DMPlexCopy_Internal(dm, PETSC_TRUE, vol);CHKERRQ(ierr);
766   ierr = DMPlexReplace_Static(dm, &vol);CHKERRQ(ierr);
767   ierr = DMDestroy(&boundary);CHKERRQ(ierr);
768   PetscFunctionReturn(0);
769 }
770 
771 static PetscErrorCode DMPlexCreateCubeMesh_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[], DMBoundaryType bdX, DMBoundaryType bdY, DMBoundaryType bdZ)
772 {
773   DMLabel        cutLabel = NULL;
774   PetscInt       markerTop      = 1, faceMarkerTop      = 1;
775   PetscInt       markerBottom   = 1, faceMarkerBottom   = 1;
776   PetscInt       markerFront    = 1, faceMarkerFront    = 1;
777   PetscInt       markerBack     = 1, faceMarkerBack     = 1;
778   PetscInt       markerRight    = 1, faceMarkerRight    = 1;
779   PetscInt       markerLeft     = 1, faceMarkerLeft     = 1;
780   PetscInt       dim;
781   PetscBool      markerSeparate = PETSC_FALSE, cutMarker = PETSC_FALSE;
782   PetscMPIInt    rank;
783   PetscErrorCode ierr;
784 
785   PetscFunctionBegin;
786   ierr = DMGetDimension(dm,&dim);CHKERRQ(ierr);
787   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRMPI(ierr);
788   ierr = DMCreateLabel(dm,"marker");CHKERRQ(ierr);
789   ierr = DMCreateLabel(dm,"Face Sets");CHKERRQ(ierr);
790   ierr = PetscOptionsGetBool(((PetscObject) dm)->options,((PetscObject) dm)->prefix, "-dm_plex_periodic_cut", &cutMarker, NULL);CHKERRQ(ierr);
791   if (bdX == DM_BOUNDARY_PERIODIC || bdX == DM_BOUNDARY_TWIST ||
792       bdY == DM_BOUNDARY_PERIODIC || bdY == DM_BOUNDARY_TWIST ||
793       bdZ == DM_BOUNDARY_PERIODIC || bdZ == DM_BOUNDARY_TWIST) {
794 
795     if (cutMarker) {ierr = DMCreateLabel(dm, "periodic_cut");CHKERRQ(ierr); ierr = DMGetLabel(dm, "periodic_cut", &cutLabel);CHKERRQ(ierr);}
796   }
797   switch (dim) {
798   case 2:
799     faceMarkerTop    = 3;
800     faceMarkerBottom = 1;
801     faceMarkerRight  = 2;
802     faceMarkerLeft   = 4;
803     break;
804   case 3:
805     faceMarkerBottom = 1;
806     faceMarkerTop    = 2;
807     faceMarkerFront  = 3;
808     faceMarkerBack   = 4;
809     faceMarkerRight  = 5;
810     faceMarkerLeft   = 6;
811     break;
812   default:
813     SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"Dimension %D not supported",dim);
814   }
815   ierr = PetscOptionsGetBool(((PetscObject) dm)->options,((PetscObject) dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL);CHKERRQ(ierr);
816   if (markerSeparate) {
817     markerBottom = faceMarkerBottom;
818     markerTop    = faceMarkerTop;
819     markerFront  = faceMarkerFront;
820     markerBack   = faceMarkerBack;
821     markerRight  = faceMarkerRight;
822     markerLeft   = faceMarkerLeft;
823   }
824   {
825     const PetscInt numXEdges    = rank == 0 ? edges[0] : 0;
826     const PetscInt numYEdges    = rank == 0 ? edges[1] : 0;
827     const PetscInt numZEdges    = rank == 0 ? edges[2] : 0;
828     const PetscInt numXVertices = rank == 0 ? (bdX == DM_BOUNDARY_PERIODIC || bdX == DM_BOUNDARY_TWIST ? edges[0] : edges[0]+1) : 0;
829     const PetscInt numYVertices = rank == 0 ? (bdY == DM_BOUNDARY_PERIODIC || bdY == DM_BOUNDARY_TWIST ? edges[1] : edges[1]+1) : 0;
830     const PetscInt numZVertices = rank == 0 ? (bdZ == DM_BOUNDARY_PERIODIC || bdZ == DM_BOUNDARY_TWIST ? edges[2] : edges[2]+1) : 0;
831     const PetscInt numCells     = numXEdges*numYEdges*numZEdges;
832     const PetscInt numXFaces    = numYEdges*numZEdges;
833     const PetscInt numYFaces    = numXEdges*numZEdges;
834     const PetscInt numZFaces    = numXEdges*numYEdges;
835     const PetscInt numTotXFaces = numXVertices*numXFaces;
836     const PetscInt numTotYFaces = numYVertices*numYFaces;
837     const PetscInt numTotZFaces = numZVertices*numZFaces;
838     const PetscInt numFaces     = numTotXFaces + numTotYFaces + numTotZFaces;
839     const PetscInt numTotXEdges = numXEdges*numYVertices*numZVertices;
840     const PetscInt numTotYEdges = numYEdges*numXVertices*numZVertices;
841     const PetscInt numTotZEdges = numZEdges*numXVertices*numYVertices;
842     const PetscInt numVertices  = numXVertices*numYVertices*numZVertices;
843     const PetscInt numEdges     = numTotXEdges + numTotYEdges + numTotZEdges;
844     const PetscInt firstVertex  = (dim == 2) ? numFaces : numCells;
845     const PetscInt firstXFace   = (dim == 2) ? 0 : numCells + numVertices;
846     const PetscInt firstYFace   = firstXFace + numTotXFaces;
847     const PetscInt firstZFace   = firstYFace + numTotYFaces;
848     const PetscInt firstXEdge   = numCells + numFaces + numVertices;
849     const PetscInt firstYEdge   = firstXEdge + numTotXEdges;
850     const PetscInt firstZEdge   = firstYEdge + numTotYEdges;
851     Vec            coordinates;
852     PetscSection   coordSection;
853     PetscScalar   *coords;
854     PetscInt       coordSize;
855     PetscInt       v, vx, vy, vz;
856     PetscInt       c, f, fx, fy, fz, e, ex, ey, ez;
857 
858     ierr = DMPlexSetChart(dm, 0, numCells+numFaces+numEdges+numVertices);CHKERRQ(ierr);
859     for (c = 0; c < numCells; c++) {
860       ierr = DMPlexSetConeSize(dm, c, 6);CHKERRQ(ierr);
861     }
862     for (f = firstXFace; f < firstXFace+numFaces; ++f) {
863       ierr = DMPlexSetConeSize(dm, f, 4);CHKERRQ(ierr);
864     }
865     for (e = firstXEdge; e < firstXEdge+numEdges; ++e) {
866       ierr = DMPlexSetConeSize(dm, e, 2);CHKERRQ(ierr);
867     }
868     ierr = DMSetUp(dm);CHKERRQ(ierr); /* Allocate space for cones */
869     /* Build cells */
870     for (fz = 0; fz < numZEdges; ++fz) {
871       for (fy = 0; fy < numYEdges; ++fy) {
872         for (fx = 0; fx < numXEdges; ++fx) {
873           PetscInt cell    = (fz*numYEdges + fy)*numXEdges + fx;
874           PetscInt faceB   = firstZFace + (fy*numXEdges+fx)*numZVertices +   fz;
875           PetscInt faceT   = firstZFace + (fy*numXEdges+fx)*numZVertices + ((fz+1)%numZVertices);
876           PetscInt faceF   = firstYFace + (fz*numXEdges+fx)*numYVertices +   fy;
877           PetscInt faceK   = firstYFace + (fz*numXEdges+fx)*numYVertices + ((fy+1)%numYVertices);
878           PetscInt faceL   = firstXFace + (fz*numYEdges+fy)*numXVertices +   fx;
879           PetscInt faceR   = firstXFace + (fz*numYEdges+fy)*numXVertices + ((fx+1)%numXVertices);
880                             /* B,  T,  F,  K,  R,  L */
881           PetscInt ornt[6] = {-2,  0,  0, -3,  0, -2}; /* ??? */
882           PetscInt cone[6];
883 
884           /* no boundary twisting in 3D */
885           cone[0] = faceB; cone[1] = faceT; cone[2] = faceF; cone[3] = faceK; cone[4] = faceR; cone[5] = faceL;
886           ierr    = DMPlexSetCone(dm, cell, cone);CHKERRQ(ierr);
887           ierr    = DMPlexSetConeOrientation(dm, cell, ornt);CHKERRQ(ierr);
888           if (bdX != DM_BOUNDARY_NONE && fx == numXEdges-1 && cutLabel) {ierr = DMLabelSetValue(cutLabel, cell, 2);CHKERRQ(ierr);}
889           if (bdY != DM_BOUNDARY_NONE && fy == numYEdges-1 && cutLabel) {ierr = DMLabelSetValue(cutLabel, cell, 2);CHKERRQ(ierr);}
890           if (bdZ != DM_BOUNDARY_NONE && fz == numZEdges-1 && cutLabel) {ierr = DMLabelSetValue(cutLabel, cell, 2);CHKERRQ(ierr);}
891         }
892       }
893     }
894     /* Build x faces */
895     for (fz = 0; fz < numZEdges; ++fz) {
896       for (fy = 0; fy < numYEdges; ++fy) {
897         for (fx = 0; fx < numXVertices; ++fx) {
898           PetscInt face    = firstXFace + (fz*numYEdges+fy)     *numXVertices+fx;
899           PetscInt edgeL   = firstZEdge + (fy                   *numXVertices+fx)*numZEdges + fz;
900           PetscInt edgeR   = firstZEdge + (((fy+1)%numYVertices)*numXVertices+fx)*numZEdges + fz;
901           PetscInt edgeB   = firstYEdge + (fz                   *numXVertices+fx)*numYEdges + fy;
902           PetscInt edgeT   = firstYEdge + (((fz+1)%numZVertices)*numXVertices+fx)*numYEdges + fy;
903           PetscInt ornt[4] = {0, 0, -1, -1};
904           PetscInt cone[4];
905 
906           if (dim == 3) {
907             /* markers */
908             if (bdX != DM_BOUNDARY_PERIODIC) {
909               if (fx == numXVertices-1) {
910                 ierr = DMSetLabelValue(dm, "Face Sets", face, faceMarkerRight);CHKERRQ(ierr);
911                 ierr = DMSetLabelValue(dm, "marker", face, markerRight);CHKERRQ(ierr);
912               }
913               else if (fx == 0) {
914                 ierr = DMSetLabelValue(dm, "Face Sets", face, faceMarkerLeft);CHKERRQ(ierr);
915                 ierr = DMSetLabelValue(dm, "marker", face, markerLeft);CHKERRQ(ierr);
916               }
917             }
918           }
919           cone[0] = edgeB; cone[1] = edgeR; cone[2] = edgeT; cone[3] = edgeL;
920           ierr    = DMPlexSetCone(dm, face, cone);CHKERRQ(ierr);
921           ierr    = DMPlexSetConeOrientation(dm, face, ornt);CHKERRQ(ierr);
922         }
923       }
924     }
925     /* Build y faces */
926     for (fz = 0; fz < numZEdges; ++fz) {
927       for (fx = 0; fx < numXEdges; ++fx) {
928         for (fy = 0; fy < numYVertices; ++fy) {
929           PetscInt face    = firstYFace + (fz*numXEdges+fx)*numYVertices + fy;
930           PetscInt edgeL   = firstZEdge + (fy*numXVertices+  fx)*numZEdges + fz;
931           PetscInt edgeR   = firstZEdge + (fy*numXVertices+((fx+1)%numXVertices))*numZEdges + fz;
932           PetscInt edgeB   = firstXEdge + (fz                   *numYVertices+fy)*numXEdges + fx;
933           PetscInt edgeT   = firstXEdge + (((fz+1)%numZVertices)*numYVertices+fy)*numXEdges + fx;
934           PetscInt ornt[4] = {0, 0, -1, -1};
935           PetscInt cone[4];
936 
937           if (dim == 3) {
938             /* markers */
939             if (bdY != DM_BOUNDARY_PERIODIC) {
940               if (fy == numYVertices-1) {
941                 ierr = DMSetLabelValue(dm, "Face Sets", face, faceMarkerBack);CHKERRQ(ierr);
942                 ierr = DMSetLabelValue(dm, "marker", face, markerBack);CHKERRQ(ierr);
943               }
944               else if (fy == 0) {
945                 ierr = DMSetLabelValue(dm, "Face Sets", face, faceMarkerFront);CHKERRQ(ierr);
946                 ierr = DMSetLabelValue(dm, "marker", face, markerFront);CHKERRQ(ierr);
947               }
948             }
949           }
950           cone[0] = edgeB; cone[1] = edgeR; cone[2] = edgeT; cone[3] = edgeL;
951           ierr    = DMPlexSetCone(dm, face, cone);CHKERRQ(ierr);
952           ierr    = DMPlexSetConeOrientation(dm, face, ornt);CHKERRQ(ierr);
953         }
954       }
955     }
956     /* Build z faces */
957     for (fy = 0; fy < numYEdges; ++fy) {
958       for (fx = 0; fx < numXEdges; ++fx) {
959         for (fz = 0; fz < numZVertices; fz++) {
960           PetscInt face    = firstZFace + (fy*numXEdges+fx)*numZVertices + fz;
961           PetscInt edgeL   = firstYEdge + (fz*numXVertices+  fx)*numYEdges + fy;
962           PetscInt edgeR   = firstYEdge + (fz*numXVertices+((fx+1)%numXVertices))*numYEdges + fy;
963           PetscInt edgeB   = firstXEdge + (fz*numYVertices+  fy)*numXEdges + fx;
964           PetscInt edgeT   = firstXEdge + (fz*numYVertices+((fy+1)%numYVertices))*numXEdges + fx;
965           PetscInt ornt[4] = {0, 0, -1, -1};
966           PetscInt cone[4];
967 
968           if (dim == 2) {
969             if (bdX == DM_BOUNDARY_TWIST && fx == numXEdges-1) {edgeR += numYEdges-1-2*fy; ornt[1] = -1;}
970             if (bdY == DM_BOUNDARY_TWIST && fy == numYEdges-1) {edgeT += numXEdges-1-2*fx; ornt[2] =  0;}
971             if (bdX != DM_BOUNDARY_NONE && fx == numXEdges-1 && cutLabel) {ierr = DMLabelSetValue(cutLabel, face, 2);CHKERRQ(ierr);}
972             if (bdY != DM_BOUNDARY_NONE && fy == numYEdges-1 && cutLabel) {ierr = DMLabelSetValue(cutLabel, face, 2);CHKERRQ(ierr);}
973           } else {
974             /* markers */
975             if (bdZ != DM_BOUNDARY_PERIODIC) {
976               if (fz == numZVertices-1) {
977                 ierr = DMSetLabelValue(dm, "Face Sets", face, faceMarkerTop);CHKERRQ(ierr);
978                 ierr = DMSetLabelValue(dm, "marker", face, markerTop);CHKERRQ(ierr);
979               }
980               else if (fz == 0) {
981                 ierr = DMSetLabelValue(dm, "Face Sets", face, faceMarkerBottom);CHKERRQ(ierr);
982                 ierr = DMSetLabelValue(dm, "marker", face, markerBottom);CHKERRQ(ierr);
983               }
984             }
985           }
986           cone[0] = edgeB; cone[1] = edgeR; cone[2] = edgeT; cone[3] = edgeL;
987           ierr    = DMPlexSetCone(dm, face, cone);CHKERRQ(ierr);
988           ierr    = DMPlexSetConeOrientation(dm, face, ornt);CHKERRQ(ierr);
989         }
990       }
991     }
992     /* Build Z edges*/
993     for (vy = 0; vy < numYVertices; vy++) {
994       for (vx = 0; vx < numXVertices; vx++) {
995         for (ez = 0; ez < numZEdges; ez++) {
996           const PetscInt edge    = firstZEdge  + (vy*numXVertices+vx)*numZEdges + ez;
997           const PetscInt vertexB = firstVertex + (ez                   *numYVertices+vy)*numXVertices + vx;
998           const PetscInt vertexT = firstVertex + (((ez+1)%numZVertices)*numYVertices+vy)*numXVertices + vx;
999           PetscInt       cone[2];
1000 
1001           if (dim == 3) {
1002             if (bdX != DM_BOUNDARY_PERIODIC) {
1003               if (vx == numXVertices-1) {
1004                 ierr = DMSetLabelValue(dm, "marker", edge, markerRight);CHKERRQ(ierr);
1005               }
1006               else if (vx == 0) {
1007                 ierr = DMSetLabelValue(dm, "marker", edge, markerLeft);CHKERRQ(ierr);
1008               }
1009             }
1010             if (bdY != DM_BOUNDARY_PERIODIC) {
1011               if (vy == numYVertices-1) {
1012                 ierr = DMSetLabelValue(dm, "marker", edge, markerBack);CHKERRQ(ierr);
1013               }
1014               else if (vy == 0) {
1015                 ierr = DMSetLabelValue(dm, "marker", edge, markerFront);CHKERRQ(ierr);
1016               }
1017             }
1018           }
1019           cone[0] = vertexB; cone[1] = vertexT;
1020           ierr = DMPlexSetCone(dm, edge, cone);CHKERRQ(ierr);
1021         }
1022       }
1023     }
1024     /* Build Y edges*/
1025     for (vz = 0; vz < numZVertices; vz++) {
1026       for (vx = 0; vx < numXVertices; vx++) {
1027         for (ey = 0; ey < numYEdges; ey++) {
1028           const PetscInt nextv   = (dim == 2 && bdY == DM_BOUNDARY_TWIST && ey == numYEdges-1) ? (numXVertices-vx-1) : (vz*numYVertices+((ey+1)%numYVertices))*numXVertices + vx;
1029           const PetscInt edge    = firstYEdge  + (vz*numXVertices+vx)*numYEdges + ey;
1030           const PetscInt vertexF = firstVertex + (vz*numYVertices+ey)*numXVertices + vx;
1031           const PetscInt vertexK = firstVertex + nextv;
1032           PetscInt       cone[2];
1033 
1034           cone[0] = vertexF; cone[1] = vertexK;
1035           ierr = DMPlexSetCone(dm, edge, cone);CHKERRQ(ierr);
1036           if (dim == 2) {
1037             if ((bdX != DM_BOUNDARY_PERIODIC) && (bdX != DM_BOUNDARY_TWIST)) {
1038               if (vx == numXVertices-1) {
1039                 ierr = DMSetLabelValue(dm, "Face Sets", edge, faceMarkerRight);CHKERRQ(ierr);
1040                 ierr = DMSetLabelValue(dm, "marker", edge,    markerRight);CHKERRQ(ierr);
1041                 ierr = DMSetLabelValue(dm, "marker", cone[0], markerRight);CHKERRQ(ierr);
1042                 if (ey == numYEdges-1) {
1043                   ierr = DMSetLabelValue(dm, "marker", cone[1], markerRight);CHKERRQ(ierr);
1044                 }
1045               } else if (vx == 0) {
1046                 ierr = DMSetLabelValue(dm, "Face Sets", edge, faceMarkerLeft);CHKERRQ(ierr);
1047                 ierr = DMSetLabelValue(dm, "marker", edge,    markerLeft);CHKERRQ(ierr);
1048                 ierr = DMSetLabelValue(dm, "marker", cone[0], markerLeft);CHKERRQ(ierr);
1049                 if (ey == numYEdges-1) {
1050                   ierr = DMSetLabelValue(dm, "marker", cone[1], markerLeft);CHKERRQ(ierr);
1051                 }
1052               }
1053             } else {
1054               if (vx == 0 && cutLabel) {
1055                 ierr = DMLabelSetValue(cutLabel, edge,    1);CHKERRQ(ierr);
1056                 ierr = DMLabelSetValue(cutLabel, cone[0], 1);CHKERRQ(ierr);
1057                 if (ey == numYEdges-1) {
1058                   ierr = DMLabelSetValue(cutLabel, cone[1], 1);CHKERRQ(ierr);
1059                 }
1060               }
1061             }
1062           } else {
1063             if (bdX != DM_BOUNDARY_PERIODIC) {
1064               if (vx == numXVertices-1) {
1065                 ierr = DMSetLabelValue(dm, "marker", edge, markerRight);CHKERRQ(ierr);
1066               } else if (vx == 0) {
1067                 ierr = DMSetLabelValue(dm, "marker", edge, markerLeft);CHKERRQ(ierr);
1068               }
1069             }
1070             if (bdZ != DM_BOUNDARY_PERIODIC) {
1071               if (vz == numZVertices-1) {
1072                 ierr = DMSetLabelValue(dm, "marker", edge, markerTop);CHKERRQ(ierr);
1073               } else if (vz == 0) {
1074                 ierr = DMSetLabelValue(dm, "marker", edge, markerBottom);CHKERRQ(ierr);
1075               }
1076             }
1077           }
1078         }
1079       }
1080     }
1081     /* Build X edges*/
1082     for (vz = 0; vz < numZVertices; vz++) {
1083       for (vy = 0; vy < numYVertices; vy++) {
1084         for (ex = 0; ex < numXEdges; ex++) {
1085           const PetscInt nextv   = (dim == 2 && bdX == DM_BOUNDARY_TWIST && ex == numXEdges-1) ? (numYVertices-vy-1)*numXVertices : (vz*numYVertices+vy)*numXVertices + (ex+1)%numXVertices;
1086           const PetscInt edge    = firstXEdge  + (vz*numYVertices+vy)*numXEdges + ex;
1087           const PetscInt vertexL = firstVertex + (vz*numYVertices+vy)*numXVertices + ex;
1088           const PetscInt vertexR = firstVertex + nextv;
1089           PetscInt       cone[2];
1090 
1091           cone[0] = vertexL; cone[1] = vertexR;
1092           ierr = DMPlexSetCone(dm, edge, cone);CHKERRQ(ierr);
1093           if (dim == 2) {
1094             if ((bdY != DM_BOUNDARY_PERIODIC) && (bdY != DM_BOUNDARY_TWIST)) {
1095               if (vy == numYVertices-1) {
1096                 ierr = DMSetLabelValue(dm, "Face Sets", edge, faceMarkerTop);CHKERRQ(ierr);
1097                 ierr = DMSetLabelValue(dm, "marker", edge,    markerTop);CHKERRQ(ierr);
1098                 ierr = DMSetLabelValue(dm, "marker", cone[0], markerTop);CHKERRQ(ierr);
1099                 if (ex == numXEdges-1) {
1100                   ierr = DMSetLabelValue(dm, "marker", cone[1], markerTop);CHKERRQ(ierr);
1101                 }
1102               } else if (vy == 0) {
1103                 ierr = DMSetLabelValue(dm, "Face Sets", edge, faceMarkerBottom);CHKERRQ(ierr);
1104                 ierr = DMSetLabelValue(dm, "marker", edge,    markerBottom);CHKERRQ(ierr);
1105                 ierr = DMSetLabelValue(dm, "marker", cone[0], markerBottom);CHKERRQ(ierr);
1106                 if (ex == numXEdges-1) {
1107                   ierr = DMSetLabelValue(dm, "marker", cone[1], markerBottom);CHKERRQ(ierr);
1108                 }
1109               }
1110             } else {
1111               if (vy == 0 && cutLabel) {
1112                 ierr = DMLabelSetValue(cutLabel, edge,    1);CHKERRQ(ierr);
1113                 ierr = DMLabelSetValue(cutLabel, cone[0], 1);CHKERRQ(ierr);
1114                 if (ex == numXEdges-1) {
1115                   ierr = DMLabelSetValue(cutLabel, cone[1], 1);CHKERRQ(ierr);
1116                 }
1117               }
1118             }
1119           } else {
1120             if (bdY != DM_BOUNDARY_PERIODIC) {
1121               if (vy == numYVertices-1) {
1122                 ierr = DMSetLabelValue(dm, "marker", edge, markerBack);CHKERRQ(ierr);
1123               }
1124               else if (vy == 0) {
1125                 ierr = DMSetLabelValue(dm, "marker", edge, markerFront);CHKERRQ(ierr);
1126               }
1127             }
1128             if (bdZ != DM_BOUNDARY_PERIODIC) {
1129               if (vz == numZVertices-1) {
1130                 ierr = DMSetLabelValue(dm, "marker", edge, markerTop);CHKERRQ(ierr);
1131               }
1132               else if (vz == 0) {
1133                 ierr = DMSetLabelValue(dm, "marker", edge, markerBottom);CHKERRQ(ierr);
1134               }
1135             }
1136           }
1137         }
1138       }
1139     }
1140     ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
1141     ierr = DMPlexStratify(dm);CHKERRQ(ierr);
1142     /* Build coordinates */
1143     ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
1144     ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
1145     ierr = PetscSectionSetFieldComponents(coordSection, 0, dim);CHKERRQ(ierr);
1146     ierr = PetscSectionSetChart(coordSection, firstVertex, firstVertex+numVertices);CHKERRQ(ierr);
1147     for (v = firstVertex; v < firstVertex+numVertices; ++v) {
1148       ierr = PetscSectionSetDof(coordSection, v, dim);CHKERRQ(ierr);
1149       ierr = PetscSectionSetFieldDof(coordSection, v, 0, dim);CHKERRQ(ierr);
1150     }
1151     ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
1152     ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
1153     ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
1154     ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
1155     ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
1156     ierr = VecSetBlockSize(coordinates, dim);CHKERRQ(ierr);
1157     ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
1158     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
1159     for (vz = 0; vz < numZVertices; ++vz) {
1160       for (vy = 0; vy < numYVertices; ++vy) {
1161         for (vx = 0; vx < numXVertices; ++vx) {
1162           coords[((vz*numYVertices+vy)*numXVertices+vx)*dim+0] = lower[0] + ((upper[0] - lower[0])/numXEdges)*vx;
1163           coords[((vz*numYVertices+vy)*numXVertices+vx)*dim+1] = lower[1] + ((upper[1] - lower[1])/numYEdges)*vy;
1164           if (dim == 3) {
1165             coords[((vz*numYVertices+vy)*numXVertices+vx)*dim+2] = lower[2] + ((upper[2] - lower[2])/numZEdges)*vz;
1166           }
1167         }
1168       }
1169     }
1170     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
1171     ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
1172     ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
1173   }
1174   PetscFunctionReturn(0);
1175 }
1176 
1177 static PetscErrorCode DMPlexCreateBoxMesh_Tensor_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[])
1178 {
1179   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1180   PetscInt       fac[3] = {0, 0, 0}, d;
1181   PetscErrorCode ierr;
1182 
1183   PetscFunctionBegin;
1184   PetscValidPointer(dm, 1);
1185   PetscValidLogicalCollectiveInt(dm, dim, 2);
1186   ierr = DMSetDimension(dm, dim);CHKERRQ(ierr);
1187   for (d = 0; d < dim; ++d) {fac[d] = faces[d]; bdt[d] = periodicity[d];}
1188   ierr = DMPlexCreateCubeMesh_Internal(dm, lower, upper, fac, bdt[0], bdt[1], bdt[2]);CHKERRQ(ierr);
1189   if (periodicity[0] == DM_BOUNDARY_PERIODIC || periodicity[0] == DM_BOUNDARY_TWIST ||
1190       periodicity[1] == DM_BOUNDARY_PERIODIC || periodicity[1] == DM_BOUNDARY_TWIST ||
1191       (dim > 2 && (periodicity[2] == DM_BOUNDARY_PERIODIC || periodicity[2] == DM_BOUNDARY_TWIST))) {
1192     PetscReal L[3];
1193     PetscReal maxCell[3];
1194 
1195     for (d = 0; d < dim; ++d) {
1196       L[d]       = upper[d] - lower[d];
1197       maxCell[d] = 1.1 * (L[d] / PetscMax(1, faces[d]));
1198     }
1199     ierr = DMSetPeriodicity(dm, PETSC_TRUE, maxCell, L, periodicity);CHKERRQ(ierr);
1200   }
1201   ierr = DMPlexSetRefinementUniform(dm, PETSC_TRUE);CHKERRQ(ierr);
1202   PetscFunctionReturn(0);
1203 }
1204 
1205 static PetscErrorCode DMPlexCreateBoxMesh_Internal(DM dm, PetscInt dim, PetscBool simplex, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate)
1206 {
1207   PetscErrorCode ierr;
1208 
1209   PetscFunctionBegin;
1210   if (dim == 1)      {ierr = DMPlexCreateLineMesh_Internal(dm, faces[0], lower[0], upper[0], periodicity[0]);CHKERRQ(ierr);}
1211   else if (simplex)  {ierr = DMPlexCreateBoxMesh_Simplex_Internal(dm, dim, faces, lower, upper, periodicity, interpolate);CHKERRQ(ierr);}
1212   else               {ierr = DMPlexCreateBoxMesh_Tensor_Internal(dm, dim, faces, lower, upper, periodicity);CHKERRQ(ierr);}
1213   if (!interpolate && dim > 1 && !simplex) {
1214     DM udm;
1215 
1216     ierr = DMPlexUninterpolate(dm, &udm);CHKERRQ(ierr);
1217     ierr = DMPlexCopyCoordinates(dm, udm);CHKERRQ(ierr);
1218     ierr = DMPlexReplace_Static(dm, &udm);CHKERRQ(ierr);
1219   }
1220   PetscFunctionReturn(0);
1221 }
1222 
1223 /*@C
1224   DMPlexCreateBoxMesh - Creates a mesh on the tensor product of unit intervals (box) using simplices or tensor cells (hexahedra).
1225 
1226   Collective
1227 
1228   Input Parameters:
1229 + comm        - The communicator for the DM object
1230 . dim         - The spatial dimension
1231 . simplex     - PETSC_TRUE for simplices, PETSC_FALSE for tensor cells
1232 . faces       - Number of faces per dimension, or NULL for (1,) in 1D and (2, 2) in 2D and (1, 1, 1) in 3D
1233 . lower       - The lower left corner, or NULL for (0, 0, 0)
1234 . upper       - The upper right corner, or NULL for (1, 1, 1)
1235 . periodicity - The boundary type for the X,Y,Z direction, or NULL for DM_BOUNDARY_NONE
1236 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
1237 
1238   Output Parameter:
1239 . dm  - The DM object
1240 
1241   Note: If you want to customize this mesh using options, you just need to
1242 $  DMCreate(comm, &dm);
1243 $  DMSetType(dm, DMPLEX);
1244 $  DMSetFromOptions(dm);
1245 and use the options on the DMSetFromOptions() page.
1246 
1247   Here is the numbering returned for 2 faces in each direction for tensor cells:
1248 $ 10---17---11---18----12
1249 $  |         |         |
1250 $  |         |         |
1251 $ 20    2   22    3    24
1252 $  |         |         |
1253 $  |         |         |
1254 $  7---15----8---16----9
1255 $  |         |         |
1256 $  |         |         |
1257 $ 19    0   21    1   23
1258 $  |         |         |
1259 $  |         |         |
1260 $  4---13----5---14----6
1261 
1262 and for simplicial cells
1263 
1264 $ 14----8---15----9----16
1265 $  |\     5  |\      7 |
1266 $  | \       | \       |
1267 $ 13   2    14    3    15
1268 $  | 4   \   | 6   \   |
1269 $  |       \ |       \ |
1270 $ 11----6---12----7----13
1271 $  |\        |\        |
1272 $  | \    1  | \     3 |
1273 $ 10   0    11    1    12
1274 $  | 0   \   | 2   \   |
1275 $  |       \ |       \ |
1276 $  8----4----9----5----10
1277 
1278   Level: beginner
1279 
1280 .seealso: DMSetFromOptions(), DMPlexCreateFromFile(), DMPlexCreateHexCylinderMesh(), DMSetType(), DMCreate()
1281 @*/
1282 PetscErrorCode DMPlexCreateBoxMesh(MPI_Comm comm, PetscInt dim, PetscBool simplex, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate, DM *dm)
1283 {
1284   PetscInt       fac[3] = {1, 1, 1};
1285   PetscReal      low[3] = {0, 0, 0};
1286   PetscReal      upp[3] = {1, 1, 1};
1287   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1288   PetscErrorCode ierr;
1289 
1290   PetscFunctionBegin;
1291   ierr = DMCreate(comm,dm);CHKERRQ(ierr);
1292   ierr = DMSetType(*dm,DMPLEX);CHKERRQ(ierr);
1293   ierr = DMPlexCreateBoxMesh_Internal(*dm, dim, simplex, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, periodicity ? periodicity : bdt, interpolate);CHKERRQ(ierr);
1294   PetscFunctionReturn(0);
1295 }
1296 
1297 static PetscErrorCode DMPlexCreateWedgeBoxMesh_Internal(DM dm, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[])
1298 {
1299   DM             bdm, vol;
1300   PetscInt       i;
1301   PetscErrorCode ierr;
1302 
1303   PetscFunctionBegin;
1304   for (i = 0; i < 3; ++i) PetscCheckFalse(periodicity[i] != DM_BOUNDARY_NONE,PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Periodicity not yet supported");
1305   ierr = DMCreate(PetscObjectComm((PetscObject) dm), &bdm);CHKERRQ(ierr);
1306   ierr = DMSetType(bdm, DMPLEX);CHKERRQ(ierr);
1307   ierr = DMSetDimension(bdm, 2);CHKERRQ(ierr);
1308   ierr = DMPlexCreateBoxMesh_Simplex_Internal(bdm, 2, faces, lower, upper, periodicity, PETSC_TRUE);CHKERRQ(ierr);
1309   ierr = DMPlexExtrude(bdm, faces[2], upper[2] - lower[2], PETSC_TRUE, PETSC_FALSE, NULL, NULL, &vol);CHKERRQ(ierr);
1310   ierr = DMDestroy(&bdm);CHKERRQ(ierr);
1311   ierr = DMPlexReplace_Static(dm, &vol);CHKERRQ(ierr);
1312   if (lower[2] != 0.0) {
1313     Vec          v;
1314     PetscScalar *x;
1315     PetscInt     cDim, n;
1316 
1317     ierr = DMGetCoordinatesLocal(dm, &v);CHKERRQ(ierr);
1318     ierr = VecGetBlockSize(v, &cDim);CHKERRQ(ierr);
1319     ierr = VecGetLocalSize(v, &n);CHKERRQ(ierr);
1320     ierr = VecGetArray(v, &x);CHKERRQ(ierr);
1321     x   += cDim;
1322     for (i = 0; i < n; i += cDim) x[i] += lower[2];
1323     ierr = VecRestoreArray(v,&x);CHKERRQ(ierr);
1324     ierr = DMSetCoordinatesLocal(dm, v);CHKERRQ(ierr);
1325   }
1326   PetscFunctionReturn(0);
1327 }
1328 
1329 /*@
1330   DMPlexCreateWedgeBoxMesh - Creates a 3-D mesh tesselating the (x,y) plane and extruding in the third direction using wedge cells.
1331 
1332   Collective
1333 
1334   Input Parameters:
1335 + comm        - The communicator for the DM object
1336 . faces       - Number of faces per dimension, or NULL for (1, 1, 1)
1337 . lower       - The lower left corner, or NULL for (0, 0, 0)
1338 . upper       - The upper right corner, or NULL for (1, 1, 1)
1339 . periodicity - The boundary type for the X,Y,Z direction, or NULL for DM_BOUNDARY_NONE
1340 . orderHeight - If PETSC_TRUE, orders the extruded cells in the height first. Otherwise, orders the cell on the layers first
1341 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
1342 
1343   Output Parameter:
1344 . dm  - The DM object
1345 
1346   Level: beginner
1347 
1348 .seealso: DMPlexCreateHexCylinderMesh(), DMPlexCreateWedgeCylinderMesh(), DMExtrude(), DMPlexCreateBoxMesh(), DMSetType(), DMCreate()
1349 @*/
1350 PetscErrorCode DMPlexCreateWedgeBoxMesh(MPI_Comm comm, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool orderHeight, PetscBool interpolate, DM *dm)
1351 {
1352   PetscInt       fac[3] = {1, 1, 1};
1353   PetscReal      low[3] = {0, 0, 0};
1354   PetscReal      upp[3] = {1, 1, 1};
1355   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1356   PetscErrorCode ierr;
1357 
1358   PetscFunctionBegin;
1359   ierr = DMCreate(comm,dm);CHKERRQ(ierr);
1360   ierr = DMSetType(*dm,DMPLEX);CHKERRQ(ierr);
1361   ierr = DMPlexCreateWedgeBoxMesh_Internal(*dm, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, periodicity ? periodicity : bdt);CHKERRQ(ierr);
1362   if (!interpolate) {
1363     DM udm;
1364 
1365     ierr = DMPlexUninterpolate(*dm, &udm);CHKERRQ(ierr);
1366     ierr = DMPlexReplace_Static(*dm, &udm);CHKERRQ(ierr);
1367   }
1368   PetscFunctionReturn(0);
1369 }
1370 
1371 /*@C
1372   DMPlexSetOptionsPrefix - Sets the prefix used for searching for all DM options in the database.
1373 
1374   Logically Collective on dm
1375 
1376   Input Parameters:
1377 + dm - the DM context
1378 - prefix - the prefix to prepend to all option names
1379 
1380   Notes:
1381   A hyphen (-) must NOT be given at the beginning of the prefix name.
1382   The first character of all runtime options is AUTOMATICALLY the hyphen.
1383 
1384   Level: advanced
1385 
1386 .seealso: SNESSetFromOptions()
1387 @*/
1388 PetscErrorCode DMPlexSetOptionsPrefix(DM dm, const char prefix[])
1389 {
1390   DM_Plex       *mesh = (DM_Plex *) dm->data;
1391   PetscErrorCode ierr;
1392 
1393   PetscFunctionBegin;
1394   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1395   ierr = PetscObjectSetOptionsPrefix((PetscObject) dm, prefix);CHKERRQ(ierr);
1396   ierr = PetscObjectSetOptionsPrefix((PetscObject) mesh->partitioner, prefix);CHKERRQ(ierr);
1397   PetscFunctionReturn(0);
1398 }
1399 
1400 /* Remap geometry to cylinder
1401    TODO: This only works for a single refinement, then it is broken
1402 
1403      Interior square: Linear interpolation is correct
1404      The other cells all have vertices on rays from the origin. We want to uniformly expand the spacing
1405      such that the last vertex is on the unit circle. So the closest and farthest vertices are at distance
1406 
1407        phi     = arctan(y/x)
1408        d_close = sqrt(1/8 + 1/4 sin^2(phi))
1409        d_far   = sqrt(1/2 + sin^2(phi))
1410 
1411      so we remap them using
1412 
1413        x_new = x_close + (x - x_close) (1 - d_close) / (d_far - d_close)
1414        y_new = y_close + (y - y_close) (1 - d_close) / (d_far - d_close)
1415 
1416      If pi/4 < phi < 3pi/4 or -3pi/4 < phi < -pi/4, then we switch x and y.
1417 */
1418 static void snapToCylinder(PetscInt dim, PetscInt Nf, PetscInt NfAux,
1419                            const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[],
1420                            const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[],
1421                            PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
1422 {
1423   const PetscReal dis = 1.0/PetscSqrtReal(2.0);
1424   const PetscReal ds2 = 0.5*dis;
1425 
1426   if ((PetscAbsScalar(u[0]) <= ds2) && (PetscAbsScalar(u[1]) <= ds2)) {
1427     f0[0] = u[0];
1428     f0[1] = u[1];
1429   } else {
1430     PetscReal phi, sinp, cosp, dc, df, x, y, xc, yc;
1431 
1432     x    = PetscRealPart(u[0]);
1433     y    = PetscRealPart(u[1]);
1434     phi  = PetscAtan2Real(y, x);
1435     sinp = PetscSinReal(phi);
1436     cosp = PetscCosReal(phi);
1437     if ((PetscAbsReal(phi) > PETSC_PI/4.0) && (PetscAbsReal(phi) < 3.0*PETSC_PI/4.0)) {
1438       dc = PetscAbsReal(ds2/sinp);
1439       df = PetscAbsReal(dis/sinp);
1440       xc = ds2*x/PetscAbsReal(y);
1441       yc = ds2*PetscSignReal(y);
1442     } else {
1443       dc = PetscAbsReal(ds2/cosp);
1444       df = PetscAbsReal(dis/cosp);
1445       xc = ds2*PetscSignReal(x);
1446       yc = ds2*y/PetscAbsReal(x);
1447     }
1448     f0[0] = xc + (u[0] - xc)*(1.0 - dc)/(df - dc);
1449     f0[1] = yc + (u[1] - yc)*(1.0 - dc)/(df - dc);
1450   }
1451   f0[2] = u[2];
1452 }
1453 
1454 static PetscErrorCode DMPlexCreateHexCylinderMesh_Internal(DM dm, DMBoundaryType periodicZ)
1455 {
1456   const PetscInt dim = 3;
1457   PetscInt       numCells, numVertices;
1458   PetscMPIInt    rank;
1459   PetscErrorCode ierr;
1460 
1461   PetscFunctionBegin;
1462   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRMPI(ierr);
1463   ierr = DMSetDimension(dm, dim);CHKERRQ(ierr);
1464   /* Create topology */
1465   {
1466     PetscInt cone[8], c;
1467 
1468     numCells    = rank == 0 ?  5 : 0;
1469     numVertices = rank == 0 ? 16 : 0;
1470     if (periodicZ == DM_BOUNDARY_PERIODIC) {
1471       numCells   *= 3;
1472       numVertices = rank == 0 ? 24 : 0;
1473     }
1474     ierr = DMPlexSetChart(dm, 0, numCells+numVertices);CHKERRQ(ierr);
1475     for (c = 0; c < numCells; c++) {ierr = DMPlexSetConeSize(dm, c, 8);CHKERRQ(ierr);}
1476     ierr = DMSetUp(dm);CHKERRQ(ierr);
1477     if (rank == 0) {
1478       if (periodicZ == DM_BOUNDARY_PERIODIC) {
1479         cone[0] = 15; cone[1] = 18; cone[2] = 17; cone[3] = 16;
1480         cone[4] = 31; cone[5] = 32; cone[6] = 33; cone[7] = 34;
1481         ierr = DMPlexSetCone(dm, 0, cone);CHKERRQ(ierr);
1482         cone[0] = 16; cone[1] = 17; cone[2] = 24; cone[3] = 23;
1483         cone[4] = 32; cone[5] = 36; cone[6] = 37; cone[7] = 33; /* 22 25 26 21 */
1484         ierr = DMPlexSetCone(dm, 1, cone);CHKERRQ(ierr);
1485         cone[0] = 18; cone[1] = 27; cone[2] = 24; cone[3] = 17;
1486         cone[4] = 34; cone[5] = 33; cone[6] = 37; cone[7] = 38;
1487         ierr = DMPlexSetCone(dm, 2, cone);CHKERRQ(ierr);
1488         cone[0] = 29; cone[1] = 27; cone[2] = 18; cone[3] = 15;
1489         cone[4] = 35; cone[5] = 31; cone[6] = 34; cone[7] = 38;
1490         ierr = DMPlexSetCone(dm, 3, cone);CHKERRQ(ierr);
1491         cone[0] = 29; cone[1] = 15; cone[2] = 16; cone[3] = 23;
1492         cone[4] = 35; cone[5] = 36; cone[6] = 32; cone[7] = 31;
1493         ierr = DMPlexSetCone(dm, 4, cone);CHKERRQ(ierr);
1494 
1495         cone[0] = 31; cone[1] = 34; cone[2] = 33; cone[3] = 32;
1496         cone[4] = 19; cone[5] = 22; cone[6] = 21; cone[7] = 20;
1497         ierr = DMPlexSetCone(dm, 5, cone);CHKERRQ(ierr);
1498         cone[0] = 32; cone[1] = 33; cone[2] = 37; cone[3] = 36;
1499         cone[4] = 22; cone[5] = 25; cone[6] = 26; cone[7] = 21;
1500         ierr = DMPlexSetCone(dm, 6, cone);CHKERRQ(ierr);
1501         cone[0] = 34; cone[1] = 38; cone[2] = 37; cone[3] = 33;
1502         cone[4] = 20; cone[5] = 21; cone[6] = 26; cone[7] = 28;
1503         ierr = DMPlexSetCone(dm, 7, cone);CHKERRQ(ierr);
1504         cone[0] = 35; cone[1] = 38; cone[2] = 34; cone[3] = 31;
1505         cone[4] = 30; cone[5] = 19; cone[6] = 20; cone[7] = 28;
1506         ierr = DMPlexSetCone(dm, 8, cone);CHKERRQ(ierr);
1507         cone[0] = 35; cone[1] = 31; cone[2] = 32; cone[3] = 36;
1508         cone[4] = 30; cone[5] = 25; cone[6] = 22; cone[7] = 19;
1509         ierr = DMPlexSetCone(dm, 9, cone);CHKERRQ(ierr);
1510 
1511         cone[0] = 19; cone[1] = 20; cone[2] = 21; cone[3] = 22;
1512         cone[4] = 15; cone[5] = 16; cone[6] = 17; cone[7] = 18;
1513         ierr = DMPlexSetCone(dm, 10, cone);CHKERRQ(ierr);
1514         cone[0] = 22; cone[1] = 21; cone[2] = 26; cone[3] = 25;
1515         cone[4] = 16; cone[5] = 23; cone[6] = 24; cone[7] = 17;
1516         ierr = DMPlexSetCone(dm, 11, cone);CHKERRQ(ierr);
1517         cone[0] = 20; cone[1] = 28; cone[2] = 26; cone[3] = 21;
1518         cone[4] = 18; cone[5] = 17; cone[6] = 24; cone[7] = 27;
1519         ierr = DMPlexSetCone(dm, 12, cone);CHKERRQ(ierr);
1520         cone[0] = 30; cone[1] = 28; cone[2] = 20; cone[3] = 19;
1521         cone[4] = 29; cone[5] = 15; cone[6] = 18; cone[7] = 27;
1522         ierr = DMPlexSetCone(dm, 13, cone);CHKERRQ(ierr);
1523         cone[0] = 30; cone[1] = 19; cone[2] = 22; cone[3] = 25;
1524         cone[4] = 29; cone[5] = 23; cone[6] = 16; cone[7] = 15;
1525         ierr = DMPlexSetCone(dm, 14, cone);CHKERRQ(ierr);
1526       } else {
1527         cone[0] =  5; cone[1] =  8; cone[2] =  7; cone[3] =  6;
1528         cone[4] =  9; cone[5] = 12; cone[6] = 11; cone[7] = 10;
1529         ierr = DMPlexSetCone(dm, 0, cone);CHKERRQ(ierr);
1530         cone[0] =  6; cone[1] =  7; cone[2] = 14; cone[3] = 13;
1531         cone[4] = 12; cone[5] = 15; cone[6] = 16; cone[7] = 11;
1532         ierr = DMPlexSetCone(dm, 1, cone);CHKERRQ(ierr);
1533         cone[0] =  8; cone[1] = 17; cone[2] = 14; cone[3] =  7;
1534         cone[4] = 10; cone[5] = 11; cone[6] = 16; cone[7] = 18;
1535         ierr = DMPlexSetCone(dm, 2, cone);CHKERRQ(ierr);
1536         cone[0] = 19; cone[1] = 17; cone[2] =  8; cone[3] =  5;
1537         cone[4] = 20; cone[5] =  9; cone[6] = 10; cone[7] = 18;
1538         ierr = DMPlexSetCone(dm, 3, cone);CHKERRQ(ierr);
1539         cone[0] = 19; cone[1] =  5; cone[2] =  6; cone[3] = 13;
1540         cone[4] = 20; cone[5] = 15; cone[6] = 12; cone[7] =  9;
1541         ierr = DMPlexSetCone(dm, 4, cone);CHKERRQ(ierr);
1542       }
1543     }
1544     ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
1545     ierr = DMPlexStratify(dm);CHKERRQ(ierr);
1546   }
1547   /* Create cube geometry */
1548   {
1549     Vec             coordinates;
1550     PetscSection    coordSection;
1551     PetscScalar    *coords;
1552     PetscInt        coordSize, v;
1553     const PetscReal dis = 1.0/PetscSqrtReal(2.0);
1554     const PetscReal ds2 = dis/2.0;
1555 
1556     /* Build coordinates */
1557     ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
1558     ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
1559     ierr = PetscSectionSetFieldComponents(coordSection, 0, dim);CHKERRQ(ierr);
1560     ierr = PetscSectionSetChart(coordSection, numCells, numCells+numVertices);CHKERRQ(ierr);
1561     for (v = numCells; v < numCells+numVertices; ++v) {
1562       ierr = PetscSectionSetDof(coordSection, v, dim);CHKERRQ(ierr);
1563       ierr = PetscSectionSetFieldDof(coordSection, v, 0, dim);CHKERRQ(ierr);
1564     }
1565     ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
1566     ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
1567     ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
1568     ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
1569     ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
1570     ierr = VecSetBlockSize(coordinates, dim);CHKERRQ(ierr);
1571     ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
1572     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
1573     if (rank == 0) {
1574       coords[0*dim+0] = -ds2; coords[0*dim+1] = -ds2; coords[0*dim+2] = 0.0;
1575       coords[1*dim+0] =  ds2; coords[1*dim+1] = -ds2; coords[1*dim+2] = 0.0;
1576       coords[2*dim+0] =  ds2; coords[2*dim+1] =  ds2; coords[2*dim+2] = 0.0;
1577       coords[3*dim+0] = -ds2; coords[3*dim+1] =  ds2; coords[3*dim+2] = 0.0;
1578       coords[4*dim+0] = -ds2; coords[4*dim+1] = -ds2; coords[4*dim+2] = 1.0;
1579       coords[5*dim+0] = -ds2; coords[5*dim+1] =  ds2; coords[5*dim+2] = 1.0;
1580       coords[6*dim+0] =  ds2; coords[6*dim+1] =  ds2; coords[6*dim+2] = 1.0;
1581       coords[7*dim+0] =  ds2; coords[7*dim+1] = -ds2; coords[7*dim+2] = 1.0;
1582       coords[ 8*dim+0] =  dis; coords[ 8*dim+1] = -dis; coords[ 8*dim+2] = 0.0;
1583       coords[ 9*dim+0] =  dis; coords[ 9*dim+1] =  dis; coords[ 9*dim+2] = 0.0;
1584       coords[10*dim+0] =  dis; coords[10*dim+1] = -dis; coords[10*dim+2] = 1.0;
1585       coords[11*dim+0] =  dis; coords[11*dim+1] =  dis; coords[11*dim+2] = 1.0;
1586       coords[12*dim+0] = -dis; coords[12*dim+1] =  dis; coords[12*dim+2] = 0.0;
1587       coords[13*dim+0] = -dis; coords[13*dim+1] =  dis; coords[13*dim+2] = 1.0;
1588       coords[14*dim+0] = -dis; coords[14*dim+1] = -dis; coords[14*dim+2] = 0.0;
1589       coords[15*dim+0] = -dis; coords[15*dim+1] = -dis; coords[15*dim+2] = 1.0;
1590       if (periodicZ == DM_BOUNDARY_PERIODIC) {
1591         /* 15 31 19 */ coords[16*dim+0] = -ds2; coords[16*dim+1] = -ds2; coords[16*dim+2] = 0.5;
1592         /* 16 32 22 */ coords[17*dim+0] =  ds2; coords[17*dim+1] = -ds2; coords[17*dim+2] = 0.5;
1593         /* 17 33 21 */ coords[18*dim+0] =  ds2; coords[18*dim+1] =  ds2; coords[18*dim+2] = 0.5;
1594         /* 18 34 20 */ coords[19*dim+0] = -ds2; coords[19*dim+1] =  ds2; coords[19*dim+2] = 0.5;
1595         /* 29 35 30 */ coords[20*dim+0] = -dis; coords[20*dim+1] = -dis; coords[20*dim+2] = 0.5;
1596         /* 23 36 25 */ coords[21*dim+0] =  dis; coords[21*dim+1] = -dis; coords[21*dim+2] = 0.5;
1597         /* 24 37 26 */ coords[22*dim+0] =  dis; coords[22*dim+1] =  dis; coords[22*dim+2] = 0.5;
1598         /* 27 38 28 */ coords[23*dim+0] = -dis; coords[23*dim+1] =  dis; coords[23*dim+2] = 0.5;
1599       }
1600     }
1601     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
1602     ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
1603     ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
1604   }
1605   /* Create periodicity */
1606   if (periodicZ == DM_BOUNDARY_PERIODIC || periodicZ == DM_BOUNDARY_TWIST) {
1607     PetscReal      L[3];
1608     PetscReal      maxCell[3];
1609     DMBoundaryType bdType[3];
1610     PetscReal      lower[3] = {0.0, 0.0, 0.0};
1611     PetscReal      upper[3] = {1.0, 1.0, 1.5};
1612     PetscInt       i, numZCells = 3;
1613 
1614     bdType[0] = DM_BOUNDARY_NONE;
1615     bdType[1] = DM_BOUNDARY_NONE;
1616     bdType[2] = periodicZ;
1617     for (i = 0; i < dim; i++) {
1618       L[i]       = upper[i] - lower[i];
1619       maxCell[i] = 1.1 * (L[i] / numZCells);
1620     }
1621     ierr = DMSetPeriodicity(dm, PETSC_TRUE, maxCell, L, bdType);CHKERRQ(ierr);
1622   }
1623   {
1624     DM          cdm;
1625     PetscDS     cds;
1626     PetscScalar c[2] = {1.0, 1.0};
1627 
1628     ierr = DMPlexCreateCoordinateSpace(dm, 1, snapToCylinder);CHKERRQ(ierr);
1629     ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
1630     ierr = DMGetDS(cdm, &cds);CHKERRQ(ierr);
1631     ierr = PetscDSSetConstants(cds, 2, c);CHKERRQ(ierr);
1632   }
1633   /* Wait for coordinate creation before doing in-place modification */
1634   ierr = DMPlexInterpolateInPlace_Internal(dm);CHKERRQ(ierr);
1635   PetscFunctionReturn(0);
1636 }
1637 
1638 /*@
1639   DMPlexCreateHexCylinderMesh - Creates a mesh on the tensor product of the unit interval with the circle (cylinder) using hexahedra.
1640 
1641   Collective
1642 
1643   Input Parameters:
1644 + comm      - The communicator for the DM object
1645 - periodicZ - The boundary type for the Z direction
1646 
1647   Output Parameter:
1648 . dm  - The DM object
1649 
1650   Note:
1651   Here is the output numbering looking from the bottom of the cylinder:
1652 $       17-----14
1653 $        |     |
1654 $        |  2  |
1655 $        |     |
1656 $ 17-----8-----7-----14
1657 $  |     |     |     |
1658 $  |  3  |  0  |  1  |
1659 $  |     |     |     |
1660 $ 19-----5-----6-----13
1661 $        |     |
1662 $        |  4  |
1663 $        |     |
1664 $       19-----13
1665 $
1666 $ and up through the top
1667 $
1668 $       18-----16
1669 $        |     |
1670 $        |  2  |
1671 $        |     |
1672 $ 18----10----11-----16
1673 $  |     |     |     |
1674 $  |  3  |  0  |  1  |
1675 $  |     |     |     |
1676 $ 20-----9----12-----15
1677 $        |     |
1678 $        |  4  |
1679 $        |     |
1680 $       20-----15
1681 
1682   Level: beginner
1683 
1684 .seealso: DMPlexCreateBoxMesh(), DMSetType(), DMCreate()
1685 @*/
1686 PetscErrorCode DMPlexCreateHexCylinderMesh(MPI_Comm comm, DMBoundaryType periodicZ, DM *dm)
1687 {
1688   PetscErrorCode ierr;
1689 
1690   PetscFunctionBegin;
1691   PetscValidPointer(dm, 3);
1692   ierr = DMCreate(comm, dm);CHKERRQ(ierr);
1693   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
1694   ierr = DMPlexCreateHexCylinderMesh_Internal(*dm, periodicZ);CHKERRQ(ierr);
1695   PetscFunctionReturn(0);
1696 }
1697 
1698 static PetscErrorCode DMPlexCreateWedgeCylinderMesh_Internal(DM dm, PetscInt n, PetscBool interpolate)
1699 {
1700   const PetscInt dim = 3;
1701   PetscInt       numCells, numVertices, v;
1702   PetscMPIInt    rank;
1703   PetscErrorCode ierr;
1704 
1705   PetscFunctionBegin;
1706   PetscCheckFalse(n < 0,PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of wedges %D cannot be negative", n);
1707   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRMPI(ierr);
1708   ierr = DMSetDimension(dm, dim);CHKERRQ(ierr);
1709   /* Must create the celltype label here so that we do not automatically try to compute the types */
1710   ierr = DMCreateLabel(dm, "celltype");CHKERRQ(ierr);
1711   /* Create topology */
1712   {
1713     PetscInt cone[6], c;
1714 
1715     numCells    = rank == 0 ?        n : 0;
1716     numVertices = rank == 0 ?  2*(n+1) : 0;
1717     ierr = DMPlexSetChart(dm, 0, numCells+numVertices);CHKERRQ(ierr);
1718     for (c = 0; c < numCells; c++) {ierr = DMPlexSetConeSize(dm, c, 6);CHKERRQ(ierr);}
1719     ierr = DMSetUp(dm);CHKERRQ(ierr);
1720     for (c = 0; c < numCells; c++) {
1721       cone[0] =  c+n*1; cone[1] = (c+1)%n+n*1; cone[2] = 0+3*n;
1722       cone[3] =  c+n*2; cone[4] = (c+1)%n+n*2; cone[5] = 1+3*n;
1723       ierr = DMPlexSetCone(dm, c, cone);CHKERRQ(ierr);
1724       ierr = DMPlexSetCellType(dm, c, DM_POLYTOPE_TRI_PRISM_TENSOR);CHKERRQ(ierr);
1725     }
1726     ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
1727     ierr = DMPlexStratify(dm);CHKERRQ(ierr);
1728   }
1729   for (v = numCells; v < numCells+numVertices; ++v) {
1730     ierr = DMPlexSetCellType(dm, v, DM_POLYTOPE_POINT);CHKERRQ(ierr);
1731   }
1732   /* Create cylinder geometry */
1733   {
1734     Vec          coordinates;
1735     PetscSection coordSection;
1736     PetscScalar *coords;
1737     PetscInt     coordSize, c;
1738 
1739     /* Build coordinates */
1740     ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
1741     ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
1742     ierr = PetscSectionSetFieldComponents(coordSection, 0, dim);CHKERRQ(ierr);
1743     ierr = PetscSectionSetChart(coordSection, numCells, numCells+numVertices);CHKERRQ(ierr);
1744     for (v = numCells; v < numCells+numVertices; ++v) {
1745       ierr = PetscSectionSetDof(coordSection, v, dim);CHKERRQ(ierr);
1746       ierr = PetscSectionSetFieldDof(coordSection, v, 0, dim);CHKERRQ(ierr);
1747     }
1748     ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
1749     ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
1750     ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
1751     ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
1752     ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
1753     ierr = VecSetBlockSize(coordinates, dim);CHKERRQ(ierr);
1754     ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
1755     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
1756     for (c = 0; c < numCells; c++) {
1757       coords[(c+0*n)*dim+0] = PetscCosReal(2.0*c*PETSC_PI/n); coords[(c+0*n)*dim+1] = PetscSinReal(2.0*c*PETSC_PI/n); coords[(c+0*n)*dim+2] = 1.0;
1758       coords[(c+1*n)*dim+0] = PetscCosReal(2.0*c*PETSC_PI/n); coords[(c+1*n)*dim+1] = PetscSinReal(2.0*c*PETSC_PI/n); coords[(c+1*n)*dim+2] = 0.0;
1759     }
1760     if (rank == 0) {
1761       coords[(2*n+0)*dim+0] = 0.0; coords[(2*n+0)*dim+1] = 0.0; coords[(2*n+0)*dim+2] = 1.0;
1762       coords[(2*n+1)*dim+0] = 0.0; coords[(2*n+1)*dim+1] = 0.0; coords[(2*n+1)*dim+2] = 0.0;
1763     }
1764     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
1765     ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
1766     ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
1767   }
1768   /* Interpolate */
1769   if (interpolate) {ierr = DMPlexInterpolateInPlace_Internal(dm);CHKERRQ(ierr);}
1770   PetscFunctionReturn(0);
1771 }
1772 
1773 /*@
1774   DMPlexCreateWedgeCylinderMesh - Creates a mesh on the tensor product of the unit interval with the circle (cylinder) using wedges.
1775 
1776   Collective
1777 
1778   Input Parameters:
1779 + comm - The communicator for the DM object
1780 . n    - The number of wedges around the origin
1781 - interpolate - Create edges and faces
1782 
1783   Output Parameter:
1784 . dm  - The DM object
1785 
1786   Level: beginner
1787 
1788 .seealso: DMPlexCreateHexCylinderMesh(), DMPlexCreateBoxMesh(), DMSetType(), DMCreate()
1789 @*/
1790 PetscErrorCode DMPlexCreateWedgeCylinderMesh(MPI_Comm comm, PetscInt n, PetscBool interpolate, DM *dm)
1791 {
1792   PetscErrorCode ierr;
1793 
1794   PetscFunctionBegin;
1795   PetscValidPointer(dm, 4);
1796   ierr = DMCreate(comm, dm);CHKERRQ(ierr);
1797   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
1798   ierr = DMPlexCreateWedgeCylinderMesh_Internal(*dm, n, interpolate);CHKERRQ(ierr);
1799   PetscFunctionReturn(0);
1800 }
1801 
1802 static inline PetscReal DiffNormReal(PetscInt dim, const PetscReal x[], const PetscReal y[])
1803 {
1804   PetscReal prod = 0.0;
1805   PetscInt  i;
1806   for (i = 0; i < dim; ++i) prod += PetscSqr(x[i] - y[i]);
1807   return PetscSqrtReal(prod);
1808 }
1809 static inline PetscReal DotReal(PetscInt dim, const PetscReal x[], const PetscReal y[])
1810 {
1811   PetscReal prod = 0.0;
1812   PetscInt  i;
1813   for (i = 0; i < dim; ++i) prod += x[i]*y[i];
1814   return prod;
1815 }
1816 
1817 /* The first constant is the sphere radius */
1818 static void snapToSphere(PetscInt dim, PetscInt Nf, PetscInt NfAux,
1819                          const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[],
1820                          const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[],
1821                          PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
1822 {
1823   PetscReal r = PetscRealPart(constants[0]);
1824   PetscReal norm2 = 0.0, fac;
1825   PetscInt  n = uOff[1] - uOff[0], d;
1826 
1827   for (d = 0; d < n; ++d) norm2 += PetscSqr(PetscRealPart(u[d]));
1828   fac = r/PetscSqrtReal(norm2);
1829   for (d = 0; d < n; ++d) f0[d] = u[d]*fac;
1830 }
1831 
1832 static PetscErrorCode DMPlexCreateSphereMesh_Internal(DM dm, PetscInt dim, PetscBool simplex, PetscReal R)
1833 {
1834   const PetscInt  embedDim = dim+1;
1835   PetscSection    coordSection;
1836   Vec             coordinates;
1837   PetscScalar    *coords;
1838   PetscReal      *coordsIn;
1839   PetscInt        numCells, numEdges, numVerts, firstVertex, v, firstEdge, coordSize, d, c, e;
1840   PetscMPIInt     rank;
1841   PetscErrorCode  ierr;
1842 
1843   PetscFunctionBegin;
1844   PetscValidLogicalCollectiveBool(dm, simplex, 3);
1845   ierr = DMSetDimension(dm, dim);CHKERRQ(ierr);
1846   ierr = DMSetCoordinateDim(dm, dim+1);CHKERRQ(ierr);
1847   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRMPI(ierr);
1848   switch (dim) {
1849   case 2:
1850     if (simplex) {
1851       const PetscReal radius    = PetscSqrtReal(1 + PETSC_PHI*PETSC_PHI)/(1.0 + PETSC_PHI);
1852       const PetscReal edgeLen   = 2.0/(1.0 + PETSC_PHI) * (R/radius);
1853       const PetscInt  degree    = 5;
1854       PetscReal       vertex[3] = {0.0, 1.0/(1.0 + PETSC_PHI), PETSC_PHI/(1.0 + PETSC_PHI)};
1855       PetscInt        s[3]      = {1, 1, 1};
1856       PetscInt        cone[3];
1857       PetscInt       *graph, p, i, j, k;
1858 
1859       vertex[0] *= R/radius; vertex[1] *= R/radius; vertex[2] *= R/radius;
1860       numCells    = rank == 0 ? 20 : 0;
1861       numVerts    = rank == 0 ? 12 : 0;
1862       firstVertex = numCells;
1863       /* Use icosahedron, which for a R-sphere has coordinates which are all cyclic permutations of
1864 
1865            (0, \pm 1/\phi+1, \pm \phi/\phi+1)
1866 
1867          where \phi^2 - \phi - 1 = 0, meaning \phi is the golden ratio \frac{1 + \sqrt{5}}{2}. The edge
1868          length is then given by 2/(1+\phi) = 2 * 0.38197 = 0.76393.
1869       */
1870       /* Construct vertices */
1871       ierr = PetscCalloc1(numVerts * embedDim, &coordsIn);CHKERRQ(ierr);
1872       if (rank == 0) {
1873         for (p = 0, i = 0; p < embedDim; ++p) {
1874           for (s[1] = -1; s[1] < 2; s[1] += 2) {
1875             for (s[2] = -1; s[2] < 2; s[2] += 2) {
1876               for (d = 0; d < embedDim; ++d) coordsIn[i*embedDim+d] = s[(d+p)%embedDim]*vertex[(d+p)%embedDim];
1877               ++i;
1878             }
1879           }
1880         }
1881       }
1882       /* Construct graph */
1883       ierr = PetscCalloc1(numVerts * numVerts, &graph);CHKERRQ(ierr);
1884       for (i = 0; i < numVerts; ++i) {
1885         for (j = 0, k = 0; j < numVerts; ++j) {
1886           if (PetscAbsReal(DiffNormReal(embedDim, &coordsIn[i*embedDim], &coordsIn[j*embedDim]) - edgeLen) < PETSC_SMALL) {graph[i*numVerts+j] = 1; ++k;}
1887         }
1888         PetscCheckFalse(k != degree,PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Invalid icosahedron, vertex %D degree %D != %D", i, k, degree);
1889       }
1890       /* Build Topology */
1891       ierr = DMPlexSetChart(dm, 0, numCells+numVerts);CHKERRQ(ierr);
1892       for (c = 0; c < numCells; c++) {
1893         ierr = DMPlexSetConeSize(dm, c, embedDim);CHKERRQ(ierr);
1894       }
1895       ierr = DMSetUp(dm);CHKERRQ(ierr); /* Allocate space for cones */
1896       /* Cells */
1897       for (i = 0, c = 0; i < numVerts; ++i) {
1898         for (j = 0; j < i; ++j) {
1899           for (k = 0; k < j; ++k) {
1900             if (graph[i*numVerts+j] && graph[j*numVerts+k] && graph[k*numVerts+i]) {
1901               cone[0] = firstVertex+i; cone[1] = firstVertex+j; cone[2] = firstVertex+k;
1902               /* Check orientation */
1903               {
1904                 const PetscInt epsilon[3][3][3] = {{{0, 0, 0}, {0, 0, 1}, {0, -1, 0}}, {{0, 0, -1}, {0, 0, 0}, {1, 0, 0}}, {{0, 1, 0}, {-1, 0, 0}, {0, 0, 0}}};
1905                 PetscReal normal[3];
1906                 PetscInt  e, f;
1907 
1908                 for (d = 0; d < embedDim; ++d) {
1909                   normal[d] = 0.0;
1910                   for (e = 0; e < embedDim; ++e) {
1911                     for (f = 0; f < embedDim; ++f) {
1912                       normal[d] += epsilon[d][e][f]*(coordsIn[j*embedDim+e] - coordsIn[i*embedDim+e])*(coordsIn[k*embedDim+f] - coordsIn[i*embedDim+f]);
1913                     }
1914                   }
1915                 }
1916                 if (DotReal(embedDim, normal, &coordsIn[i*embedDim]) < 0) {PetscInt tmp = cone[1]; cone[1] = cone[2]; cone[2] = tmp;}
1917               }
1918               ierr = DMPlexSetCone(dm, c++, cone);CHKERRQ(ierr);
1919             }
1920           }
1921         }
1922       }
1923       ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
1924       ierr = DMPlexStratify(dm);CHKERRQ(ierr);
1925       ierr = PetscFree(graph);CHKERRQ(ierr);
1926     } else {
1927       /*
1928         12-21--13
1929          |     |
1930         25  4  24
1931          |     |
1932   12-25--9-16--8-24--13
1933    |     |     |     |
1934   23  5 17  0 15  3  22
1935    |     |     |     |
1936   10-20--6-14--7-19--11
1937          |     |
1938         20  1  19
1939          |     |
1940         10-18--11
1941          |     |
1942         23  2  22
1943          |     |
1944         12-21--13
1945        */
1946       PetscInt cone[4], ornt[4];
1947 
1948       numCells    = rank == 0 ?  6 : 0;
1949       numEdges    = rank == 0 ? 12 : 0;
1950       numVerts    = rank == 0 ?  8 : 0;
1951       firstVertex = numCells;
1952       firstEdge   = numCells + numVerts;
1953       /* Build Topology */
1954       ierr = DMPlexSetChart(dm, 0, numCells+numEdges+numVerts);CHKERRQ(ierr);
1955       for (c = 0; c < numCells; c++) {
1956         ierr = DMPlexSetConeSize(dm, c, 4);CHKERRQ(ierr);
1957       }
1958       for (e = firstEdge; e < firstEdge+numEdges; ++e) {
1959         ierr = DMPlexSetConeSize(dm, e, 2);CHKERRQ(ierr);
1960       }
1961       ierr = DMSetUp(dm);CHKERRQ(ierr); /* Allocate space for cones */
1962       if (rank == 0) {
1963         /* Cell 0 */
1964         cone[0] = 14; cone[1] = 15; cone[2] = 16; cone[3] = 17;
1965         ierr = DMPlexSetCone(dm, 0, cone);CHKERRQ(ierr);
1966         ornt[0] = 0; ornt[1] = 0; ornt[2] = 0; ornt[3] = 0;
1967         ierr = DMPlexSetConeOrientation(dm, 0, ornt);CHKERRQ(ierr);
1968         /* Cell 1 */
1969         cone[0] = 18; cone[1] = 19; cone[2] = 14; cone[3] = 20;
1970         ierr = DMPlexSetCone(dm, 1, cone);CHKERRQ(ierr);
1971         ornt[0] = 0; ornt[1] = 0; ornt[2] = -1; ornt[3] = 0;
1972         ierr = DMPlexSetConeOrientation(dm, 1, ornt);CHKERRQ(ierr);
1973         /* Cell 2 */
1974         cone[0] = 21; cone[1] = 22; cone[2] = 18; cone[3] = 23;
1975         ierr = DMPlexSetCone(dm, 2, cone);CHKERRQ(ierr);
1976         ornt[0] = 0; ornt[1] = 0; ornt[2] = -1; ornt[3] = 0;
1977         ierr = DMPlexSetConeOrientation(dm, 2, ornt);CHKERRQ(ierr);
1978         /* Cell 3 */
1979         cone[0] = 19; cone[1] = 22; cone[2] = 24; cone[3] = 15;
1980         ierr = DMPlexSetCone(dm, 3, cone);CHKERRQ(ierr);
1981         ornt[0] = -1; ornt[1] = -1; ornt[2] = 0; ornt[3] = -1;
1982         ierr = DMPlexSetConeOrientation(dm, 3, ornt);CHKERRQ(ierr);
1983         /* Cell 4 */
1984         cone[0] = 16; cone[1] = 24; cone[2] = 21; cone[3] = 25;
1985         ierr = DMPlexSetCone(dm, 4, cone);CHKERRQ(ierr);
1986         ornt[0] = -1; ornt[1] = -1; ornt[2] = -1; ornt[3] = 0;
1987         ierr = DMPlexSetConeOrientation(dm, 4, ornt);CHKERRQ(ierr);
1988         /* Cell 5 */
1989         cone[0] = 20; cone[1] = 17; cone[2] = 25; cone[3] = 23;
1990         ierr = DMPlexSetCone(dm, 5, cone);CHKERRQ(ierr);
1991         ornt[0] = -1; ornt[1] = -1; ornt[2] = -1; ornt[3] = -1;
1992         ierr = DMPlexSetConeOrientation(dm, 5, ornt);CHKERRQ(ierr);
1993         /* Edges */
1994         cone[0] =  6; cone[1] =  7;
1995         ierr = DMPlexSetCone(dm, 14, cone);CHKERRQ(ierr);
1996         cone[0] =  7; cone[1] =  8;
1997         ierr = DMPlexSetCone(dm, 15, cone);CHKERRQ(ierr);
1998         cone[0] =  8; cone[1] =  9;
1999         ierr = DMPlexSetCone(dm, 16, cone);CHKERRQ(ierr);
2000         cone[0] =  9; cone[1] =  6;
2001         ierr = DMPlexSetCone(dm, 17, cone);CHKERRQ(ierr);
2002         cone[0] = 10; cone[1] = 11;
2003         ierr = DMPlexSetCone(dm, 18, cone);CHKERRQ(ierr);
2004         cone[0] = 11; cone[1] =  7;
2005         ierr = DMPlexSetCone(dm, 19, cone);CHKERRQ(ierr);
2006         cone[0] =  6; cone[1] = 10;
2007         ierr = DMPlexSetCone(dm, 20, cone);CHKERRQ(ierr);
2008         cone[0] = 12; cone[1] = 13;
2009         ierr = DMPlexSetCone(dm, 21, cone);CHKERRQ(ierr);
2010         cone[0] = 13; cone[1] = 11;
2011         ierr = DMPlexSetCone(dm, 22, cone);CHKERRQ(ierr);
2012         cone[0] = 10; cone[1] = 12;
2013         ierr = DMPlexSetCone(dm, 23, cone);CHKERRQ(ierr);
2014         cone[0] = 13; cone[1] =  8;
2015         ierr = DMPlexSetCone(dm, 24, cone);CHKERRQ(ierr);
2016         cone[0] = 12; cone[1] =  9;
2017         ierr = DMPlexSetCone(dm, 25, cone);CHKERRQ(ierr);
2018       }
2019       ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
2020       ierr = DMPlexStratify(dm);CHKERRQ(ierr);
2021       /* Build coordinates */
2022       ierr = PetscCalloc1(numVerts * embedDim, &coordsIn);CHKERRQ(ierr);
2023       if (rank == 0) {
2024         coordsIn[0*embedDim+0] = -R; coordsIn[0*embedDim+1] =  R; coordsIn[0*embedDim+2] = -R;
2025         coordsIn[1*embedDim+0] =  R; coordsIn[1*embedDim+1] =  R; coordsIn[1*embedDim+2] = -R;
2026         coordsIn[2*embedDim+0] =  R; coordsIn[2*embedDim+1] = -R; coordsIn[2*embedDim+2] = -R;
2027         coordsIn[3*embedDim+0] = -R; coordsIn[3*embedDim+1] = -R; coordsIn[3*embedDim+2] = -R;
2028         coordsIn[4*embedDim+0] = -R; coordsIn[4*embedDim+1] =  R; coordsIn[4*embedDim+2] =  R;
2029         coordsIn[5*embedDim+0] =  R; coordsIn[5*embedDim+1] =  R; coordsIn[5*embedDim+2] =  R;
2030         coordsIn[6*embedDim+0] = -R; coordsIn[6*embedDim+1] = -R; coordsIn[6*embedDim+2] =  R;
2031         coordsIn[7*embedDim+0] =  R; coordsIn[7*embedDim+1] = -R; coordsIn[7*embedDim+2] =  R;
2032       }
2033     }
2034     break;
2035   case 3:
2036     if (simplex) {
2037       const PetscReal edgeLen         = 1.0/PETSC_PHI;
2038       PetscReal       vertexA[4]      = {0.5, 0.5, 0.5, 0.5};
2039       PetscReal       vertexB[4]      = {1.0, 0.0, 0.0, 0.0};
2040       PetscReal       vertexC[4]      = {0.5, 0.5*PETSC_PHI, 0.5/PETSC_PHI, 0.0};
2041       const PetscInt  degree          = 12;
2042       PetscInt        s[4]            = {1, 1, 1};
2043       PetscInt        evenPerm[12][4] = {{0, 1, 2, 3}, {0, 2, 3, 1}, {0, 3, 1, 2}, {1, 0, 3, 2}, {1, 2, 0, 3}, {1, 3, 2, 0},
2044                                          {2, 0, 1, 3}, {2, 1, 3, 0}, {2, 3, 0, 1}, {3, 0, 2, 1}, {3, 1, 0, 2}, {3, 2, 1, 0}};
2045       PetscInt        cone[4];
2046       PetscInt       *graph, p, i, j, k, l;
2047 
2048       vertexA[0] *= R; vertexA[1] *= R; vertexA[2] *= R; vertexA[3] *= R;
2049       vertexB[0] *= R; vertexB[1] *= R; vertexB[2] *= R; vertexB[3] *= R;
2050       vertexC[0] *= R; vertexC[1] *= R; vertexC[2] *= R; vertexC[3] *= R;
2051       numCells    = rank == 0 ? 600 : 0;
2052       numVerts    = rank == 0 ? 120 : 0;
2053       firstVertex = numCells;
2054       /* Use the 600-cell, which for a unit sphere has coordinates which are
2055 
2056            1/2 (\pm 1, \pm 1,    \pm 1, \pm 1)                          16
2057                (\pm 1,    0,       0,      0)  all cyclic permutations   8
2058            1/2 (\pm 1, \pm phi, \pm 1/phi, 0)  all even permutations    96
2059 
2060          where \phi^2 - \phi - 1 = 0, meaning \phi is the golden ratio \frac{1 + \sqrt{5}}{2}. The edge
2061          length is then given by 1/\phi = 0.61803.
2062 
2063          http://buzzard.pugetsound.edu/sage-practice/ch03s03.html
2064          http://mathworld.wolfram.com/600-Cell.html
2065       */
2066       /* Construct vertices */
2067       ierr = PetscCalloc1(numVerts * embedDim, &coordsIn);CHKERRQ(ierr);
2068       i    = 0;
2069       if (rank == 0) {
2070         for (s[0] = -1; s[0] < 2; s[0] += 2) {
2071           for (s[1] = -1; s[1] < 2; s[1] += 2) {
2072             for (s[2] = -1; s[2] < 2; s[2] += 2) {
2073               for (s[3] = -1; s[3] < 2; s[3] += 2) {
2074                 for (d = 0; d < embedDim; ++d) coordsIn[i*embedDim+d] = s[d]*vertexA[d];
2075                 ++i;
2076               }
2077             }
2078           }
2079         }
2080         for (p = 0; p < embedDim; ++p) {
2081           s[1] = s[2] = s[3] = 1;
2082           for (s[0] = -1; s[0] < 2; s[0] += 2) {
2083             for (d = 0; d < embedDim; ++d) coordsIn[i*embedDim+d] = s[(d+p)%embedDim]*vertexB[(d+p)%embedDim];
2084             ++i;
2085           }
2086         }
2087         for (p = 0; p < 12; ++p) {
2088           s[3] = 1;
2089           for (s[0] = -1; s[0] < 2; s[0] += 2) {
2090             for (s[1] = -1; s[1] < 2; s[1] += 2) {
2091               for (s[2] = -1; s[2] < 2; s[2] += 2) {
2092                 for (d = 0; d < embedDim; ++d) coordsIn[i*embedDim+d] = s[evenPerm[p][d]]*vertexC[evenPerm[p][d]];
2093                 ++i;
2094               }
2095             }
2096           }
2097         }
2098       }
2099       PetscCheckFalse(i != numVerts,PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Invalid 600-cell, vertices %D != %D", i, numVerts);
2100       /* Construct graph */
2101       ierr = PetscCalloc1(numVerts * numVerts, &graph);CHKERRQ(ierr);
2102       for (i = 0; i < numVerts; ++i) {
2103         for (j = 0, k = 0; j < numVerts; ++j) {
2104           if (PetscAbsReal(DiffNormReal(embedDim, &coordsIn[i*embedDim], &coordsIn[j*embedDim]) - edgeLen) < PETSC_SMALL) {graph[i*numVerts+j] = 1; ++k;}
2105         }
2106         PetscCheckFalse(k != degree,PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Invalid 600-cell, vertex %D degree %D != %D", i, k, degree);
2107       }
2108       /* Build Topology */
2109       ierr = DMPlexSetChart(dm, 0, numCells+numVerts);CHKERRQ(ierr);
2110       for (c = 0; c < numCells; c++) {
2111         ierr = DMPlexSetConeSize(dm, c, embedDim);CHKERRQ(ierr);
2112       }
2113       ierr = DMSetUp(dm);CHKERRQ(ierr); /* Allocate space for cones */
2114       /* Cells */
2115       if (rank == 0) {
2116         for (i = 0, c = 0; i < numVerts; ++i) {
2117           for (j = 0; j < i; ++j) {
2118             for (k = 0; k < j; ++k) {
2119               for (l = 0; l < k; ++l) {
2120                 if (graph[i*numVerts+j] && graph[j*numVerts+k] && graph[k*numVerts+i] &&
2121                     graph[l*numVerts+i] && graph[l*numVerts+j] && graph[l*numVerts+k]) {
2122                   cone[0] = firstVertex+i; cone[1] = firstVertex+j; cone[2] = firstVertex+k; cone[3] = firstVertex+l;
2123                   /* Check orientation: https://ef.gy/linear-algebra:normal-vectors-in-higher-dimensional-spaces */
2124                   {
2125                     const PetscInt epsilon[4][4][4][4] = {{{{0,  0,  0,  0}, { 0, 0,  0,  0}, { 0,  0, 0,  0}, { 0,  0,  0, 0}},
2126                                                            {{0,  0,  0,  0}, { 0, 0,  0,  0}, { 0,  0, 0,  1}, { 0,  0, -1, 0}},
2127                                                            {{0,  0,  0,  0}, { 0, 0,  0, -1}, { 0,  0, 0,  0}, { 0,  1,  0, 0}},
2128                                                            {{0,  0,  0,  0}, { 0, 0,  1,  0}, { 0, -1, 0,  0}, { 0,  0,  0, 0}}},
2129 
2130                                                           {{{0,  0,  0,  0}, { 0, 0,  0,  0}, { 0,  0, 0, -1}, { 0,  0,  1, 0}},
2131                                                            {{0,  0,  0,  0}, { 0, 0,  0,  0}, { 0,  0, 0,  0}, { 0,  0,  0, 0}},
2132                                                            {{0,  0,  0,  1}, { 0, 0,  0,  0}, { 0,  0, 0,  0}, {-1,  0,  0, 0}},
2133                                                            {{0,  0, -1,  0}, { 0, 0,  0,  0}, { 1,  0, 0,  0}, { 0,  0,  0, 0}}},
2134 
2135                                                           {{{0,  0,  0,  0}, { 0, 0,  0,  1}, { 0,  0, 0,  0}, { 0, -1,  0, 0}},
2136                                                            {{0,  0,  0, -1}, { 0, 0,  0,  0}, { 0,  0, 0,  0}, { 1,  0,  0, 0}},
2137                                                            {{0,  0,  0,  0}, { 0, 0,  0,  0}, { 0,  0, 0,  0}, { 0,  0,  0, 0}},
2138                                                            {{0,  1,  0,  0}, {-1, 0,  0,  0}, { 0,  0, 0,  0}, { 0,  0,  0, 0}}},
2139 
2140                                                           {{{0,  0,  0,  0}, { 0, 0, -1,  0}, { 0,  1, 0,  0}, { 0,  0,  0, 0}},
2141                                                            {{0,  0,  1,  0}, { 0, 0,  0,  0}, {-1,  0, 0,  0}, { 0,  0,  0, 0}},
2142                                                            {{0, -1,  0,  0}, { 1, 0,  0,  0}, { 0,  0, 0,  0}, { 0,  0,  0, 0}},
2143                                                            {{0,  0,  0,  0}, { 0, 0,  0,  0}, { 0,  0, 0,  0}, { 0,  0,  0, 0}}}};
2144                     PetscReal normal[4];
2145                     PetscInt  e, f, g;
2146 
2147                     for (d = 0; d < embedDim; ++d) {
2148                       normal[d] = 0.0;
2149                       for (e = 0; e < embedDim; ++e) {
2150                         for (f = 0; f < embedDim; ++f) {
2151                           for (g = 0; g < embedDim; ++g) {
2152                             normal[d] += epsilon[d][e][f][g]*(coordsIn[j*embedDim+e] - coordsIn[i*embedDim+e])*(coordsIn[k*embedDim+f] - coordsIn[i*embedDim+f])*(coordsIn[l*embedDim+f] - coordsIn[i*embedDim+f]);
2153                           }
2154                         }
2155                       }
2156                     }
2157                     if (DotReal(embedDim, normal, &coordsIn[i*embedDim]) < 0) {PetscInt tmp = cone[1]; cone[1] = cone[2]; cone[2] = tmp;}
2158                   }
2159                   ierr = DMPlexSetCone(dm, c++, cone);CHKERRQ(ierr);
2160                 }
2161               }
2162             }
2163           }
2164         }
2165       }
2166       ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
2167       ierr = DMPlexStratify(dm);CHKERRQ(ierr);
2168       ierr = PetscFree(graph);CHKERRQ(ierr);
2169       break;
2170     }
2171   default: SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Unsupported dimension for sphere: %D", dim);
2172   }
2173   /* Create coordinates */
2174   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
2175   ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
2176   ierr = PetscSectionSetFieldComponents(coordSection, 0, embedDim);CHKERRQ(ierr);
2177   ierr = PetscSectionSetChart(coordSection, firstVertex, firstVertex+numVerts);CHKERRQ(ierr);
2178   for (v = firstVertex; v < firstVertex+numVerts; ++v) {
2179     ierr = PetscSectionSetDof(coordSection, v, embedDim);CHKERRQ(ierr);
2180     ierr = PetscSectionSetFieldDof(coordSection, v, 0, embedDim);CHKERRQ(ierr);
2181   }
2182   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
2183   ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
2184   ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
2185   ierr = VecSetBlockSize(coordinates, embedDim);CHKERRQ(ierr);
2186   ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
2187   ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
2188   ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
2189   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
2190   for (v = 0; v < numVerts; ++v) for (d = 0; d < embedDim; ++d) {coords[v*embedDim+d] = coordsIn[v*embedDim+d];}
2191   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
2192   ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
2193   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
2194   ierr = PetscFree(coordsIn);CHKERRQ(ierr);
2195   {
2196     DM          cdm;
2197     PetscDS     cds;
2198     PetscScalar c = R;
2199 
2200     ierr = DMPlexCreateCoordinateSpace(dm, 1, snapToSphere);CHKERRQ(ierr);
2201     ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
2202     ierr = DMGetDS(cdm, &cds);CHKERRQ(ierr);
2203     ierr = PetscDSSetConstants(cds, 1, &c);CHKERRQ(ierr);
2204   }
2205   /* Wait for coordinate creation before doing in-place modification */
2206   if (simplex) {ierr = DMPlexInterpolateInPlace_Internal(dm);CHKERRQ(ierr);}
2207   PetscFunctionReturn(0);
2208 }
2209 
2210 typedef void (*TPSEvaluateFunc)(const PetscReal[], PetscReal*, PetscReal[], PetscReal(*)[3]);
2211 
2212 /*
2213  The Schwarz P implicit surface is
2214 
2215      f(x) = cos(x0) + cos(x1) + cos(x2) = 0
2216 */
2217 static void TPSEvaluate_SchwarzP(const PetscReal y[3], PetscReal *f, PetscReal grad[], PetscReal (*hess)[3])
2218 {
2219   PetscReal c[3] = {PetscCosReal(y[0] * PETSC_PI), PetscCosReal(y[1] * PETSC_PI), PetscCosReal(y[2] * PETSC_PI)};
2220   PetscReal g[3] = {-PetscSinReal(y[0] * PETSC_PI), -PetscSinReal(y[1] * PETSC_PI), -PetscSinReal(y[2] * PETSC_PI)};
2221   f[0] = c[0] + c[1] + c[2];
2222   for (PetscInt i=0; i<3; i++) {
2223     grad[i] = PETSC_PI * g[i];
2224     for (PetscInt j=0; j<3; j++) {
2225       hess[i][j] = (i == j) ? -PetscSqr(PETSC_PI) * c[i] : 0.;
2226     }
2227   }
2228 }
2229 
2230 /*
2231  The Gyroid implicit surface is
2232 
2233  f(x,y,z) = sin(pi * x) * cos (pi * (y + 1/2))  + sin(pi * (y + 1/2)) * cos(pi * (z + 1/4)) + sin(pi * (z + 1/4)) * cos(pi * x)
2234 
2235 */
2236 static void TPSEvaluate_Gyroid(const PetscReal y[3], PetscReal *f, PetscReal grad[], PetscReal (*hess)[3])
2237 {
2238   PetscReal s[3] = {PetscSinReal(PETSC_PI * y[0]), PetscSinReal(PETSC_PI * (y[1] + .5)), PetscSinReal(PETSC_PI * (y[2] + .25))};
2239   PetscReal c[3] = {PetscCosReal(PETSC_PI * y[0]), PetscCosReal(PETSC_PI * (y[1] + .5)), PetscCosReal(PETSC_PI * (y[2] + .25))};
2240   f[0] = s[0] * c[1] + s[1] * c[2] + s[2] * c[0];
2241   grad[0] = PETSC_PI * (c[0] * c[1] - s[2] * s[0]);
2242   grad[1] = PETSC_PI * (c[1] * c[2] - s[0] * s[1]);
2243   grad[2] = PETSC_PI * (c[2] * c[0] - s[1] * s[2]);
2244   hess[0][0] = -PetscSqr(PETSC_PI) * (s[0] * c[1] + s[2] * c[0]);
2245   hess[0][1] = -PetscSqr(PETSC_PI) * (c[0] * s[1]);
2246   hess[0][2] = -PetscSqr(PETSC_PI) * (c[2] * s[0]);
2247   hess[1][0] = -PetscSqr(PETSC_PI) * (s[1] * c[2] + s[0] * c[1]);
2248   hess[1][1] = -PetscSqr(PETSC_PI) * (c[1] * s[2]);
2249   hess[2][2] = -PetscSqr(PETSC_PI) * (c[0] * s[1]);
2250   hess[2][0] = -PetscSqr(PETSC_PI) * (s[2] * c[0] + s[1] * c[2]);
2251   hess[2][1] = -PetscSqr(PETSC_PI) * (c[2] * s[0]);
2252   hess[2][2] = -PetscSqr(PETSC_PI) * (c[1] * s[2]);
2253 }
2254 
2255 /*
2256    We wish to solve
2257 
2258          min_y || y - x ||^2  subject to f(y) = 0
2259 
2260    Let g(y) = grad(f).  The minimization problem is equivalent to asking to satisfy
2261    f(y) = 0 and (y-x) is parallel to g(y).  We do this by using Householder QR to obtain a basis for the
2262    tangent space and ask for both components in the tangent space to be zero.
2263 
2264    Take g to be a column vector and compute the "full QR" factorization Q R = g,
2265    where Q = I - 2 n n^T is a symmetric orthogonal matrix.
2266    The first column of Q is parallel to g so the remaining two columns span the null space.
2267    Let Qn = Q[:,1:] be those remaining columns.  Then Qn Qn^T is an orthogonal projector into the tangent space.
2268    Since Q is symmetric, this is equivalent to multipyling by Q and taking the last two entries.
2269    In total, we have a system of 3 equations in 3 unknowns:
2270 
2271      f(y) = 0                       1 equation
2272      Qn^T (y - x) = 0               2 equations
2273 
2274    Here, we compute the residual and Jacobian of this system.
2275 */
2276 static void TPSNearestPointResJac(TPSEvaluateFunc feval, const PetscScalar x[], const PetscScalar y[], PetscScalar res[], PetscScalar J[])
2277 {
2278   PetscReal yreal[3] = {PetscRealPart(y[0]), PetscRealPart(y[1]), PetscRealPart(y[2])};
2279   PetscReal d[3] = {PetscRealPart(y[0] - x[0]), PetscRealPart(y[1] - x[1]), PetscRealPart(y[2] - x[2])};
2280   PetscReal f, grad[3], n[3], n_y[3][3], norm, norm_y[3], nd, nd_y[3], sign;
2281 
2282   feval(yreal, &f, grad, n_y);
2283 
2284   for (PetscInt i=0; i<3; i++) n[i] = grad[i];
2285   norm = PetscSqrtReal(PetscSqr(n[0]) + PetscSqr(n[1]) + PetscSqr(n[2]));
2286   for (PetscInt i=0; i<3; i++) {
2287     norm_y[i] = 1. / norm * n[i] * n_y[i][i];
2288   }
2289 
2290   // Define the Householder reflector
2291   sign = n[0] >= 0 ? 1. : -1.;
2292   n[0] += norm * sign;
2293   for (PetscInt i=0; i<3; i++) n_y[0][i] += norm_y[i] * sign;
2294 
2295   norm = PetscSqrtReal(PetscSqr(n[0]) + PetscSqr(n[1]) + PetscSqr(n[2]));
2296   norm_y[0] = 1. / norm * (n[0] * n_y[0][0]);
2297   norm_y[1] = 1. / norm * (n[0] * n_y[0][1] + n[1] * n_y[1][1]);
2298   norm_y[2] = 1. / norm * (n[0] * n_y[0][2] + n[2] * n_y[2][2]);
2299 
2300   for (PetscInt i=0; i<3; i++) {
2301     n[i] /= norm;
2302     for (PetscInt j=0; j<3; j++) {
2303       // note that n[i] is n_old[i]/norm when executing the code below
2304       n_y[i][j] = n_y[i][j] / norm - n[i] / norm * norm_y[j];
2305     }
2306   }
2307 
2308   nd = n[0] * d[0] + n[1] * d[1] + n[2] * d[2];
2309   for (PetscInt i=0; i<3; i++) nd_y[i] = n[i] + n_y[0][i] * d[0] + n_y[1][i] * d[1] + n_y[2][i] * d[2];
2310 
2311   res[0] = f;
2312   res[1] = d[1] - 2 * n[1] * nd;
2313   res[2] = d[2] - 2 * n[2] * nd;
2314   // J[j][i] is J_{ij} (column major)
2315   for (PetscInt j=0; j<3; j++) {
2316     J[0 + j*3] = grad[j];
2317     J[1 + j*3] = (j == 1)*1. - 2 * (n_y[1][j] * nd + n[1] * nd_y[j]);
2318     J[2 + j*3] = (j == 2)*1. - 2 * (n_y[2][j] * nd + n[2] * nd_y[j]);
2319   }
2320 }
2321 
2322 /*
2323    Project x to the nearest point on the implicit surface using Newton's method.
2324 */
2325 static PetscErrorCode TPSNearestPoint(TPSEvaluateFunc feval, PetscScalar x[])
2326 {
2327   PetscScalar y[3] = {x[0], x[1], x[2]}; // Initial guess
2328   PetscErrorCode ierr;
2329 
2330   PetscFunctionBegin;
2331   for (PetscInt iter=0; iter<10; iter++) {
2332     PetscScalar res[3], J[9];
2333     PetscReal resnorm;
2334     TPSNearestPointResJac(feval, x, y, res, J);
2335     resnorm = PetscSqrtReal(PetscSqr(PetscRealPart(res[0])) + PetscSqr(PetscRealPart(res[1])) + PetscSqr(PetscRealPart(res[2])));
2336     if (0) { // Turn on this monitor if you need to confirm quadratic convergence
2337       ierr = PetscPrintf(PETSC_COMM_SELF, "[%D] res [%g %g %g]\n", iter, PetscRealPart(res[0]), PetscRealPart(res[1]), PetscRealPart(res[2]));CHKERRQ(ierr);
2338     }
2339     if (resnorm < PETSC_SMALL) break;
2340 
2341     // Take the Newton step
2342     ierr = PetscKernel_A_gets_inverse_A_3(J, 0., PETSC_FALSE, NULL);CHKERRQ(ierr);
2343     PetscKernel_v_gets_v_minus_A_times_w_3(y, J, res);
2344   }
2345   for (PetscInt i=0; i<3; i++) x[i] = y[i];
2346   PetscFunctionReturn(0);
2347 }
2348 
2349 const char *const DMPlexTPSTypes[] = {"SCHWARZ_P", "GYROID", "DMPlexTPSType", "DMPLEX_TPS_", NULL};
2350 
2351 static PetscErrorCode DMPlexCreateTPSMesh_Internal(DM dm, DMPlexTPSType tpstype, const PetscInt extent[], const DMBoundaryType periodic[], PetscBool tps_distribute, PetscInt refinements, PetscInt layers, PetscReal thickness)
2352 {
2353   PetscErrorCode ierr;
2354   PetscMPIInt rank;
2355   PetscInt topoDim = 2, spaceDim = 3, numFaces = 0, numVertices = 0, numEdges = 0;
2356   PetscInt (*edges)[2] = NULL, *edgeSets = NULL;
2357   PetscInt *cells_flat = NULL;
2358   PetscReal *vtxCoords = NULL;
2359   TPSEvaluateFunc evalFunc = NULL;
2360   DMLabel label;
2361 
2362   PetscFunctionBegin;
2363   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRMPI(ierr);
2364   PetscCheck((layers != 0) ^ (thickness == 0.), PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_INCOMP, "Layers %D must be nonzero iff thickness %g is nonzero", layers, (double)thickness);
2365   switch (tpstype) {
2366   case DMPLEX_TPS_SCHWARZ_P:
2367     PetscCheck(!periodic || (periodic[0] == DM_BOUNDARY_NONE && periodic[1] == DM_BOUNDARY_NONE && periodic[2] == DM_BOUNDARY_NONE), PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Schwarz P does not support periodic meshes");
2368     if (!rank) {
2369       PetscInt (*cells)[6][4][4] = NULL; // [junction, junction-face, cell, conn]
2370       PetscInt Njunctions = 0, Ncuts = 0, Npipes[3], vcount;
2371       PetscReal L = 1;
2372 
2373       Npipes[0] = (extent[0] + 1) * extent[1] * extent[2];
2374       Npipes[1] = extent[0] * (extent[1] + 1) * extent[2];
2375       Npipes[2] = extent[0] * extent[1] * (extent[2] + 1);
2376       Njunctions = extent[0] * extent[1] * extent[2];
2377       Ncuts = 2 * (extent[0] * extent[1] + extent[1] * extent[2] + extent[2] * extent[0]);
2378       numVertices = 4 * (Npipes[0] + Npipes[1] + Npipes[2]) + 8 * Njunctions;
2379       ierr = PetscMalloc1(3*numVertices, &vtxCoords);CHKERRQ(ierr);
2380       ierr = PetscMalloc1(Njunctions, &cells);CHKERRQ(ierr);
2381       ierr = PetscMalloc1(Ncuts*4, &edges);CHKERRQ(ierr);
2382       ierr = PetscMalloc1(Ncuts*4, &edgeSets);CHKERRQ(ierr);
2383       // x-normal pipes
2384       vcount = 0;
2385       for (PetscInt i=0; i<extent[0]+1; i++) {
2386         for (PetscInt j=0; j<extent[1]; j++) {
2387           for (PetscInt k=0; k<extent[2]; k++) {
2388             for (PetscInt l=0; l<4; l++) {
2389               vtxCoords[vcount++] = (2*i - 1) * L;
2390               vtxCoords[vcount++] = 2 * j * L + PetscCosReal((2*l + 1) * PETSC_PI / 4) * L / 2;
2391               vtxCoords[vcount++] = 2 * k * L + PetscSinReal((2*l + 1) * PETSC_PI / 4) * L / 2;
2392             }
2393           }
2394         }
2395       }
2396       // y-normal pipes
2397       for (PetscInt i=0; i<extent[0]; i++) {
2398         for (PetscInt j=0; j<extent[1]+1; j++) {
2399           for (PetscInt k=0; k<extent[2]; k++) {
2400             for (PetscInt l=0; l<4; l++) {
2401               vtxCoords[vcount++] = 2 * i * L + PetscSinReal((2*l + 1) * PETSC_PI / 4) * L / 2;
2402               vtxCoords[vcount++] = (2*j - 1) * L;
2403               vtxCoords[vcount++] = 2 * k * L + PetscCosReal((2*l + 1) * PETSC_PI / 4) * L / 2;
2404             }
2405           }
2406         }
2407       }
2408       // z-normal pipes
2409       for (PetscInt i=0; i<extent[0]; i++) {
2410         for (PetscInt j=0; j<extent[1]; j++) {
2411           for (PetscInt k=0; k<extent[2]+1; k++) {
2412             for (PetscInt l=0; l<4; l++) {
2413               vtxCoords[vcount++] = 2 * i * L + PetscCosReal((2*l + 1) * PETSC_PI / 4) * L / 2;
2414               vtxCoords[vcount++] = 2 * j * L + PetscSinReal((2*l + 1) * PETSC_PI / 4) * L / 2;
2415               vtxCoords[vcount++] = (2*k - 1) * L;
2416             }
2417           }
2418         }
2419       }
2420       // junctions
2421       for (PetscInt i=0; i<extent[0]; i++) {
2422         for (PetscInt j=0; j<extent[1]; j++) {
2423           for (PetscInt k=0; k<extent[2]; k++) {
2424             const PetscInt J = (i*extent[1] + j)*extent[2] + k, Jvoff = (Npipes[0] + Npipes[1] + Npipes[2])*4 + J*8;
2425             PetscCheck(vcount / 3 == Jvoff, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Unexpected vertex count");
2426             for (PetscInt ii=0; ii<2; ii++) {
2427               for (PetscInt jj=0; jj<2; jj++) {
2428                 for (PetscInt kk=0; kk<2; kk++) {
2429                   double Ls = (1 - sqrt(2) / 4) * L;
2430                   vtxCoords[vcount++] = 2*i*L + (2*ii-1) * Ls;
2431                   vtxCoords[vcount++] = 2*j*L + (2*jj-1) * Ls;
2432                   vtxCoords[vcount++] = 2*k*L + (2*kk-1) * Ls;
2433                 }
2434               }
2435             }
2436             const PetscInt jfaces[3][2][4] = {
2437               {{3,1,0,2}, {7,5,4,6}}, // x-aligned
2438               {{5,4,0,1}, {7,6,2,3}}, // y-aligned
2439               {{6,2,0,4}, {7,3,1,5}}  // z-aligned
2440             };
2441             const PetscInt pipe_lo[3] = { // vertex numbers of pipes
2442               ((i * extent[1] + j) * extent[2] + k)*4,
2443               ((i * (extent[1] + 1) + j) * extent[2] + k + Npipes[0])*4,
2444               ((i * extent[1] + j) * (extent[2]+1) + k + Npipes[0] + Npipes[1])*4
2445             };
2446             const PetscInt pipe_hi[3] = { // vertex numbers of pipes
2447               (((i + 1) * extent[1] + j) * extent[2] + k)*4,
2448               ((i * (extent[1] + 1) + j + 1) * extent[2] + k + Npipes[0])*4,
2449               ((i * extent[1] + j) * (extent[2]+1) + k + 1 + Npipes[0] + Npipes[1])*4
2450             };
2451             for (PetscInt dir=0; dir<3; dir++) { // x,y,z
2452               const PetscInt ijk[3] = {i, j, k};
2453               for (PetscInt l=0; l<4; l++) { // rotations
2454                 cells[J][dir*2+0][l][0] = pipe_lo[dir] + l;
2455                 cells[J][dir*2+0][l][1] = Jvoff + jfaces[dir][0][l];
2456                 cells[J][dir*2+0][l][2] = Jvoff + jfaces[dir][0][(l-1+4)%4];
2457                 cells[J][dir*2+0][l][3] = pipe_lo[dir] + (l-1+4)%4;
2458                 cells[J][dir*2+1][l][0] = Jvoff + jfaces[dir][1][l];
2459                 cells[J][dir*2+1][l][1] = pipe_hi[dir] + l;
2460                 cells[J][dir*2+1][l][2] = pipe_hi[dir] + (l-1+4)%4;
2461                 cells[J][dir*2+1][l][3] = Jvoff + jfaces[dir][1][(l-1+4)%4];
2462                 if (ijk[dir] == 0) {
2463                   edges[numEdges][0] = pipe_lo[dir] + l;
2464                   edges[numEdges][1] = pipe_lo[dir] + (l+1) % 4;
2465                   edgeSets[numEdges] = dir*2 + 1;
2466                   numEdges++;
2467                 }
2468                 if (ijk[dir] + 1 == extent[dir]) {
2469                   edges[numEdges][0] = pipe_hi[dir] + l;
2470                   edges[numEdges][1] = pipe_hi[dir] + (l+1) % 4;
2471                   edgeSets[numEdges] = dir*2 + 2;
2472                   numEdges++;
2473                 }
2474               }
2475             }
2476           }
2477         }
2478       }
2479       PetscCheck(numEdges == Ncuts * 4, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Edge count %D incompatible with number of cuts %D", numEdges, Ncuts);
2480       numFaces = 24 * Njunctions;
2481       cells_flat = cells[0][0][0];
2482     }
2483     evalFunc = TPSEvaluate_SchwarzP;
2484     break;
2485   case DMPLEX_TPS_GYROID:
2486     if (!rank) {
2487       // This is a coarse mesh approximation of the gyroid shifted to being the zero of the level set
2488       //
2489       //     sin(pi*x)*cos(pi*(y+1/2)) + sin(pi*(y+1/2))*cos(pi*(z+1/4)) + sin(pi*(z+1/4))*cos(x)
2490       //
2491       // on the cell [0,2]^3.
2492       //
2493       // Think about dividing that cell into four columns, and focus on the column [0,1]x[0,1]x[0,2].
2494       // If you looked at the gyroid in that column at different slices of z you would see that it kind of spins
2495       // like a boomerang:
2496       //
2497       //     z = 0          z = 1/4        z = 1/2        z = 3/4     //
2498       //     -----          -------        -------        -------     //
2499       //                                                              //
2500       //     +       +      +       +      +       +      +   \   +   //
2501       //      \                                   /            \      //
2502       //       \            `-_   _-'            /              }     //
2503       //        *-_            `-'            _-'              /      //
2504       //     +     `-+      +       +      +-'     +      +   /   +   //
2505       //                                                              //
2506       //                                                              //
2507       //     z = 1          z = 5/4        z = 3/2        z = 7/4     //
2508       //     -----          -------        -------        -------     //
2509       //                                                              //
2510       //     +-_     +      +       +      +     _-+      +   /   +   //
2511       //        `-_            _-_            _-`            /        //
2512       //           \        _-'   `-_        /              {         //
2513       //            \                       /                \        //
2514       //     +       +      +       +      +       +      +   \   +   //
2515       //
2516       //
2517       // This course mesh approximates each of these slices by two line segments,
2518       // and then connects the segments in consecutive layers with quadrilateral faces.
2519       // All of the end points of the segments are multiples of 1/4 except for the
2520       // point * in the picture for z = 0 above and the similar points in other layers.
2521       // That point is at (gamma, gamma, 0), where gamma is calculated below.
2522       //
2523       // The column  [1,2]x[1,2]x[0,2] looks the same as this column;
2524       // The columns [1,2]x[0,1]x[0,2] and [0,1]x[1,2]x[0,2] are mirror images.
2525       //
2526       // As for how this method turned into the names given to the vertices:
2527       // that was not systematic, it was just the way it worked out in my handwritten notes.
2528 
2529       PetscInt facesPerBlock = 64;
2530       PetscInt vertsPerBlock = 56;
2531       PetscInt extentPlus[3];
2532       PetscInt numBlocks, numBlocksPlus;
2533       const PetscInt A =  0,   B =  1,   C =  2,   D =  3,   E =  4,   F =  5,   G =  6,   H =  7,
2534         II =  8,   J =  9,   K = 10,   L = 11,   M = 12,   N = 13,   O = 14,   P = 15,
2535         Q = 16,   R = 17,   S = 18,   T = 19,   U = 20,   V = 21,   W = 22,   X = 23,
2536         Y = 24,   Z = 25,  Ap = 26,  Bp = 27,  Cp = 28,  Dp = 29,  Ep = 30,  Fp = 31,
2537         Gp = 32,  Hp = 33,  Ip = 34,  Jp = 35,  Kp = 36,  Lp = 37,  Mp = 38,  Np = 39,
2538         Op = 40,  Pp = 41,  Qp = 42,  Rp = 43,  Sp = 44,  Tp = 45,  Up = 46,  Vp = 47,
2539         Wp = 48,  Xp = 49,  Yp = 50,  Zp = 51,  Aq = 52,  Bq = 53,  Cq = 54,  Dq = 55;
2540       const PetscInt pattern[64][4] =
2541         { /* face to vertex within the coarse discretization of a single gyroid block */
2542           /* layer 0 */
2543           {A,C,K,G},{C,B,II,K},{D,A,H,L},{B+56*1,D,L,J},{E,B+56*1,J,N},{A+56*2,E,N,H+56*2},{F,A+56*2,G+56*2,M},{B,F,M,II},
2544           /* layer 1 */
2545           {G,K,Q,O},{K,II,P,Q},{L,H,O+56*1,R},{J,L,R,P},{N,J,P,S},{H+56*2,N,S,O+56*3},{M,G+56*2,O+56*2,T},{II,M,T,P},
2546           /* layer 2 */
2547           {O,Q,Y,U},{Q,P,W,Y},{R,O+56*1,U+56*1,Ap},{P,R,Ap,W},{S,P,X,Bp},{O+56*3,S,Bp,V+56*1},{T,O+56*2,V,Z},{P,T,Z,X},
2548           /* layer 3 */
2549           {U,Y,Ep,Dp},{Y,W,Cp,Ep},{Ap,U+56*1,Dp+56*1,Gp},{W,Ap,Gp,Cp},{Bp,X,Cp+56*2,Fp},{V+56*1,Bp,Fp,Dp+56*1},{Z,V,Dp,Hp},{X,Z,Hp,Cp+56*2},
2550           /* layer 4 */
2551           {Dp,Ep,Mp,Kp},{Ep,Cp,Ip,Mp},{Gp,Dp+56*1,Lp,Np},{Cp,Gp,Np,Jp},{Fp,Cp+56*2,Jp+56*2,Pp},{Dp+56*1,Fp,Pp,Lp},{Hp,Dp,Kp,Op},{Cp+56*2,Hp,Op,Ip+56*2},
2552           /* layer 5 */
2553           {Kp,Mp,Sp,Rp},{Mp,Ip,Qp,Sp},{Np,Lp,Rp,Tp},{Jp,Np,Tp,Qp+56*1},{Pp,Jp+56*2,Qp+56*3,Up},{Lp,Pp,Up,Rp},{Op,Kp,Rp,Vp},{Ip+56*2,Op,Vp,Qp+56*2},
2554           /* layer 6 */
2555           {Rp,Sp,Aq,Yp},{Sp,Qp,Wp,Aq},{Tp,Rp,Yp,Cq},{Qp+56*1,Tp,Cq,Wp+56*1},{Up,Qp+56*3,Xp+56*1,Dq},{Rp,Up,Dq,Zp},{Vp,Rp,Zp,Bq},{Qp+56*2,Vp,Bq,Xp},
2556           /* layer 7 (the top is the periodic image of the bottom of layer 0) */
2557           {Yp,Aq,C+56*4,A+56*4},{Aq,Wp,B+56*4,C+56*4},{Cq,Yp,A+56*4,D+56*4},{Wp+56*1,Cq,D+56*4,B+56*5},{Dq,Xp+56*1,B+56*5,E+56*4},{Zp,Dq,E+56*4,A+56*6},{Bq,Zp,A+56*6,F+56*4},{Xp,Bq,F+56*4,B+56*4}
2558         };
2559       const PetscReal gamma = PetscAcosReal((PetscSqrtReal(3.)-1.) / PetscSqrtReal(2.)) / PETSC_PI;
2560       const PetscReal patternCoords[56][3] =
2561         {
2562           /* A  */ {1.,0.,0.},
2563           /* B  */ {0.,1.,0.},
2564           /* C  */ {gamma,gamma,0.},
2565           /* D  */ {1+gamma,1-gamma,0.},
2566           /* E  */ {2-gamma,2-gamma,0.},
2567           /* F  */ {1-gamma,1+gamma,0.},
2568 
2569           /* G  */ {.5,0,.25},
2570           /* H  */ {1.5,0.,.25},
2571           /* II */ {.5,1.,.25},
2572           /* J  */ {1.5,1.,.25},
2573           /* K  */ {.25,.5,.25},
2574           /* L  */ {1.25,.5,.25},
2575           /* M  */ {.75,1.5,.25},
2576           /* N  */ {1.75,1.5,.25},
2577 
2578           /* O  */ {0.,0.,.5},
2579           /* P  */ {1.,1.,.5},
2580           /* Q  */ {gamma,1-gamma,.5},
2581           /* R  */ {1+gamma,gamma,.5},
2582           /* S  */ {2-gamma,1+gamma,.5},
2583           /* T  */ {1-gamma,2-gamma,.5},
2584 
2585           /* U  */ {0.,.5,.75},
2586           /* V  */ {0.,1.5,.75},
2587           /* W  */ {1.,.5,.75},
2588           /* X  */ {1.,1.5,.75},
2589           /* Y  */ {.5,.75,.75},
2590           /* Z  */ {.5,1.75,.75},
2591           /* Ap */ {1.5,.25,.75},
2592           /* Bp */ {1.5,1.25,.75},
2593 
2594           /* Cp */ {1.,0.,1.},
2595           /* Dp */ {0.,1.,1.},
2596           /* Ep */ {1-gamma,1-gamma,1.},
2597           /* Fp */ {1+gamma,1+gamma,1.},
2598           /* Gp */ {2-gamma,gamma,1.},
2599           /* Hp */ {gamma,2-gamma,1.},
2600 
2601           /* Ip */ {.5,0.,1.25},
2602           /* Jp */ {1.5,0.,1.25},
2603           /* Kp */ {.5,1.,1.25},
2604           /* Lp */ {1.5,1.,1.25},
2605           /* Mp */ {.75,.5,1.25},
2606           /* Np */ {1.75,.5,1.25},
2607           /* Op */ {.25,1.5,1.25},
2608           /* Pp */ {1.25,1.5,1.25},
2609 
2610           /* Qp */ {0.,0.,1.5},
2611           /* Rp */ {1.,1.,1.5},
2612           /* Sp */ {1-gamma,gamma,1.5},
2613           /* Tp */ {2-gamma,1-gamma,1.5},
2614           /* Up */ {1+gamma,2-gamma,1.5},
2615           /* Vp */ {gamma,1+gamma,1.5},
2616 
2617           /* Wp */ {0.,.5,1.75},
2618           /* Xp */ {0.,1.5,1.75},
2619           /* Yp */ {1.,.5,1.75},
2620           /* Zp */ {1.,1.5,1.75},
2621           /* Aq */ {.5,.25,1.75},
2622           /* Bq */ {.5,1.25,1.75},
2623           /* Cq */ {1.5,.75,1.75},
2624           /* Dq */ {1.5,1.75,1.75},
2625         };
2626       PetscInt  (*cells)[64][4] = NULL;
2627       PetscBool *seen;
2628       PetscInt  *vertToTrueVert;
2629       PetscInt  count;
2630 
2631       for (PetscInt i = 0; i < 3; i++) extentPlus[i]  = extent[i] + 1;
2632       numBlocks = 1;
2633       for (PetscInt i = 0; i < 3; i++)     numBlocks *= extent[i];
2634       numBlocksPlus = 1;
2635       for (PetscInt i = 0; i < 3; i++) numBlocksPlus *= extentPlus[i];
2636       numFaces = numBlocks * facesPerBlock;
2637       ierr = PetscMalloc1(numBlocks, &cells);CHKERRQ(ierr);
2638       ierr = PetscCalloc1(numBlocksPlus * vertsPerBlock,&seen);CHKERRQ(ierr);
2639       for (PetscInt k = 0; k < extent[2]; k++) {
2640         for (PetscInt j = 0; j < extent[1]; j++) {
2641           for (PetscInt i = 0; i < extent[0]; i++) {
2642             for (PetscInt f = 0; f < facesPerBlock; f++) {
2643               for (PetscInt v = 0; v < 4; v++) {
2644                 PetscInt vertRaw = pattern[f][v];
2645                 PetscInt blockidx = vertRaw / 56;
2646                 PetscInt patternvert = vertRaw % 56;
2647                 PetscInt xplus = (blockidx & 1);
2648                 PetscInt yplus = (blockidx & 2) >> 1;
2649                 PetscInt zplus = (blockidx & 4) >> 2;
2650                 PetscInt zcoord = (periodic && periodic[2] == DM_BOUNDARY_PERIODIC) ? ((k + zplus) % extent[2]) : (k + zplus);
2651                 PetscInt ycoord = (periodic && periodic[1] == DM_BOUNDARY_PERIODIC) ? ((j + yplus) % extent[1]) : (j + yplus);
2652                 PetscInt xcoord = (periodic && periodic[0] == DM_BOUNDARY_PERIODIC) ? ((i + xplus) % extent[0]) : (i + xplus);
2653                 PetscInt vert = ((zcoord * extentPlus[1] + ycoord) * extentPlus[0] + xcoord) * 56 + patternvert;
2654 
2655                 cells[(k * extent[1] + j) * extent[0] + i][f][v] = vert;
2656                 seen[vert] = PETSC_TRUE;
2657               }
2658             }
2659           }
2660         }
2661       }
2662       for (PetscInt i = 0; i < numBlocksPlus * vertsPerBlock; i++) if (seen[i]) numVertices++;
2663       count = 0;
2664       ierr = PetscMalloc1(numBlocksPlus * vertsPerBlock, &vertToTrueVert);CHKERRQ(ierr);
2665       ierr = PetscMalloc1(numVertices * 3, &vtxCoords);CHKERRQ(ierr);
2666       for (PetscInt i = 0; i < numBlocksPlus * vertsPerBlock; i++) vertToTrueVert[i] = -1;
2667       for (PetscInt k = 0; k < extentPlus[2]; k++) {
2668         for (PetscInt j = 0; j < extentPlus[1]; j++) {
2669           for (PetscInt i = 0; i < extentPlus[0]; i++) {
2670             for (PetscInt v = 0; v < vertsPerBlock; v++) {
2671               PetscInt vIdx = ((k * extentPlus[1] + j) * extentPlus[0] + i) * vertsPerBlock + v;
2672 
2673               if (seen[vIdx]) {
2674                 PetscInt thisVert;
2675 
2676                 vertToTrueVert[vIdx] = thisVert = count++;
2677 
2678                 for (PetscInt d = 0; d < 3; d++) vtxCoords[3 * thisVert + d] = patternCoords[v][d];
2679                 vtxCoords[3 * thisVert + 0] += i * 2;
2680                 vtxCoords[3 * thisVert + 1] += j * 2;
2681                 vtxCoords[3 * thisVert + 2] += k * 2;
2682               }
2683             }
2684           }
2685         }
2686       }
2687       for (PetscInt i = 0; i < numBlocks; i++) {
2688         for (PetscInt f = 0; f < facesPerBlock; f++) {
2689           for (PetscInt v = 0; v < 4; v++) {
2690             cells[i][f][v] = vertToTrueVert[cells[i][f][v]];
2691           }
2692         }
2693       }
2694       ierr = PetscFree(vertToTrueVert);CHKERRQ(ierr);
2695       ierr = PetscFree(seen);CHKERRQ(ierr);
2696       cells_flat = cells[0][0];
2697       numEdges = 0;
2698       for (PetscInt i = 0; i < numFaces; i++) {
2699         for (PetscInt e = 0; e < 4; e++) {
2700           PetscInt ev[] = {cells_flat[i*4 + e], cells_flat[i*4 + ((e+1)%4)]};
2701           const PetscReal *evCoords[] = {&vtxCoords[3*ev[0]], &vtxCoords[3*ev[1]]};
2702 
2703           for (PetscInt d = 0; d < 3; d++) {
2704             if (!periodic || periodic[0] != DM_BOUNDARY_PERIODIC) {
2705               if (evCoords[0][d] == 0. && evCoords[1][d] == 0.) numEdges++;
2706               if (evCoords[0][d] == 2.*extent[d] && evCoords[1][d] == 2.*extent[d]) numEdges++;
2707             }
2708           }
2709         }
2710       }
2711       ierr = PetscMalloc1(numEdges, &edges);CHKERRQ(ierr);
2712       ierr = PetscMalloc1(numEdges, &edgeSets);CHKERRQ(ierr);
2713       for (PetscInt edge = 0, i = 0; i < numFaces; i++) {
2714         for (PetscInt e = 0; e < 4; e++) {
2715           PetscInt ev[] = {cells_flat[i*4 + e], cells_flat[i*4 + ((e+1)%4)]};
2716           const PetscReal *evCoords[] = {&vtxCoords[3*ev[0]], &vtxCoords[3*ev[1]]};
2717 
2718           for (PetscInt d = 0; d < 3; d++) {
2719             if (!periodic || periodic[d] != DM_BOUNDARY_PERIODIC) {
2720               if (evCoords[0][d] == 0. && evCoords[1][d] == 0.) {
2721                 edges[edge][0] = ev[0];
2722                 edges[edge][1] = ev[1];
2723                 edgeSets[edge++] = 2 * d;
2724               }
2725               if (evCoords[0][d] == 2.*extent[d] && evCoords[1][d] == 2.*extent[d]) {
2726                 edges[edge][0] = ev[0];
2727                 edges[edge][1] = ev[1];
2728                 edgeSets[edge++] = 2 * d + 1;
2729               }
2730             }
2731           }
2732         }
2733       }
2734     }
2735     evalFunc = TPSEvaluate_Gyroid;
2736     break;
2737   }
2738 
2739   ierr = DMSetDimension(dm, topoDim);CHKERRQ(ierr);
2740   if (!rank) {ierr = DMPlexBuildFromCellList(dm, numFaces, numVertices, 4, cells_flat);CHKERRQ(ierr);}
2741   else       {ierr = DMPlexBuildFromCellList(dm, 0, 0, 0, NULL);CHKERRQ(ierr);}
2742   ierr = PetscFree(cells_flat);CHKERRQ(ierr);
2743   {
2744     DM idm;
2745     ierr = DMPlexInterpolate(dm, &idm);CHKERRQ(ierr);
2746     ierr = DMPlexReplace_Static(dm, &idm);CHKERRQ(ierr);
2747   }
2748   if (!rank) {ierr = DMPlexBuildCoordinatesFromCellList(dm, spaceDim, vtxCoords);CHKERRQ(ierr);}
2749   else       {ierr = DMPlexBuildCoordinatesFromCellList(dm, spaceDim, NULL);CHKERRQ(ierr);}
2750   ierr = PetscFree(vtxCoords);CHKERRQ(ierr);
2751 
2752   ierr = DMCreateLabel(dm, "Face Sets");CHKERRQ(ierr);
2753   ierr = DMGetLabel(dm, "Face Sets", &label);CHKERRQ(ierr);
2754   for (PetscInt e=0; e<numEdges; e++) {
2755     PetscInt njoin;
2756     const PetscInt *join, verts[] = {numFaces + edges[e][0], numFaces + edges[e][1]};
2757     ierr = DMPlexGetJoin(dm, 2, verts, &njoin, &join);CHKERRQ(ierr);
2758     PetscCheck(njoin == 1, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Expected unique join of vertices %D and %D", edges[e][0], edges[e][1]);
2759     ierr = DMLabelSetValue(label, join[0], edgeSets[e]);CHKERRQ(ierr);
2760     ierr = DMPlexRestoreJoin(dm, 2, verts, &njoin, &join);CHKERRQ(ierr);
2761   }
2762   ierr = PetscFree(edges);CHKERRQ(ierr);
2763   ierr = PetscFree(edgeSets);CHKERRQ(ierr);
2764   if (tps_distribute) {
2765     DM               pdm = NULL;
2766     PetscPartitioner part;
2767 
2768     ierr = DMPlexGetPartitioner(dm, &part);CHKERRQ(ierr);
2769     ierr = PetscPartitionerSetFromOptions(part);CHKERRQ(ierr);
2770     ierr = DMPlexDistribute(dm, 0, NULL, &pdm);CHKERRQ(ierr);
2771     if (pdm) {
2772       ierr = DMPlexReplace_Static(dm, &pdm);CHKERRQ(ierr);
2773     }
2774     // Do not auto-distribute again
2775     ierr = DMPlexDistributeSetDefault(dm, PETSC_FALSE);CHKERRQ(ierr);
2776   }
2777 
2778   ierr = DMPlexSetRefinementUniform(dm, PETSC_TRUE);CHKERRQ(ierr);
2779   for (PetscInt refine=0; refine<refinements; refine++) {
2780     PetscInt m;
2781     DM dmf;
2782     Vec X;
2783     PetscScalar *x;
2784     ierr = DMRefine(dm, MPI_COMM_NULL, &dmf);CHKERRQ(ierr);
2785     ierr = DMPlexReplace_Static(dm, &dmf);CHKERRQ(ierr);
2786 
2787     ierr = DMGetCoordinatesLocal(dm, &X);CHKERRQ(ierr);
2788     ierr = VecGetLocalSize(X, &m);CHKERRQ(ierr);
2789     ierr = VecGetArray(X, &x);CHKERRQ(ierr);
2790     for (PetscInt i=0; i<m; i+=3) {
2791       ierr = TPSNearestPoint(evalFunc, &x[i]);CHKERRQ(ierr);
2792     }
2793     ierr = VecRestoreArray(X, &x);CHKERRQ(ierr);
2794   }
2795 
2796   // Face Sets has already been propagated to new vertices during refinement; this propagates to the initial vertices.
2797   ierr = DMGetLabel(dm, "Face Sets", &label);CHKERRQ(ierr);
2798   ierr = DMPlexLabelComplete(dm, label);CHKERRQ(ierr);
2799 
2800   if (thickness > 0) {
2801     DM dm3;
2802     ierr = DMPlexExtrude(dm, layers, thickness, PETSC_FALSE, PETSC_TRUE, NULL, NULL, &dm3);CHKERRQ(ierr);
2803     ierr = DMPlexReplace_Static(dm, &dm3);CHKERRQ(ierr);
2804   }
2805   PetscFunctionReturn(0);
2806 }
2807 
2808 /*@
2809   DMPlexCreateTPSMesh - Create a distributed, interpolated mesh of a triply-periodic surface
2810 
2811   Collective
2812 
2813   Input Parameters:
2814 + comm   - The communicator for the DM object
2815 . tpstype - Type of triply-periodic surface
2816 . extent - Array of length 3 containing number of periods in each direction
2817 . periodic - array of length 3 with periodicity, or NULL for non-periodic
2818 . tps_distribute - Distribute 2D manifold mesh prior to refinement and extrusion (more scalable)
2819 . refinements - Number of factor-of-2 refinements of 2D manifold mesh
2820 . layers - Number of cell layers extruded in normal direction
2821 - thickness - Thickness in normal direction
2822 
2823   Output Parameter:
2824 . dm  - The DM object
2825 
2826   Notes:
2827   This meshes the surface of the Schwarz P or Gyroid surfaces.  Schwarz P is is the simplest member of the triply-periodic minimal surfaces.
2828   https://en.wikipedia.org/wiki/Schwarz_minimal_surface#Schwarz_P_(%22Primitive%22) and can be cut with "clean" boundaries.
2829   The Gyroid (https://en.wikipedia.org/wiki/Gyroid) is another triply-periodic minimal surface with applications in additive manufacturing; it is much more difficult to "cut" since there are no planes of symmetry.
2830   Our implementation creates a very coarse mesh of the surface and refines (by 4-way splitting) as many times as requested.
2831   On each refinement, all vertices are projected to their nearest point on the surface.
2832   This projection could readily be extended to related surfaces.
2833 
2834   The face (edge) sets for the Schwarz P surface are numbered 1(-x), 2(+x), 3(-y), 4(+y), 5(-z), 6(+z).
2835   When the mesh is refined, "Face Sets" contain the new vertices (created during refinement).  Use DMPlexLabelComplete() to propagate to coarse-level vertices.
2836 
2837   References:
2838 . * - Maskery et al, Insights into the mechanical properties of several triply periodic minimal surface lattice structures made by polymer additive manufacturing, 2017. https://doi.org/10.1016/j.polymer.2017.11.049
2839 
2840   Developer Notes:
2841   The Gyroid mesh does not currently mark boundary sets.
2842 
2843   Level: beginner
2844 
2845 .seealso: DMPlexCreateSphereMesh(), DMSetType(), DMCreate()
2846 @*/
2847 PetscErrorCode DMPlexCreateTPSMesh(MPI_Comm comm, DMPlexTPSType tpstype, const PetscInt extent[], const DMBoundaryType periodic[], PetscBool tps_distribute, PetscInt refinements, PetscInt layers, PetscReal thickness, DM *dm)
2848 {
2849   PetscErrorCode ierr;
2850 
2851   PetscFunctionBegin;
2852   ierr = DMCreate(comm, dm);CHKERRQ(ierr);
2853   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
2854   ierr = DMPlexCreateTPSMesh_Internal(*dm, tpstype, extent, periodic, tps_distribute, refinements, layers, thickness);CHKERRQ(ierr);
2855   PetscFunctionReturn(0);
2856 }
2857 
2858 /*@
2859   DMPlexCreateSphereMesh - Creates a mesh on the d-dimensional sphere, S^d.
2860 
2861   Collective
2862 
2863   Input Parameters:
2864 + comm    - The communicator for the DM object
2865 . dim     - The dimension
2866 . simplex - Use simplices, or tensor product cells
2867 - R       - The radius
2868 
2869   Output Parameter:
2870 . dm  - The DM object
2871 
2872   Level: beginner
2873 
2874 .seealso: DMPlexCreateBallMesh(), DMPlexCreateBoxMesh(), DMSetType(), DMCreate()
2875 @*/
2876 PetscErrorCode DMPlexCreateSphereMesh(MPI_Comm comm, PetscInt dim, PetscBool simplex, PetscReal R, DM *dm)
2877 {
2878   PetscErrorCode ierr;
2879 
2880   PetscFunctionBegin;
2881   PetscValidPointer(dm, 5);
2882   ierr = DMCreate(comm, dm);CHKERRQ(ierr);
2883   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
2884   ierr = DMPlexCreateSphereMesh_Internal(*dm, dim, simplex, R);CHKERRQ(ierr);
2885   PetscFunctionReturn(0);
2886 }
2887 
2888 static PetscErrorCode DMPlexCreateBallMesh_Internal(DM dm, PetscInt dim, PetscReal R)
2889 {
2890   DM             sdm, vol;
2891   DMLabel        bdlabel;
2892   PetscErrorCode ierr;
2893 
2894   PetscFunctionBegin;
2895   ierr = DMCreate(PetscObjectComm((PetscObject) dm), &sdm);CHKERRQ(ierr);
2896   ierr = DMSetType(sdm, DMPLEX);CHKERRQ(ierr);
2897   ierr = PetscObjectSetOptionsPrefix((PetscObject) sdm, "bd_");CHKERRQ(ierr);
2898   ierr = DMPlexCreateSphereMesh_Internal(sdm, dim-1, PETSC_TRUE, R);CHKERRQ(ierr);
2899   ierr = DMSetFromOptions(sdm);CHKERRQ(ierr);
2900   ierr = DMViewFromOptions(sdm, NULL, "-dm_view");CHKERRQ(ierr);
2901   ierr = DMPlexGenerate(sdm, NULL, PETSC_TRUE, &vol);CHKERRQ(ierr);
2902   ierr = DMDestroy(&sdm);CHKERRQ(ierr);
2903   ierr = DMPlexReplace_Static(dm, &vol);CHKERRQ(ierr);
2904   ierr = DMCreateLabel(dm, "marker");CHKERRQ(ierr);
2905   ierr = DMGetLabel(dm, "marker", &bdlabel);CHKERRQ(ierr);
2906   ierr = DMPlexMarkBoundaryFaces(dm, PETSC_DETERMINE, bdlabel);CHKERRQ(ierr);
2907   ierr = DMPlexLabelComplete(dm, bdlabel);CHKERRQ(ierr);
2908   PetscFunctionReturn(0);
2909 }
2910 
2911 /*@
2912   DMPlexCreateBallMesh - Creates a simplex mesh on the d-dimensional ball, B^d.
2913 
2914   Collective
2915 
2916   Input Parameters:
2917 + comm  - The communicator for the DM object
2918 . dim   - The dimension
2919 - R     - The radius
2920 
2921   Output Parameter:
2922 . dm  - The DM object
2923 
2924   Options Database Keys:
2925 - bd_dm_refine - This will refine the surface mesh preserving the sphere geometry
2926 
2927   Level: beginner
2928 
2929 .seealso: DMPlexCreateSphereMesh(), DMPlexCreateBoxMesh(), DMSetType(), DMCreate()
2930 @*/
2931 PetscErrorCode DMPlexCreateBallMesh(MPI_Comm comm, PetscInt dim, PetscReal R, DM *dm)
2932 {
2933   PetscErrorCode ierr;
2934 
2935   PetscFunctionBegin;
2936   ierr = DMCreate(comm, dm);CHKERRQ(ierr);
2937   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
2938   ierr = DMPlexCreateBallMesh_Internal(*dm, dim, R);CHKERRQ(ierr);
2939   PetscFunctionReturn(0);
2940 }
2941 
2942 static PetscErrorCode DMPlexCreateReferenceCell_Internal(DM rdm, DMPolytopeType ct)
2943 {
2944   PetscErrorCode ierr;
2945 
2946   PetscFunctionBegin;
2947   switch (ct) {
2948     case DM_POLYTOPE_POINT:
2949     {
2950       PetscInt    numPoints[1]        = {1};
2951       PetscInt    coneSize[1]         = {0};
2952       PetscInt    cones[1]            = {0};
2953       PetscInt    coneOrientations[1] = {0};
2954       PetscScalar vertexCoords[1]     = {0.0};
2955 
2956       ierr = DMSetDimension(rdm, 0);CHKERRQ(ierr);
2957       ierr = DMPlexCreateFromDAG(rdm, 0, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
2958     }
2959     break;
2960     case DM_POLYTOPE_SEGMENT:
2961     {
2962       PetscInt    numPoints[2]        = {2, 1};
2963       PetscInt    coneSize[3]         = {2, 0, 0};
2964       PetscInt    cones[2]            = {1, 2};
2965       PetscInt    coneOrientations[2] = {0, 0};
2966       PetscScalar vertexCoords[2]     = {-1.0,  1.0};
2967 
2968       ierr = DMSetDimension(rdm, 1);CHKERRQ(ierr);
2969       ierr = DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
2970     }
2971     break;
2972     case DM_POLYTOPE_POINT_PRISM_TENSOR:
2973     {
2974       PetscInt    numPoints[2]        = {2, 1};
2975       PetscInt    coneSize[3]         = {2, 0, 0};
2976       PetscInt    cones[2]            = {1, 2};
2977       PetscInt    coneOrientations[2] = {0, 0};
2978       PetscScalar vertexCoords[2]     = {-1.0,  1.0};
2979 
2980       ierr = DMSetDimension(rdm, 1);CHKERRQ(ierr);
2981       ierr = DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
2982     }
2983     break;
2984     case DM_POLYTOPE_TRIANGLE:
2985     {
2986       PetscInt    numPoints[2]        = {3, 1};
2987       PetscInt    coneSize[4]         = {3, 0, 0, 0};
2988       PetscInt    cones[3]            = {1, 2, 3};
2989       PetscInt    coneOrientations[3] = {0, 0, 0};
2990       PetscScalar vertexCoords[6]     = {-1.0, -1.0,  1.0, -1.0,  -1.0, 1.0};
2991 
2992       ierr = DMSetDimension(rdm, 2);CHKERRQ(ierr);
2993       ierr = DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
2994     }
2995     break;
2996     case DM_POLYTOPE_QUADRILATERAL:
2997     {
2998       PetscInt    numPoints[2]        = {4, 1};
2999       PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
3000       PetscInt    cones[4]            = {1, 2, 3, 4};
3001       PetscInt    coneOrientations[4] = {0, 0, 0, 0};
3002       PetscScalar vertexCoords[8]     = {-1.0, -1.0,  1.0, -1.0,  1.0, 1.0,  -1.0, 1.0};
3003 
3004       ierr = DMSetDimension(rdm, 2);CHKERRQ(ierr);
3005       ierr = DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
3006     }
3007     break;
3008     case DM_POLYTOPE_SEG_PRISM_TENSOR:
3009     {
3010       PetscInt    numPoints[2]        = {4, 1};
3011       PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
3012       PetscInt    cones[4]            = {1, 2, 3, 4};
3013       PetscInt    coneOrientations[4] = {0, 0, 0, 0};
3014       PetscScalar vertexCoords[8]     = {-1.0, -1.0,  1.0, -1.0,  -1.0, 1.0,  1.0, 1.0};
3015 
3016       ierr = DMSetDimension(rdm, 2);CHKERRQ(ierr);
3017       ierr = DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
3018     }
3019     break;
3020     case DM_POLYTOPE_TETRAHEDRON:
3021     {
3022       PetscInt    numPoints[2]        = {4, 1};
3023       PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
3024       PetscInt    cones[4]            = {1, 2, 3, 4};
3025       PetscInt    coneOrientations[4] = {0, 0, 0, 0};
3026       PetscScalar vertexCoords[12]    = {-1.0, -1.0, -1.0,  -1.0, 1.0, -1.0,  1.0, -1.0, -1.0,  -1.0, -1.0, 1.0};
3027 
3028       ierr = DMSetDimension(rdm, 3);CHKERRQ(ierr);
3029       ierr = DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
3030     }
3031     break;
3032     case DM_POLYTOPE_HEXAHEDRON:
3033     {
3034       PetscInt    numPoints[2]        = {8, 1};
3035       PetscInt    coneSize[9]         = {8, 0, 0, 0, 0, 0, 0, 0, 0};
3036       PetscInt    cones[8]            = {1, 2, 3, 4, 5, 6, 7, 8};
3037       PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
3038       PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0,  -1.0,  1.0, -1.0,  1.0, 1.0, -1.0,   1.0, -1.0, -1.0,
3039                                          -1.0, -1.0,  1.0,   1.0, -1.0,  1.0,  1.0, 1.0,  1.0,  -1.0,  1.0,  1.0};
3040 
3041       ierr = DMSetDimension(rdm, 3);CHKERRQ(ierr);
3042       ierr = DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
3043     }
3044     break;
3045     case DM_POLYTOPE_TRI_PRISM:
3046     {
3047       PetscInt    numPoints[2]        = {6, 1};
3048       PetscInt    coneSize[7]         = {6, 0, 0, 0, 0, 0, 0};
3049       PetscInt    cones[6]            = {1, 2, 3, 4, 5, 6};
3050       PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
3051       PetscScalar vertexCoords[18]    = {-1.0, -1.0, -1.0, -1.0,  1.0, -1.0,   1.0, -1.0, -1.0,
3052                                          -1.0, -1.0,  1.0,  1.0, -1.0,  1.0,  -1.0,  1.0,  1.0};
3053 
3054       ierr = DMSetDimension(rdm, 3);CHKERRQ(ierr);
3055       ierr = DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
3056     }
3057     break;
3058     case DM_POLYTOPE_TRI_PRISM_TENSOR:
3059     {
3060       PetscInt    numPoints[2]        = {6, 1};
3061       PetscInt    coneSize[7]         = {6, 0, 0, 0, 0, 0, 0};
3062       PetscInt    cones[6]            = {1, 2, 3, 4, 5, 6};
3063       PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
3064       PetscScalar vertexCoords[18]    = {-1.0, -1.0, -1.0,  1.0, -1.0, -1.0,  -1.0, 1.0, -1.0,
3065                                          -1.0, -1.0,  1.0,  1.0, -1.0,  1.0,  -1.0, 1.0,  1.0};
3066 
3067       ierr = DMSetDimension(rdm, 3);CHKERRQ(ierr);
3068       ierr = DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
3069     }
3070     break;
3071     case DM_POLYTOPE_QUAD_PRISM_TENSOR:
3072     {
3073       PetscInt    numPoints[2]        = {8, 1};
3074       PetscInt    coneSize[9]         = {8, 0, 0, 0, 0, 0, 0, 0, 0};
3075       PetscInt    cones[8]            = {1, 2, 3, 4, 5, 6, 7, 8};
3076       PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
3077       PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0,  1.0, -1.0, -1.0,  1.0, 1.0, -1.0,  -1.0, 1.0, -1.0,
3078                                          -1.0, -1.0,  1.0,  1.0, -1.0,  1.0,  1.0, 1.0,  1.0,  -1.0, 1.0,  1.0};
3079 
3080       ierr = DMSetDimension(rdm, 3);CHKERRQ(ierr);
3081       ierr = DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
3082     }
3083     break;
3084     case DM_POLYTOPE_PYRAMID:
3085     {
3086       PetscInt    numPoints[2]        = {5, 1};
3087       PetscInt    coneSize[6]         = {5, 0, 0, 0, 0, 0};
3088       PetscInt    cones[5]            = {1, 2, 3, 4, 5};
3089       PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
3090       PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0,  -1.0, 1.0, -1.0,  1.0, 1.0, -1.0,  1.0, -1.0, -1.0,
3091                                           0.0,  0.0,  1.0};
3092 
3093       ierr = DMSetDimension(rdm, 3);CHKERRQ(ierr);
3094       ierr = DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
3095     }
3096     break;
3097     default: SETERRQ(PetscObjectComm((PetscObject) rdm), PETSC_ERR_ARG_WRONG, "Cannot create reference cell for cell type %s", DMPolytopeTypes[ct]);
3098   }
3099   {
3100     PetscInt Nv, v;
3101 
3102     /* Must create the celltype label here so that we do not automatically try to compute the types */
3103     ierr = DMCreateLabel(rdm, "celltype");CHKERRQ(ierr);
3104     ierr = DMPlexSetCellType(rdm, 0, ct);CHKERRQ(ierr);
3105     ierr = DMPlexGetChart(rdm, NULL, &Nv);CHKERRQ(ierr);
3106     for (v = 1; v < Nv; ++v) {ierr = DMPlexSetCellType(rdm, v, DM_POLYTOPE_POINT);CHKERRQ(ierr);}
3107   }
3108   ierr = DMPlexInterpolateInPlace_Internal(rdm);CHKERRQ(ierr);
3109   ierr = PetscObjectSetName((PetscObject) rdm, DMPolytopeTypes[ct]);CHKERRQ(ierr);
3110   PetscFunctionReturn(0);
3111 }
3112 
3113 /*@
3114   DMPlexCreateReferenceCell - Create a DMPLEX with the appropriate FEM reference cell
3115 
3116   Collective
3117 
3118   Input Parameters:
3119 + comm - The communicator
3120 - ct   - The cell type of the reference cell
3121 
3122   Output Parameter:
3123 . refdm - The reference cell
3124 
3125   Level: intermediate
3126 
3127 .seealso: DMPlexCreateReferenceCell(), DMPlexCreateBoxMesh()
3128 @*/
3129 PetscErrorCode DMPlexCreateReferenceCell(MPI_Comm comm, DMPolytopeType ct, DM *refdm)
3130 {
3131   PetscErrorCode ierr;
3132 
3133   PetscFunctionBegin;
3134   ierr = DMCreate(comm, refdm);CHKERRQ(ierr);
3135   ierr = DMSetType(*refdm, DMPLEX);CHKERRQ(ierr);
3136   ierr = DMPlexCreateReferenceCell_Internal(*refdm, ct);CHKERRQ(ierr);
3137   PetscFunctionReturn(0);
3138 }
3139 
3140 static PetscErrorCode DMPlexCreateBoundaryLabel_Private(DM dm, const char name[])
3141 {
3142   DM             plex;
3143   DMLabel        label;
3144   PetscBool      hasLabel;
3145   PetscErrorCode ierr;
3146 
3147   PetscFunctionBeginUser;
3148   ierr = DMHasLabel(dm, name, &hasLabel);CHKERRQ(ierr);
3149   if (hasLabel) PetscFunctionReturn(0);
3150   ierr = DMCreateLabel(dm, name);CHKERRQ(ierr);
3151   ierr = DMGetLabel(dm, name, &label);CHKERRQ(ierr);
3152   ierr = DMConvert(dm, DMPLEX, &plex);CHKERRQ(ierr);
3153   ierr = DMPlexMarkBoundaryFaces(plex, 1, label);CHKERRQ(ierr);
3154   ierr = DMDestroy(&plex);CHKERRQ(ierr);
3155   PetscFunctionReturn(0);
3156 }
3157 
3158 const char * const DMPlexShapes[] = {"box", "box_surface", "ball", "sphere", "cylinder", "schwarz_p", "gyroid", "unknown", "DMPlexShape", "DM_SHAPE_", NULL};
3159 
3160 static PetscErrorCode DMPlexCreateFromOptions_Internal(PetscOptionItems *PetscOptionsObject, PetscBool *useCoordSpace, DM dm)
3161 {
3162   DMPlexShape    shape = DM_SHAPE_BOX;
3163   DMPolytopeType cell  = DM_POLYTOPE_TRIANGLE;
3164   PetscInt       dim   = 2;
3165   PetscBool      simplex = PETSC_TRUE, interpolate = PETSC_TRUE, adjCone = PETSC_FALSE, adjClosure = PETSC_TRUE, refDomain = PETSC_FALSE;
3166   PetscBool      flg, flg2, fflg, bdfflg, nameflg;
3167   MPI_Comm       comm;
3168   char           filename[PETSC_MAX_PATH_LEN]   = "<unspecified>";
3169   char           bdFilename[PETSC_MAX_PATH_LEN] = "<unspecified>";
3170   char           plexname[PETSC_MAX_PATH_LEN]   = "";
3171   PetscErrorCode ierr;
3172 
3173   PetscFunctionBegin;
3174   ierr = PetscObjectGetComm((PetscObject) dm, &comm);CHKERRQ(ierr);
3175   /* TODO Turn this into a registration interface */
3176   ierr = PetscOptionsString("-dm_plex_filename", "File containing a mesh", "DMPlexCreateFromFile", filename, filename, sizeof(filename), &fflg);CHKERRQ(ierr);
3177   ierr = PetscOptionsString("-dm_plex_boundary_filename", "File containing a mesh boundary", "DMPlexCreateFromFile", bdFilename, bdFilename, sizeof(bdFilename), &bdfflg);CHKERRQ(ierr);
3178   ierr = PetscOptionsString("-dm_plex_name", "Name of the mesh in the file", "DMPlexCreateFromFile", plexname, plexname, sizeof(plexname), &nameflg);CHKERRQ(ierr);
3179   ierr = PetscOptionsEnum("-dm_plex_cell", "Cell shape", "", DMPolytopeTypes, (PetscEnum) cell, (PetscEnum *) &cell, NULL);CHKERRQ(ierr);
3180   ierr = PetscOptionsBool("-dm_plex_reference_cell_domain", "Use a reference cell domain", "", refDomain, &refDomain, NULL);CHKERRQ(ierr);
3181   ierr = PetscOptionsEnum("-dm_plex_shape", "Shape for built-in mesh", "", DMPlexShapes, (PetscEnum) shape, (PetscEnum *) &shape, &flg);CHKERRQ(ierr);
3182   ierr = PetscOptionsBoundedInt("-dm_plex_dim", "Topological dimension of the mesh", "DMGetDimension", dim, &dim, &flg, 0);CHKERRQ(ierr);
3183   PetscCheckFalse((dim < 0) || (dim > 3),comm, PETSC_ERR_ARG_OUTOFRANGE, "Dimension %D should be in [1, 3]", dim);
3184   ierr = PetscOptionsBool("-dm_plex_simplex", "Mesh cell shape", "", simplex,  &simplex, &flg);CHKERRQ(ierr);
3185   ierr = PetscOptionsBool("-dm_plex_interpolate", "Flag to create edges and faces automatically", "", interpolate, &interpolate, &flg);CHKERRQ(ierr);
3186   ierr = PetscOptionsBool("-dm_plex_adj_cone", "Set adjacency direction", "DMSetBasicAdjacency", adjCone,  &adjCone, &flg);CHKERRQ(ierr);
3187   ierr = PetscOptionsBool("-dm_plex_adj_closure", "Set adjacency size", "DMSetBasicAdjacency", adjClosure,  &adjClosure, &flg2);CHKERRQ(ierr);
3188   if (flg || flg2) {ierr = DMSetBasicAdjacency(dm, adjCone, adjClosure);CHKERRQ(ierr);}
3189 
3190   switch (cell) {
3191     case DM_POLYTOPE_POINT:
3192     case DM_POLYTOPE_SEGMENT:
3193     case DM_POLYTOPE_POINT_PRISM_TENSOR:
3194     case DM_POLYTOPE_TRIANGLE:
3195     case DM_POLYTOPE_QUADRILATERAL:
3196     case DM_POLYTOPE_TETRAHEDRON:
3197     case DM_POLYTOPE_HEXAHEDRON:
3198       *useCoordSpace = PETSC_TRUE;break;
3199     default: *useCoordSpace = PETSC_FALSE;break;
3200   }
3201 
3202   if (fflg) {
3203     DM dmnew;
3204 
3205     ierr = DMPlexCreateFromFile(PetscObjectComm((PetscObject) dm), filename, plexname, interpolate, &dmnew);CHKERRQ(ierr);
3206     ierr = DMPlexCopy_Internal(dm, PETSC_FALSE, dmnew);CHKERRQ(ierr);
3207     ierr = DMPlexReplace_Static(dm, &dmnew);CHKERRQ(ierr);
3208   } else if (refDomain) {
3209     ierr = DMPlexCreateReferenceCell_Internal(dm, cell);CHKERRQ(ierr);
3210   } else if (bdfflg) {
3211     DM bdm, dmnew;
3212 
3213     ierr = DMPlexCreateFromFile(PetscObjectComm((PetscObject) dm), bdFilename, plexname, interpolate, &bdm);CHKERRQ(ierr);
3214     ierr = PetscObjectSetOptionsPrefix((PetscObject) bdm, "bd_");CHKERRQ(ierr);
3215     ierr = DMSetFromOptions(bdm);CHKERRQ(ierr);
3216     ierr = DMPlexGenerate(bdm, NULL, interpolate, &dmnew);CHKERRQ(ierr);
3217     ierr = DMDestroy(&bdm);CHKERRQ(ierr);
3218     ierr = DMPlexCopy_Internal(dm, PETSC_FALSE, dmnew);CHKERRQ(ierr);
3219     ierr = DMPlexReplace_Static(dm, &dmnew);CHKERRQ(ierr);
3220   } else {
3221     ierr = PetscObjectSetName((PetscObject) dm, DMPlexShapes[shape]);CHKERRQ(ierr);
3222     switch (shape) {
3223       case DM_SHAPE_BOX:
3224       {
3225         PetscInt       faces[3] = {0, 0, 0};
3226         PetscReal      lower[3] = {0, 0, 0};
3227         PetscReal      upper[3] = {1, 1, 1};
3228         DMBoundaryType bdt[3]   = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
3229         PetscInt       i, n;
3230 
3231         n    = dim;
3232         for (i = 0; i < dim; ++i) faces[i] = (dim == 1 ? 1 : 4-dim);
3233         ierr = PetscOptionsIntArray("-dm_plex_box_faces", "Number of faces along each dimension", "", faces, &n, &flg);CHKERRQ(ierr);
3234         n    = 3;
3235         ierr = PetscOptionsRealArray("-dm_plex_box_lower", "Lower left corner of box", "", lower, &n, &flg);CHKERRQ(ierr);
3236         PetscCheckFalse(flg && (n != dim),comm, PETSC_ERR_ARG_SIZ, "Lower box point had %D values, should have been %D", n, dim);
3237         n    = 3;
3238         ierr = PetscOptionsRealArray("-dm_plex_box_upper", "Upper right corner of box", "", upper, &n, &flg);CHKERRQ(ierr);
3239         PetscCheckFalse(flg && (n != dim),comm, PETSC_ERR_ARG_SIZ, "Upper box point had %D values, should have been %D", n, dim);
3240         n    = 3;
3241         ierr = PetscOptionsEnumArray("-dm_plex_box_bd", "Boundary type for each dimension", "", DMBoundaryTypes, (PetscEnum *) bdt, &n, &flg);CHKERRQ(ierr);
3242         PetscCheckFalse(flg && (n != dim),comm, PETSC_ERR_ARG_SIZ, "Box boundary types had %D values, should have been %D", n, dim);
3243         switch (cell) {
3244           case DM_POLYTOPE_TRI_PRISM_TENSOR:
3245             ierr = DMPlexCreateWedgeBoxMesh_Internal(dm, faces, lower, upper, bdt);CHKERRQ(ierr);
3246             if (!interpolate) {
3247               DM udm;
3248 
3249               ierr = DMPlexUninterpolate(dm, &udm);CHKERRQ(ierr);
3250               ierr = DMPlexReplace_Static(dm, &udm);CHKERRQ(ierr);
3251             }
3252             break;
3253           default:
3254             ierr = DMPlexCreateBoxMesh_Internal(dm, dim, simplex, faces, lower, upper, bdt, interpolate);CHKERRQ(ierr);
3255             break;
3256         }
3257       }
3258       break;
3259       case DM_SHAPE_BOX_SURFACE:
3260       {
3261         PetscInt  faces[3] = {0, 0, 0};
3262         PetscReal lower[3] = {0, 0, 0};
3263         PetscReal upper[3] = {1, 1, 1};
3264         PetscInt  i, n;
3265 
3266         n    = dim+1;
3267         for (i = 0; i < dim+1; ++i) faces[i] = (dim+1 == 1 ? 1 : 4-(dim+1));
3268         ierr = PetscOptionsIntArray("-dm_plex_box_faces", "Number of faces along each dimension", "", faces, &n, &flg);CHKERRQ(ierr);
3269         n    = 3;
3270         ierr = PetscOptionsRealArray("-dm_plex_box_lower", "Lower left corner of box", "", lower, &n, &flg);CHKERRQ(ierr);
3271         PetscCheckFalse(flg && (n != dim+1),comm, PETSC_ERR_ARG_SIZ, "Lower box point had %D values, should have been %D", n, dim+1);
3272         n    = 3;
3273         ierr = PetscOptionsRealArray("-dm_plex_box_upper", "Upper right corner of box", "", upper, &n, &flg);CHKERRQ(ierr);
3274         PetscCheckFalse(flg && (n != dim+1),comm, PETSC_ERR_ARG_SIZ, "Upper box point had %D values, should have been %D", n, dim+1);
3275         ierr = DMPlexCreateBoxSurfaceMesh_Internal(dm, dim+1, faces, lower, upper, interpolate);CHKERRQ(ierr);
3276       }
3277       break;
3278       case DM_SHAPE_SPHERE:
3279       {
3280         PetscReal R = 1.0;
3281 
3282         ierr = PetscOptionsReal("-dm_plex_sphere_radius", "Radius of the sphere", "", R,  &R, &flg);CHKERRQ(ierr);
3283         ierr = DMPlexCreateSphereMesh_Internal(dm, dim, simplex, R);CHKERRQ(ierr);
3284       }
3285       break;
3286       case DM_SHAPE_BALL:
3287       {
3288         PetscReal R = 1.0;
3289 
3290         ierr = PetscOptionsReal("-dm_plex_ball_radius", "Radius of the ball", "", R,  &R, &flg);CHKERRQ(ierr);
3291         ierr = DMPlexCreateBallMesh_Internal(dm, dim, R);CHKERRQ(ierr);
3292       }
3293       break;
3294       case DM_SHAPE_CYLINDER:
3295       {
3296         DMBoundaryType bdt = DM_BOUNDARY_NONE;
3297         PetscInt       Nw  = 6;
3298 
3299         ierr = PetscOptionsEnum("-dm_plex_cylinder_bd", "Boundary type in the z direction", "", DMBoundaryTypes, (PetscEnum) bdt, (PetscEnum *) &bdt, NULL);CHKERRQ(ierr);
3300         ierr = PetscOptionsInt("-dm_plex_cylinder_num_wedges", "Number of wedges around the cylinder", "", Nw, &Nw, NULL);CHKERRQ(ierr);
3301         switch (cell) {
3302           case DM_POLYTOPE_TRI_PRISM_TENSOR:
3303             ierr = DMPlexCreateWedgeCylinderMesh_Internal(dm, Nw, interpolate);CHKERRQ(ierr);
3304             break;
3305           default:
3306             ierr = DMPlexCreateHexCylinderMesh_Internal(dm, bdt);CHKERRQ(ierr);
3307             break;
3308         }
3309       }
3310       break;
3311       case DM_SHAPE_SCHWARZ_P: // fallthrough
3312       case DM_SHAPE_GYROID:
3313       {
3314         PetscInt       extent[3] = {1,1,1}, refine = 0, layers = 0, three;
3315         PetscReal      thickness = 0.;
3316         DMBoundaryType periodic[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
3317         DMPlexTPSType  tps_type = shape == DM_SHAPE_SCHWARZ_P ? DMPLEX_TPS_SCHWARZ_P : DMPLEX_TPS_GYROID;
3318         PetscBool      tps_distribute;
3319         ierr = PetscOptionsIntArray("-dm_plex_tps_extent", "Number of replicas for each of three dimensions", NULL, extent, (three=3, &three), NULL);CHKERRQ(ierr);
3320         ierr = PetscOptionsInt("-dm_plex_tps_refine", "Number of refinements", NULL, refine, &refine, NULL);CHKERRQ(ierr);
3321         ierr = PetscOptionsEnumArray("-dm_plex_tps_periodic", "Periodicity in each of three dimensions", NULL, DMBoundaryTypes, (PetscEnum*)periodic, (three=3, &three), NULL);CHKERRQ(ierr);
3322         ierr = PetscOptionsInt("-dm_plex_tps_layers", "Number of layers in volumetric extrusion (or zero to not extrude)", NULL, layers, &layers, NULL);CHKERRQ(ierr);
3323         ierr = PetscOptionsReal("-dm_plex_tps_thickness", "Thickness of volumetric extrusion", NULL, thickness, &thickness, NULL);CHKERRQ(ierr);
3324         ierr = DMPlexDistributeGetDefault(dm, &tps_distribute);CHKERRQ(ierr);
3325         ierr = PetscOptionsBool("-dm_plex_tps_distribute", "Distribute the 2D mesh prior to refinement and extrusion", NULL, tps_distribute, &tps_distribute, NULL);CHKERRQ(ierr);
3326         ierr = DMPlexCreateTPSMesh_Internal(dm, tps_type, extent, periodic, tps_distribute, refine, layers, thickness);CHKERRQ(ierr);
3327       }
3328       break;
3329       default: SETERRQ(comm, PETSC_ERR_SUP, "Domain shape %s is unsupported", DMPlexShapes[shape]);
3330     }
3331   }
3332   ierr = DMPlexSetRefinementUniform(dm, PETSC_TRUE);CHKERRQ(ierr);
3333   if (!((PetscObject)dm)->name && nameflg) {
3334     ierr = PetscObjectSetName((PetscObject)dm, plexname);CHKERRQ(ierr);
3335   }
3336   PetscFunctionReturn(0);
3337 }
3338 
3339 PetscErrorCode DMSetFromOptions_NonRefinement_Plex(PetscOptionItems *PetscOptionsObject, DM dm)
3340 {
3341   DM_Plex       *mesh = (DM_Plex*) dm->data;
3342   PetscBool      flg;
3343   char           bdLabel[PETSC_MAX_PATH_LEN];
3344   PetscErrorCode ierr;
3345 
3346   PetscFunctionBegin;
3347   /* Handle viewing */
3348   ierr = PetscOptionsBool("-dm_plex_print_set_values", "Output all set values info", "DMPlexMatSetClosure", PETSC_FALSE, &mesh->printSetValues, NULL);CHKERRQ(ierr);
3349   ierr = PetscOptionsBoundedInt("-dm_plex_print_fem", "Debug output level all fem computations", "DMPlexSNESComputeResidualFEM", 0, &mesh->printFEM, NULL,0);CHKERRQ(ierr);
3350   ierr = PetscOptionsReal("-dm_plex_print_tol", "Tolerance for FEM output", "DMPlexSNESComputeResidualFEM", mesh->printTol, &mesh->printTol, NULL);CHKERRQ(ierr);
3351   ierr = PetscOptionsBoundedInt("-dm_plex_print_l2", "Debug output level all L2 diff computations", "DMComputeL2Diff", 0, &mesh->printL2, NULL,0);CHKERRQ(ierr);
3352   ierr = DMMonitorSetFromOptions(dm, "-dm_plex_monitor_throughput", "Monitor the simulation throughput", "DMPlexMonitorThroughput", DMPlexMonitorThroughput, NULL, &flg);CHKERRQ(ierr);
3353   if (flg) {ierr = PetscLogDefaultBegin();CHKERRQ(ierr);}
3354   /* Labeling */
3355   ierr = PetscOptionsString("-dm_plex_boundary_label", "Label to mark the mesh boundary", "", bdLabel, bdLabel, sizeof(bdLabel), &flg);CHKERRQ(ierr);
3356   if (flg) {ierr = DMPlexCreateBoundaryLabel_Private(dm, bdLabel);CHKERRQ(ierr);}
3357   /* Point Location */
3358   ierr = PetscOptionsBool("-dm_plex_hash_location", "Use grid hashing for point location", "DMInterpolate", PETSC_FALSE, &mesh->useHashLocation, NULL);CHKERRQ(ierr);
3359   /* Partitioning and distribution */
3360   ierr = PetscOptionsBool("-dm_plex_partition_balance", "Attempt to evenly divide points on partition boundary between processes", "DMPlexSetPartitionBalance", PETSC_FALSE, &mesh->partitionBalance, NULL);CHKERRQ(ierr);
3361   /* Generation and remeshing */
3362   ierr = PetscOptionsBool("-dm_plex_remesh_bd", "Allow changes to the boundary on remeshing", "DMAdapt", PETSC_FALSE, &mesh->remeshBd, NULL);CHKERRQ(ierr);
3363   /* Projection behavior */
3364   ierr = PetscOptionsBoundedInt("-dm_plex_max_projection_height", "Maxmimum mesh point height used to project locally", "DMPlexSetMaxProjectionHeight", 0, &mesh->maxProjectionHeight, NULL,0);CHKERRQ(ierr);
3365   ierr = PetscOptionsBool("-dm_plex_regular_refinement", "Use special nested projection algorithm for regular refinement", "DMPlexSetRegularRefinement", mesh->regularRefinement, &mesh->regularRefinement, NULL);CHKERRQ(ierr);
3366   /* Checking structure */
3367   {
3368     PetscBool   flg = PETSC_FALSE, flg2 = PETSC_FALSE, all = PETSC_FALSE;
3369 
3370     ierr = PetscOptionsBool("-dm_plex_check_all", "Perform all checks", NULL, PETSC_FALSE, &all, &flg2);CHKERRQ(ierr);
3371     ierr = PetscOptionsBool("-dm_plex_check_symmetry", "Check that the adjacency information in the mesh is symmetric", "DMPlexCheckSymmetry", PETSC_FALSE, &flg, &flg2);CHKERRQ(ierr);
3372     if (all || (flg && flg2)) {ierr = DMPlexCheckSymmetry(dm);CHKERRQ(ierr);}
3373     ierr = PetscOptionsBool("-dm_plex_check_skeleton", "Check that each cell has the correct number of vertices (only for homogeneous simplex or tensor meshes)", "DMPlexCheckSkeleton", PETSC_FALSE, &flg, &flg2);CHKERRQ(ierr);
3374     if (all || (flg && flg2)) {ierr = DMPlexCheckSkeleton(dm, 0);CHKERRQ(ierr);}
3375     ierr = PetscOptionsBool("-dm_plex_check_faces", "Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type", "DMPlexCheckFaces", PETSC_FALSE, &flg, &flg2);CHKERRQ(ierr);
3376     if (all || (flg && flg2)) {ierr = DMPlexCheckFaces(dm, 0);CHKERRQ(ierr);}
3377     ierr = PetscOptionsBool("-dm_plex_check_geometry", "Check that cells have positive volume", "DMPlexCheckGeometry", PETSC_FALSE, &flg, &flg2);CHKERRQ(ierr);
3378     if (all || (flg && flg2)) {ierr = DMPlexCheckGeometry(dm);CHKERRQ(ierr);}
3379     ierr = PetscOptionsBool("-dm_plex_check_pointsf", "Check some necessary conditions for PointSF", "DMPlexCheckPointSF", PETSC_FALSE, &flg, &flg2);CHKERRQ(ierr);
3380     if (all || (flg && flg2)) {ierr = DMPlexCheckPointSF(dm);CHKERRQ(ierr);}
3381     ierr = PetscOptionsBool("-dm_plex_check_interface_cones", "Check points on inter-partition interfaces have conforming order of cone points", "DMPlexCheckInterfaceCones", PETSC_FALSE, &flg, &flg2);CHKERRQ(ierr);
3382     if (all || (flg && flg2)) {ierr = DMPlexCheckInterfaceCones(dm);CHKERRQ(ierr);}
3383     ierr = PetscOptionsBool("-dm_plex_check_cell_shape", "Check cell shape", "DMPlexCheckCellShape", PETSC_FALSE, &flg, &flg2);CHKERRQ(ierr);
3384     if (flg && flg2) {ierr = DMPlexCheckCellShape(dm, PETSC_TRUE, PETSC_DETERMINE);CHKERRQ(ierr);}
3385   }
3386   {
3387     PetscReal scale = 1.0;
3388 
3389     ierr = PetscOptionsReal("-dm_plex_scale", "Scale factor for mesh coordinates", "DMPlexScale", scale, &scale, &flg);CHKERRQ(ierr);
3390     if (flg) {
3391       Vec coordinates, coordinatesLocal;
3392 
3393       ierr = DMGetCoordinates(dm, &coordinates);CHKERRQ(ierr);
3394       ierr = DMGetCoordinatesLocal(dm, &coordinatesLocal);CHKERRQ(ierr);
3395       ierr = VecScale(coordinates, scale);CHKERRQ(ierr);
3396       ierr = VecScale(coordinatesLocal, scale);CHKERRQ(ierr);
3397     }
3398   }
3399   ierr = PetscPartitionerSetFromOptions(mesh->partitioner);CHKERRQ(ierr);
3400   PetscFunctionReturn(0);
3401 }
3402 
3403 static PetscErrorCode DMSetFromOptions_Plex(PetscOptionItems *PetscOptionsObject,DM dm)
3404 {
3405   PetscFunctionList ordlist;
3406   char              oname[256];
3407   PetscReal         volume = -1.0;
3408   PetscInt          prerefine = 0, refine = 0, r, coarsen = 0, overlap = 0, extLayers = 0, dim;
3409   PetscBool         uniformOrig, created = PETSC_FALSE, uniform = PETSC_TRUE, distribute, interpolate = PETSC_TRUE, coordSpace = PETSC_TRUE, remap = PETSC_TRUE, ghostCells = PETSC_FALSE, isHierarchy, ignoreModel = PETSC_FALSE, flg;
3410   PetscErrorCode ierr;
3411 
3412   PetscFunctionBegin;
3413   PetscValidHeaderSpecific(dm, DM_CLASSID, 2);
3414   ierr = PetscOptionsHead(PetscOptionsObject,"DMPlex Options");CHKERRQ(ierr);
3415   /* Handle automatic creation */
3416   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
3417   if (dim < 0) {ierr = DMPlexCreateFromOptions_Internal(PetscOptionsObject, &coordSpace, dm);CHKERRQ(ierr);created = PETSC_TRUE;}
3418   /* Handle interpolation before distribution */
3419   ierr = PetscOptionsBool("-dm_plex_interpolate_pre", "Flag to interpolate mesh before distribution", "", interpolate, &interpolate, &flg);CHKERRQ(ierr);
3420   if (flg) {
3421     DMPlexInterpolatedFlag interpolated;
3422 
3423     ierr = DMPlexIsInterpolated(dm, &interpolated);CHKERRQ(ierr);
3424     if (interpolated == DMPLEX_INTERPOLATED_FULL && !interpolate) {
3425       DM udm;
3426 
3427       ierr = DMPlexUninterpolate(dm, &udm);CHKERRQ(ierr);
3428       ierr = DMPlexReplace_Static(dm, &udm);CHKERRQ(ierr);
3429     } else if (interpolated != DMPLEX_INTERPOLATED_FULL && interpolate) {
3430       DM idm;
3431 
3432       ierr = DMPlexInterpolate(dm, &idm);CHKERRQ(ierr);
3433       ierr = DMPlexReplace_Static(dm, &idm);CHKERRQ(ierr);
3434     }
3435   }
3436   /* Handle DMPlex refinement before distribution */
3437   ierr = PetscOptionsBool("-dm_refine_ignore_model", "Flag to ignore the geometry model when refining", "DMCreate", ignoreModel, &ignoreModel, &flg);CHKERRQ(ierr);
3438   if (flg) {((DM_Plex *) dm->data)->ignoreModel = ignoreModel;}
3439   ierr = DMPlexGetRefinementUniform(dm, &uniformOrig);CHKERRQ(ierr);
3440   ierr = PetscOptionsBoundedInt("-dm_refine_pre", "The number of refinements before distribution", "DMCreate", prerefine, &prerefine, NULL,0);CHKERRQ(ierr);
3441   ierr = PetscOptionsBool("-dm_refine_remap_pre", "Flag to control coordinate remapping", "DMCreate", remap, &remap, NULL);CHKERRQ(ierr);
3442   ierr = PetscOptionsBool("-dm_refine_uniform_pre", "Flag for uniform refinement before distribution", "DMCreate", uniform, &uniform, &flg);CHKERRQ(ierr);
3443   if (flg) {ierr = DMPlexSetRefinementUniform(dm, uniform);CHKERRQ(ierr);}
3444   ierr = PetscOptionsReal("-dm_refine_volume_limit_pre", "The maximum cell volume after refinement before distribution", "DMCreate", volume, &volume, &flg);CHKERRQ(ierr);
3445   if (flg) {
3446     ierr = DMPlexSetRefinementUniform(dm, PETSC_FALSE);CHKERRQ(ierr);
3447     ierr = DMPlexSetRefinementLimit(dm, volume);CHKERRQ(ierr);
3448     prerefine = PetscMax(prerefine, 1);
3449   }
3450   for (r = 0; r < prerefine; ++r) {
3451     DM             rdm;
3452     PetscPointFunc coordFunc = ((DM_Plex*) dm->data)->coordFunc;
3453 
3454     ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dm);CHKERRQ(ierr);
3455     ierr = DMRefine(dm, PetscObjectComm((PetscObject) dm), &rdm);CHKERRQ(ierr);
3456     ierr = DMPlexReplace_Static(dm, &rdm);CHKERRQ(ierr);
3457     ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dm);CHKERRQ(ierr);
3458     if (coordFunc && remap) {
3459       ierr = DMPlexRemapGeometry(dm, 0.0, coordFunc);CHKERRQ(ierr);
3460       ((DM_Plex*) dm->data)->coordFunc = coordFunc;
3461     }
3462   }
3463   ierr = DMPlexSetRefinementUniform(dm, uniformOrig);CHKERRQ(ierr);
3464   /* Handle DMPlex extrusion before distribution */
3465   ierr = PetscOptionsBoundedInt("-dm_extrude", "The number of layers to extrude", "", extLayers, &extLayers, NULL, 0);CHKERRQ(ierr);
3466   if (extLayers) {
3467     DM edm;
3468 
3469     ierr = DMExtrude(dm, extLayers, &edm);CHKERRQ(ierr);
3470     ierr = DMPlexReplace_Static(dm, &edm);CHKERRQ(ierr);
3471     ((DM_Plex *) dm->data)->coordFunc = NULL;
3472     ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dm);CHKERRQ(ierr);
3473     extLayers = 0;
3474   }
3475   /* Handle DMPlex reordering before distribution */
3476   ierr = MatGetOrderingList(&ordlist);CHKERRQ(ierr);
3477   ierr = PetscOptionsFList("-dm_plex_reorder", "Set mesh reordering type", "DMPlexGetOrdering", ordlist, MATORDERINGNATURAL, oname, sizeof(oname), &flg);CHKERRQ(ierr);
3478   if (flg) {
3479     DM pdm;
3480     IS perm;
3481 
3482     ierr = DMPlexGetOrdering(dm, oname, NULL, &perm);CHKERRQ(ierr);
3483     ierr = DMPlexPermute(dm, perm, &pdm);CHKERRQ(ierr);
3484     ierr = ISDestroy(&perm);CHKERRQ(ierr);
3485     ierr = DMPlexReplace_Static(dm, &pdm);CHKERRQ(ierr);
3486     ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dm);CHKERRQ(ierr);
3487   }
3488   /* Handle DMPlex distribution */
3489   ierr = DMPlexDistributeGetDefault(dm, &distribute);CHKERRQ(ierr);
3490   ierr = PetscOptionsBool("-dm_distribute", "Flag to redistribute a mesh among processes", "DMCreate", distribute, &distribute, NULL);CHKERRQ(ierr);
3491   ierr = PetscOptionsBoundedInt("-dm_distribute_overlap", "The size of the overlap halo", "DMCreate", overlap, &overlap, NULL, 0);CHKERRQ(ierr);
3492   if (distribute) {
3493     DM               pdm = NULL;
3494     PetscPartitioner part;
3495 
3496     ierr = DMPlexGetPartitioner(dm, &part);CHKERRQ(ierr);
3497     ierr = PetscPartitionerSetFromOptions(part);CHKERRQ(ierr);
3498     ierr = DMPlexDistribute(dm, overlap, NULL, &pdm);CHKERRQ(ierr);
3499     if (pdm) {
3500       ierr = DMPlexReplace_Static(dm, &pdm);CHKERRQ(ierr);
3501     }
3502   }
3503   /* Create coordinate space */
3504   if (created) {
3505     DM_Plex  *mesh = (DM_Plex *) dm->data;
3506     PetscInt  degree = 1;
3507     PetscBool periodic, flg;
3508 
3509     ierr = PetscOptionsBool("-dm_coord_space", "Use an FEM space for coordinates", "", coordSpace, &coordSpace, &flg);CHKERRQ(ierr);
3510     ierr = PetscOptionsInt("-dm_coord_petscspace_degree", "FEM degree for coordinate space", "", degree, &degree, NULL);CHKERRQ(ierr);
3511     if (coordSpace) {ierr = DMPlexCreateCoordinateSpace(dm, degree, mesh->coordFunc);CHKERRQ(ierr);}
3512     if (flg && !coordSpace) {
3513       DM           cdm;
3514       PetscDS      cds;
3515       PetscObject  obj;
3516       PetscClassId id;
3517 
3518       ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
3519       ierr = DMGetDS(cdm, &cds);CHKERRQ(ierr);
3520       ierr = PetscDSGetDiscretization(cds, 0, &obj);CHKERRQ(ierr);
3521       ierr = PetscObjectGetClassId(obj, &id);CHKERRQ(ierr);
3522       if (id == PETSCFE_CLASSID) {
3523         PetscContainer dummy;
3524 
3525         ierr = PetscContainerCreate(PETSC_COMM_SELF, &dummy);CHKERRQ(ierr);
3526         ierr = PetscObjectSetName((PetscObject) dummy, "coordinates");CHKERRQ(ierr);
3527         ierr = DMSetField(cdm, 0, NULL, (PetscObject) dummy);CHKERRQ(ierr);
3528         ierr = PetscContainerDestroy(&dummy);CHKERRQ(ierr);
3529         ierr = DMClearDS(cdm);CHKERRQ(ierr);
3530       }
3531       mesh->coordFunc = NULL;
3532     }
3533     ierr = DMLocalizeCoordinates(dm);CHKERRQ(ierr);
3534     ierr = DMGetPeriodicity(dm, &periodic, NULL, NULL, NULL);CHKERRQ(ierr);
3535     if (periodic) {ierr = DMSetPeriodicity(dm, PETSC_TRUE, NULL, NULL, NULL);CHKERRQ(ierr);}
3536   }
3537   /* Handle DMPlex refinement */
3538   remap = PETSC_TRUE;
3539   ierr = PetscOptionsBoundedInt("-dm_refine", "The number of uniform refinements", "DMCreate", refine, &refine, NULL,0);CHKERRQ(ierr);
3540   ierr = PetscOptionsBool("-dm_refine_remap", "Flag to control coordinate remapping", "DMCreate", remap, &remap, NULL);CHKERRQ(ierr);
3541   ierr = PetscOptionsBoundedInt("-dm_refine_hierarchy", "The number of uniform refinements", "DMCreate", refine, &refine, &isHierarchy,0);CHKERRQ(ierr);
3542   if (refine) {ierr = DMPlexSetRefinementUniform(dm, PETSC_TRUE);CHKERRQ(ierr);}
3543   if (refine && isHierarchy) {
3544     DM *dms, coarseDM;
3545 
3546     ierr = DMGetCoarseDM(dm, &coarseDM);CHKERRQ(ierr);
3547     ierr = PetscObjectReference((PetscObject)coarseDM);CHKERRQ(ierr);
3548     ierr = PetscMalloc1(refine,&dms);CHKERRQ(ierr);
3549     ierr = DMRefineHierarchy(dm, refine, dms);CHKERRQ(ierr);
3550     /* Total hack since we do not pass in a pointer */
3551     ierr = DMPlexSwap_Static(dm, dms[refine-1]);CHKERRQ(ierr);
3552     if (refine == 1) {
3553       ierr = DMSetCoarseDM(dm, dms[0]);CHKERRQ(ierr);
3554       ierr = DMPlexSetRegularRefinement(dm, PETSC_TRUE);CHKERRQ(ierr);
3555     } else {
3556       ierr = DMSetCoarseDM(dm, dms[refine-2]);CHKERRQ(ierr);
3557       ierr = DMPlexSetRegularRefinement(dm, PETSC_TRUE);CHKERRQ(ierr);
3558       ierr = DMSetCoarseDM(dms[0], dms[refine-1]);CHKERRQ(ierr);
3559       ierr = DMPlexSetRegularRefinement(dms[0], PETSC_TRUE);CHKERRQ(ierr);
3560     }
3561     ierr = DMSetCoarseDM(dms[refine-1], coarseDM);CHKERRQ(ierr);
3562     ierr = PetscObjectDereference((PetscObject)coarseDM);CHKERRQ(ierr);
3563     /* Free DMs */
3564     for (r = 0; r < refine; ++r) {
3565       ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dms[r]);CHKERRQ(ierr);
3566       ierr = DMDestroy(&dms[r]);CHKERRQ(ierr);
3567     }
3568     ierr = PetscFree(dms);CHKERRQ(ierr);
3569   } else {
3570     for (r = 0; r < refine; ++r) {
3571       DM             rdm;
3572       PetscPointFunc coordFunc = ((DM_Plex*) dm->data)->coordFunc;
3573 
3574       ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dm);CHKERRQ(ierr);
3575       ierr = DMRefine(dm, PetscObjectComm((PetscObject) dm), &rdm);CHKERRQ(ierr);
3576       /* Total hack since we do not pass in a pointer */
3577       ierr = DMPlexReplace_Static(dm, &rdm);CHKERRQ(ierr);
3578       ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dm);CHKERRQ(ierr);
3579       if (coordFunc && remap) {
3580         ierr = DMPlexRemapGeometry(dm, 0.0, coordFunc);CHKERRQ(ierr);
3581         ((DM_Plex*) dm->data)->coordFunc = coordFunc;
3582       }
3583     }
3584   }
3585   /* Handle DMPlex coarsening */
3586   ierr = PetscOptionsBoundedInt("-dm_coarsen", "Coarsen the mesh", "DMCreate", coarsen, &coarsen, NULL,0);CHKERRQ(ierr);
3587   ierr = PetscOptionsBoundedInt("-dm_coarsen_hierarchy", "The number of coarsenings", "DMCreate", coarsen, &coarsen, &isHierarchy,0);CHKERRQ(ierr);
3588   if (coarsen && isHierarchy) {
3589     DM *dms;
3590 
3591     ierr = PetscMalloc1(coarsen, &dms);CHKERRQ(ierr);
3592     ierr = DMCoarsenHierarchy(dm, coarsen, dms);CHKERRQ(ierr);
3593     /* Free DMs */
3594     for (r = 0; r < coarsen; ++r) {
3595       ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dms[r]);CHKERRQ(ierr);
3596       ierr = DMDestroy(&dms[r]);CHKERRQ(ierr);
3597     }
3598     ierr = PetscFree(dms);CHKERRQ(ierr);
3599   } else {
3600     for (r = 0; r < coarsen; ++r) {
3601       DM             cdm;
3602       PetscPointFunc coordFunc = ((DM_Plex*) dm->data)->coordFunc;
3603 
3604       ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dm);CHKERRQ(ierr);
3605       ierr = DMCoarsen(dm, PetscObjectComm((PetscObject) dm), &cdm);CHKERRQ(ierr);
3606       /* Total hack since we do not pass in a pointer */
3607       ierr = DMPlexReplace_Static(dm, &cdm);CHKERRQ(ierr);
3608       ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dm);CHKERRQ(ierr);
3609       if (coordFunc) {
3610         ierr = DMPlexRemapGeometry(dm, 0.0, coordFunc);CHKERRQ(ierr);
3611         ((DM_Plex*) dm->data)->coordFunc = coordFunc;
3612       }
3613     }
3614   }
3615   /* Handle ghost cells */
3616   ierr = PetscOptionsBool("-dm_plex_create_fv_ghost_cells", "Flag to create finite volume ghost cells on the boundary", "DMCreate", ghostCells, &ghostCells, NULL);CHKERRQ(ierr);
3617   if (ghostCells) {
3618     DM   gdm;
3619     char lname[PETSC_MAX_PATH_LEN];
3620 
3621     lname[0] = '\0';
3622     ierr = PetscOptionsString("-dm_plex_fv_ghost_cells_label", "Label name for ghost cells boundary", "DMCreate", lname, lname, sizeof(lname), &flg);CHKERRQ(ierr);
3623     ierr = DMPlexConstructGhostCells(dm, flg ? lname : NULL, NULL, &gdm);CHKERRQ(ierr);
3624     ierr = DMPlexReplace_Static(dm, &gdm);CHKERRQ(ierr);
3625   }
3626   /* Handle */
3627   ierr = DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dm);CHKERRQ(ierr);
3628   ierr = PetscOptionsTail();CHKERRQ(ierr);
3629   PetscFunctionReturn(0);
3630 }
3631 
3632 static PetscErrorCode DMCreateGlobalVector_Plex(DM dm,Vec *vec)
3633 {
3634   PetscErrorCode ierr;
3635 
3636   PetscFunctionBegin;
3637   ierr = DMCreateGlobalVector_Section_Private(dm,vec);CHKERRQ(ierr);
3638   /* ierr = VecSetOperation(*vec, VECOP_DUPLICATE, (void(*)(void)) VecDuplicate_MPI_DM);CHKERRQ(ierr); */
3639   ierr = VecSetOperation(*vec, VECOP_VIEW, (void (*)(void)) VecView_Plex);CHKERRQ(ierr);
3640   ierr = VecSetOperation(*vec, VECOP_VIEWNATIVE, (void (*)(void)) VecView_Plex_Native);CHKERRQ(ierr);
3641   ierr = VecSetOperation(*vec, VECOP_LOAD, (void (*)(void)) VecLoad_Plex);CHKERRQ(ierr);
3642   ierr = VecSetOperation(*vec, VECOP_LOADNATIVE, (void (*)(void)) VecLoad_Plex_Native);CHKERRQ(ierr);
3643   PetscFunctionReturn(0);
3644 }
3645 
3646 static PetscErrorCode DMCreateLocalVector_Plex(DM dm,Vec *vec)
3647 {
3648   PetscErrorCode ierr;
3649 
3650   PetscFunctionBegin;
3651   ierr = DMCreateLocalVector_Section_Private(dm,vec);CHKERRQ(ierr);
3652   ierr = VecSetOperation(*vec, VECOP_VIEW, (void (*)(void)) VecView_Plex_Local);CHKERRQ(ierr);
3653   ierr = VecSetOperation(*vec, VECOP_LOAD, (void (*)(void)) VecLoad_Plex_Local);CHKERRQ(ierr);
3654   PetscFunctionReturn(0);
3655 }
3656 
3657 static PetscErrorCode DMGetDimPoints_Plex(DM dm, PetscInt dim, PetscInt *pStart, PetscInt *pEnd)
3658 {
3659   PetscInt       depth, d;
3660   PetscErrorCode ierr;
3661 
3662   PetscFunctionBegin;
3663   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3664   if (depth == 1) {
3665     ierr = DMGetDimension(dm, &d);CHKERRQ(ierr);
3666     if (dim == 0)      {ierr = DMPlexGetDepthStratum(dm, dim, pStart, pEnd);CHKERRQ(ierr);}
3667     else if (dim == d) {ierr = DMPlexGetDepthStratum(dm, 1, pStart, pEnd);CHKERRQ(ierr);}
3668     else               {*pStart = 0; *pEnd = 0;}
3669   } else {
3670     ierr = DMPlexGetDepthStratum(dm, dim, pStart, pEnd);CHKERRQ(ierr);
3671   }
3672   PetscFunctionReturn(0);
3673 }
3674 
3675 static PetscErrorCode DMGetNeighbors_Plex(DM dm, PetscInt *nranks, const PetscMPIInt *ranks[])
3676 {
3677   PetscSF           sf;
3678   PetscInt          niranks, njranks, n;
3679   const PetscMPIInt *iranks, *jranks;
3680   DM_Plex           *data = (DM_Plex*) dm->data;
3681   PetscErrorCode    ierr;
3682 
3683   PetscFunctionBegin;
3684   ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr);
3685   if (!data->neighbors) {
3686     ierr = PetscSFSetUp(sf);CHKERRQ(ierr);
3687     ierr = PetscSFGetRootRanks(sf, &njranks, &jranks, NULL, NULL, NULL);CHKERRQ(ierr);
3688     ierr = PetscSFGetLeafRanks(sf, &niranks, &iranks, NULL, NULL);CHKERRQ(ierr);
3689     ierr = PetscMalloc1(njranks + niranks + 1, &data->neighbors);CHKERRQ(ierr);
3690     ierr = PetscArraycpy(data->neighbors + 1, jranks, njranks);CHKERRQ(ierr);
3691     ierr = PetscArraycpy(data->neighbors + njranks + 1, iranks, niranks);CHKERRQ(ierr);
3692     n = njranks + niranks;
3693     ierr = PetscSortRemoveDupsMPIInt(&n, data->neighbors + 1);CHKERRQ(ierr);
3694     /* The following cast should never fail: can't have more neighbors than PETSC_MPI_INT_MAX */
3695     ierr = PetscMPIIntCast(n, data->neighbors);CHKERRQ(ierr);
3696   }
3697   if (nranks) *nranks = data->neighbors[0];
3698   if (ranks) {
3699     if (data->neighbors[0]) *ranks = data->neighbors + 1;
3700     else                    *ranks = NULL;
3701   }
3702   PetscFunctionReturn(0);
3703 }
3704 
3705 PETSC_INTERN PetscErrorCode DMInterpolateSolution_Plex(DM, DM, Mat, Vec, Vec);
3706 
3707 static PetscErrorCode DMInitialize_Plex(DM dm)
3708 {
3709   PetscErrorCode ierr;
3710 
3711   PetscFunctionBegin;
3712   dm->ops->view                            = DMView_Plex;
3713   dm->ops->load                            = DMLoad_Plex;
3714   dm->ops->setfromoptions                  = DMSetFromOptions_Plex;
3715   dm->ops->clone                           = DMClone_Plex;
3716   dm->ops->setup                           = DMSetUp_Plex;
3717   dm->ops->createlocalsection              = DMCreateLocalSection_Plex;
3718   dm->ops->createdefaultconstraints        = DMCreateDefaultConstraints_Plex;
3719   dm->ops->createglobalvector              = DMCreateGlobalVector_Plex;
3720   dm->ops->createlocalvector               = DMCreateLocalVector_Plex;
3721   dm->ops->getlocaltoglobalmapping         = NULL;
3722   dm->ops->createfieldis                   = NULL;
3723   dm->ops->createcoordinatedm              = DMCreateCoordinateDM_Plex;
3724   dm->ops->createcoordinatefield           = DMCreateCoordinateField_Plex;
3725   dm->ops->getcoloring                     = NULL;
3726   dm->ops->creatematrix                    = DMCreateMatrix_Plex;
3727   dm->ops->createinterpolation             = DMCreateInterpolation_Plex;
3728   dm->ops->createmassmatrix                = DMCreateMassMatrix_Plex;
3729   dm->ops->createmassmatrixlumped          = DMCreateMassMatrixLumped_Plex;
3730   dm->ops->createinjection                 = DMCreateInjection_Plex;
3731   dm->ops->refine                          = DMRefine_Plex;
3732   dm->ops->coarsen                         = DMCoarsen_Plex;
3733   dm->ops->refinehierarchy                 = DMRefineHierarchy_Plex;
3734   dm->ops->coarsenhierarchy                = DMCoarsenHierarchy_Plex;
3735   dm->ops->extrude                         = DMExtrude_Plex;
3736   dm->ops->globaltolocalbegin              = NULL;
3737   dm->ops->globaltolocalend                = NULL;
3738   dm->ops->localtoglobalbegin              = NULL;
3739   dm->ops->localtoglobalend                = NULL;
3740   dm->ops->destroy                         = DMDestroy_Plex;
3741   dm->ops->createsubdm                     = DMCreateSubDM_Plex;
3742   dm->ops->createsuperdm                   = DMCreateSuperDM_Plex;
3743   dm->ops->getdimpoints                    = DMGetDimPoints_Plex;
3744   dm->ops->locatepoints                    = DMLocatePoints_Plex;
3745   dm->ops->projectfunctionlocal            = DMProjectFunctionLocal_Plex;
3746   dm->ops->projectfunctionlabellocal       = DMProjectFunctionLabelLocal_Plex;
3747   dm->ops->projectfieldlocal               = DMProjectFieldLocal_Plex;
3748   dm->ops->projectfieldlabellocal          = DMProjectFieldLabelLocal_Plex;
3749   dm->ops->projectbdfieldlabellocal        = DMProjectBdFieldLabelLocal_Plex;
3750   dm->ops->computel2diff                   = DMComputeL2Diff_Plex;
3751   dm->ops->computel2gradientdiff           = DMComputeL2GradientDiff_Plex;
3752   dm->ops->computel2fielddiff              = DMComputeL2FieldDiff_Plex;
3753   dm->ops->getneighbors                    = DMGetNeighbors_Plex;
3754   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMPlexInsertBoundaryValues_C",DMPlexInsertBoundaryValues_Plex);CHKERRQ(ierr);
3755   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMPlexInsertTimeDerviativeBoundaryValues_C",DMPlexInsertTimeDerivativeBoundaryValues_Plex);CHKERRQ(ierr);
3756   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMSetUpGLVisViewer_C",DMSetUpGLVisViewer_Plex);CHKERRQ(ierr);
3757   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMCreateNeumannOverlap_C",DMCreateNeumannOverlap_Plex);CHKERRQ(ierr);
3758   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMPlexGetOverlap_C",DMPlexGetOverlap_Plex);CHKERRQ(ierr);
3759   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMPlexDistributeGetDefault_C",DMPlexDistributeGetDefault_Plex);CHKERRQ(ierr);
3760   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMPlexDistributeSetDefault_C",DMPlexDistributeSetDefault_Plex);CHKERRQ(ierr);
3761   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMInterpolateSolution_C",DMInterpolateSolution_Plex);CHKERRQ(ierr);
3762   PetscFunctionReturn(0);
3763 }
3764 
3765 PETSC_INTERN PetscErrorCode DMClone_Plex(DM dm, DM *newdm)
3766 {
3767   DM_Plex        *mesh = (DM_Plex *) dm->data;
3768   PetscErrorCode ierr;
3769 
3770   PetscFunctionBegin;
3771   mesh->refct++;
3772   (*newdm)->data = mesh;
3773   ierr = PetscObjectChangeTypeName((PetscObject) *newdm, DMPLEX);CHKERRQ(ierr);
3774   ierr = DMInitialize_Plex(*newdm);CHKERRQ(ierr);
3775   PetscFunctionReturn(0);
3776 }
3777 
3778 /*MC
3779   DMPLEX = "plex" - A DM object that encapsulates an unstructured mesh, or CW Complex, which can be expressed using a Hasse Diagram.
3780                     In the local representation, Vecs contain all unknowns in the interior and shared boundary. This is
3781                     specified by a PetscSection object. Ownership in the global representation is determined by
3782                     ownership of the underlying DMPlex points. This is specified by another PetscSection object.
3783 
3784   Options Database Keys:
3785 + -dm_refine_pre                     - Refine mesh before distribution
3786 + -dm_refine_uniform_pre             - Choose uniform or generator-based refinement
3787 + -dm_refine_volume_limit_pre        - Cell volume limit after pre-refinement using generator
3788 . -dm_distribute                     - Distribute mesh across processes
3789 . -dm_distribute_overlap             - Number of cells to overlap for distribution
3790 . -dm_refine                         - Refine mesh after distribution
3791 . -dm_plex_hash_location             - Use grid hashing for point location
3792 . -dm_plex_hash_box_faces <n,m,p>    - The number of divisions in each direction of the grid hash
3793 . -dm_plex_partition_balance         - Attempt to evenly divide points on partition boundary between processes
3794 . -dm_plex_remesh_bd                 - Allow changes to the boundary on remeshing
3795 . -dm_plex_max_projection_height     - Maxmimum mesh point height used to project locally
3796 . -dm_plex_regular_refinement        - Use special nested projection algorithm for regular refinement
3797 . -dm_plex_check_all                 - Perform all shecks below
3798 . -dm_plex_check_symmetry            - Check that the adjacency information in the mesh is symmetric
3799 . -dm_plex_check_skeleton <celltype> - Check that each cell has the correct number of vertices
3800 . -dm_plex_check_faces <celltype>    - Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type
3801 . -dm_plex_check_geometry            - Check that cells have positive volume
3802 . -dm_view :mesh.tex:ascii_latex     - View the mesh in LaTeX/TikZ
3803 . -dm_plex_view_scale <num>          - Scale the TikZ
3804 - -dm_plex_print_fem <num>           - View FEM assembly information, such as element vectors and matrices
3805 
3806   Level: intermediate
3807 
3808 .seealso: DMType, DMPlexCreate(), DMCreate(), DMSetType()
3809 M*/
3810 
3811 PETSC_EXTERN PetscErrorCode DMCreate_Plex(DM dm)
3812 {
3813   DM_Plex       *mesh;
3814   PetscInt       unit;
3815   PetscErrorCode ierr;
3816 
3817   PetscFunctionBegin;
3818   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3819   ierr     = PetscNewLog(dm,&mesh);CHKERRQ(ierr);
3820   dm->data = mesh;
3821 
3822   mesh->refct             = 1;
3823   ierr                    = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &mesh->coneSection);CHKERRQ(ierr);
3824   mesh->maxConeSize       = 0;
3825   mesh->cones             = NULL;
3826   mesh->coneOrientations  = NULL;
3827   ierr                    = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &mesh->supportSection);CHKERRQ(ierr);
3828   mesh->maxSupportSize    = 0;
3829   mesh->supports          = NULL;
3830   mesh->refinementUniform = PETSC_TRUE;
3831   mesh->refinementLimit   = -1.0;
3832   mesh->distDefault       = PETSC_TRUE;
3833   mesh->interpolated      = DMPLEX_INTERPOLATED_INVALID;
3834   mesh->interpolatedCollective = DMPLEX_INTERPOLATED_INVALID;
3835 
3836   mesh->facesTmp = NULL;
3837 
3838   mesh->tetgenOpts   = NULL;
3839   mesh->triangleOpts = NULL;
3840   ierr = PetscPartitionerCreate(PetscObjectComm((PetscObject)dm), &mesh->partitioner);CHKERRQ(ierr);
3841   mesh->remeshBd     = PETSC_FALSE;
3842 
3843   mesh->subpointMap = NULL;
3844 
3845   for (unit = 0; unit < NUM_PETSC_UNITS; ++unit) mesh->scale[unit] = 1.0;
3846 
3847   mesh->regularRefinement   = PETSC_FALSE;
3848   mesh->depthState          = -1;
3849   mesh->celltypeState       = -1;
3850   mesh->globalVertexNumbers = NULL;
3851   mesh->globalCellNumbers   = NULL;
3852   mesh->anchorSection       = NULL;
3853   mesh->anchorIS            = NULL;
3854   mesh->createanchors       = NULL;
3855   mesh->computeanchormatrix = NULL;
3856   mesh->parentSection       = NULL;
3857   mesh->parents             = NULL;
3858   mesh->childIDs            = NULL;
3859   mesh->childSection        = NULL;
3860   mesh->children            = NULL;
3861   mesh->referenceTree       = NULL;
3862   mesh->getchildsymmetry    = NULL;
3863   mesh->vtkCellHeight       = 0;
3864   mesh->useAnchors          = PETSC_FALSE;
3865 
3866   mesh->maxProjectionHeight = 0;
3867 
3868   mesh->neighbors           = NULL;
3869 
3870   mesh->printSetValues = PETSC_FALSE;
3871   mesh->printFEM       = 0;
3872   mesh->printTol       = 1.0e-10;
3873 
3874   ierr = DMInitialize_Plex(dm);CHKERRQ(ierr);
3875   PetscFunctionReturn(0);
3876 }
3877 
3878 /*@
3879   DMPlexCreate - Creates a DMPlex object, which encapsulates an unstructured mesh, or CW complex, which can be expressed using a Hasse Diagram.
3880 
3881   Collective
3882 
3883   Input Parameter:
3884 . comm - The communicator for the DMPlex object
3885 
3886   Output Parameter:
3887 . mesh  - The DMPlex object
3888 
3889   Level: beginner
3890 
3891 @*/
3892 PetscErrorCode DMPlexCreate(MPI_Comm comm, DM *mesh)
3893 {
3894   PetscErrorCode ierr;
3895 
3896   PetscFunctionBegin;
3897   PetscValidPointer(mesh,2);
3898   ierr = DMCreate(comm, mesh);CHKERRQ(ierr);
3899   ierr = DMSetType(*mesh, DMPLEX);CHKERRQ(ierr);
3900   PetscFunctionReturn(0);
3901 }
3902 
3903 /*@C
3904   DMPlexBuildFromCellListParallel - Build distributed DMPLEX topology from a list of vertices for each cell (common mesh generator output)
3905 
3906   Input Parameters:
3907 + dm - The DM
3908 . numCells - The number of cells owned by this process
3909 . numVertices - The number of vertices to be owned by this process, or PETSC_DECIDE
3910 . NVertices - The global number of vertices, or PETSC_DETERMINE
3911 . numCorners - The number of vertices for each cell
3912 - cells - An array of numCells*numCorners numbers, the global vertex numbers for each cell
3913 
3914   Output Parameters:
3915 + vertexSF - (Optional) SF describing complete vertex ownership
3916 - verticesAdjSaved - (Optional) vertex adjacency array
3917 
3918   Notes:
3919   Two triangles sharing a face
3920 $
3921 $        2
3922 $      / | \
3923 $     /  |  \
3924 $    /   |   \
3925 $   0  0 | 1  3
3926 $    \   |   /
3927 $     \  |  /
3928 $      \ | /
3929 $        1
3930 would have input
3931 $  numCells = 2, numVertices = 4
3932 $  cells = [0 1 2  1 3 2]
3933 $
3934 which would result in the DMPlex
3935 $
3936 $        4
3937 $      / | \
3938 $     /  |  \
3939 $    /   |   \
3940 $   2  0 | 1  5
3941 $    \   |   /
3942 $     \  |  /
3943 $      \ | /
3944 $        3
3945 
3946   Vertices are implicitly numbered consecutively 0,...,NVertices.
3947   Each rank owns a chunk of numVertices consecutive vertices.
3948   If numVertices is PETSC_DECIDE, PETSc will distribute them as evenly as possible using PetscLayout.
3949   If NVertices is PETSC_DETERMINE and numVertices is PETSC_DECIDE, NVertices is computed by PETSc as the maximum vertex index in cells + 1.
3950   If only NVertices is PETSC_DETERMINE, it is computed as the sum of numVertices over all ranks.
3951 
3952   The cell distribution is arbitrary non-overlapping, independent of the vertex distribution.
3953 
3954   Not currently supported in Fortran.
3955 
3956   Level: advanced
3957 
3958 .seealso: DMPlexBuildFromCellList(), DMPlexCreateFromCellListParallelPetsc(), DMPlexBuildCoordinatesFromCellListParallel()
3959 @*/
3960 PetscErrorCode DMPlexBuildFromCellListParallel(DM dm, PetscInt numCells, PetscInt numVertices, PetscInt NVertices, PetscInt numCorners, const PetscInt cells[], PetscSF *vertexSF, PetscInt **verticesAdjSaved)
3961 {
3962   PetscSF         sfPoint;
3963   PetscLayout     layout;
3964   PetscInt        numVerticesAdj, *verticesAdj, *cones, c, p;
3965   PetscErrorCode  ierr;
3966 
3967   PetscFunctionBegin;
3968   PetscValidLogicalCollectiveInt(dm,NVertices,4);
3969   ierr = PetscLogEventBegin(DMPLEX_BuildFromCellList,dm,0,0,0);CHKERRQ(ierr);
3970   /* Get/check global number of vertices */
3971   {
3972     PetscInt NVerticesInCells, i;
3973     const PetscInt len = numCells * numCorners;
3974 
3975     /* NVerticesInCells = max(cells) + 1 */
3976     NVerticesInCells = PETSC_MIN_INT;
3977     for (i=0; i<len; i++) if (cells[i] > NVerticesInCells) NVerticesInCells = cells[i];
3978     ++NVerticesInCells;
3979     ierr = MPI_Allreduce(MPI_IN_PLACE, &NVerticesInCells, 1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject) dm));CHKERRMPI(ierr);
3980 
3981     if (numVertices == PETSC_DECIDE && NVertices == PETSC_DECIDE) NVertices = NVerticesInCells;
3982     else PetscCheckFalse(NVertices != PETSC_DECIDE && NVertices < NVerticesInCells,PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Specified global number of vertices %D must be greater than or equal to the number of vertices in cells %D",NVertices,NVerticesInCells);
3983   }
3984   /* Count locally unique vertices */
3985   {
3986     PetscHSetI vhash;
3987     PetscInt off = 0;
3988 
3989     ierr = PetscHSetICreate(&vhash);CHKERRQ(ierr);
3990     for (c = 0; c < numCells; ++c) {
3991       for (p = 0; p < numCorners; ++p) {
3992         ierr = PetscHSetIAdd(vhash, cells[c*numCorners+p]);CHKERRQ(ierr);
3993       }
3994     }
3995     ierr = PetscHSetIGetSize(vhash, &numVerticesAdj);CHKERRQ(ierr);
3996     if (!verticesAdjSaved) { ierr = PetscMalloc1(numVerticesAdj, &verticesAdj);CHKERRQ(ierr); }
3997     else { verticesAdj = *verticesAdjSaved; }
3998     ierr = PetscHSetIGetElems(vhash, &off, verticesAdj);CHKERRQ(ierr);
3999     ierr = PetscHSetIDestroy(&vhash);CHKERRQ(ierr);
4000     PetscCheckFalse(off != numVerticesAdj,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid number of local vertices %D should be %D", off, numVerticesAdj);
4001   }
4002   ierr = PetscSortInt(numVerticesAdj, verticesAdj);CHKERRQ(ierr);
4003   /* Create cones */
4004   ierr = DMPlexSetChart(dm, 0, numCells+numVerticesAdj);CHKERRQ(ierr);
4005   for (c = 0; c < numCells; ++c) {ierr = DMPlexSetConeSize(dm, c, numCorners);CHKERRQ(ierr);}
4006   ierr = DMSetUp(dm);CHKERRQ(ierr);
4007   ierr = DMPlexGetCones(dm,&cones);CHKERRQ(ierr);
4008   for (c = 0; c < numCells; ++c) {
4009     for (p = 0; p < numCorners; ++p) {
4010       const PetscInt gv = cells[c*numCorners+p];
4011       PetscInt       lv;
4012 
4013       /* Positions within verticesAdj form 0-based local vertex numbering;
4014          we need to shift it by numCells to get correct DAG points (cells go first) */
4015       ierr = PetscFindInt(gv, numVerticesAdj, verticesAdj, &lv);CHKERRQ(ierr);
4016       PetscCheckFalse(lv < 0,PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Could not find global vertex %D in local connectivity", gv);
4017       cones[c*numCorners+p] = lv+numCells;
4018     }
4019   }
4020   /* Build point sf */
4021   ierr = PetscLayoutCreate(PetscObjectComm((PetscObject)dm), &layout);CHKERRQ(ierr);
4022   ierr = PetscLayoutSetSize(layout, NVertices);CHKERRQ(ierr);
4023   ierr = PetscLayoutSetLocalSize(layout, numVertices);CHKERRQ(ierr);
4024   ierr = PetscLayoutSetBlockSize(layout, 1);CHKERRQ(ierr);
4025   ierr = PetscSFCreateByMatchingIndices(layout, numVerticesAdj, verticesAdj, NULL, numCells, numVerticesAdj, verticesAdj, NULL, numCells, vertexSF, &sfPoint);CHKERRQ(ierr);
4026   ierr = PetscLayoutDestroy(&layout);CHKERRQ(ierr);
4027   if (!verticesAdjSaved) { ierr = PetscFree(verticesAdj);CHKERRQ(ierr); }
4028   ierr = PetscObjectSetName((PetscObject) sfPoint, "point SF");CHKERRQ(ierr);
4029   if (dm->sf) {
4030     const char *prefix;
4031 
4032     ierr = PetscObjectGetOptionsPrefix((PetscObject)dm->sf, &prefix);CHKERRQ(ierr);
4033     ierr = PetscObjectSetOptionsPrefix((PetscObject)sfPoint, prefix);CHKERRQ(ierr);
4034   }
4035   ierr = DMSetPointSF(dm, sfPoint);CHKERRQ(ierr);
4036   ierr = PetscSFDestroy(&sfPoint);CHKERRQ(ierr);
4037   if (vertexSF) {ierr = PetscObjectSetName((PetscObject)(*vertexSF), "Vertex Ownership SF");CHKERRQ(ierr);}
4038   /* Fill in the rest of the topology structure */
4039   ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
4040   ierr = DMPlexStratify(dm);CHKERRQ(ierr);
4041   ierr = PetscLogEventEnd(DMPLEX_BuildFromCellList,dm,0,0,0);CHKERRQ(ierr);
4042   PetscFunctionReturn(0);
4043 }
4044 
4045 /*@C
4046   DMPlexBuildCoordinatesFromCellListParallel - Build DM coordinates from a list of coordinates for each owned vertex (common mesh generator output)
4047 
4048   Input Parameters:
4049 + dm - The DM
4050 . spaceDim - The spatial dimension used for coordinates
4051 . sfVert - SF describing complete vertex ownership
4052 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
4053 
4054   Level: advanced
4055 
4056   Notes:
4057   Not currently supported in Fortran.
4058 
4059 .seealso: DMPlexBuildCoordinatesFromCellList(), DMPlexCreateFromCellListParallelPetsc(), DMPlexBuildFromCellListParallel()
4060 @*/
4061 PetscErrorCode DMPlexBuildCoordinatesFromCellListParallel(DM dm, PetscInt spaceDim, PetscSF sfVert, const PetscReal vertexCoords[])
4062 {
4063   PetscSection   coordSection;
4064   Vec            coordinates;
4065   PetscScalar   *coords;
4066   PetscInt       numVertices, numVerticesAdj, coordSize, v, vStart, vEnd;
4067   PetscErrorCode ierr;
4068 
4069   PetscFunctionBegin;
4070   ierr = PetscLogEventBegin(DMPLEX_BuildCoordinatesFromCellList,dm,0,0,0);CHKERRQ(ierr);
4071   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
4072   PetscCheckFalse(vStart < 0 || vEnd < 0,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM is not set up properly. DMPlexBuildFromCellList() should be called first.");
4073   ierr = DMSetCoordinateDim(dm, spaceDim);CHKERRQ(ierr);
4074   ierr = PetscSFGetGraph(sfVert, &numVertices, &numVerticesAdj, NULL, NULL);CHKERRQ(ierr);
4075   PetscCheckFalse(vEnd - vStart != numVerticesAdj,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Supplied sfVert has wrong number of leaves = %D != %D = vEnd - vStart",numVerticesAdj,vEnd - vStart);
4076   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
4077   ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
4078   ierr = PetscSectionSetFieldComponents(coordSection, 0, spaceDim);CHKERRQ(ierr);
4079   ierr = PetscSectionSetChart(coordSection, vStart, vEnd);CHKERRQ(ierr);
4080   for (v = vStart; v < vEnd; ++v) {
4081     ierr = PetscSectionSetDof(coordSection, v, spaceDim);CHKERRQ(ierr);
4082     ierr = PetscSectionSetFieldDof(coordSection, v, 0, spaceDim);CHKERRQ(ierr);
4083   }
4084   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
4085   ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
4086   ierr = VecCreate(PetscObjectComm((PetscObject)dm), &coordinates);CHKERRQ(ierr);
4087   ierr = VecSetBlockSize(coordinates, spaceDim);CHKERRQ(ierr);
4088   ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
4089   ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
4090   ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
4091   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
4092   {
4093     MPI_Datatype coordtype;
4094 
4095     /* Need a temp buffer for coords if we have complex/single */
4096     ierr = MPI_Type_contiguous(spaceDim, MPIU_SCALAR, &coordtype);CHKERRMPI(ierr);
4097     ierr = MPI_Type_commit(&coordtype);CHKERRMPI(ierr);
4098 #if defined(PETSC_USE_COMPLEX)
4099     {
4100     PetscScalar *svertexCoords;
4101     PetscInt    i;
4102     ierr = PetscMalloc1(numVertices*spaceDim,&svertexCoords);CHKERRQ(ierr);
4103     for (i=0; i<numVertices*spaceDim; i++) svertexCoords[i] = vertexCoords[i];
4104     ierr = PetscSFBcastBegin(sfVert, coordtype, svertexCoords, coords,MPI_REPLACE);CHKERRQ(ierr);
4105     ierr = PetscSFBcastEnd(sfVert, coordtype, svertexCoords, coords,MPI_REPLACE);CHKERRQ(ierr);
4106     ierr = PetscFree(svertexCoords);CHKERRQ(ierr);
4107     }
4108 #else
4109     ierr = PetscSFBcastBegin(sfVert, coordtype, vertexCoords, coords,MPI_REPLACE);CHKERRQ(ierr);
4110     ierr = PetscSFBcastEnd(sfVert, coordtype, vertexCoords, coords,MPI_REPLACE);CHKERRQ(ierr);
4111 #endif
4112     ierr = MPI_Type_free(&coordtype);CHKERRMPI(ierr);
4113   }
4114   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
4115   ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
4116   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
4117   ierr = PetscLogEventEnd(DMPLEX_BuildCoordinatesFromCellList,dm,0,0,0);CHKERRQ(ierr);
4118   PetscFunctionReturn(0);
4119 }
4120 
4121 /*@
4122   DMPlexCreateFromCellListParallelPetsc - Create distributed DMPLEX from a list of vertices for each cell (common mesh generator output)
4123 
4124   Input Parameters:
4125 + comm - The communicator
4126 . dim - The topological dimension of the mesh
4127 . numCells - The number of cells owned by this process
4128 . numVertices - The number of vertices owned by this process, or PETSC_DECIDE
4129 . NVertices - The global number of vertices, or PETSC_DECIDE
4130 . numCorners - The number of vertices for each cell
4131 . interpolate - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
4132 . cells - An array of numCells*numCorners numbers, the global vertex numbers for each cell
4133 . spaceDim - The spatial dimension used for coordinates
4134 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
4135 
4136   Output Parameters:
4137 + dm - The DM
4138 . vertexSF - (Optional) SF describing complete vertex ownership
4139 - verticesAdjSaved - (Optional) vertex adjacency array
4140 
4141   Notes:
4142   This function is just a convenient sequence of DMCreate(), DMSetType(), DMSetDimension(),
4143   DMPlexBuildFromCellListParallel(), DMPlexInterpolate(), DMPlexBuildCoordinatesFromCellListParallel()
4144 
4145   See DMPlexBuildFromCellListParallel() for an example and details about the topology-related parameters.
4146   See DMPlexBuildCoordinatesFromCellListParallel() for details about the geometry-related parameters.
4147 
4148   Level: intermediate
4149 
4150 .seealso: DMPlexCreateFromCellListPetsc(), DMPlexBuildFromCellListParallel(), DMPlexBuildCoordinatesFromCellListParallel(), DMPlexCreateFromDAG(), DMPlexCreate()
4151 @*/
4152 PetscErrorCode DMPlexCreateFromCellListParallelPetsc(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt NVertices, PetscInt numCorners, PetscBool interpolate, const PetscInt cells[], PetscInt spaceDim, const PetscReal vertexCoords[], PetscSF *vertexSF, PetscInt **verticesAdj, DM *dm)
4153 {
4154   PetscSF        sfVert;
4155   PetscErrorCode ierr;
4156 
4157   PetscFunctionBegin;
4158   ierr = DMCreate(comm, dm);CHKERRQ(ierr);
4159   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
4160   PetscValidLogicalCollectiveInt(*dm, dim, 2);
4161   PetscValidLogicalCollectiveInt(*dm, spaceDim, 9);
4162   ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
4163   ierr = DMPlexBuildFromCellListParallel(*dm, numCells, numVertices, NVertices, numCorners, cells, &sfVert, verticesAdj);CHKERRQ(ierr);
4164   if (interpolate) {
4165     DM idm;
4166 
4167     ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
4168     ierr = DMDestroy(dm);CHKERRQ(ierr);
4169     *dm  = idm;
4170   }
4171   ierr = DMPlexBuildCoordinatesFromCellListParallel(*dm, spaceDim, sfVert, vertexCoords);CHKERRQ(ierr);
4172   if (vertexSF) *vertexSF = sfVert;
4173   else {ierr = PetscSFDestroy(&sfVert);CHKERRQ(ierr);}
4174   PetscFunctionReturn(0);
4175 }
4176 
4177 /*@C
4178   DMPlexBuildFromCellList - Build DMPLEX topology from a list of vertices for each cell (common mesh generator output)
4179 
4180   Input Parameters:
4181 + dm - The DM
4182 . numCells - The number of cells owned by this process
4183 . numVertices - The number of vertices owned by this process, or PETSC_DETERMINE
4184 . numCorners - The number of vertices for each cell
4185 - cells - An array of numCells*numCorners numbers, the global vertex numbers for each cell
4186 
4187   Level: advanced
4188 
4189   Notes:
4190   Two triangles sharing a face
4191 $
4192 $        2
4193 $      / | \
4194 $     /  |  \
4195 $    /   |   \
4196 $   0  0 | 1  3
4197 $    \   |   /
4198 $     \  |  /
4199 $      \ | /
4200 $        1
4201 would have input
4202 $  numCells = 2, numVertices = 4
4203 $  cells = [0 1 2  1 3 2]
4204 $
4205 which would result in the DMPlex
4206 $
4207 $        4
4208 $      / | \
4209 $     /  |  \
4210 $    /   |   \
4211 $   2  0 | 1  5
4212 $    \   |   /
4213 $     \  |  /
4214 $      \ | /
4215 $        3
4216 
4217   If numVertices is PETSC_DETERMINE, it is computed by PETSc as the maximum vertex index in cells + 1.
4218 
4219   Not currently supported in Fortran.
4220 
4221 .seealso: DMPlexBuildFromCellListParallel(), DMPlexBuildCoordinatesFromCellList(), DMPlexCreateFromCellListPetsc()
4222 @*/
4223 PetscErrorCode DMPlexBuildFromCellList(DM dm, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, const PetscInt cells[])
4224 {
4225   PetscInt      *cones, c, p, dim;
4226   PetscErrorCode ierr;
4227 
4228   PetscFunctionBegin;
4229   ierr = PetscLogEventBegin(DMPLEX_BuildFromCellList,dm,0,0,0);CHKERRQ(ierr);
4230   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
4231   /* Get/check global number of vertices */
4232   {
4233     PetscInt NVerticesInCells, i;
4234     const PetscInt len = numCells * numCorners;
4235 
4236     /* NVerticesInCells = max(cells) + 1 */
4237     NVerticesInCells = PETSC_MIN_INT;
4238     for (i=0; i<len; i++) if (cells[i] > NVerticesInCells) NVerticesInCells = cells[i];
4239     ++NVerticesInCells;
4240 
4241     if (numVertices == PETSC_DECIDE) numVertices = NVerticesInCells;
4242     else PetscCheckFalse(numVertices < NVerticesInCells,PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Specified number of vertices %D must be greater than or equal to the number of vertices in cells %D",numVertices,NVerticesInCells);
4243   }
4244   ierr = DMPlexSetChart(dm, 0, numCells+numVertices);CHKERRQ(ierr);
4245   for (c = 0; c < numCells; ++c) {
4246     ierr = DMPlexSetConeSize(dm, c, numCorners);CHKERRQ(ierr);
4247   }
4248   ierr = DMSetUp(dm);CHKERRQ(ierr);
4249   ierr = DMPlexGetCones(dm,&cones);CHKERRQ(ierr);
4250   for (c = 0; c < numCells; ++c) {
4251     for (p = 0; p < numCorners; ++p) {
4252       cones[c*numCorners+p] = cells[c*numCorners+p]+numCells;
4253     }
4254   }
4255   ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
4256   ierr = DMPlexStratify(dm);CHKERRQ(ierr);
4257   ierr = PetscLogEventEnd(DMPLEX_BuildFromCellList,dm,0,0,0);CHKERRQ(ierr);
4258   PetscFunctionReturn(0);
4259 }
4260 
4261 /*@C
4262   DMPlexBuildCoordinatesFromCellList - Build DM coordinates from a list of coordinates for each owned vertex (common mesh generator output)
4263 
4264   Input Parameters:
4265 + dm - The DM
4266 . spaceDim - The spatial dimension used for coordinates
4267 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
4268 
4269   Level: advanced
4270 
4271   Notes:
4272   Not currently supported in Fortran.
4273 
4274 .seealso: DMPlexBuildCoordinatesFromCellListParallel(), DMPlexCreateFromCellListPetsc(), DMPlexBuildFromCellList()
4275 @*/
4276 PetscErrorCode DMPlexBuildCoordinatesFromCellList(DM dm, PetscInt spaceDim, const PetscReal vertexCoords[])
4277 {
4278   PetscSection   coordSection;
4279   Vec            coordinates;
4280   DM             cdm;
4281   PetscScalar   *coords;
4282   PetscInt       v, vStart, vEnd, d;
4283   PetscErrorCode ierr;
4284 
4285   PetscFunctionBegin;
4286   ierr = PetscLogEventBegin(DMPLEX_BuildCoordinatesFromCellList,dm,0,0,0);CHKERRQ(ierr);
4287   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
4288   PetscCheckFalse(vStart < 0 || vEnd < 0,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM is not set up properly. DMPlexBuildFromCellList() should be called first.");
4289   ierr = DMSetCoordinateDim(dm, spaceDim);CHKERRQ(ierr);
4290   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
4291   ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
4292   ierr = PetscSectionSetFieldComponents(coordSection, 0, spaceDim);CHKERRQ(ierr);
4293   ierr = PetscSectionSetChart(coordSection, vStart, vEnd);CHKERRQ(ierr);
4294   for (v = vStart; v < vEnd; ++v) {
4295     ierr = PetscSectionSetDof(coordSection, v, spaceDim);CHKERRQ(ierr);
4296     ierr = PetscSectionSetFieldDof(coordSection, v, 0, spaceDim);CHKERRQ(ierr);
4297   }
4298   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
4299 
4300   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
4301   ierr = DMCreateLocalVector(cdm, &coordinates);CHKERRQ(ierr);
4302   ierr = VecSetBlockSize(coordinates, spaceDim);CHKERRQ(ierr);
4303   ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
4304   ierr = VecGetArrayWrite(coordinates, &coords);CHKERRQ(ierr);
4305   for (v = 0; v < vEnd-vStart; ++v) {
4306     for (d = 0; d < spaceDim; ++d) {
4307       coords[v*spaceDim+d] = vertexCoords[v*spaceDim+d];
4308     }
4309   }
4310   ierr = VecRestoreArrayWrite(coordinates, &coords);CHKERRQ(ierr);
4311   ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
4312   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
4313   ierr = PetscLogEventEnd(DMPLEX_BuildCoordinatesFromCellList,dm,0,0,0);CHKERRQ(ierr);
4314   PetscFunctionReturn(0);
4315 }
4316 
4317 /*@
4318   DMPlexCreateFromCellListPetsc - Create DMPLEX from a list of vertices for each cell (common mesh generator output), but only process 0 takes in the input
4319 
4320   Collective on comm
4321 
4322   Input Parameters:
4323 + comm - The communicator
4324 . dim - The topological dimension of the mesh
4325 . numCells - The number of cells, only on process 0
4326 . numVertices - The number of vertices owned by this process, or PETSC_DECIDE, only on process 0
4327 . numCorners - The number of vertices for each cell, only on process 0
4328 . interpolate - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
4329 . cells - An array of numCells*numCorners numbers, the vertices for each cell, only on process 0
4330 . spaceDim - The spatial dimension used for coordinates
4331 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex, only on process 0
4332 
4333   Output Parameter:
4334 . dm - The DM, which only has points on process 0
4335 
4336   Notes:
4337   This function is just a convenient sequence of DMCreate(), DMSetType(), DMSetDimension(), DMPlexBuildFromCellList(),
4338   DMPlexInterpolate(), DMPlexBuildCoordinatesFromCellList()
4339 
4340   See DMPlexBuildFromCellList() for an example and details about the topology-related parameters.
4341   See DMPlexBuildCoordinatesFromCellList() for details about the geometry-related parameters.
4342   See DMPlexCreateFromCellListParallelPetsc() for parallel input
4343 
4344   Level: intermediate
4345 
4346 .seealso: DMPlexCreateFromCellListParallelPetsc(), DMPlexBuildFromCellList(), DMPlexBuildCoordinatesFromCellList(), DMPlexCreateFromDAG(), DMPlexCreate()
4347 @*/
4348 PetscErrorCode DMPlexCreateFromCellListPetsc(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, PetscBool interpolate, const PetscInt cells[], PetscInt spaceDim, const PetscReal vertexCoords[], DM *dm)
4349 {
4350   PetscMPIInt    rank;
4351   PetscErrorCode ierr;
4352 
4353   PetscFunctionBegin;
4354   PetscCheckFalse(!dim,comm, PETSC_ERR_ARG_OUTOFRANGE, "This is not appropriate for 0-dimensional meshes. Consider either creating the DM using DMPlexCreateFromDAG(), by hand, or using DMSwarm.");
4355   ierr = MPI_Comm_rank(comm, &rank);CHKERRMPI(ierr);
4356   ierr = DMCreate(comm, dm);CHKERRQ(ierr);
4357   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
4358   ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
4359   if (!rank) {ierr = DMPlexBuildFromCellList(*dm, numCells, numVertices, numCorners, cells);CHKERRQ(ierr);}
4360   else       {ierr = DMPlexBuildFromCellList(*dm, 0, 0, 0, NULL);CHKERRQ(ierr);}
4361   if (interpolate) {
4362     DM idm;
4363 
4364     ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
4365     ierr = DMDestroy(dm);CHKERRQ(ierr);
4366     *dm  = idm;
4367   }
4368   if (!rank) {ierr = DMPlexBuildCoordinatesFromCellList(*dm, spaceDim, vertexCoords);CHKERRQ(ierr);}
4369   else       {ierr = DMPlexBuildCoordinatesFromCellList(*dm, spaceDim, NULL);CHKERRQ(ierr);}
4370   PetscFunctionReturn(0);
4371 }
4372 
4373 /*@
4374   DMPlexCreateFromDAG - This takes as input the adjacency-list representation of the Directed Acyclic Graph (Hasse Diagram) encoding a mesh, and produces a DM
4375 
4376   Input Parameters:
4377 + dm - The empty DM object, usually from DMCreate() and DMSetDimension()
4378 . depth - The depth of the DAG
4379 . numPoints - Array of size depth + 1 containing the number of points at each depth
4380 . coneSize - The cone size of each point
4381 . cones - The concatenation of the cone points for each point, the cone list must be oriented correctly for each point
4382 . coneOrientations - The orientation of each cone point
4383 - vertexCoords - An array of numPoints[0]*spacedim numbers representing the coordinates of each vertex, with spacedim the value set via DMSetCoordinateDim()
4384 
4385   Output Parameter:
4386 . dm - The DM
4387 
4388   Note: Two triangles sharing a face would have input
4389 $  depth = 1, numPoints = [4 2], coneSize = [3 3 0 0 0 0]
4390 $  cones = [2 3 4  3 5 4], coneOrientations = [0 0 0  0 0 0]
4391 $ vertexCoords = [-1.0 0.0  0.0 -1.0  0.0 1.0  1.0 0.0]
4392 $
4393 which would result in the DMPlex
4394 $
4395 $        4
4396 $      / | \
4397 $     /  |  \
4398 $    /   |   \
4399 $   2  0 | 1  5
4400 $    \   |   /
4401 $     \  |  /
4402 $      \ | /
4403 $        3
4404 $
4405 $ Notice that all points are numbered consecutively, unlike DMPlexCreateFromCellListPetsc()
4406 
4407   Level: advanced
4408 
4409 .seealso: DMPlexCreateFromCellListPetsc(), DMPlexCreate()
4410 @*/
4411 PetscErrorCode DMPlexCreateFromDAG(DM dm, PetscInt depth, const PetscInt numPoints[], const PetscInt coneSize[], const PetscInt cones[], const PetscInt coneOrientations[], const PetscScalar vertexCoords[])
4412 {
4413   Vec            coordinates;
4414   PetscSection   coordSection;
4415   PetscScalar    *coords;
4416   PetscInt       coordSize, firstVertex = -1, pStart = 0, pEnd = 0, p, v, dim, dimEmbed, d, off;
4417   PetscErrorCode ierr;
4418 
4419   PetscFunctionBegin;
4420   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
4421   ierr = DMGetCoordinateDim(dm, &dimEmbed);CHKERRQ(ierr);
4422   PetscCheckFalse(dimEmbed < dim,PETSC_COMM_SELF,PETSC_ERR_PLIB,"Embedding dimension %D cannot be less than intrinsic dimension %d",dimEmbed,dim);
4423   for (d = 0; d <= depth; ++d) pEnd += numPoints[d];
4424   ierr = DMPlexSetChart(dm, pStart, pEnd);CHKERRQ(ierr);
4425   for (p = pStart; p < pEnd; ++p) {
4426     ierr = DMPlexSetConeSize(dm, p, coneSize[p-pStart]);CHKERRQ(ierr);
4427     if (firstVertex < 0 && !coneSize[p - pStart]) {
4428       firstVertex = p - pStart;
4429     }
4430   }
4431   PetscCheckFalse(firstVertex < 0 && numPoints[0],PETSC_COMM_SELF,PETSC_ERR_ARG_WRONG,"Expected %D vertices but could not find any", numPoints[0]);
4432   ierr = DMSetUp(dm);CHKERRQ(ierr); /* Allocate space for cones */
4433   for (p = pStart, off = 0; p < pEnd; off += coneSize[p-pStart], ++p) {
4434     ierr = DMPlexSetCone(dm, p, &cones[off]);CHKERRQ(ierr);
4435     ierr = DMPlexSetConeOrientation(dm, p, &coneOrientations[off]);CHKERRQ(ierr);
4436   }
4437   ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
4438   ierr = DMPlexStratify(dm);CHKERRQ(ierr);
4439   /* Build coordinates */
4440   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
4441   ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
4442   ierr = PetscSectionSetFieldComponents(coordSection, 0, dimEmbed);CHKERRQ(ierr);
4443   ierr = PetscSectionSetChart(coordSection, firstVertex, firstVertex+numPoints[0]);CHKERRQ(ierr);
4444   for (v = firstVertex; v < firstVertex+numPoints[0]; ++v) {
4445     ierr = PetscSectionSetDof(coordSection, v, dimEmbed);CHKERRQ(ierr);
4446     ierr = PetscSectionSetFieldDof(coordSection, v, 0, dimEmbed);CHKERRQ(ierr);
4447   }
4448   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
4449   ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
4450   ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
4451   ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
4452   ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
4453   ierr = VecSetBlockSize(coordinates, dimEmbed);CHKERRQ(ierr);
4454   ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
4455   if (vertexCoords) {
4456     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
4457     for (v = 0; v < numPoints[0]; ++v) {
4458       PetscInt off;
4459 
4460       ierr = PetscSectionGetOffset(coordSection, v+firstVertex, &off);CHKERRQ(ierr);
4461       for (d = 0; d < dimEmbed; ++d) {
4462         coords[off+d] = vertexCoords[v*dimEmbed+d];
4463       }
4464     }
4465   }
4466   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
4467   ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
4468   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
4469   PetscFunctionReturn(0);
4470 }
4471 
4472 /*@C
4473   DMPlexCreateCellVertexFromFile - Create a DMPlex mesh from a simple cell-vertex file.
4474 
4475 + comm        - The MPI communicator
4476 . filename    - Name of the .dat file
4477 - interpolate - Create faces and edges in the mesh
4478 
4479   Output Parameter:
4480 . dm  - The DM object representing the mesh
4481 
4482   Note: The format is the simplest possible:
4483 $ Ne
4484 $ v0 v1 ... vk
4485 $ Nv
4486 $ x y z marker
4487 
4488   Level: beginner
4489 
4490 .seealso: DMPlexCreateFromFile(), DMPlexCreateMedFromFile(), DMPlexCreateGmsh(), DMPlexCreate()
4491 @*/
4492 PetscErrorCode DMPlexCreateCellVertexFromFile(MPI_Comm comm, const char filename[], PetscBool interpolate, DM *dm)
4493 {
4494   DMLabel         marker;
4495   PetscViewer     viewer;
4496   Vec             coordinates;
4497   PetscSection    coordSection;
4498   PetscScalar    *coords;
4499   char            line[PETSC_MAX_PATH_LEN];
4500   PetscInt        dim = 3, cdim = 3, coordSize, v, c, d;
4501   PetscMPIInt     rank;
4502   int             snum, Nv, Nc, Ncn, Nl;
4503   PetscErrorCode  ierr;
4504 
4505   PetscFunctionBegin;
4506   ierr = MPI_Comm_rank(comm, &rank);CHKERRMPI(ierr);
4507   ierr = PetscViewerCreate(comm, &viewer);CHKERRQ(ierr);
4508   ierr = PetscViewerSetType(viewer, PETSCVIEWERASCII);CHKERRQ(ierr);
4509   ierr = PetscViewerFileSetMode(viewer, FILE_MODE_READ);CHKERRQ(ierr);
4510   ierr = PetscViewerFileSetName(viewer, filename);CHKERRQ(ierr);
4511   if (rank == 0) {
4512     ierr = PetscViewerRead(viewer, line, 4, NULL, PETSC_STRING);CHKERRQ(ierr);
4513     snum = sscanf(line, "%d %d %d %d", &Nc, &Nv, &Ncn, &Nl);
4514     PetscCheckFalse(snum != 4,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
4515   } else {
4516     Nc = Nv = Ncn = Nl = 0;
4517   }
4518   ierr = DMCreate(comm, dm);CHKERRQ(ierr);
4519   ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
4520   ierr = DMPlexSetChart(*dm, 0, Nc+Nv);CHKERRQ(ierr);
4521   ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
4522   ierr = DMSetCoordinateDim(*dm, cdim);CHKERRQ(ierr);
4523   /* Read topology */
4524   if (rank == 0) {
4525     char     format[PETSC_MAX_PATH_LEN];
4526     PetscInt cone[8];
4527     int      vbuf[8], v;
4528 
4529     for (c = 0; c < Ncn; ++c) {format[c*3+0] = '%'; format[c*3+1] = 'd'; format[c*3+2] = ' ';}
4530     format[Ncn*3-1] = '\0';
4531     for (c = 0; c < Nc; ++c) {ierr = DMPlexSetConeSize(*dm, c, Ncn);CHKERRQ(ierr);}
4532     ierr = DMSetUp(*dm);CHKERRQ(ierr);
4533     for (c = 0; c < Nc; ++c) {
4534       ierr = PetscViewerRead(viewer, line, Ncn, NULL, PETSC_STRING);CHKERRQ(ierr);
4535       switch (Ncn) {
4536         case 2: snum = sscanf(line, format, &vbuf[0], &vbuf[1]);break;
4537         case 3: snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2]);break;
4538         case 4: snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3]);break;
4539         case 6: snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3], &vbuf[4], &vbuf[5]);break;
4540         case 8: snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3], &vbuf[4], &vbuf[5], &vbuf[6], &vbuf[7]);break;
4541         default: SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "No cell shape with %D vertices", Ncn);
4542       }
4543       PetscCheckFalse(snum != Ncn,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
4544       for (v = 0; v < Ncn; ++v) cone[v] = vbuf[v] + Nc;
4545       /* Hexahedra are inverted */
4546       if (Ncn == 8) {
4547         PetscInt tmp = cone[1];
4548         cone[1] = cone[3];
4549         cone[3] = tmp;
4550       }
4551       ierr = DMPlexSetCone(*dm, c, cone);CHKERRQ(ierr);
4552     }
4553   }
4554   ierr = DMPlexSymmetrize(*dm);CHKERRQ(ierr);
4555   ierr = DMPlexStratify(*dm);CHKERRQ(ierr);
4556   /* Read coordinates */
4557   ierr = DMGetCoordinateSection(*dm, &coordSection);CHKERRQ(ierr);
4558   ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
4559   ierr = PetscSectionSetFieldComponents(coordSection, 0, cdim);CHKERRQ(ierr);
4560   ierr = PetscSectionSetChart(coordSection, Nc, Nc + Nv);CHKERRQ(ierr);
4561   for (v = Nc; v < Nc+Nv; ++v) {
4562     ierr = PetscSectionSetDof(coordSection, v, cdim);CHKERRQ(ierr);
4563     ierr = PetscSectionSetFieldDof(coordSection, v, 0, cdim);CHKERRQ(ierr);
4564   }
4565   ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
4566   ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
4567   ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
4568   ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
4569   ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
4570   ierr = VecSetBlockSize(coordinates, cdim);CHKERRQ(ierr);
4571   ierr = VecSetType(coordinates, VECSTANDARD);CHKERRQ(ierr);
4572   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
4573   if (rank == 0) {
4574     char   format[PETSC_MAX_PATH_LEN];
4575     double x[3];
4576     int    l, val[3];
4577 
4578     if (Nl) {
4579       for (l = 0; l < Nl; ++l) {format[l*3+0] = '%'; format[l*3+1] = 'd'; format[l*3+2] = ' ';}
4580       format[Nl*3-1] = '\0';
4581       ierr = DMCreateLabel(*dm, "marker");CHKERRQ(ierr);
4582       ierr = DMGetLabel(*dm, "marker", &marker);CHKERRQ(ierr);
4583     }
4584     for (v = 0; v < Nv; ++v) {
4585       ierr = PetscViewerRead(viewer, line, 3+Nl, NULL, PETSC_STRING);CHKERRQ(ierr);
4586       snum = sscanf(line, "%lg %lg %lg", &x[0], &x[1], &x[2]);
4587       PetscCheckFalse(snum != 3,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
4588       switch (Nl) {
4589         case 0: snum = 0;break;
4590         case 1: snum = sscanf(line, format, &val[0]);break;
4591         case 2: snum = sscanf(line, format, &val[0], &val[1]);break;
4592         case 3: snum = sscanf(line, format, &val[0], &val[1], &val[2]);break;
4593         default: SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Request support for %D labels", Nl);
4594       }
4595       PetscCheckFalse(snum != Nl,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
4596       for (d = 0; d < cdim; ++d) coords[v*cdim+d] = x[d];
4597       for (l = 0; l < Nl; ++l) {ierr = DMLabelSetValue(marker, v+Nc, val[l]);CHKERRQ(ierr);}
4598     }
4599   }
4600   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
4601   ierr = DMSetCoordinatesLocal(*dm, coordinates);CHKERRQ(ierr);
4602   ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
4603   ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
4604   if (interpolate) {
4605     DM      idm;
4606     DMLabel bdlabel;
4607 
4608     ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
4609     ierr = DMDestroy(dm);CHKERRQ(ierr);
4610     *dm  = idm;
4611 
4612     if (!Nl) {
4613       ierr = DMCreateLabel(*dm, "marker");CHKERRQ(ierr);
4614       ierr = DMGetLabel(*dm, "marker", &bdlabel);CHKERRQ(ierr);
4615       ierr = DMPlexMarkBoundaryFaces(*dm, PETSC_DETERMINE, bdlabel);CHKERRQ(ierr);
4616       ierr = DMPlexLabelComplete(*dm, bdlabel);CHKERRQ(ierr);
4617     }
4618   }
4619   PetscFunctionReturn(0);
4620 }
4621 
4622 /*@C
4623   DMPlexCreateFromFile - This takes a filename and produces a DM
4624 
4625   Input Parameters:
4626 + comm - The communicator
4627 . filename - A file name
4628 . plexname - The object name of the resulting DM, also used for intra-datafile lookup by some formats
4629 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
4630 
4631   Output Parameter:
4632 . dm - The DM
4633 
4634   Options Database Keys:
4635 . -dm_plex_create_from_hdf5_xdmf - use the PETSC_VIEWER_HDF5_XDMF format for reading HDF5
4636 
4637   Use -dm_plex_create_ prefix to pass options to the internal PetscViewer, e.g.
4638 $ -dm_plex_create_viewer_hdf5_collective
4639 
4640   Notes:
4641   Using PETSCVIEWERHDF5 type with PETSC_VIEWER_HDF5_PETSC format, one can save multiple DMPlex
4642   meshes in a single HDF5 file. This in turn requires one to name the DMPlex object with PetscObjectSetName()
4643   before saving it with DMView() and before loading it with DMLoad() for identification of the mesh object.
4644   The input parameter name is thus used to name the DMPlex object when DMPlexCreateFromFile() internally
4645   calls DMLoad(). Currently, name is ignored for other viewer types and/or formats.
4646 
4647   Level: beginner
4648 
4649 .seealso: DMPlexCreateFromDAG(), DMPlexCreateFromCellListPetsc(), DMPlexCreate(), PetscObjectSetName(), DMView(), DMLoad()
4650 @*/
4651 PetscErrorCode DMPlexCreateFromFile(MPI_Comm comm, const char filename[], const char plexname[], PetscBool interpolate, DM *dm)
4652 {
4653   const char    *extGmsh      = ".msh";
4654   const char    *extGmsh2     = ".msh2";
4655   const char    *extGmsh4     = ".msh4";
4656   const char    *extCGNS      = ".cgns";
4657   const char    *extExodus    = ".exo";
4658   const char    *extExodus_e  = ".e";
4659   const char    *extGenesis   = ".gen";
4660   const char    *extFluent    = ".cas";
4661   const char    *extHDF5      = ".h5";
4662   const char    *extMed       = ".med";
4663   const char    *extPLY       = ".ply";
4664   const char    *extEGADSLite = ".egadslite";
4665   const char    *extEGADS     = ".egads";
4666   const char    *extIGES      = ".igs";
4667   const char    *extSTEP      = ".stp";
4668   const char    *extCV        = ".dat";
4669   size_t         len;
4670   PetscBool      isGmsh, isGmsh2, isGmsh4, isCGNS, isExodus, isGenesis, isFluent, isHDF5, isMed, isPLY, isEGADSLite, isEGADS, isIGES, isSTEP, isCV;
4671   PetscMPIInt    rank;
4672   PetscErrorCode ierr;
4673 
4674   PetscFunctionBegin;
4675   PetscValidCharPointer(filename, 2);
4676   if (plexname) PetscValidCharPointer(plexname, 3);
4677   PetscValidPointer(dm, 5);
4678   ierr = DMInitializePackage();CHKERRQ(ierr);
4679   ierr = PetscLogEventBegin(DMPLEX_CreateFromFile,0,0,0,0);CHKERRQ(ierr);
4680   ierr = MPI_Comm_rank(comm, &rank);CHKERRMPI(ierr);
4681   ierr = PetscStrlen(filename, &len);CHKERRQ(ierr);
4682   PetscCheckFalse(!len,comm, PETSC_ERR_ARG_WRONG, "Filename must be a valid path");
4683   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)],  extGmsh,      4, &isGmsh);CHKERRQ(ierr);
4684   ierr = PetscStrncmp(&filename[PetscMax(0,len-5)],  extGmsh2,     5, &isGmsh2);CHKERRQ(ierr);
4685   ierr = PetscStrncmp(&filename[PetscMax(0,len-5)],  extGmsh4,     5, &isGmsh4);CHKERRQ(ierr);
4686   ierr = PetscStrncmp(&filename[PetscMax(0,len-5)],  extCGNS,      5, &isCGNS);CHKERRQ(ierr);
4687   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)],  extExodus,    4, &isExodus);CHKERRQ(ierr);
4688   if (!isExodus) {
4689     ierr = PetscStrncmp(&filename[PetscMax(0,len-2)],  extExodus_e,    2, &isExodus);CHKERRQ(ierr);
4690   }
4691   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)],  extGenesis,   4, &isGenesis);CHKERRQ(ierr);
4692   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)],  extFluent,    4, &isFluent);CHKERRQ(ierr);
4693   ierr = PetscStrncmp(&filename[PetscMax(0,len-3)],  extHDF5,      3, &isHDF5);CHKERRQ(ierr);
4694   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)],  extMed,       4, &isMed);CHKERRQ(ierr);
4695   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)],  extPLY,       4, &isPLY);CHKERRQ(ierr);
4696   ierr = PetscStrncmp(&filename[PetscMax(0,len-10)], extEGADSLite, 10, &isEGADSLite);CHKERRQ(ierr);
4697   ierr = PetscStrncmp(&filename[PetscMax(0,len-6)],  extEGADS,     6, &isEGADS);CHKERRQ(ierr);
4698   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)],  extIGES,      4, &isIGES);CHKERRQ(ierr);
4699   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)],  extSTEP,      4, &isSTEP);CHKERRQ(ierr);
4700   ierr = PetscStrncmp(&filename[PetscMax(0,len-4)],  extCV,        4, &isCV);CHKERRQ(ierr);
4701   if (isGmsh || isGmsh2 || isGmsh4) {
4702     ierr = DMPlexCreateGmshFromFile(comm, filename, interpolate, dm);CHKERRQ(ierr);
4703   } else if (isCGNS) {
4704     ierr = DMPlexCreateCGNSFromFile(comm, filename, interpolate, dm);CHKERRQ(ierr);
4705   } else if (isExodus || isGenesis) {
4706     ierr = DMPlexCreateExodusFromFile(comm, filename, interpolate, dm);CHKERRQ(ierr);
4707   } else if (isFluent) {
4708     ierr = DMPlexCreateFluentFromFile(comm, filename, interpolate, dm);CHKERRQ(ierr);
4709   } else if (isHDF5) {
4710     PetscBool      load_hdf5_xdmf = PETSC_FALSE;
4711     PetscViewer viewer;
4712 
4713     /* PETSC_VIEWER_HDF5_XDMF is used if the filename ends with .xdmf.h5, or if -dm_plex_create_from_hdf5_xdmf option is present */
4714     ierr = PetscStrncmp(&filename[PetscMax(0,len-8)], ".xdmf",  5, &load_hdf5_xdmf);CHKERRQ(ierr);
4715     ierr = PetscOptionsGetBool(NULL, NULL, "-dm_plex_create_from_hdf5_xdmf", &load_hdf5_xdmf, NULL);CHKERRQ(ierr);
4716     ierr = PetscViewerCreate(comm, &viewer);CHKERRQ(ierr);
4717     ierr = PetscViewerSetType(viewer, PETSCVIEWERHDF5);CHKERRQ(ierr);
4718     ierr = PetscViewerSetOptionsPrefix(viewer, "dm_plex_create_");CHKERRQ(ierr);
4719     ierr = PetscViewerSetFromOptions(viewer);CHKERRQ(ierr);
4720     ierr = PetscViewerFileSetMode(viewer, FILE_MODE_READ);CHKERRQ(ierr);
4721     ierr = PetscViewerFileSetName(viewer, filename);CHKERRQ(ierr);
4722 
4723     ierr = DMCreate(comm, dm);CHKERRQ(ierr);
4724     ierr = PetscObjectSetName((PetscObject)(*dm), plexname);CHKERRQ(ierr);
4725     ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
4726     if (load_hdf5_xdmf) {ierr = PetscViewerPushFormat(viewer, PETSC_VIEWER_HDF5_XDMF);CHKERRQ(ierr);}
4727     ierr = DMLoad(*dm, viewer);CHKERRQ(ierr);
4728     if (load_hdf5_xdmf) {ierr = PetscViewerPopFormat(viewer);CHKERRQ(ierr);}
4729     ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
4730 
4731     if (interpolate) {
4732       DM idm;
4733 
4734       ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
4735       ierr = DMDestroy(dm);CHKERRQ(ierr);
4736       *dm  = idm;
4737     }
4738   } else if (isMed) {
4739     ierr = DMPlexCreateMedFromFile(comm, filename, interpolate, dm);CHKERRQ(ierr);
4740   } else if (isPLY) {
4741     ierr = DMPlexCreatePLYFromFile(comm, filename, interpolate, dm);CHKERRQ(ierr);
4742   } else if (isEGADSLite || isEGADS || isIGES || isSTEP) {
4743     if (isEGADSLite) {ierr = DMPlexCreateEGADSLiteFromFile(comm, filename, dm);CHKERRQ(ierr);}
4744     else             {ierr = DMPlexCreateEGADSFromFile(comm, filename, dm);CHKERRQ(ierr);}
4745     if (!interpolate) {
4746       DM udm;
4747 
4748       ierr = DMPlexUninterpolate(*dm, &udm);CHKERRQ(ierr);
4749       ierr = DMDestroy(dm);CHKERRQ(ierr);
4750       *dm  = udm;
4751     }
4752   } else if (isCV) {
4753     ierr = DMPlexCreateCellVertexFromFile(comm, filename, interpolate, dm);CHKERRQ(ierr);
4754   } else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cannot load file %s: unrecognized extension", filename);
4755   ierr = PetscStrlen(plexname, &len);CHKERRQ(ierr);
4756   if (len) {ierr = PetscObjectSetName((PetscObject)(*dm), plexname);CHKERRQ(ierr);}
4757   ierr = PetscLogEventEnd(DMPLEX_CreateFromFile,0,0,0,0);CHKERRQ(ierr);
4758   PetscFunctionReturn(0);
4759 }
4760